diff --git a/.gitignore b/.gitignore index 13ae2767ce2844c65a5d716a9763cecc36326501..74ec91aceac9bc0d9a07df1bd692854fa2f2b2d3 100644 --- a/.gitignore +++ b/.gitignore @@ -38,9 +38,6 @@ test/st/functions/pkg test/st/functions/**/*.so test/st/functions/**/*.zip test/st/functions/**/*.xml -functionsystem -datasystem -metrics #thirdparty thirdparty go/pkg/mod/ diff --git a/frontend/build/build.sh b/frontend/build/build.sh new file mode 100644 index 0000000000000000000000000000000000000000..880b801e98fcb9c48f96dc5fc2d4c662d89be9a9 --- /dev/null +++ b/frontend/build/build.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +BASE_DIR=$(cd "$(dirname "$0")"; pwd) +PROJECT_DIR=$(cd "$(dirname "$0")"/..; pwd) +OUTPUT_DIR="${BASE_DIR}/../output/yuanrong/pattern/pattern_faas" +TAR_OUT_DIR="${PROJECT_DIR}/build/_output" +TAR_OUT_FILE="faasfunctions.tar.gz" +EXECUTOR_DIR="${PROJECT_DIR}/build/faas/executor-meta" +TEST_CERT_PATH="${GOROOT}/src/net/http/internal/testcert.go" +BUILD_TAG_FUNCTION="function" +echo LD_LIBRARY_PATH=$LD_LIBRARY_PATH +MODULE_NAME_LIST=("faasfrontend") + +# go module prepare +export GO111MODULE=on +export GONOSUMDB=* +export CGO_ENABLED=1 + +# resolve missing go.sum entry +go env -w "GOFLAGS"="-mod=mod" + +# remove hard coded cert file in net/http +[ -f "${TEST_CERT_PATH}" ] && rm -f "${TEST_CERT_PATH}" + +# clean build history +bash "${BASE_DIR}"/clean.sh + +cd "${PROJECT_DIR}" +. "${BASE_DIR}"/compile_functions.sh + +# zip function file +FLAGS='-extldflags "-fPIC -fstack-protector-strong -Wl,-z,now,-z,relro,-z,noexecstack,-s -Wall -Werror"' +export CGO_CFLAGS="$CGO_CFLAGS -fstack-protector-strong -D_FORTIFY_SOURCE=2 -O2" +MODULE_NAME="faasfrontend" +cd "${TAR_OUT_DIR}" +mkdir -p "${MODULE_NAME}" +SO_PATH="${TAR_OUT_DIR}/${MODULE_NAME}/${MODULE_NAME}.so" +BIN_PATH="${TAR_OUT_DIR}/${MODULE_NAME}/${MODULE_NAME}" + +CC='gcc -fstack-protector-strong -D_FORTIFY_SOURCE=2 -O2' +go build -tags "module" -buildmode=pie -ldflags "${FLAGS}" \ +-o ${BIN_PATH} "${PROJECT_DIR}/cmd/${MODULE_NAME}/module_main.go" + +CC='gcc -fstack-protector-strong -D_FORTIFY_SOURCE=2 -O2' +go build -tags "${BUILD_TAG_FUNCTION}" -buildmode=plugin -ldflags "${FLAGS}" \ +-o ${SO_PATH} "${PROJECT_DIR}/cmd/${MODULE_NAME}/function_main.go" + +chmod -R 500 ${SO_PATH} +cd "${MODULE_NAME}" +zip -r "${MODULE_NAME}.zip" * + +cp "${PROJECT_DIR}/build/function_meta.json" "${TAR_OUT_DIR}/${MODULE_NAME}/${MODULE_NAME}_meta.json" + +sed -i "s/moduleName/${MODULE_NAME}/g" "${TAR_OUT_DIR}/${MODULE_NAME}/${MODULE_NAME}_meta.json" + +code_size_line=11 +code_size_old=0 +code_size_new=$(stat --format=%s "${TAR_OUT_DIR}/${MODULE_NAME}/${MODULE_NAME}.zip") +sed -i "${code_size_line} s@${code_size_old}@${code_size_new}@" "${TAR_OUT_DIR}/${MODULE_NAME}/${MODULE_NAME}_meta.json" + +# get the final tar package. +chmod -R 700 "${TAR_OUT_DIR}" + +cp -ar "${TAR_OUT_DIR}/"* "${OUTPUT_DIR}" +mkdir -p "$OUTPUT_DIR/templates/" +cp -arf "${PROJECT_DIR}/build/system-function-config.yaml" "${OUTPUT_DIR}/templates/system-function-config.yaml" +cp -arf "${PROJECT_DIR}/build/faasfrontend-function-config.yaml" "${OUTPUT_DIR}/templates/faasfrontend-function-config.yaml" +cp -arf "${PROJECT_DIR}/build/faasfrontend-function-meta.yaml" "${OUTPUT_DIR}/templates/faasfrontend-function-meta.yaml" +cp -arf "${PROJECT_DIR}/build/fassfrontend-service.yaml" "${OUTPUT_DIR}/templates/fassfrontend-service.yaml" +chmod -R 700 "${OUTPUT_DIR}" \ No newline at end of file diff --git a/frontend/build/clean.sh b/frontend/build/clean.sh new file mode 100644 index 0000000000000000000000000000000000000000..f6d609f192fa8a1829e7973032f116b7a38cfc06 --- /dev/null +++ b/frontend/build/clean.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +BASE_DIR=$(cd "$(dirname "$0")"; pwd) +PROJECT_DIR=$(cd "$(dirname "$0")"/..; pwd) +OUTPUT_DIR="${BASE_DIR}/../output/yuanrong/pattern/pattern_faas" +TAR_OUT_DIR="${PROJECT_DIR}/build/_output" + +# cleanup and rebuild folders +cd "${PROJECT_DIR}" || die "${PROJECT_DIR} not exist" +[ -d "${OUTPUT_DIR}" ] && rm -rf "${OUTPUT_DIR}" +mkdir -p "${OUTPUT_DIR}" +[ -d "${TAR_OUT_DIR}" ] && rm -rf "${TAR_OUT_DIR}" +mkdir -p "${TAR_OUT_DIR}" \ No newline at end of file diff --git a/frontend/build/compile_functions.sh b/frontend/build/compile_functions.sh new file mode 100644 index 0000000000000000000000000000000000000000..d34890742d6e94991d82e726e9bf665b12846551 --- /dev/null +++ b/frontend/build/compile_functions.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +function generate_pb() { + # generate pb files + if [ -z "${GOPATH}" ] || [ ! -d "${GOPATH}" ]; then + log_error "GOPATH ${GOPATH} not exist!" + return 1 + fi + cd "${PROJECT_DIR}"/pkg + [ -d "${GOPATH}/src/frontend" ] && rm -rf "${GOPATH}/src/frontend" + mkdir -p "${GOPATH}"/src/ + ln -s "${PROJECT_DIR}" "${GOPATH}"/src/frontend + if ! bash "${PROJECT_DIR}"/build/gen_grpc_pb.sh; then + log_error "Failed to generate pb files!" + return 1 + fi +} + +go mod tidy +generate_pb \ No newline at end of file diff --git a/frontend/build/faasfrontend-function-config.yaml b/frontend/build/faasfrontend-function-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..661ede8dc3d612385ce927cf4d0ce65f9dc8acb3 --- /dev/null +++ b/frontend/build/faasfrontend-function-config.yaml @@ -0,0 +1,132 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: faasfrontend-function-config + namespace: {{ .Values.global.namespace }} +data: + faasfrontend_config.json: |- + { + "0-system-faasfrontend": { + "systemFunctionName":"0-system-faascontroller", + "signal":65, + "payload":{ + "instanceNum": {{ .Values.global.systemFunc.faasFrontend.instanceNum }}, + "azID": "1", + "cpu": {{ .Values.global.systemFunc.faasFrontend.cpu }}, + "memory": {{ .Values.global.systemFunc.faasFrontend.memory }}, + "image": "", + "version":"", + "clusterID":"cluster001", + "clusterName": "dsweb_cceturbo_bj4_auto_az1", + "regionName": "beijing4", + "affinity": {{ quote .Values.global.systemFunc.faasFrontend.affinity | default "\"\"" }}, + "alarmConfig": { + "enableAlarm": false, + "minInsStartInterval": 15, + "minInsCheckInterval": 15, + "alarmLogConfig": { + "filepath": "/opt/logs/alarms", + "level": "Info", + "tick": 0, + "first": 0, + "thereafter": 0, + "singlesize": 500, + "threshold": 3, + "disable": false + }, + "xiangYunFourConfig": { + "site": "cn_dev_default", + "tenantID": "T014", + "applicationID": "", + "serviceID": "" + } + }, + "http": { + "resptimeout": 5, + "workerInstanceReadTimeOut": 5, + "maxRequestBodySize": 6, + "serverReadTimeout": {{ .Values.global.systemFunc.faasFrontend.serverReadTimeout }}, + "serverWriteTimeout": {{ .Values.global.systemFunc.faasFrontend.serverWriteTimeout }}, + "clientIdleTimeout": {{ .Values.global.systemFunc.faasFrontend.clientIdleTimeout }} + }, + "businessType": 0, + "diskMonitorEnable": false, + "authenticationEnable": false, + "trafficLimitDisable": true, + "functionCapability": 1, + "dynamicPoolEnable": {{ .Values.global.systemFunc.faasFrontend.pool.enabled }}, + "rawStsConfig": { + "stsEnable": {{ .Values.global.sts.enable }}, + "serverConfig": { + "domain": {{ quote .Values.global.sts.serverDomain }}, + "path": "" + } + }, + "dataSystemConfig": { + "uploadWriteMode": "NoneL2Cache", + "executeWriteMode": "NoneL2Cache", + "uploadTTLSec": 86400, + "executeTTLSec": 1800, + "timeoutMs": 60000 + }, + "slaQuota": 1000, + "scaleDownTime": 60000, + "burstScaleNum": 1000, + "leaseSpan": 5000, + "functionLimitRate": 5000, + "dockerRootPath":"/var/lib/docker", + "invokeMaxRetryTimes": {{ .Values.global.systemFunc.faasFrontend.invokeMaxRetryTimes }}, + "retry": { + "instanceExceptionRetry": {{ .Values.global.systemFunc.faasFrontend.retry.instanceExceptionRetry }} + }, + "httpsConfig": { + "httpsEnable": {{ .Values.global.systemFunc.httpsEnabled }}, + "tlsProtocol": "TLSv1.2", + "tlsCiphers": "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", + "sslBasePath": {{ quote .Values.global.mutualSSLConfig.sslBasePath }}, + "rootCAFile": "ca.crt", + "moduleCertFile": "module.crt", + "moduleKeyFile": "module.key", + "pwdFile": "cert_pwd", + "secretName": {{ quote .Values.global.mutualSSLConfig.secretName }}, + "sslDecryptTool": {{ quote .Values.global.mutualSSLConfig.sslDecryptTool }} + }, + "routerEtcd": { + "servers": ["{{ .Values.global.etcdManagement.detcd }}"], + {{- if eq .Values.global.etcdManagement.authType "TLS" }} + "sslEnable": true, + {{- else }} + "sslEnable": false, + {{- end}} + "user":"", + "password":"", + "authType": {{ quote .Values.global.etcdManagement.authType }}, + "useSecret": {{ .Values.global.etcdManagement.useSecret }}, + "secretName": {{ quote .Values.global.etcdManagement.secretName }} + }, + "metaEtcd": { + "servers": ["{{ .Values.global.etcdManagement.metcd }}"], + {{- if eq .Values.global.etcdManagement.authType "TLS" }} + "sslEnable": true, + {{- else }} + "sslEnable": false, + {{- end}} + "user":"", + "password":"", + "authType": {{ quote .Values.global.etcdManagement.authType }}, + "useSecret": {{ .Values.global.etcdManagement.useSecret }}, + "secretName": {{ quote .Values.global.etcdManagement.secretName }} + }, + "tlsConfig": { + "caContent": {{ quote .Values.global.systemFunc.tlsConfig.caContent }}, + "keyContent": {{ quote .Values.global.systemFunc.tlsConfig.keyContent }}, + "certContent": {{ quote .Values.global.systemFunc.tlsConfig.certContent }} + }, + "sccConfig": { + "enable": {{ .Values.global.scc.enable }}, + "secretName": {{ quote .Values.global.scc.secretName }}, + "algorithm": {{ quote .Values.global.scc.algorithm }} + } + } + } + } diff --git a/frontend/build/faasfrontend-function-meta.yaml b/frontend/build/faasfrontend-function-meta.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6d05aa827c266eb16034d972b5245e9f24208382 --- /dev/null +++ b/frontend/build/faasfrontend-function-meta.yaml @@ -0,0 +1,99 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: faasfrontend-function-meta + namespace: {{ .Values.global.namespace }} +data: + faasfrontend_meta.json: |- + { + "funcMetaData": { + "layers": [], + "name": "0-system-faasfrontend", + "description": "", + "functionUrn": "sn:cn:yrk:0:function:0-system-faasfrontend", + "reversedConcurrency": 0, + "tags": null, + "functionUpdateTime": "", + "functionVersionUrn": "sn:cn:yrk:0:function:0-system-faasfrontend:$latest", + "codeSize": 10503830, + "codeSha256": "0", + "handler": "", + "runtime": "go1.13", + "timeout": 900, + "version": "$latest", + "versionDescription": "$latest", + "deadLetterConfig": "", + "latestVersionUpdateTime": "", + "publishTime": "", + "businessId": "yrk", + "tenantId": "0", + "domain_id": "", + "project_name": "", + "revisionId": "20230116102015135", + "created": "2023-01-16 10:20:15.135 UTC", + "statefulFlag": false, + "hookHandler": { + "call": "faasfrontend.CallHandler", + "init": "faasfrontend.InitHandler", + "checkpoint": "faasfrontend.CheckpointHandler", + "recover": "faasfrontend.RecoverHandler", + "shutdown": "faasfrontend.ShutdownHandler" + } + }, + "codeMetaData": { + "storage_type": "local", + "code_path": "/home/sn/system-function-packages/faasfrontend" + }, + "envMetaData": { + "envKey": "", + "environment": "", + "encrypted_user_data": "" + }, + "resourceMetaData": { + "cpu": {{ .Values.global.systemFunc.faasFrontend.cpu }}, + "memory": {{ .Values.global.systemFunc.faasFrontend.memory }}, + "customResources": "" + }, + "extendedMetaData": { + "image_name": "", + "role": { + "xrole": "", + "app_xrole": "" + }, + "mount_config": { + "mount_user": { + "user_id": 0, + "user_group_id": 0 + }, + "func_mounts": null + }, + "strategy_config": { + "concurrency": 0 + }, + "extend_config": "", + "initializer": { + "initializer_handler": "", + "initializer_timeout": 0 + }, + "enterprise_project_id": "", + "log_tank_service": { + "logGroupId": "", + "logStreamId": "" + }, + "tracing_config": { + "tracing_ak": "", + "tracing_sk": "", + "project_name": "" + }, + "user_type": "", + "instance_meta_data": { + "maxInstance": 100, + "minInstance": 0, + "concurrentNum": 100, + "cacheInstance": 0 + }, + "extended_handler": null, + "extended_timeout": null + } + } + diff --git a/frontend/build/fassfrontend-service.yaml b/frontend/build/fassfrontend-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..483392b9a28d64b041360bb63175d8edba051d06 --- /dev/null +++ b/frontend/build/fassfrontend-service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + name: faasfrontend + namespace: {{ .Values.global.namespace }} +spec: + externalTrafficPolicy: Cluster + ports: + - name: http + nodePort: {{ .Values.global.port.frontend }} + port: 8888 + protocol: TCP + targetPort: 8888 + selector: + systemFuncName: faasfrontend + type: NodePort + sessionAffinity: None \ No newline at end of file diff --git a/frontend/build/function_meta.json b/frontend/build/function_meta.json new file mode 100644 index 0000000000000000000000000000000000000000..a8caeb73c973a49ff141685750e7fa3c74838cdf --- /dev/null +++ b/frontend/build/function_meta.json @@ -0,0 +1,95 @@ +{ + "funcMetaData": { + "layers": [], + "name": "0-system-moduleName", + "description": "", + "functionUrn": "sn:cn:yrk:12345678901234561234567890123456:function:0-system-moduleName", + "reversedConcurrency": 0, + "tags": null, + "functionUpdateTime": "", + "functionVersionUrn": "sn:cn:yrk:12345678901234561234567890123456:function:0-system-moduleName:$latest", + "codeSize": 0, + "codeSha256": "0", + "handler": "", + "runtime": "go1.13", + "timeout": 900, + "version": "$latest", + "versionDescription": "$latest", + "deadLetterConfig": "", + "latestVersionUpdateTime": "", + "publishTime": "", + "businessId": "yrk", + "tenantId": "12345678901234561234567890123456", + "domain_id": "", + "project_name": "", + "revisionId": "20230116102015135", + "created": "2023-01-16 10:20:15.135 UTC", + "statefulFlag": false, + "hookHandler": { + "call": "moduleName.CallHandler", + "init": "moduleName.InitHandler", + "checkpoint": "moduleName.CheckpointHandler", + "recover": "moduleName.RecoverHandler", + "shutdown": "moduleName.ShutdownHandler" + } + }, + "codeMetaData": { + "storage_type": "local", + "code_path": "/home/sn/system-function-packages/moduleName" + }, + "envMetaData": { + "envKey": "", + "environment": "", + "encrypted_user_data": "" + }, + "resourceMetaData": { + "cpu": 500, + "memory": 500, + "customResources": "" + }, + "extendedMetaData": { + "image_name": "", + "role": { + "xrole": "", + "app_xrole": "" + }, + "pre_stop":{ + "pre_stop_handler": "handler.prestop", + "pre_stop_timeout": 10 + }, + "mount_config": { + "mount_user": { + "user_id": 0, + "user_group_id": 0 + }, + "func_mounts": null + }, + "strategy_config": { + "concurrency": 0 + }, + "extend_config": "", + "initializer": { + "initializer_handler": "", + "initializer_timeout": 0 + }, + "enterprise_project_id": "", + "log_tank_service": { + "logGroupId": "", + "logStreamId": "" + }, + "tracing_config": { + "tracing_ak": "", + "tracing_sk": "", + "project_name": "" + }, + "user_type": "", + "instance_meta_data": { + "maxInstance": 100, + "minInstance": 0, + "concurrentNum": 100, + "cacheInstance": 0 + }, + "extended_handler": null, + "extended_timeout": null + } +} \ No newline at end of file diff --git a/frontend/build/gen_grpc_pb.sh b/frontend/build/gen_grpc_pb.sh new file mode 100644 index 0000000000000000000000000000000000000000..de9ef7870ef9707e35f64a8238101dce9cb91f1c --- /dev/null +++ b/frontend/build/gen_grpc_pb.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +BASE_DIR=$( + cd "$(dirname "$0")" + pwd +) +PROJECT_DIR="${BASE_DIR}"/../ +DST_DIR="${PROJECT_DIR}"pkg/common/faas_common + +function gen_code() { + cd "${DST_DIR}" + RPC_SERVICE_PROTO_PATH="${PROJECT_DIR}"pkg/common/faas_common/protobuf/ + if [ -d "${DST_DIR}/grpc" ]; then + rm "${DST_DIR}/grpc" -rf + fi + + protoc --proto_path="${RPC_SERVICE_PROTO_PATH}" --go_out="${GOPATH}/src" --go-grpc_out="${GOPATH}/src" \ + "${RPC_SERVICE_PROTO_PATH}"*.proto + + echo "generate gRPC: Done!" +} + +function gen_posix_code() { + cd "${DST_DIR}" + RPC_PROTO_PATH="${PROJECT_DIR}"posix/proto/ + if [ ! -d "${RPC_PROTO_PATH}" ]; then + echo "posix directory doesn't exist" + return + fi + + sed -i 's#"grpc/pb#"frontend/pkg/common/faas_common/grpc/pb#g' "${RPC_PROTO_PATH}"*.proto + protoc --proto_path="${RPC_PROTO_PATH}" --go_out="${GOPATH}/src" --go-grpc_out="${GOPATH}/src" \ + "${RPC_PROTO_PATH}"*.proto + + echo "generate posix gRPC: Done!" +} + +if [ -d "${DST_DIR}/grpc" ]; then + rm "${DST_DIR}/grpc" -rf +fi +gen_code +gen_posix_code +exit 0 \ No newline at end of file diff --git a/frontend/build/system-function-config.yaml b/frontend/build/system-function-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a60fcee642494da8a1a2a8a603fcb3cbf60b55b --- /dev/null +++ b/frontend/build/system-function-config.yaml @@ -0,0 +1,102 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: system-function-config + namespace: {{ .Values.global.namespace }} +data: + system-function-config.json: |- + { + "0-system-faascontroller": { + "tenantID": "0", + "version": "$latest", + "memory": {{ .Values.global.systemFunc.faasController.memory }}, + "cpu": {{ .Values.global.systemFunc.faasController.cpu }}, + "instanceNum": {{ .Values.global.systemFunc.faasController.instanceNum }}, + "schedulingOps": {"extension":{"schedule_policy":"monopoly"}}, + "createOptions": { + "concurrentNum": "10", + "DELEGATE_ENCRYPT": "{\"metaEtcdPwd\":\"\"}", + "DELEGATE_POD_LABELS": "{\"systemFuncName\":\"faascontroller\"}", + "DELEGATE_RUNTIME_MANAGER": "{\"image\":\"\"}", + "DELEGATE_AFFINITY": {{ quote .Values.global.systemFunc.faasController.affinity | default "\"\"" }} + }, + "args": { + "faasmanagerConfig": { + "affinity": {{ quote .Values.global.systemFunc.faasManager.affinity | default "\"\"" }}, + "cpu": {{ .Values.global.systemFunc.faasManager.cpu | default 500 }}, + "memory": {{ .Values.global.systemFunc.faasManager.memory | default 500 }}, + "nodeSelector": {} + }, + "managerInstanceNum": {{ .Values.global.systemFunc.faasManager.instanceNum }}, + "enableRetry": true, + "nameSpace": {{ quote .Values.global.namespace }}, + "clusterID":"cluster001", + "clusterName": "dsweb_cceturbo_bj4_auto_az1", + "regionName": "beijing4", + "alarmConfig": { + "enableAlarm": false, + "minInsStartInterval": 15, + "minInsCheckInterval": 15, + "alarmLogConfig": { + "filepath": "/opt/logs/alarms", + "level": "Info", + "tick": 0, + "first": 0, + "thereafter": 0, + "singlesize": 500, + "threshold": 3, + "disable": false + }, + "xiangYunFourConfig": { + "site": "cn_dev_default", + "tenantID": "T014", + "applicationID": "", + "serviceID": "" + } + }, + "rawStsConfig": { + "stsEnable": {{ .Values.global.sts.enable }}, + "serverConfig": { + "domain": {{ quote .Values.global.sts.serverDomain }}, + "path": "" + } + }, + "routerEtcd": { + "servers": ["{{ .Values.global.etcdManagement.detcd }}"], + {{- if eq .Values.global.etcdManagement.authType "TLS" }} + "sslEnable": true, + {{- else }} + "sslEnable": false, + {{- end}} + "user":"", + "password":"", + "authType": {{ quote .Values.global.etcdManagement.authType }}, + "useSecret": {{ .Values.global.etcdManagement.useSecret }}, + "secretName": {{ quote .Values.global.etcdManagement.secretName }} + }, + "metaEtcd": { + "servers": ["{{ .Values.global.etcdManagement.metcd }}"], + {{- if eq .Values.global.etcdManagement.authType "TLS" }} + "sslEnable": true, + {{- else }} + "sslEnable": false, + {{- end}} + "user":"", + "password":"", + "authType": {{ quote .Values.global.etcdManagement.authType }}, + "useSecret": {{ .Values.global.etcdManagement.useSecret }}, + "secretName": {{ quote .Values.global.etcdManagement.secretName }} + }, + "tlsConfig": { + "caContent": {{ quote .Values.global.systemFunc.tlsConfig.caContent }}, + "keyContent": {{ quote .Values.global.systemFunc.tlsConfig.keyContent }}, + "certContent": {{ quote .Values.global.systemFunc.tlsConfig.certContent }} + }, + "sccConfig": { + "enable": {{ .Values.global.scc.enable }}, + "secretName": {{ quote .Values.global.scc.secretName }}, + "algorithm": {{ quote .Values.global.scc.algorithm }} + } + } + } + } diff --git a/frontend/cmd/faasfrontend/function_main.go b/frontend/cmd/faasfrontend/function_main.go new file mode 100644 index 0000000000000000000000000000000000000000..d5a2f6ce4e2cc84dd7651d2314cda2db38ff0505 --- /dev/null +++ b/frontend/cmd/faasfrontend/function_main.go @@ -0,0 +1,230 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package main - +package main + +import ( + "encoding/json" + "errors" + "fmt" + "sync" + + _ "go.uber.org/automaxprocs" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/datasystemclient" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/monitor" + "frontend/pkg/common/faas_common/urnutils" + "frontend/pkg/common/faas_common/utils" + "frontend/pkg/frontend/common/util" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/invocation" + "frontend/pkg/frontend/middleware" + "frontend/pkg/frontend/responsehandler" + "frontend/pkg/frontend/schedulerproxy" + "frontend/pkg/frontend/server" + "frontend/pkg/frontend/state" +) + +var ( + stopCh = make(chan struct{}) +) + +const ( + defaultArgsLength = 5 + defaultFileMode = 0640 + serverNum = 2 +) + +// InitHandlerLibruntime is the init handler called by runtime +func InitHandlerLibruntime(args []api.Arg, rt api.LibruntimeAPI) ([]byte, error) { + log.SetupLoggerLibruntime(rt.GetFormatLogger()) + var err error + defer func() { + if err != nil { + fmt.Printf("panic, module: faasfrontend, err: %s\n", err.Error()) + log.GetLogger().Errorf("panic, module: faasfrontend, err: %s", err.Error()) + } + log.GetLogger().Sync() + }() + if err = config.InitFunctionConfig(args[0].Data); err != nil { + log.GetLogger().Errorf("init frontend config fail, err: %s", err) + return []byte{}, err + } + if err = config.InitEtcd(stopCh); err != nil { + log.GetLogger().Errorf("failed to init etcd ,err:%s", err.Error()) + return []byte{}, err + } + state.InitState() + var stateByte []byte + stateByte, err = state.GetStateByte() + if err == nil && len(stateByte) != 0 { + return []byte{}, RecoverHandlerLibruntime(stateByte, rt) + } + cfg := config.GetConfig() + state.Update(cfg) + if err = setupFaaSFrontendLibruntime(rt, stopCh); err != nil { + return []byte{}, err + } + config.ClearSensitiveInfo() + return []byte{}, nil +} + +// CallHandlerLibruntime handles the invoke request between in-cloud faas functions +// the posix args are all value only (type=0, no object ref) args, the convention: +// args[0]: target faas function name +// args[1]: target faas service name +// args[2]: target tenant id +// args[3]: target function version +// args[4]: invoke payload to the target function +func CallHandlerLibruntime(argsLibrt []api.Arg) ([]byte, error) { + if len(argsLibrt) < defaultArgsLength { + return nil, fmt.Errorf("invalid call with num of argsLibrt %d", len(argsLibrt)) + } + + req := InCloudFunctionInvokeRequest{ + functionName: string(argsLibrt[0].Data), + serviceName: string(argsLibrt[1].Data), + tenantID: string(argsLibrt[2].Data), + functionVersion: string(argsLibrt[3].Data), + invokePayload: argsLibrt[4].Data, + } + resp := innerInvoke(req) + b, err := json.Marshal(resp) + if err != nil { + return []byte{}, err + } + return b, nil +} + +// InCloudFunctionInvokeRequest - +type InCloudFunctionInvokeRequest struct { + functionName string + serviceName string + tenantID string + functionVersion string + invokePayload []byte +} + +// InCloudFunctionInvokeResponse - +type InCloudFunctionInvokeResponse struct { + Code int + Message string +} + +func innerInvoke(request InCloudFunctionInvokeRequest) InCloudFunctionInvokeResponse { + return InCloudFunctionInvokeResponse{ + Code: 0, + Message: "Successful in-cloud invoke", + } +} + +// CheckpointHandlerLibruntime is the checkpoint handler called by runtime +func CheckpointHandlerLibruntime(checkpointID string) ([]byte, error) { + return state.GetStateByte() +} + +func initStateAndConfig(stateData []byte) error { + var err error + log.GetLogger().Infof("trigger: faasfrontend.RecoverHandler") + if err = state.SetState(stateData); err != nil { + log.GetLogger().Errorf("recover frontend error:%s", err.Error()) + return fmt.Errorf("faaS frontend recover error:%s", err.Error()) + } + state.InitState() + cfg := config.GetConfig() + state.Update(cfg) + return nil +} + +// RecoverHandlerLibruntime is the recover handler called by runtime +func RecoverHandlerLibruntime(stateData []byte, rt api.LibruntimeAPI) error { + var err error + log.SetupLoggerLibruntime(rt.GetFormatLogger()) + err = initStateAndConfig(stateData) + if err != nil { + return err + } + if err = setupFaaSFrontendLibruntime(rt, stopCh); err != nil { + log.GetLogger().Errorf("restart initHandler error:%s", err.Error()) + return fmt.Errorf("faaS frontend restart initHandler error:%s", err.Error()) + } + config.ClearSensitiveInfo() + return nil +} + +// ShutdownHandlerLibruntime is the shutdown handler called by runtime +func ShutdownHandlerLibruntime(gracePeriodSecond uint64) error { + log.GetLogger().Infof("trigger: faasfrontend.ShutdownHandler") + utils.SafeCloseChannel(stopCh) + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + server.GracefulShutdown(server.GetHTTPServer()) + wg.Done() + }() + wg.Wait() + log.GetLogger().Infof("faasfrontendLibruntime exit") + log.GetLogger().Sync() + return nil +} + +// SignalHandlerLibruntime is the signal handler called by runtime +func SignalHandlerLibruntime(signal int, payload []byte) error { + return nil +} + +func setupFaaSFrontendLibruntime(rt api.LibruntimeAPI, stopChLibrt <-chan struct{}) error { + util.SetAPIClientLibruntime(rt) + schedulerproxy.Proxy.RTAPI = rt + cfg := config.GetConfig() + datasystemclient.InitDataSystemLibruntime(cfg.DataSystemConfig, rt, stopChLibrt) + monitor.SetMemoryControlConfig(cfg.MemoryControlConfig) + if err := monitor.InitMemMonitor(stopCh); err != nil { + log.GetLogger().Errorf("failed to init mem monitor") + return err + } + if err := assembleAdapter(); err != nil { + return err + } + httpServer := server.CreateHTTPServer() + go func() { + err := server.Start(httpServer, stopCh) + if err != nil { + log.GetLogger().Errorf("start faas frontend server failed will exit, err:%s", err.Error()) + rt.Exit(0, "") + } + }() + return nil +} + +func assembleAdapter() error { + switch config.GetConfig().BusinessType { + case constant.BusinessTypeFG: + urnutils.SetSeparator(urnutils.TenantProductSplitStr) + fgAdapter := &invocation.FGAdapter{} + responsehandler.Handler = fgAdapter.MakeResponseHandler() + middleware.Invoker = fgAdapter.MakeInvoker() + default: + log.GetLogger().Errorf("Not support businessType") + return errors.New("assembleAdapter error,not support businessType") + } + return nil +} diff --git a/frontend/cmd/faasfrontend/function_main_test.go b/frontend/cmd/faasfrontend/function_main_test.go new file mode 100644 index 0000000000000000000000000000000000000000..5e57b700889acb6fca816c35404b683fec6e0a99 --- /dev/null +++ b/frontend/cmd/faasfrontend/function_main_test.go @@ -0,0 +1,203 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "net/http" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/frontend/server" + "frontend/pkg/frontend/state" + "github.com/stretchr/testify/assert" +) + +var cfg = `{ + "slaQuota": 1000, + "functionCapability": 1, + "authenticationEnable": false, + "trafficLimitDisable": true, + "http": { + "resptimeout": 5, + "workerInstanceReadTimeOut": 5, + "maxRequestBodySize": 6 + }, + "routerEtcd": { + "servers": ["1.2.3.4:1234"], + "user": "tom", + "password": "**" + }, + "metaEtcd": { + "servers": ["1.2.3.4:5678"], + "user": "tom", + "password": "**" + } + }` +var invalidCfg = `{"abc":"123"` + +func TestCheckpointHandler(t *testing.T) { + state.SetState([]byte(`{}`)) + type args struct { + checkpointID string + } + tests := []struct { + name string + args args + want []byte + wantErr bool + }{ + { + name: "success", + args: args{"123"}, + want: []byte(`{"Config":null}`), + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := CheckpointHandlerLibruntime(tt.args.checkpointID) + if (err != nil) != tt.wantErr { + t.Errorf("CheckpointHandler() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("CheckpointHandler() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestCallHandler(t *testing.T) { + state.InitState() + type args struct { + args []api.Arg + } + tests := []struct { + name string + args args + want interface{} + wantErr bool + }{ + { + name: "args error", + args: args{ + args: []api.Arg{}, + }, + want: nil, + wantErr: true, + }, + { + name: "success", + args: args{ + args: []api.Arg{ + {Data: []byte("1")}, + {Data: []byte("2")}, + {Data: []byte("3")}, + {Data: []byte("4")}, + {Data: []byte("5")}, + }, + }, + want: InCloudFunctionInvokeResponse{ + Code: 0, + Message: "Successful in-cloud invoke", + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := CallHandlerLibruntime(tt.args.args) + if (err != nil) != tt.wantErr { + t.Errorf("CallHandler() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("CallHandler() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestInitHandlerError(t *testing.T) { + applyFunc := gomonkey.ApplyFunc(state.InitState, func() { + return + }) + defer applyFunc.Reset() + patches := []*gomonkey.Patches{ + gomonkey.ApplyFunc(server.GracefulShutdown, func(httpServer *http.Server) { + return + }), + } + defer func() { + for _, patch := range patches { + patch.Reset() + } + }() + res, err := InitHandlerLibruntime([]api.Arg{{Data: []byte(invalidCfg)}}, nil) + assert.NotNil(t, err) + assert.Equal(t, nil, res) + + res, err = InitHandlerLibruntime([]api.Arg{{Data: []byte(cfg)}}, nil) + assert.Nil(t, err) + assert.Equal(t, "", res) +} + +func TestRecoverHandler(t *testing.T) { + applyFunc := gomonkey.ApplyFunc(state.InitState, func() { + return + }) + defer applyFunc.Reset() + patches := gomonkey.ApplyFunc(server.GracefulShutdown, func(httpServer *http.Server) { + return + }) + defer patches.Reset() + type args struct { + stateData []byte + client api.LibruntimeAPI + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "success", + args: args{ + stateData: []byte(`{"Config":` + cfg + `}`), + client: nil, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := RecoverHandlerLibruntime(tt.args.stateData, tt.args.client) + if (err != nil) != tt.wantErr { + t.Errorf("CallHandler() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} + +func TestShutdownHandler(t *testing.T) { + err := ShutdownHandlerLibruntime(0) + assert.Nil(t, err) +} diff --git a/frontend/cmd/faasfrontend/module_main.go b/frontend/cmd/faasfrontend/module_main.go new file mode 100644 index 0000000000000000000000000000000000000000..860723c4effadb557156e2f9f95ae13a38e1396e --- /dev/null +++ b/frontend/cmd/faasfrontend/module_main.go @@ -0,0 +1,140 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "context" + "errors" + "fmt" + "net/http" + + "frontend/pkg/common/faas_common/autogc" + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/monitor" + "frontend/pkg/common/faas_common/signals" + "frontend/pkg/common/faas_common/tracer" + "frontend/pkg/common/faas_common/urnutils" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/functiontask" + "frontend/pkg/frontend/invocation" + "frontend/pkg/frontend/metrics" + "frontend/pkg/frontend/middleware" + "frontend/pkg/frontend/responsehandler" + "frontend/pkg/frontend/server" +) + +const ( + logFileName = "frontend" +) + +func main() { + defer func() { + log.GetLogger().Sync() + }() + // init logger config + err := log.InitRunLog(logFileName, true) + if err != nil { + fmt.Print("init logger error: " + err.Error()) + return + } + autogc.InitAutoGOGC() + shutdown := func() { fmt.Println("common tracer is not initialized") } + go tracer.InitCommonTracer(shutdown, "frontend") + defer func() { + if shutdown != nil { + shutdown() + } + }() + // init constant + err = config.InitModuleConfig() + if err != nil { + logAndPrintError(fmt.Sprintf("init module config error: %s", err.Error())) + return + } + urnutils.SetSeparator(config.GetConfig().FunctionNameSeparator) + stopCh := signals.WaitForSignal() + if err = config.InitEtcd(stopCh); err != nil { + logAndPrintError(fmt.Sprintf("init etcd error: %s", err.Error())) + return + } + + go metrics.StartReportMetrics(stopCh) + err = setupModuleFrontend(stopCh) + if err != nil { + logAndPrintError(fmt.Sprintf("setup module frontend error: %s", err.Error())) + return + } + errChan := make(chan error, 1) + httpServer := server.CreateHTTPServer() + go func() { + err = server.Start(httpServer, stopCh) + if err != nil { + errChan <- err + logAndPrintError(fmt.Sprintf("start http server error: %s", err.Error())) + } + }() + if err := waitShutdown(httpServer, stopCh, errChan); err != nil { + logAndPrintError(fmt.Sprintf("wait http server error: %s", err.Error())) + } +} +func logAndPrintError(errMessage string) { + log.GetLogger().Errorf(errMessage) + fmt.Println(errMessage) +} + +func waitShutdown(server *http.Server, stopCh <-chan struct{}, errChan <-chan error) error { + if server == nil { + return errors.New("http server is nil") + } + if stopCh == nil || errChan == nil { + return errors.New("input channel is nil") + } + select { + case <-stopCh: + log.GetLogger().Infof("received termination signal") + ctx, cancel := context.WithTimeout(context.Background(), constant.DefaultServerWriteTimeOut) + defer cancel() + return server.Shutdown(ctx) + case err := <-errChan: + return err + } +} + +func setupModuleFrontend(stopCh <-chan struct{}) error { + updateConfig() + if err := config.WatchConfig(config.ConfigFilePath, stopCh, updateConfig); err != nil { + log.GetLogger().Warnf("WatchConfig %s failed, err %s", config.ConfigFilePath, err.Error()) + } + if err := monitor.InitMemMonitor(stopCh); err != nil { + log.GetLogger().Errorf("failed to init mem monitor") + return err + } + fgAdapter := &invocation.FGAdapter{} + responsehandler.Handler = fgAdapter.MakeResponseHandler() + middleware.Invoker = fgAdapter.MakeInvoker() + return nil +} + +func updateConfig() { + cfg := config.GetConfig() + monitor.SetMemoryControlConfig(cfg.MemoryControlConfig) + if cfg.FunctionInvokeBackend == constant.BackendTypeFG { + functiontask.GetBusProxies().UpdateConfig() + } + +} diff --git a/frontend/go.mod b/frontend/go.mod new file mode 100644 index 0000000000000000000000000000000000000000..db3733a370b9cbb1b13edc5c90ff761ecbd00e17 --- /dev/null +++ b/frontend/go.mod @@ -0,0 +1,58 @@ +module frontend + +go 1.24.1 + +require ( + frontend/pkg/common v1.0.0 + frontend/pkg/frontend v1.0.0 + github.com/agiledragon/gomonkey/v2 v2.11.0 + github.com/stretchr/testify v1.10.0 + go.uber.org/automaxprocs v1.6.0 + yuanrong.org/kernel/runtime v1.0.0 +) + +replace ( + frontend/pkg/common => ./pkg/common + frontend/pkg/frontend => ./pkg/frontend + github.com/agiledragon/gomonkey => github.com/agiledragon/gomonkey v2.0.1+incompatible + github.com/asaskevich/govalidator/v11 => github.com/asaskevich/govalidator/v11 v11.0.1-0.20250122183457-e11347878e23 + github.com/fsnotify/fsnotify => github.com/fsnotify/fsnotify v1.7.0 + // for test or internal use + github.com/gin-gonic/gin => github.com/gin-gonic/gin v1.10.0 + github.com/golang/mock => github.com/golang/mock v1.3.1 + github.com/google/uuid => github.com/google/uuid v1.6.0 + github.com/olekukonko/tablewriter => github.com/olekukonko/tablewriter v0.0.5 + github.com/operator-framework/operator-lib => github.com/operator-framework/operator-lib v0.4.0 + github.com/prashantv/gostub => github.com/prashantv/gostub v1.0.0 + github.com/robfig/cron/v3 => github.com/robfig/cron/v3 v3.0.1 + github.com/smartystreets/goconvey => github.com/smartystreets/goconvey v1.6.4 + github.com/spf13/cobra => github.com/spf13/cobra v1.8.1 + github.com/stretchr/testify => github.com/stretchr/testify v1.5.1 + github.com/valyala/fasthttp => github.com/valyala/fasthttp v1.58.0 + go.etcd.io/etcd/api/v3 => go.etcd.io/etcd/api/v3 v3.5.11 + go.etcd.io/etcd/client/v3 => go.etcd.io/etcd/client/v3 v3.5.11 + go.opentelemetry.io/otel => go.opentelemetry.io/otel v1.24.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace => go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc => go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 + go.opentelemetry.io/otel/metric => go.opentelemetry.io/otel/metric v1.24.0 + go.opentelemetry.io/otel/sdk => go.opentelemetry.io/otel/sdk v1.24.0 + go.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace v1.24.0 + go.uber.org/automaxprocs => go.uber.org/automaxprocs v1.6.0 + go.uber.org/zap => go.uber.org/zap v1.27.0 + golang.org/x/crypto => golang.org/x/crypto v0.24.0 + // affects VPC plugin building, will cause error if not pinned + golang.org/x/net => golang.org/x/net v0.26.0 + golang.org/x/sync => golang.org/x/sync v0.0.0-20190423024810-112230192c58 + golang.org/x/sys => golang.org/x/sys v0.21.0 + golang.org/x/text => golang.org/x/text v0.16.0 + golang.org/x/time => golang.org/x/time v0.10.0 + google.golang.org/genproto => google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e + google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d + google.golang.org/grpc => google.golang.org/grpc v1.67.0 + google.golang.org/protobuf => google.golang.org/protobuf v1.36.6 + gopkg.in/yaml.v3 => gopkg.in/yaml.v3 v3.0.1 + yuanrong.org/kernel/runtime => ../runtime/api/go + k8s.io/api => k8s.io/api v0.31.2 + k8s.io/apimachinery => k8s.io/apimachinery v0.31.2 + k8s.io/client-go => k8s.io/client-go v0.31.2 +) \ No newline at end of file diff --git a/frontend/pkg/common/constants/constant_test.go b/frontend/pkg/common/constants/constant_test.go new file mode 100644 index 0000000000000000000000000000000000000000..42f631590e9ccd6a63748484798c68d565b65561 --- /dev/null +++ b/frontend/pkg/common/constants/constant_test.go @@ -0,0 +1,17 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package constants diff --git a/frontend/pkg/common/constants/constants.go b/frontend/pkg/common/constants/constants.go new file mode 100644 index 0000000000000000000000000000000000000000..cd49c74f0c53558980bb86afd12640ee9d04c04e --- /dev/null +++ b/frontend/pkg/common/constants/constants.go @@ -0,0 +1,395 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package constants implements vars of all +package constants + +import ( + "os" + "strconv" + "time" +) + +const ( + // ZoneKey zone key + ZoneKey = "KUBERNETES_IO_AVAILABLEZONE" + // ZoneNameLen define zone length + ZoneNameLen = 255 + // DefaultAZ default az + DefaultAZ = "defaultaz" + + // PodIPEnvKey define pod ip env key + PodIPEnvKey = "POD_IP" + + // HostNameEnvKey defines the hostname env key + HostNameEnvKey = "HOSTNAME" + + // NodeID defines the node name env key + NodeID = "NODE_ID" + + // HostIPEnvKey defines the host ip env key + HostIPEnvKey = "HOST_IP" + + // PodNamespaceEnvKey define pod namespace env key + PodNamespaceEnvKey = "POD_NAMESPACE" + + // ResourceLimitsMemory Memory limit, in bytes + ResourceLimitsMemory = "MEMORY_LIMIT_BYTES" + + // ResourceLimitsCPU CPU limit, in m(1/1000) + ResourceLimitsCPU = "CPU_LIMIT" + + // FuncBranchEnvKey is branch env key + FuncBranchEnvKey = "FUNC_BRANCH" + + // DataSystemBranchEnvKey is branch env key + DataSystemBranchEnvKey = "DATASYSTEM_CAPABILITY" + + // HTTPort busproxy httpserver listen port + HTTPort = "22423" + // GRPCPort busproxy gRPCserver listen port + GRPCPort = "22769" + // WorkerAgentPort is the listen port of worker agent grpc server + WorkerAgentPort = "22888" + // DataSystemPort is the port of data system + DataSystemPort = "31501" + // LocalSchedulerPort is the listen port string of local scheduler grpc server + LocalSchedulerPort = GRPCPort + // DomainSchedulerPort is the listen port of domain scheduler grpc server + DomainSchedulerPort = 22771 + // MaxPort maximum number of ports + MaxPort = 65535 + // SchedulerAddressSeparator is the separator of domain scheduler address + SchedulerAddressSeparator = ":" + // PlatformTenantID is tenant ID of platform function + PlatformTenantID = "0" + + // RuntimeLogOptTail - + RuntimeLogOptTail = "Tail" + // RuntimeLayerDirName - + RuntimeLayerDirName = "layer" + // RuntimeFuncDirName - + RuntimeFuncDirName = "func" + + // FunctionTaskAppID - + FunctionTaskAppID = "function-task" + + // TenantID config from function task + TenantID = "0" + + // BackpressureCode indicate that frontend should choose another proxy/worker and retry + BackpressureCode = 211429 + // HeaderBackpressure indicate that proxy can backpressure this request + HeaderBackpressure = "X-Backpressure" + + // SrcInstanceID gRPC context of metadata + SrcInstanceID = "src_instance_id" + // ReturnObjID gRPC context of metadata + ReturnObjID = "return_obj_id" + + // DelWorkerAgentEvent delete workerAgent + DelWorkerAgentEvent = "WorkerAgent-Del" + // UpdWorkerAgentEvent update workerAgent + UpdWorkerAgentEvent = "WorkerAgent-Upd" + + // DefaultLatestVersion is default function name + DefaultLatestVersion = "$latest" + // DefaultLatestFaaSVersion is default faas function name + DefaultLatestFaaSVersion = "latest" + // DefaultJavaRuntimeName is default java runtime name + DefaultJavaRuntimeName = "java1.8" + // DefaultJavaRuntimeNameForFaas is defualt + DefaultJavaRuntimeNameForFaas = "java8" +) + +// grpc parameters +const ( + // MaxMsgSize grpc client max message size(bit) + MaxMsgSize = 1024 * 1024 * 2 + // MaxWindowSize grpc flow control window size(bit) + MaxWindowSize = 1024 * 1024 * 2 + // MaxBufferSize grpc read/write buffer size(bit) + MaxBufferSize = 1024 * 1024 * 2 +) + +// functionBus userData key flag +const ( + // FrontendCallFlag invoke from task + FrontendCallFlag = "FrontendCallFlag" +) + +const ( + // DynamicRouterParamPrefix 动态路由参数前缀 + DynamicRouterParamPrefix = "/:" +) + +// HTTP invoke request header key +const ( + // HeaderExecutedDuration - + HeaderExecutedDuration = "X-Executed-Duration" + // HeaderTraceID - + HeaderTraceID = "X-Trace-Id" + // HeaderEventSourceID - + HeaderEventSourceID = "X-Event-Source-Id" + // HeaderBusinessID - + HeaderBusinessID = "X-Business-ID" + // HeaderTenantID - + HeaderTenantID = "X-Tenant-ID" + // HeaderTenantId - + HeaderTenantId = "X-Tenant-Id" + // HeaderPoolLabel - + HeaderPoolLabel = "X-Pool-Label" + // HeaderLogType - + HeaderLogType = "X-Log-Type" + // HeaderLogResult - + HeaderLogResult = "X-Log-Result" + // HeaderTriggerFlag - + HeaderTriggerFlag = "X-Trigger-Flag" + // HeaderInnerCode - + HeaderInnerCode = "X-Inner-Code" + // HeaderInvokeURN - + HeaderInvokeURN = "X-Tag-VersionUrn" + // HeaderStateKey - + HeaderStateKey = "X-State-Key" + // HeaderCallType is the request type + HeaderCallType = "X-Call-Type" + // HeaderLoadDuration duration of loading function + HeaderLoadDuration = "X-Load-Duration" + // HeaderNodeLabel is node label + HeaderNodeLabel = "X-Node-Label" + // HeaderForceDeploy is Force Deploy + HeaderForceDeploy = "X-Force-Deploy" + // HeaderAuthorization is authorization + HeaderAuthorization = "authorization" + // HeaderFutureID is futureID of invocation + HeaderFutureID = "X-Future-ID" + // HeaderAsync indicate whether it is an async request + HeaderAsync = "X-ASYNC" + // HeaderRuntimeID represents runtime instance identification + HeaderRuntimeID = "X-Runtime-ID" + // HeaderRuntimePort represents runtime rpc port + HeaderRuntimePort = "X-Runtime-Port" + // HeaderCPUSize is cpu size specified by invoke + HeaderCPUSize = "X-Instance-CPU" + // HeaderMemorySize is cpu memory specified by invoke + HeaderMemorySize = "X-Instance-Memory" + HeaderFileDigest = "X-File-Digest" + HeaderProductID = "X-Product-Id" + HeaderPrivilege = "X-Privilege" + HeaderUserID = "X-User-Id" + HeaderVersion = "X-Version" + HeaderKind = "X-Kind" + // HeaderCompatibleRuntimes - + HeaderCompatibleRuntimes = "X-Header-Compatible-Runtimes" + // HeaderDescription - + HeaderDescription = "X-Description" + // HeaderLicenseInfo - + HeaderLicenseInfo = "X-License-Info" + // HeaderGroupID is group id + HeaderGroupID = "X-Group-ID" + // ApplicationJSON - + ApplicationJSON = "application/json" + // ContentType - + ContentType = "Content-Type" + // PriorityHeader - + PriorityHeader = "priority" + // HeaderDataContentType - + HeaderDataContentType = "X-Content-Type" + // ErrorDuration duration when error happened, + // used with key $LoadDuration + ErrorDuration = -1 +) + +// Extra Request Header +const ( + // HeaderRequestID - + HeaderRequestID = "x-request-id" + // HeaderAccessKey - + HeaderAccessKey = "x-access-key" + // HeaderSecretKey - + HeaderSecretKey = "x-secret-key" + // HeaderAuthToken - + HeaderAuthToken = "x-auth-token" + // HeaderSecurityToken - + HeaderSecurityToken = "x-security-token" + // HeaderStorageType code storage type + HeaderStorageType = "x-storage-type" +) + +const ( + // FunctionStatusUnavailable function status is unavailable + FunctionStatusUnavailable = "unavailable" + + // FunctionStatusAvailable function status is available + FunctionStatusAvailable = "available" +) + +const ( + // OndemandKey is used in ondemand scenario + OndemandKey = "ondemand" +) + +// stage +const ( + InitializeStage = "initialize" +) + +// default UIDs and GIDs +const ( + DefaultWorkerGID = 1002 + DefaultRuntimeUID = 1003 + DefaultRuntimeUName = "snuser" + DefaultRuntimeGID = 1003 +) + +const ( + // WorkerManagerApplier mark the instance is created by minInstance + WorkerManagerApplier = "worker-manager" +) + +const ( + DialBaseDelay = 300 * time.Millisecond + DialMultiplier = 1.2 + DialJitter = 0.1 + DialMaxDelay = 15 * time.Second + RuntimeDialMaxDelay = 100 * time.Second +) + +// constants of network connection +const ( + // DefaultConnectInterval is the default connect interval + DefaultConnectInterval = 3 * time.Second + // DefaultDialInterval is the default grpc dial request interval + DefaultDialInterval = 3 * time.Second + // DefaultRetryTimes is the default request retry times + DefaultRetryTimes = 3 + ConnectIntervalTime = 1 * time.Second +) + +// request message +const ( + // RequestCPU - + RequestCPU = "CPU" + // RequestMemory - + RequestMemory = "Memory" + // MinCustomResourcesSize is min gpu size of invoke + MinCustomResourcesSize = 0 + + // CpuUnitConvert - + CpuUnitConvert = 1000 + // MemoryUnitConvert - + MemoryUnitConvert = 1024 + + // minInvokeCPUSize is default min cpu size of invoke (One CPU core corresponds to 1000) + minInvokeCPUSize = 300 + // MaxInvokeCPUSize is max cpu size of invoke (One CPU core corresponds to 1000) + MaxInvokeCPUSize = 16000 + // minInvokeMemorySize is default min memory size of invoke (MB) + minInvokeMemorySize = 128 + // MaxInvokeMemorySize is max memory size of invoke (MB) + MaxInvokeMemorySize = 1024 * 1024 * 1024 + // InstanceConcurrency - + InstanceConcurrency = "Concurrency" + // DefaultMapSize default map size + DefaultMapSize = 2 + // DefaultSliceSize default slice size + DefaultSliceSize = 16 + // MaxUploadMemorySize is max memory size of upload (MB) + MaxUploadMemorySize = 10 * 1024 * 1024 + // S3StorageType the code is stored in the minio + S3StorageType = "s3" + // LocalStorageType the code is stored in the disk + LocalStorageType = "local" + // CopyStorageType the code is stored in the disk and need to copy to container path + CopyStorageType = "copy" + // Faas kind of function creation + Faas = "faas" +) + +// prefixes of ETCD keys +const ( + WorkerETCDKeyPrefix = "/sn/workeragent" + NodeETCDKeyPrefix = "/sn/node" + // InstanceETCDKeyPrefix is the prefix of etcd key for instance + InstanceETCDKeyPrefix = "/sn/instance" + // ResourceGroupETCDKeyPrefix is the prefix of etcd key for resource group + ResourceGroupETCDKeyPrefix = "/sn/resourcegroup" + // WorkersEtcdKeyPrefix is the prefix of etcd key for workers + WorkersEtcdKeyPrefix = "/sn/workers" + // AliasEtcdKeyPrefix is the key prefix of aliases in etcd + AliasEtcdKeyPrefix = "/sn/aliases" +) + +// constants of posix custom runtime +const ( + PosixCustomRuntime = "posix-custom-runtime" + GORuntime = "go" + JavaRuntime = "java" + _ +) + +const ( + // OriginSchedulePolicy use origin scheduler policy + OriginSchedulePolicy = 0 + // NewSchedulePolicy use new scheduler policy + NewSchedulePolicy = 1 +) + +const ( + // LocalSchedulerLevel local scheduler level is 0 + LocalSchedulerLevel = iota + // LowDomainSchedulerLevel low domain scheduler level is 0 + LowDomainSchedulerLevel +) + +const ( + // Base10 is the decimal base number when use FormatInt + Base10 = 10 +) + +// MinInvokeCPUSize is min cpu size of invoke (One CPU core corresponds to 1000) +// Return default minInvokeCPUSize or system env[MinInvokeCPUSize] +var MinInvokeCPUSize = func() float64 { + minInvokeCPUSizeStr := os.Getenv("MinInvokeCPUSize") + if minInvokeCPUSizeStr != "" { + value, err := strconv.Atoi(minInvokeCPUSizeStr) + if err != nil { + return minInvokeCPUSize + } + return float64(value) + } + return minInvokeCPUSize +}() + +// MinInvokeMemorySize is min memory size of invoke (MB) +// Return default minInvokeMemorySize or system env[MinInvokeMemorySize] +var MinInvokeMemorySize = func() float64 { + minInvokeMemorySizeStr := os.Getenv("MinInvokeMemorySize") + if minInvokeMemorySizeStr != "" { + value, err := strconv.Atoi(minInvokeMemorySizeStr) + if err != nil { + return minInvokeMemorySize + } + return float64(value) + } + return minInvokeMemorySize +}() + +// SelfNodeIP - node IP +var SelfNodeIP = os.Getenv(HostIPEnvKey) + +// SelfNodeID - node ID +var SelfNodeID = os.Getenv(NodeID) diff --git a/frontend/pkg/common/crypto/crypto.go b/frontend/pkg/common/crypto/crypto.go new file mode 100644 index 0000000000000000000000000000000000000000..8612e2ca73b23063f647795646ea61a6ae88bb6e --- /dev/null +++ b/frontend/pkg/common/crypto/crypto.go @@ -0,0 +1,267 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package crypto for auth +package crypto + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "math" + "strings" + "sync" + + "golang.org/x/crypto/pbkdf2" + + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/utils" +) + +const ( + randomNumberMaxLength = 16 + randomNumberMinLength = 12 + defaultSliceLen = 1024 + cipherTextsLen = 2 +) + +var ( + decryptAlgorithm string = "GCM" + decryptAlgorithmMutex sync.RWMutex +) + +// SetDecryptAlgorithm - +func SetDecryptAlgorithm(algorithm string) { + decryptAlgorithmMutex.Lock() + defer decryptAlgorithmMutex.Unlock() + decryptAlgorithm = algorithm +} + +// GetDecryptAlgorithm returns global decryptAlgorithm +func GetDecryptAlgorithm() string { + decryptAlgorithmMutex.RLock() + defer decryptAlgorithmMutex.RUnlock() + return decryptAlgorithm +} + +// Encrypt encrypts data by GCM algorithm +func Encrypt(content string, secret []byte) ([]byte, error) { + if GetDecryptAlgorithm() == "NO_CRYPTO" { + log.GetLogger().Debug("decrypt algorithm is NO_CRYPTO, return plain text directly") + return []byte(content), nil + } + textByte := []byte(content) + cipherByte, salt, err := encryptGcmDataFromBody(textByte, secret) + if err != nil { + return nil, err + } + ciperText := fmt.Sprintf("%s:%s", salt, hex.EncodeToString(cipherByte)) + return []byte(ciperText), nil +} + +func encryptPBKDF2WithSHA256(f *RootKeyFactor) *RootKey { + minLen := math.Min(float64(len(f.k1Data)), math.Min(float64(len(f.k2Data)), float64(len(f.component3byte)))) + bytePsd := make([]byte, int(minLen), int(minLen)) + + for i := 0; i < int(minLen); i++ { + bytePsd[i] = f.k1Data[i] ^ f.k2Data[i] ^ f.component3[i] + } + + rootKeyByte := pbkdf2.Key(bytePsd, f.saltData, f.iterCount, byteSize, sha256.New) + sliceLen := len(rootKeyByte) + if sliceLen <= 0 || sliceLen > defaultSliceLen { + sliceLen = defaultSliceLen + } + + byteMac := make([]byte, sliceLen) + macSecretKeyByte := pbkdf2.Key(byteMac, f.macData, f.iterCount, byteSize, sha256.New) + + rootKey := &RootKey{} + rootKey.RootKey = rootKeyByte + rootKey.MacSecretKey = macSecretKeyByte + + return rootKey +} + +func hmacHash(data []byte, key []byte) string { + hm := hmac.New(sha256.New, key) + _, err := hm.Write(data) + if err != nil { + log.GetLogger().Errorf("failed to hmacHash write data: %s ", err.Error()) + return "" + } + return hex.EncodeToString(hm.Sum([]byte{})) +} + +// encryptGcmDataFromBody encrypts data +func encryptGcmDataFromBody(body []byte, secret []byte) ([]byte, string, error) { + if len(body) == 0 { + return nil, "", fmt.Errorf("body is empty") + } + secretBytes, err := hex.DecodeString(string(secret)) + if err != nil { + return nil, "", err + } + + aesBlock, err := aes.NewCipher(secretBytes) + if err != nil { + return nil, "", err + } + aesgcm, err := cipher.NewGCM(aesBlock) + if err != nil { + return nil, "", err + } + + // generate salt value + nonceSize := aesgcm.NonceSize() + if nonceSize > randomNumberMaxLength || nonceSize < randomNumberMinLength { + err = errors.New("nonceSize out of bound") + return nil, "", err + } + salt := make([]byte, nonceSize, nonceSize) + if _, err := io.ReadFull(rand.Reader, salt); err != nil { + return nil, "", err + } + + cipherByte := aesgcm.Seal(nil, salt, body, nil) + return cipherByte, hex.EncodeToString(salt), nil +} + +// Decrypt returns string cipher bytes by AES and GCM algorithms +func Decrypt(cipherText []byte, secret []byte) (string, error) { + if GetDecryptAlgorithm() == "NO_CRYPTO" { + log.GetLogger().Debug("decrypt algorithm is NO_CRYPTO, return plain text directly") + return string(cipherText), nil + } + cipherTexts := strings.Split(string(cipherText), ":") + if len(cipherTexts) != cipherTextsLen { + return "", fmt.Errorf("wrong cipher text") + } + + saltStr := cipherTexts[0] + encryptStr := cipherTexts[1] + + salt, err := hex.DecodeString(saltStr) + if err != nil { + return "", err + } + + encrypt, err := hex.DecodeString(encryptStr) + if err != nil { + return "", err + } + + secretData := secret + if utils.IsHexString(string(secret)) { + var err error + secretData, err = hex.DecodeString(string(secret)) + if err != nil { + return "", err + } + } + + cipherBytes, err := decryptGcmData(encrypt, secretData, salt) + if err != nil { + return "", err + } + + if cipherBytes == nil { + return "", fmt.Errorf("decrypt error") + } + + return string(cipherBytes), nil +} + +// DecryptByte returns string cipher bytes by AES and GCM algorithms +func DecryptByte(cipherText []byte, secret []byte) ([]byte, error) { + cipherTexts := strings.Split(string(cipherText), ":") + if len(cipherTexts) != cipherTextsLen { + return nil, fmt.Errorf("wrong cipher text") + } + + saltStr := cipherTexts[0] + encryptStr := cipherTexts[1] + salt, err := hex.DecodeString(saltStr) + if err != nil { + return nil, err + } + + encryptByte, err := hex.DecodeString(encryptStr) + if err != nil { + return nil, err + } + + secretData := secret + if utils.IsHexString(string(secret)) { + var err error + secretData, err = hex.DecodeString(string(secret)) + if err != nil { + return nil, err + } + } + + cipherBytes, err := decryptGcmData(encryptByte, secretData, salt) + if err != nil { + return nil, err + } + + if cipherBytes == nil { + return nil, fmt.Errorf("decrypt error") + } + + return cipherBytes, nil +} + +// decryptGcmData decrypt data with aes gcm mode +func decryptGcmData(encrypt []byte, secret []byte, salt []byte) ([]byte, error) { + block, err := aes.NewCipher(secret) + if err != nil { + return nil, err + } + + aesGcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + + decrypted, err := aesGcm.Open(nil, salt, encrypt, nil) + if err != nil { + return nil, err + } + + return decrypted, nil +} + +// decryptWorkKey Decrypt Work Key +func decryptWorkKey(workKey string, workMac string, rootKey *RootKey) (string, error) { + workKeyDecrypt, err := Decrypt([]byte(workKey), rootKey.RootKey) + if err != nil { + return "", err + } + + workKeyMac := hmacHash([]byte(workKeyDecrypt), rootKey.MacSecretKey) + if workKeyMac == workMac { + return workKeyDecrypt, nil + } + + return "", fmt.Errorf("workKey is changed") +} diff --git a/frontend/pkg/common/crypto/crypto_test.go b/frontend/pkg/common/crypto/crypto_test.go new file mode 100644 index 0000000000000000000000000000000000000000..81bdc7737108e2e44fe3d1523baf53624f5965af --- /dev/null +++ b/frontend/pkg/common/crypto/crypto_test.go @@ -0,0 +1,143 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This test file can also be used as a tool to create, encrypt and decrypt our secrets and cipher texts +package crypto + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestAll tests all processes, including creating random numbers, encryption and decryption +func TestAll(t *testing.T) { + rootKey := RootKey{} + randNum := hex.EncodeToString(createRandNum()) + fmt.Println(randNum) + rootKey.RootKey = []byte(randNum) + content := "abcd" + secret := hex.EncodeToString(createRandNum()) + fmt.Println(secret) + cipherText, err := Encrypt(content, []byte(secret)) + if err != nil { + t.Fatal(err) + } + fmt.Println(cipherText) + result, err := Decrypt(cipherText, []byte(secret)) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, content, result) +} + +// TestRandNum is also a tool to create the random number for encryption +func TestRandNum(t *testing.T) { + randNum := hex.EncodeToString(createRandNum()) + fmt.Println("randNum: " + randNum) +} + +// TestEncrypt is also a tool to generate a cipher text from a plain text and a secret +func TestEncrypt(t *testing.T) { + content := "7b83a1e330ccb177048671182f5ce1fde59c4c1c8167e8cf56190c4a5dd2c434" + secret := "f7de29fa800605cd7f490ff1d1607fffc1387f05ad8ca059868ab605d6bb6b6b" + cipherText, err := Encrypt(content, []byte(secret)) + if err != nil { + t.Fatal(err) + } + fmt.Println(string(cipherText)) +} + +// TestDecrypt is also a tool to decrypt a cipher text with a secret and get the plain text +func TestDecrypt(t *testing.T) { + cipherText := "b53df10229eead59476ae034:1feb0793e5b021511f064681827dbb8660594b31dfd90e665fa9664fdf02f1aa64304b1db66328e0b87f19c188d9e0d6487049b19a3b3aab25e3c3dcdcd22d390e020dce27af51b94ac154d137a9ce19" + secret := "f7de29fa800605cd7f490ff1d1607fffc1387f05ad8ca059868ab605d6bb6b6b" + content, err := Decrypt([]byte(cipherText), []byte(secret)) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, content, "7b83a1e330ccb177048671182f5ce1fde59c4c1c8167e8cf56190c4a5dd2c434") + fmt.Println(content) +} + +func TestDecryptError(t *testing.T) { + _, err := Decrypt([]byte("1A"), []byte("1C")) + assert.NotNil(t, err) + + _, err = Decrypt([]byte("1A:1B"), []byte("1C")) + assert.NotNil(t, err) + + _, err = Decrypt([]byte("1Z:1B"), []byte("1C")) + assert.NotNil(t, err) + + _, err = Decrypt([]byte("1A:1Z"), []byte("1C")) + assert.NotNil(t, err) + + _, err = Decrypt([]byte("1A:1B"), []byte("1Z")) + assert.NotNil(t, err) +} + +func createRandNum() []byte { + var keyLengthAES256 = 32 + initNum := make([]byte, keyLengthAES256) + _, err := rand.Read(initNum) + if err != nil { + return nil + } + return initNum +} + +func TestDecryptByte(t *testing.T) { + _, err := DecryptByte([]byte("1A"), []byte("1C")) + assert.NotNil(t, err) + + _, err = DecryptByte([]byte("1A:1B"), []byte("1C")) + assert.NotNil(t, err) +} + +func Test_encryptPBKDF2WithSHA256(t *testing.T) { + data3 := "0B6AA66FADD74F59F019109582E1AAED1EEEEA14CEDFAFCA6DB384D8C3360D5E34087FD513B16929A2567E5E184" + + "AE2B49A71B9E25E6371C91227D8CE114957D3D383EBC4899DBA7C43F6D80273E57F60B8FC918C2474CA687F1C5DBD7A71" + + "B1DC0A1EA455C7F2304A4846FD05FFD9FDD96B606546C51241A190EF8B70382ABE55" + + f := &RootKeyFactor{ + iterCount: IterKeyFactoryIter, + component3: data3, + component3byte: []byte(data3), + } + rKey := encryptPBKDF2WithSHA256(f) + assert.NotNil(t, rKey.RootKey) + + rootKey = rKey + + s := &SecretWorkKey{ + Key: "123", + Mac: "abc", + } + + data, err := s.MarshalJSON() + assert.Nil(t, err) + + err = s.UnmarshalJSON(data) + assert.Nil(t, err) +} + +func Test_EncryptGcmDataFromBody(t *testing.T) { + encryptGcmDataFromBody([]byte{}, []byte{}) +} diff --git a/frontend/pkg/common/crypto/pem_crypto.go b/frontend/pkg/common/crypto/pem_crypto.go new file mode 100644 index 0000000000000000000000000000000000000000..1db417ce8e58621de5d799ebb6762c3b93fb6f34 --- /dev/null +++ b/frontend/pkg/common/crypto/pem_crypto.go @@ -0,0 +1,180 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package crypto for auth +package crypto + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/md5" + "encoding/hex" + "encoding/pem" + "errors" + "strings" + + "frontend/pkg/common/faas_common/logger/log" +) + +// PEMCipher - +type PEMCipher int + +// Possible values for the EncryptPEMBlock encryption algorithm. +const ( + _ PEMCipher = iota + PEMCipherAES128 + PEMCipherAES192 + PEMCipherAES256 +) + +const ( + saltLength = 8 + aes128Cbc = "AES-128-CBC" + aes192Cbc = "AES-192-CBC" + aes256Cbc = "AES-256-CBC" +) + +// cipherUnit holds a method for enciphering a PEM block. +type cipherUnit struct { + cipher PEMCipher + name string + cipherFunc func(key []byte) (cipher.Block, error) + keySize int + blockSize int +} + +// cipherUnits holds a slice of cipherUnit. +var cipherUnits = []cipherUnit{{ + name: aes256Cbc, + cipher: PEMCipherAES256, + cipherFunc: aes.NewCipher, + keySize: 32, + blockSize: aes.BlockSize, +}, { + name: aes192Cbc, + cipher: PEMCipherAES192, + cipherFunc: aes.NewCipher, + keySize: 24, + blockSize: aes.BlockSize, +}, { + name: aes128Cbc, + cipher: PEMCipherAES128, + cipherFunc: aes.NewCipher, + keySize: 16, + blockSize: aes.BlockSize, +}, +} + +// deriveKey uses a key derivation function to stretch the password into a key with +// the number of bits our cipher requires. +func (c cipherUnit) deriveKey(password, salt []byte) []byte { + hash := md5.New() + out := make([]byte, c.keySize) + var digest []byte + + for i := 0; i < len(out); i += len(digest) { + hash.Reset() + _, err := hash.Write(digest) + if err != nil { + log.GetLogger().Warnf("write digest failed, err: %s", err) + } + _, err = hash.Write(password) + if err != nil { + log.GetLogger().Warnf("write password failed, err: %s", err) + } + _, err = hash.Write(salt) + if err != nil { + log.GetLogger().Warnf("write salt failed, err: %s", err) + } + digest = hash.Sum(digest[:0]) + copy(out[i:], digest) + } + return out +} + +func cipherByName(name string) *cipherUnit { + for i := range cipherUnits { + alg := &cipherUnits[i] + if alg.name == name { + return alg + } + } + return nil +} + +// IsEncryptedPEMBlock returns whether the PEM block is password encrypted according to RFC 1423. +func IsEncryptedPEMBlock(b *pem.Block) bool { + _, ok := b.Headers["DEK-Info"] + return ok +} + +// DecryptPEMBlock takes a PEM block encrypted according to RFC 1423 and the password used to encrypt +// it and returns a slice of decrypted DER encoded bytes. +func DecryptPEMBlock(b *pem.Block, pwd []byte) ([]byte, error) { + dekInfo, ok := b.Headers["DEK-Info"] + if !ok { + return nil, errors.New("crypto: no DEK-Info header in block") + } + + mode, hexIV, ok := strings.Cut(dekInfo, ",") + if !ok { + return nil, errors.New("crypto: malformed DEK-Info header") + } + + ciph := cipherByName(mode) + if ciph == nil { + return nil, errors.New("crypto: unknown encryption mode") + } + iv, err := hex.DecodeString(hexIV) + if err != nil { + return nil, err + } + if len(iv) != ciph.blockSize { + return nil, errors.New("crypto: incorrect IV size") + } + + key := ciph.deriveKey(pwd, iv[:saltLength]) + block, err := ciph.cipherFunc(key) + if err != nil { + return nil, err + } + + if len(b.Bytes)%block.BlockSize() != 0 { + return nil, errors.New("crypto: encrypted PEM data is not a multiple of the block size") + } + + data := make([]byte, len(b.Bytes)) + dec := cipher.NewCBCDecrypter(block, iv) + dec.CryptBlocks(data, b.Bytes) + + dataLen := len(data) + if dataLen == 0 || dataLen%ciph.blockSize != 0 { + return nil, errors.New("crypto: invalid padding") + } + last := int(data[dataLen-1]) + if dataLen < last { + return nil, errors.New("crypto: decryption password incorrect") + } + if last == 0 || last > ciph.blockSize { + return nil, errors.New("crypto: decryption password incorrect") + } + for _, val := range data[dataLen-last:] { + if int(val) != last { + return nil, errors.New("crypto: decryption password incorrect") + } + } + return data[:dataLen-last], nil +} diff --git a/frontend/pkg/common/crypto/scc_constants.go b/frontend/pkg/common/crypto/scc_constants.go new file mode 100644 index 0000000000000000000000000000000000000000..d5e5586ec371b8df41c164541679c21e82493334 --- /dev/null +++ b/frontend/pkg/common/crypto/scc_constants.go @@ -0,0 +1,47 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package crypto for auth +package crypto + +import ( + "sync" +) + +var ( + sccInitialized bool = false + m sync.RWMutex +) + +const ( + // Aes128Gcm - + Aes128Gcm = "AES128_GCM" + // Aes256Gcm - + Aes256Gcm = "AES256_GCM" + // Aes256Cbc - + Aes256Cbc = "AES256_CBC" + // Sm4Cbc - + Sm4Cbc = "SM4_CBC" + // Sm4Ctr - + Sm4Ctr = "SM4_CTR" +) + +// SccConfig - +type SccConfig struct { + Enable bool `json:"enable" valid:"optional"` + Algorithm string `json:"algorithm" valid:"optional"` + SccConfigPath string `json:"sccConfigPath" valid:"optional"` +} diff --git a/frontend/pkg/common/crypto/scc_crypto.go b/frontend/pkg/common/crypto/scc_crypto.go new file mode 100644 index 0000000000000000000000000000000000000000..03f3438341069eaabbef82e77f601f4c48dc2e57 --- /dev/null +++ b/frontend/pkg/common/crypto/scc_crypto.go @@ -0,0 +1,115 @@ +//go:build cryptoapi +// +build cryptoapi + +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package crypto for auth +package crypto + +import ( + "cryptoapi" + "fmt" + "path" + + "frontend/pkg/common/faas_common/logger/log" +) + +// SCCInitialized - +func SCCInitialized() bool { + m.RLock() + defer m.RUnlock() + return sccInitialized +} + +// GetSCCAlgorithm - +func GetSCCAlgorithm(algorithm string) int { + switch algorithm { + case Aes128Gcm: + return cryptoapi.ALG_AES128_GCM + case Aes256Gcm: + return cryptoapi.ALG_AES256_GCM + case Aes256Cbc: + return cryptoapi.ALG_AES256_CBC + case Sm4Cbc: + return cryptoapi.ALG_SM4_CBC + case Sm4Ctr: + return cryptoapi.ALG_SM4_CTR + default: + return cryptoapi.ALG_AES256_GCM + } +} + +// InitializeSCC - +func InitializeSCC(config SccConfig) bool { + m.Lock() + defer m.Unlock() + if !config.Enable { + return true + } + options := cryptoapi.NewSccOptions() + const configPath = "/home/sn/resource/scc" + sccConfigPath := config.SccConfigPath + if sccConfigPath == "" { + sccConfigPath = configPath + } + options.PrimaryKeyFile = path.Join(sccConfigPath, "primary.ks") + options.StandbyKeyFile = path.Join(sccConfigPath, "standby.ks") + options.LogPath = "/tmp/log/" + options.LogFile = "scc" + options.DefaultAlgorithm = GetSCCAlgorithm(config.Algorithm) + options.RandomDevice = "/dev/random" + options.EnableChangeFilePermission = 0 + cryptoapi.Finalize() + err := cryptoapi.InitializeWithConfig(options) + if err != nil { + fmt.Printf("failed to initialize crypto, Error = [%s]\n", err.Error()) + log.GetLogger().Errorf("Initialize SCC Error = [%s]", err.Error()) + return false + } + sccInitialized = true + return true +} + +// FinalizeSCC - +func FinalizeSCC() { + m.Lock() + defer m.Unlock() + sccInitialized = false + cryptoapi.Finalize() +} + +// SCCDecrypt - +func SCCDecrypt(cipher []byte) (string, error) { + plain, err := cryptoapi.Decrypt(string(cipher)) + if err != nil { + log.GetLogger().Errorf("SCC Decrypt Error = [%s]", err.Error()) + return "", err + } + + return plain, nil +} + +// SCCEncrypt - +func SCCEncrypt(plainInput string) ([]byte, error) { + cipher, err := cryptoapi.Encrypt(plainInput) + if err != nil { + log.GetLogger().Errorf("SCC Encrypt Error = [%s]", err.Error()) + return nil, err + } + + return []byte(cipher), nil +} diff --git a/frontend/pkg/common/crypto/scc_crypto_fake.go b/frontend/pkg/common/crypto/scc_crypto_fake.go new file mode 100644 index 0000000000000000000000000000000000000000..bdf6219775010010da4f06ee4c1df6f0f9d30438 --- /dev/null +++ b/frontend/pkg/common/crypto/scc_crypto_fake.go @@ -0,0 +1,66 @@ +//go:build !cryptoapi +// +build !cryptoapi + +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package crypto for auth +package crypto + +// SCCInitialized - +func SCCInitialized() bool { + return false +} + +// GetSCCAlgorithm - +func GetSCCAlgorithm(algorithm string) int { + return 0 +} + +// InitializeSCC - +func InitializeSCC(config SccConfig) bool { + return false +} + +// FinalizeSCC - +func FinalizeSCC() { +} + +// SCCDecrypt - +func SCCDecrypt(cipher []byte) (string, error) { + return "", nil +} + +// SCCEncrypt - +func SCCEncrypt(plainInput string) ([]byte, error) { + return []byte{}, nil +} diff --git a/frontend/pkg/common/crypto/scc_crypto_test.go b/frontend/pkg/common/crypto/scc_crypto_test.go new file mode 100644 index 0000000000000000000000000000000000000000..d37d693c61d6a98703e6ddb3a2ed620fb1db1e39 --- /dev/null +++ b/frontend/pkg/common/crypto/scc_crypto_test.go @@ -0,0 +1,83 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This test file can also be used as a tool to create, encrypt and decrypt our secrets and cipher texts +package crypto + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSCCEncryptDecryptInitialized(t *testing.T) { + var c = SccConfig{ + Enable: true, + Algorithm: "AES256_GCM", + } + ret := InitializeSCC(c) + assert.True(t, ret) + input := "text to encrypt" + encrypted, err := SCCEncrypt(input) + fmt.Printf("encrypted : %s\n", string(encrypted)) + assert.Nil(t, err) + decrypt, err := SCCDecrypt(encrypted) + fmt.Printf("decrypt : %s\n", decrypt) + assert.Nil(t, err) + assert.Equal(t, input, decrypt) + assert.NotEqual(t, encrypted, input) + FinalizeSCC() +} + +func TestSCCEncryptDecryptNotInitialized(t *testing.T) { + var c = SccConfig{ + Enable: false, + Algorithm: "AES256_GCM", + } + ret := InitializeSCC(c) + assert.True(t, ret) + input := "text to encrypt" + encrypted, _ := SCCEncrypt(input) + fmt.Printf("encrypted : %s\n", string(encrypted)) + decrypt, _ := SCCDecrypt(encrypted) + fmt.Printf("decrypt : %s\n", decrypt) + FinalizeSCC() +} + +func TestSCCEncryptDecryptAlgorithms(t *testing.T) { + var c = SccConfig{ + Enable: true, + Algorithm: "AES256_GCM", + } + + algorithms := []string{"AES256_CBC", "AES128_GCM", "AES256_GCM", "SM4_CBC", "SM4_CTR", "DEFAULT"} + for _, algo := range algorithms { + FinalizeSCC() + c.Algorithm = algo + ret := InitializeSCC(c) + assert.True(t, ret) + input := "text to encrypt" + encrypted, err := SCCEncrypt(input) + fmt.Printf("encrypted : %s\n", string(encrypted)) + assert.Nil(t, err) + decrypt, err := SCCDecrypt(encrypted) + fmt.Printf("decrypt : %s\n", decrypt) + assert.Nil(t, err) + assert.Equal(t, input, decrypt) + assert.NotEqual(t, encrypted, input) + } +} diff --git a/frontend/pkg/common/crypto/types.go b/frontend/pkg/common/crypto/types.go new file mode 100644 index 0000000000000000000000000000000000000000..cff85a268454574c1d78df88113c8b707a01a80e --- /dev/null +++ b/frontend/pkg/common/crypto/types.go @@ -0,0 +1,300 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package crypto for auth +package crypto + +import ( + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "path" + "sync" + + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/reader" + "frontend/pkg/common/utils" +) + +const ( + // byteSize defines the key length + byteSize = 32 + queryElementLem = 2 + saltDataMinLength = 16 + // IterKeyFactoryIter is the iter Count of Root Key Factor + IterKeyFactoryIter = 10000 + apple = "apple" + boy = "boy" + cat = "cat" + dog = "dog" + egg = "egg" + fish = "fish" + wdo = "wdo" + KeyFactorNums = 5 +) + +var ( + rootKeyOnce sync.Once + workKeyOnce sync.Once + + workKey []byte + rootKey *RootKey +) + +// set root key factor +func buildRootKeyFactor(f *RootKeyFactor) error { + resourcePath := utils.GetResourcePath() + k1Path := path.Join(resourcePath, "rdo", "v1", apple, "a.txt") + k2Path := path.Join(resourcePath, "rdo", "v1", boy, "b.txt") + macPath := path.Join(resourcePath, "rdo", "v1", cat, "c.txt") + saltPath := path.Join(resourcePath, "rdo", "v1", dog, "d.txt") + + // k1Data + k1Data, err := reader.ReadFileWithTimeout(k1Path) + if err != nil { + return err + } + f.k1Data, err = hex.DecodeString(string(k1Data)) + if err != nil { + return err + } + + // k2Data + k2Data, err := reader.ReadFileWithTimeout(k2Path) + if err != nil { + return err + } + f.k2Data, err = hex.DecodeString(string(k2Data)) + if err != nil { + return err + } + + // macData + macData, err := reader.ReadFileWithTimeout(macPath) + if err != nil { + return err + } + f.macData, err = hex.DecodeString(string(macData)) + if err != nil { + return err + } + + // saltData + saltData, err := reader.ReadFileWithTimeout(saltPath) + if len(saltData) < saltDataMinLength { + return fmt.Errorf("invalid salt data length of %d", len(saltData)) + } + if err != nil { + return err + } + if f.saltData, err = hex.DecodeString(string(saltData)); err != nil { + return err + } + return nil +} + +// LoadRootKey Load Root Key +func LoadRootKey() (*RootKey, error) { + // k3 + resourcePath := utils.GetResourcePath() + k3Path := path.Join(resourcePath, "rdo", "v1", egg, "e.txt") + // k1Data + k3Data, err := reader.ReadFileWithTimeout(k3Path) + k3DataDecode, err := hex.DecodeString(string(k3Data)) + if err != nil { + return nil, err + } + f := &RootKeyFactor{ + // 10000 is the iter Count of Root Key Factor + iterCount: IterKeyFactoryIter, + component3: string(k3DataDecode), + component3byte: k3DataDecode, + } + err = buildRootKeyFactor(f) + if err != nil { + return nil, err + } + rootKey := encryptPBKDF2WithSHA256(f) + return rootKey, nil +} + +// LoadRootKeyWithKeyFactor Load Root Key With Key Factor +func LoadRootKeyWithKeyFactor(keyFactor []string) (*RootKey, error) { + if len(keyFactor) < KeyFactorNums { + return nil, errors.New("short key factors") + } + var err error + k3Data := keyFactor[2] + k3DataDecode, err := hex.DecodeString(k3Data) + f := &RootKeyFactor{ + // 10000 is the iter Count of Root Key Factor + iterCount: IterKeyFactoryIter, + component3: string(k3DataDecode), + component3byte: k3DataDecode, + } + f.k1Data, err = hex.DecodeString(keyFactor[0]) + if err != nil { + return nil, err + } + f.k2Data, err = hex.DecodeString(keyFactor[1]) + if err != nil { + return nil, err + } + f.macData, err = hex.DecodeString(keyFactor[3]) + if err != nil { + return nil, err + } + if f.saltData, err = hex.DecodeString(keyFactor[4]); err != nil { + return nil, err + } + rootKey := encryptPBKDF2WithSHA256(f) + return rootKey, nil +} + +// RootKey include RootKey and MacSecretKey +type RootKey struct { + RootKey []byte + MacSecretKey []byte +} + +// RootKeyFactor include Root Key Factor +type RootKeyFactor struct { + k1Data []byte + k2Data []byte + macData []byte + saltData []byte + iterCount int + component3 string + component3byte []byte +} + +// WorkKeys define Work Keys +type WorkKeys map[string]*SecretNamedWorkKeys + +// GetKeyByName Get Key By Name +func (k *WorkKeys) GetKeyByName(name string) *SecretWorkKey { + namedKey, exist := (*k)[name] + if !exist { + return nil + } + + return namedKey.Keys +} + +// SecretNamedWorkKeys include Keys and Description +type SecretNamedWorkKeys struct { + Keys *SecretWorkKey `json:"keys"` + Description string `json:"description"` +} + +// SecretWorkKey include Key and Mac +type SecretWorkKey struct { + Key string `json:"key"` + Mac string `json:"mac"` +} + +// MarshalJSON Marshal JSON +func (s *SecretWorkKey) MarshalJSON() ([]byte, error) { + if rootKey == nil || rootKey.RootKey == nil || rootKey.MacSecretKey == nil { + return nil, fmt.Errorf("rootKey is nil") + } + + key, err := Encrypt(s.Key, []byte(hex.EncodeToString(rootKey.RootKey))) + if err != nil { + return nil, err + } + mac := hmacHash([]byte(s.Key), rootKey.MacSecretKey) + + type SecretWorkKeyJSON SecretWorkKey + + return json.Marshal(SecretWorkKeyJSON(SecretWorkKey{ + Key: string(key), Mac: mac})) +} + +// UnmarshalJSON Unmarshal JSON +func (s *SecretWorkKey) UnmarshalJSON(data []byte) error { + + type SecretWorkKeyJSON SecretWorkKey + + err := json.Unmarshal(data, (*SecretWorkKeyJSON)(s)) + if err != nil { + return err + } + + key, err := decryptWorkKey(s.Key, s.Mac, rootKey) + if err != nil { + return err + } + + s.Key = key + + return nil +} + +// Signature define Signature +type Signature struct { + Method []byte + Path []byte + QueryStr string + Body []byte + AppID []byte + CurTimeTamp []byte +} + +// GetRootKey Get Root Key +func GetRootKey() []byte { + rootKeyOnce.Do(func() { + rk, err := LoadRootKey() + if err != nil { + log.GetLogger().Errorf("failed to load rootKey, err: %s", err.Error()) + return + } + rootKey = rk + }) + + if rootKey == nil { + log.GetLogger().Errorf("root key is nil") + return []byte{} + } + return []byte(hex.EncodeToString(rootKey.RootKey)) +} + +// LoadWorkKey Load work Key +func LoadWorkKey() ([]byte, error) { + resourcePath := utils.GetResourcePath() + workKeyPath := path.Join(resourcePath, "rdo", "v1", fish, "f.txt") + workKey, err := reader.ReadFileWithTimeout(workKeyPath) + return workKey, err +} + +// GetWorkKey Get Work Key +func GetWorkKey() []byte { + workKeyOnce.Do(func() { + wk, err := LoadWorkKey() + if err != nil { + log.GetLogger().Errorf("failed to load workKey, err: %s", err.Error()) + return + } + workKey = wk + }) + + if workKey == nil { + log.GetLogger().Errorf("work key is nil") + return []byte{} + } + return workKey +} diff --git a/frontend/pkg/common/crypto/types_test.go b/frontend/pkg/common/crypto/types_test.go new file mode 100644 index 0000000000000000000000000000000000000000..b8482fa84cf2ec6edddee4d8ea67c7a7096d3dd2 --- /dev/null +++ b/frontend/pkg/common/crypto/types_test.go @@ -0,0 +1,47 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package crypto + +import ( + "testing" + + "github.com/agiledragon/gomonkey" +) + +func TestGetKeyByName(t *testing.T) { + k := &WorkKeys{} + k.GetKeyByName("") + + (*k)[""] = &SecretNamedWorkKeys{} + k.GetKeyByName("") +} + +func TestLoadRootKeyWithKeyFactor(t *testing.T) { + LoadRootKeyWithKeyFactor([]string{""}) + LoadRootKeyWithKeyFactor([]string{"", "", "", "", ""}) +} + +// TestGetWorkKey is also a tool to get the work key from the pre-set resource path +func TestGetWorkKey(t *testing.T) { + GetRootKey() + + patch := gomonkey.ApplyFunc(LoadRootKey, func() (*RootKey, error) { + return nil, nil + }) + GetRootKey() + patch.Reset() +} diff --git a/frontend/pkg/common/faas_common/alarm/config.go b/frontend/pkg/common/faas_common/alarm/config.go new file mode 100644 index 0000000000000000000000000000000000000000..5ed118df0f0ace62f70db2dbdbba95c332a47a37 --- /dev/null +++ b/frontend/pkg/common/faas_common/alarm/config.go @@ -0,0 +1,32 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package alarm alarm log by filebeat +package alarm + +import ( + "frontend/pkg/common/faas_common/logger/config" + "frontend/pkg/common/faas_common/types" +) + +// Config - +type Config struct { + EnableAlarm bool `json:"enableAlarm"` + AlarmLogConfig config.CoreInfo `json:"alarmLogConfig" valid:"optional"` + XiangYunFourConfig types.XiangYunFourConfig `json:"xiangYunFourConfig" valid:"optional"` + MinInsStartInterval int `json:"minInsStartInterval"` + MinInsCheckInterval int `json:"minInsCheckInterval"` +} diff --git a/frontend/pkg/common/faas_common/alarm/logalarm.go b/frontend/pkg/common/faas_common/alarm/logalarm.go new file mode 100644 index 0000000000000000000000000000000000000000..5ef2ee9b0ebca54e2b111a13a395a2e89be8eb9c --- /dev/null +++ b/frontend/pkg/common/faas_common/alarm/logalarm.go @@ -0,0 +1,242 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package alarm alarm log by filebeat +package alarm + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strconv" + "sync" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger" + "frontend/pkg/common/faas_common/logger/config" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/types" + "frontend/pkg/common/faas_common/urnutils" +) + +const ( + // ConfigKey environment variable key of alarm config + ConfigKey = "ALARM_CONFIG" + + cacheLimit = 10 * 1 << 20 // 10 mb + + // Level3 - + Level3 = "critical" + // Level2 - + Level2 = "major" + // Level1 - + Level1 = "minor" + // Level0 - + Level0 = "notice" + + // GenerateAlarmLog - + GenerateAlarmLog = "firing" + // ClearAlarmLog - + ClearAlarmLog = "resolved" + + // InsufficientMinInstance00001 alarm id + InsufficientMinInstance00001 = "InsufficientMinInstance00001" + // MetadataEtcdConnection00001 alarm id + MetadataEtcdConnection00001 = "MetadataEtcdConnection00001" + // RouterEtcdConnection00001 alarm id + RouterEtcdConnection00001 = "RouterEtcdConnection00001" + // InitStsSdkErr00001 alarm id + InitStsSdkErr00001 = "InitStsSdkErr00001" + // PullStsConfiguration00001 alarm id + PullStsConfiguration00001 = "PullStsConfiguration00001" + // ReportToXPUManageFailed00001 alarm id + ReportToXPUManageFailed00001 = "ReportToXPUManageFailed00001" + // FaaSSchedulerRemovedFromHashRing00001 alarm id + FaaSSchedulerRemovedFromHashRing00001 = "FaaSSchedulerRemovedFromHashRing00001" + // FaaSFrontendReceiptDMQMessage00001 - + FaaSFrontendReceiptDMQMessage00001 = "FaaSFrontendReceiptDMQMessage00001" + // FaaSFrontendDequeueDMQMessage00001 - + FaaSFrontendDequeueDMQMessage00001 = "FaaSFrontendDequeueDMQMessage00001" + // NoAvailableSchedulerInstance00001 没有可用的scheduler实例的告警id + NoAvailableSchedulerInstance00001 = "NoAvailableSchedulerInstance00001" +) + +var ( + alarmLogger *zap.Logger + createLoggerErr error + createLoggerOnce sync.Once +) + +// LogAlarmInfo Custom alarm info +type LogAlarmInfo struct { + AlarmID string + AlarmName string + AlarmLevel string +} + +// Detail alarm detail +type Detail struct { + SourceTag string // 告警来源 + OpType string // 告警操作类型 + Details string // 告警详情 + StartTimestamp int // 产生时间 + EndTimestamp int // 清除时间 +} + +// GetAlarmLogger - +func GetAlarmLogger() (*zap.Logger, error) { + createLoggerOnce.Do(func() { + alarmLogger, createLoggerErr = newAlarmLogger() + if createLoggerErr != nil { + return + } + if alarmLogger == nil { + createLoggerErr = errors.New("failed to new alarmLogger") + return + } + // 祥云四元组 - 站点/租户ID/产品ID/服务ID + alarmLogger = alarmLogger.With(zapcore.Field{ + Key: "site", Type: zapcore.StringType, + String: os.Getenv(constant.WiseCloudSite), + }, zapcore.Field{ + Key: "tenant_id", Type: zapcore.StringType, + String: os.Getenv(constant.TenantID), + }, zapcore.Field{ + Key: "application_id", Type: zapcore.StringType, + String: os.Getenv(constant.ApplicationID), + }, zapcore.Field{ + Key: "service_id", Type: zapcore.StringType, + String: os.Getenv(constant.ServiceID), + }) + }) + return alarmLogger, createLoggerErr +} + +func newAlarmLogger() (*zap.Logger, error) { + coreInfo, err := config.ExtractCoreInfoFromEnv(ConfigKey) + log.GetLogger().Infof("ALARM_CONFIG is: %v", coreInfo) + if err != nil { + log.GetLogger().Errorf("failed to valid log path, err: %s", err.Error()) + return nil, err + } + + coreInfo.FilePath = filepath.Join(coreInfo.FilePath, "alarm.dat") + + sink, err := logger.CreateSink(coreInfo) + if err != nil { + log.GetLogger().Errorf("failed to create sink: %s", err.Error()) + return nil, err + } + + ws := zapcore.AddSync(sink) + priority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool { + return lvl >= zapcore.DebugLevel + }) + encoderConfig := zapcore.EncoderConfig{} + rollingFileEncoder := zapcore.NewJSONEncoder(encoderConfig) + + return zap.New(zapcore.NewCore(rollingFileEncoder, ws, priority)), nil +} + +func addAlarmLogger(rollingLogger *zap.Logger, alarmInfo *LogAlarmInfo, detail *Detail) *zap.Logger { + return rollingLogger.With(zapcore.Field{ + Key: "id", Type: zapcore.StringType, + String: alarmInfo.AlarmID, + }, zapcore.Field{ + Key: "name", Type: zapcore.StringType, + String: alarmInfo.AlarmName, + }, zapcore.Field{ + Key: "level", Type: zapcore.StringType, + String: alarmInfo.AlarmLevel, + }, zapcore.Field{ + Key: "source_tag", Type: zapcore.StringType, + String: detail.SourceTag, + }, zapcore.Field{ + Key: "op_type", Type: zapcore.StringType, + String: detail.OpType, + }, zapcore.Field{ + Key: "details", Type: zapcore.StringType, + String: detail.Details, + }, zapcore.Field{ + Key: "clear_type", Type: zapcore.StringType, + String: "ADAC", + }, zapcore.Field{ + Key: "start_timestamp", Type: zapcore.StringType, + String: strconv.Itoa(detail.StartTimestamp), + }, zapcore.Field{ + Key: "end_timestamp", Type: zapcore.StringType, + String: strconv.Itoa(detail.EndTimestamp), + }) +} + +// ReportOrClearAlarm - +func ReportOrClearAlarm(alarmInfo *LogAlarmInfo, detail *Detail) { + alarmLog, err := GetAlarmLogger() + if err != nil { + log.GetLogger().Errorf("GetAlarmLogger err %v", err) + return + } + logger := addAlarmLogger(alarmLog, alarmInfo, detail) + logger.Info("") +} + +// SetAlarmEnv - +func SetAlarmEnv(alarmConfigInfo config.CoreInfo) { + alarmConfigBytes, err := json.Marshal(alarmConfigInfo) + if err != nil { + log.GetLogger().Errorf("json marshal alarmConfigInfo err %v", err) + } + if err := os.Setenv(ConfigKey, string(alarmConfigBytes)); err != nil { + log.GetLogger().Errorf("failed to set env of %s, err: %s", ConfigKey, err.Error()) + } + log.GetLogger().Debugf("succeeded to set env of %s, value: %s", ConfigKey, string(alarmConfigBytes)) +} + +// SetXiangYunFourConfigEnv - +func SetXiangYunFourConfigEnv(xiangYunFourConfig types.XiangYunFourConfig) { + if err := os.Setenv(constant.WiseCloudSite, xiangYunFourConfig.Site); err != nil { + log.GetLogger().Errorf("failed to set env of %s, err: %s", constant.WiseCloudSite, err.Error()) + } + if err := os.Setenv(constant.TenantID, xiangYunFourConfig.TenantID); err != nil { + log.GetLogger().Errorf("failed to set env of %s, err: %s", constant.TenantID, err.Error()) + } + if err := os.Setenv(constant.ApplicationID, xiangYunFourConfig.ApplicationID); err != nil { + log.GetLogger().Errorf("failed to set env of %s, err: %s", constant.ApplicationID, err.Error()) + } + if err := os.Setenv(constant.ServiceID, xiangYunFourConfig.ServiceID); err != nil { + log.GetLogger().Errorf("failed to set env of %s, err: %s", constant.ServiceID, err.Error()) + } + log.GetLogger().Debugf("succeeded to set env, value: %v", xiangYunFourConfig) +} + +// SetPodIP - +func SetPodIP() error { + ip, err := urnutils.GetServerIP() + if err != nil { + log.GetLogger().Errorf("failed to get pod ip, err: %s", err.Error()) + return err + } + err = os.Setenv(constant.PodIPEnvKey, ip) + if err != nil { + log.GetLogger().Errorf("failed to set env of %s, err: %s", constant.PodIPEnvKey, err.Error()) + return err + } + return nil +} diff --git a/frontend/pkg/common/faas_common/alarm/logalarm_test.go b/frontend/pkg/common/faas_common/alarm/logalarm_test.go new file mode 100644 index 0000000000000000000000000000000000000000..742b3458bf27494a21154e58ca8c3c02c1216c7e --- /dev/null +++ b/frontend/pkg/common/faas_common/alarm/logalarm_test.go @@ -0,0 +1,85 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package alarm +package alarm + +import ( + "encoding/json" + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/config" + "frontend/pkg/common/faas_common/urnutils" + "github.com/smartystreets/goconvey/convey" + "os" + "sync" + "testing" +) + +func TestGetAlarmLogger(t *testing.T) { + convey.Convey("TestGetAlarmLogger", t, func() { + convey.Convey("failed to new alarmLogger", func() { + logger, err := GetAlarmLogger() + convey.So(err, convey.ShouldBeError) + convey.So(logger, convey.ShouldBeNil) + }) + + convey.Convey("success", func() { + dir, _ := os.Getwd() + defer gomonkey.ApplyFunc(config.ExtractCoreInfoFromEnv, func(env string) (config.CoreInfo, error) { + return config.CoreInfo{FilePath: dir}, nil + }).Reset() + createLoggerOnce = sync.Once{} + logger, err := GetAlarmLogger() + convey.So(err, convey.ShouldBeNil) + convey.So(logger, convey.ShouldNotBeNil) + }) + }) +} + +func TestReportOrClearAlarm(t *testing.T) { + convey.Convey("ReportOrClearAlarm", t, func() { + convey.Convey("no test assert", func() { + ReportOrClearAlarm(&LogAlarmInfo{}, &Detail{}) + }) + }) +} + +func TestSetAlarmEnv(t *testing.T) { + convey.Convey("SetAlarmEnv", t, func() { + convey.Convey("set env", func() { + dir, _ := os.Getwd() + SetAlarmEnv(config.CoreInfo{FilePath: dir}) + getenv := os.Getenv(ConfigKey) + var cfg *config.CoreInfo + err := json.Unmarshal([]byte(getenv), &cfg) + convey.So(err, convey.ShouldBeNil) + convey.So(cfg.FilePath, convey.ShouldEqual, dir) + os.Unsetenv(ConfigKey) + }) + }) +} + +func TestSetPodIP(t *testing.T) { + convey.Convey("SetPodIP", t, func() { + convey.Convey("", func() { + ip, _ := urnutils.GetServerIP() + SetPodIP() + convey.So(os.Getenv(constant.PodIPEnvKey), convey.ShouldEqual, ip) + os.Unsetenv(constant.PodIPEnvKey) + }) + }) + +} diff --git a/frontend/pkg/common/faas_common/aliasroute/alias.go b/frontend/pkg/common/faas_common/aliasroute/alias.go new file mode 100644 index 0000000000000000000000000000000000000000..629222f695ccb7c2eddd3dd4754ba9ad6d90f7d6 --- /dev/null +++ b/frontend/pkg/common/faas_common/aliasroute/alias.go @@ -0,0 +1,503 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package aliasroute alias routing in busclient +package aliasroute + +import ( + "encoding/json" + "errors" + "fmt" + "reflect" + "strconv" + "strings" + "sync" + + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/loadbalance" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/urnutils" +) + +const ( + weightRatio = 100 // max weight of a node + routingTypeRule = "rule" + // AliasKeySeparator is the separator in an alias key + AliasKeySeparator = "/" +) + +const ( + defaultVersion = "latest" + defaultBusinessID = "yrk" +) + +// example of an aliasKey: +// ///////// +const ( + ProductIDIndex = iota + 1 + AliasSignIndex + BusinessSignIndex + BusinessIDIndex + TenantSignIndex + TenantIDIndex + FunctionSignIndex + FunctionIDIndex + AliasNameIndex + aliasKeyLength +) + +// Aliases map for stateless function alias +type Aliases struct { + AliasMap *sync.Map // Key: aliasURN -- Value: *AliasElement +} + +// aliases for alias routing +var ( + aliases = &Aliases{ + AliasMap: &sync.Map{}, + } +) + +// GetAliases - +func GetAliases() *Aliases { + return aliases +} + +// AddAlias add alias to Aliases map from etcd +func (a *Aliases) AddAlias(alias *AliasElement) { + existAliasIf, exist := a.AliasMap.Load(alias.AliasURN) + var existAlias *AliasElement + var ok bool + if !exist { + // new alias, initialize RR and Mutex + existAlias = &AliasElement{ + AliasURN: alias.AliasURN, + FunctionURN: alias.FunctionURN, + FunctionVersionURN: alias.FunctionVersionURN, + Name: alias.Name, + Description: alias.Description, + FunctionVersion: alias.FunctionVersion, + RevisionID: alias.RevisionID, + RoutingConfigs: alias.RoutingConfigs, + RoutingRules: alias.RoutingRules, + RoutingType: alias.RoutingType, + + lb: loadbalance.LBFactory(loadbalance.RoundRobinNginx), + aliasLock: &sync.RWMutex{}, + } + existAlias.resetRR() + a.AliasMap.Store(alias.AliasURN, existAlias) + return + } + existAlias, ok = existAliasIf.(*AliasElement) + if ok { + aliasUpdate(existAlias, alias) + existAlias.resetRR() + } +} + +// RemoveAlias remove alias to aliases map +func (a *Aliases) RemoveAlias(aliasURN string) { + a.AliasMap.Delete(aliasURN) +} + +// GetFuncURNFromAlias If the alias exists, the weighted route version is returned. +// If the alias does not exist, the original URN is returned. +func (a *Aliases) GetFuncURNFromAlias(urn string) string { + existAliasIf, exist := a.AliasMap.Load(urn) + if !exist { + return urn + } + existAlias, ok := existAliasIf.(*AliasElement) + if !ok { + log.GetLogger().Warnf("Failed to convert the alias urn %s", urn) + return "" + } + return existAlias.getFuncVersionURN() +} + +// GetFuncVersionURNWithParams gets the routing version URN of stateless functionName with parmas for rules +func (a *Aliases) GetFuncVersionURNWithParams(aliasURN string, params map[string]string) string { + existAliasIf, exist := a.AliasMap.Load(aliasURN) + if !exist { + return aliasURN + } + existAlias, ok := existAliasIf.(*AliasElement) + if !ok { + log.GetLogger().Warnf("Failed to convert the alias urn %s", aliasURN) + return "" + } + return existAlias.GetFuncVersionURNWithParams(params) +} + +// CheckAliasRoutingChange - return false means oldURN is not equal to newURN or alise is not exist +func (a *Aliases) CheckAliasRoutingChange(aliasURN, oldURN string, params map[string]string) bool { + existAliasIf, exist := a.AliasMap.Load(aliasURN) + if !exist { + return true + } + existAlias, ok := existAliasIf.(*AliasElement) + if ok && existAlias.RoutingType == routingTypeRule { + return oldURN != existAlias.getFuncVersionURNByRule(params) + } + // routingType is weight + for _, config := range existAlias.RoutingConfigs { + if config.FunctionVersionURN == oldURN && config.Weight > 0.0 { + return false + } + } + return true +} + +// GetAliasRoutingType - +func (a *Aliases) GetAliasRoutingType(aliasURN string) string { + existAliasIf, exist := a.AliasMap.Load(aliasURN) + if !exist { + return "" + } + if existAlias, ok := existAliasIf.(*AliasElement); ok { + return existAlias.RoutingType + } + return "" +} + +// change means the following 3 conditions +func isAliasWeightTypeChange(originAlias, srcAlias *AliasElement) map[string]int { + changedURNMap := make(map[string]int) + if originAlias.RoutingType == routingTypeRule || srcAlias.RoutingType == routingTypeRule { + return map[string]int{"": NoneUpdate} + } + // 1、delete weight alias + if len(srcAlias.RoutingConfigs) == 0 { + return map[string]int{"": UpdateAllURN} + } + srcAliasMap := make(map[string]float64, len(srcAlias.RoutingConfigs)) + for _, config := range srcAlias.RoutingConfigs { + if config.Weight <= 0 { + // 2、weight decrease to 0 + changedURNMap[config.FunctionVersionURN] = UpdateWeightGreyURN + } + srcAliasMap[config.FunctionVersionURN] = config.Weight + } + + for _, config := range originAlias.RoutingConfigs { + // 3、grey functionURN in originAlias is not in srcAlias + // old device still follow old urn, new device follow the weight + if _, ok := srcAliasMap[config.FunctionVersionURN]; !ok { + changedURNMap[config.FunctionVersionURN] = UpdateWeightGreyURN + } + } + if originAlias.FunctionVersion != srcAlias.FunctionVersion { + changedURNMap[originAlias.FunctionVersion] = UpdateMainURN + } + return changedURNMap +} + +type routingRules struct { + RuleLogic string `json:"ruleLogic"` + Rules []string `json:"rules"` + GrayVersion string `json:"grayVersion"` +} + +// AliasElement struct stores an alias configs of stateless function +type AliasElement struct { + aliasLock *sync.RWMutex + lb loadbalance.LoadBalance + AliasURN string `json:"aliasUrn"` + FunctionURN string `json:"functionUrn"` + FunctionVersionURN string `json:"functionVersionUrn"` + Name string `json:"name"` + FunctionVersion string `json:"functionVersion"` + RevisionID string `json:"revisionId"` + Description string `json:"description"` + RoutingType string `json:"routingType"` + RoutingRules routingRules `json:"routingRules"` + RoutingConfigs []*routingConfig `json:"routingconfig"` +} + +type routingConfig struct { + FunctionVersionURN string `json:"functionVersionUrn"` + Weight float64 `json:"weight"` +} + +func (a *AliasElement) getFuncVersionURN() string { + a.aliasLock.RLock() + defer a.aliasLock.RUnlock() + funcVersion := a.lb.Next("", true) + if funcVersion == nil { + return "" + } + res, ok := funcVersion.(string) + if !ok { + return "" + } + return res +} + +func (a *AliasElement) resetRR() { + a.aliasLock.Lock() + defer a.aliasLock.Unlock() + a.lb.RemoveAll() + for _, v := range a.RoutingConfigs { + a.lb.Add(v.FunctionVersionURN, int(v.Weight*weightRatio)) + } +} + +func (a *AliasElement) getFuncVersionURNByRule(params map[string]string) string { + a.aliasLock.RLock() + defer a.aliasLock.RUnlock() + if len(params) == 0 { + log.GetLogger().Warnf("params is empty, use default func version") + return a.FunctionVersionURN + } + if len(a.RoutingRules.Rules) == 0 { + log.GetLogger().Warnf("rule len is 0, use default func version") + return a.FunctionVersionURN + } + + matchRules, err := parseRules(a.RoutingRules) + if err != nil { + log.GetLogger().Warnf("parse rule error, use default func version: %s", err.Error()) + return a.FunctionVersionURN + } + + // To obtain the final matching result by matching each rule and considering the "AND" or "OR"relationship of the rules + matched := matchRule(params, matchRules, a.RoutingRules.RuleLogic) + // got to default version if not matched + if matched { + return a.RoutingRules.GrayVersion + } + return a.FunctionVersionURN +} + +// GetFuncVersionURNWithParams - +func (a *AliasElement) GetFuncVersionURNWithParams(params map[string]string) string { + if a.RoutingType == routingTypeRule { + return a.getFuncVersionURNByRule(params) + } + // default to go weight + return a.getFuncVersionURN() +} + +func aliasUpdate(destAlias, srcAlias *AliasElement) { + destAlias.AliasURN = srcAlias.AliasURN + destAlias.FunctionURN = srcAlias.FunctionURN + destAlias.FunctionVersionURN = srcAlias.FunctionVersionURN + destAlias.Name = srcAlias.Name + destAlias.FunctionVersion = srcAlias.FunctionVersion + destAlias.RevisionID = srcAlias.RevisionID + destAlias.Description = srcAlias.Description + destAlias.RoutingConfigs = srcAlias.RoutingConfigs + destAlias.RoutingRules = srcAlias.RoutingRules + destAlias.RoutingType = srcAlias.RoutingType +} + +func ifAliasRoutingChanged(originAlias, srcAlias *AliasElement) map[string]int { + changedURNMap := make(map[string]int) + if originAlias.RoutingType == srcAlias.RoutingType { + if originAlias.RoutingType == routingTypeRule { + if !reflect.DeepEqual(originAlias.RoutingRules, srcAlias.RoutingRules) { + changedURNMap[originAlias.RoutingRules.GrayVersion] = UpdateAllURN + } + if originAlias.FunctionVersionURN != srcAlias.FunctionVersionURN { + changedURNMap[originAlias.FunctionVersionURN] = UpdateMainURN + } + return changedURNMap + } + return isAliasWeightTypeChange(originAlias, srcAlias) + } + // routingTypeWeight change to routingTypeRule + if srcAlias.RoutingType == routingTypeRule { + return map[string]int{"": UpdateAllURN} + } + // routingTypeRule change to routingTypeWeight + for _, config := range srcAlias.RoutingConfigs { + if config.Weight <= 0 { + return map[string]int{"": UpdateAllURN} + } + } + if len(srcAlias.RoutingConfigs) == 0 { + return map[string]int{"": UpdateAllURN} + } + return map[string]int{"": NoneUpdate} +} + +// AliasKey contains the elements of an alias key +type AliasKey struct { + ProductID string + AliasSign string + BusinessSign string + BusinessID string + TenantSign string + TenantID string + FunctionSign string + FunctionID string + AliasName string +} + +// ParseFrom parses elements from an alias key +func (a *AliasKey) ParseFrom(aliasKeyStr string) error { + elements := strings.Split(aliasKeyStr, AliasKeySeparator) + urnLen := len(elements) + if urnLen != aliasKeyLength { + return fmt.Errorf("failed to parse an alias key %s, incorrect length", aliasKeyStr) + } + a.ProductID = elements[ProductIDIndex] + a.AliasSign = elements[AliasSignIndex] + a.BusinessSign = elements[BusinessSignIndex] + a.BusinessID = elements[BusinessIDIndex] + a.TenantSign = elements[TenantSignIndex] + a.TenantID = elements[TenantIDIndex] + a.FunctionSign = elements[FunctionSignIndex] + a.FunctionID = elements[FunctionIDIndex] + a.AliasName = elements[AliasNameIndex] + return nil +} + +// FetchInfoFromAliasKey collects alias information from an alias key +func FetchInfoFromAliasKey(aliasKeyStr string) *AliasKey { + var aliasKey AliasKey + if err := aliasKey.ParseFrom(aliasKeyStr); err != nil { + log.GetLogger().Errorf("error while parsing an URN: %s", err.Error()) + return &AliasKey{} + } + return &aliasKey +} + +// BuildURNFromAliasKey builds a URN from a alias key +func BuildURNFromAliasKey(aliasKeyStr string) string { + aliasKey := FetchInfoFromAliasKey(aliasKeyStr) + productURN := &urnutils.FunctionURN{ + ProductID: urnutils.DefaultURNProductID, + RegionID: urnutils.DefaultURNRegion, + BusinessID: aliasKey.BusinessID, + TenantID: aliasKey.TenantID, + TypeSign: urnutils.DefaultURNFuncSign, + FuncName: aliasKey.FunctionID, + FuncVersion: aliasKey.AliasName, + } + return productURN.String() +} + +func parseRules(routingRules routingRules) ([]Expression, error) { + rules := routingRules.Rules + var expressions []Expression + const expressionSize = 3 + for _, value := range rules { + partition := strings.Split(value, ":") + if len(partition) != expressionSize { + return nil, fmt.Errorf("rules (%s) fields size not equal %v", value, expressionSize) + } + expression := Expression{ + leftVal: partition[0], + operator: partition[1], + rightVal: partition[2], + } + expressions = append(expressions, expression) + } + return expressions, nil +} + +func matchRule(params map[string]string, expressions []Expression, ruleLogic string) bool { + var matchResultList []bool + + for _, exp := range expressions { + matchResultList = append(matchResultList, exp.Execute(params)) + } + if len(matchResultList) > 0 { + return isMatch(matchResultList, ruleLogic) + } + return false +} + +func isMatch(matchResultList []bool, ruleLogic string) bool { + matchResult := matchResultList[0] + if len(matchResultList) > 1 { + switch ruleLogic { + case "or": + for _, value := range matchResultList { + matchResult = matchResult || value + } + case "and": + for _, value := range matchResultList { + matchResult = matchResult && value + } + default: + log.GetLogger().Warnf("unknow rulelogic: %s, return false", ruleLogic) + return false + } + } + return matchResult +} + +// MarshalTenantAliasList marshal alias map to list with specific tenant id +func MarshalTenantAliasList(tenantID string) ([]byte, error) { + var aliasList []*AliasElement + GetAliases().AliasMap.Range(func(key, value interface{}) bool { + aliasElement, _ := value.(*AliasElement) + if urnutils.CheckAliasUrnTenant(tenantID, aliasElement.AliasURN) { + aliasList = append(aliasList, aliasElement) + return true + } + return true + }) + aliasData, err := json.Marshal(aliasList) + if err != nil { + return nil, errors.New("marshal alias list error") + } + return aliasData, nil +} + +// ProcessDelete - +func ProcessDelete(event *etcd3.Event) string { + aliasURN := BuildURNFromAliasKey(event.Key) + GetAliases().RemoveAlias(aliasURN) + return aliasURN +} + +// ProcessUpdate - +func ProcessUpdate(event *etcd3.Event) (string, error) { + alias := &AliasElement{} + err := json.Unmarshal(event.Value, alias) + if err != nil { + log.GetLogger().Errorf("failed to unmarshal alias event, err: %s", err.Error()) + return "", err + } + GetAliases().AddAlias(alias) + return alias.AliasURN, nil +} + +// ResolveAliasedFunctionNameToURN - {functionName}:{alias|version} 解析别名路由 +func ResolveAliasedFunctionNameToURN(functionNameWithAlias string, tenantID string, params map[string]string) string { + splits := strings.Split(functionNameWithAlias, ":") + if len(splits) > 2 || len(splits) == 0 { // {functionName}:{alias|version} + return "" + } + + if len(splits) == 1 { + return urnutils.BuildURNOrAliasURNTemp(defaultBusinessID, tenantID, + urnutils.BuildStandardFunctionName(functionNameWithAlias), defaultVersion) + } + + functionName := urnutils.BuildStandardFunctionName(splits[0]) + versionOrAlias := splits[1] + _, err := strconv.Atoi(versionOrAlias) + if err != nil && versionOrAlias != defaultVersion { + aliasUrn := urnutils.BuildURNOrAliasURNTemp(defaultBusinessID, tenantID, functionName, versionOrAlias) + return GetAliases().GetFuncVersionURNWithParams(aliasUrn, params) + } + return urnutils.BuildURNOrAliasURNTemp(defaultBusinessID, tenantID, functionName, versionOrAlias) +} diff --git a/frontend/pkg/common/faas_common/aliasroute/alias_test.go b/frontend/pkg/common/faas_common/aliasroute/alias_test.go new file mode 100644 index 0000000000000000000000000000000000000000..e8504096624cc6649ae14113c3956b0b8ad8b0e0 --- /dev/null +++ b/frontend/pkg/common/faas_common/aliasroute/alias_test.go @@ -0,0 +1,515 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package aliasroute alias routing +package aliasroute + +import ( + "fmt" + "sync" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" +) + +const ( + aliasURN = "sn:cn:yrk:12345678901234561234567890123456:function:helloworld:myaliasv1" +) + +// TestCase init +func GetFakeAliasEle() *AliasElement { + fakeAliasEle := &AliasElement{ + AliasURN: aliasURN, + FunctionURN: "sn:cn:yrk:12345678901234561234567890123456:function:helloworld", + FunctionVersionURN: "sn:cn:yrk:12345678901234561234567890123456:function:helloworld:$latest", + Name: "myaliasv1", + FunctionVersion: "$latest", + RevisionID: "20210617023315921", + Description: "", + RoutingConfigs: []*routingConfig{ + { + FunctionVersionURN: "sn:cn:yrk:12345678901234561234567890123456:function:helloworld:$latest", + Weight: 60, + }, + { + FunctionVersionURN: "sn:cn:yrk:12345678901234561234567890123456:function:helloworld:v1", + Weight: 40, + }, + }, + } + return fakeAliasEle +} + +func GetFakeRuleAliasEle() *AliasElement { + fakeAliasEle := &AliasElement{ + AliasURN: "sn:cn:yrk:12345678901234561234567890123456:function:helloworld:myaliasrulev1", + FunctionURN: "sn:cn:yrk:12345678901234561234567890123456:function:helloworld", + FunctionVersionURN: "sn:cn:yrk:12345678901234561234567890123456:function:helloworld:$latest", + Name: "myaliasrulev1", + FunctionVersion: "$latest", + RevisionID: "20210617023315921", + Description: "", + RoutingType: "rule", + RoutingRules: routingRules{ + RuleLogic: "and", + Rules: []string{"userType:=:VIP", "age:<=:20", "devType:in:P40,P50,MATE40"}, + GrayVersion: "sn:cn:yrk:172120022620195843:function:0@default@test_func:3", + }, + } + return fakeAliasEle +} + +func GetFakeWeightAliasEle() *AliasElement { + fakeAliasEle := &AliasElement{ + AliasURN: "sn:cn:yrk:12345678901234561234567890123456:function:0@default@aliasfunc:myaliasrulev1", + FunctionURN: "sn:cn:yrk:12345678901234561234567890123456:function:0@default@aliasfunc", + FunctionVersionURN: "sn:cn:yrk:c53626012ba84727b938ca8bf03108ef:function:0@default@aliasfunc:latest", + Name: "myaliasrulev1", + FunctionVersion: "$latest", + RevisionID: "20210617023315921", + Description: "", + RoutingType: "weigh", + RoutingConfigs: []*routingConfig{{ + FunctionVersionURN: "sn:cn:yrk:c53626012ba84727b938ca8bf03108ef:function:0@default@aliasfunc:latest", + Weight: 80, + }, { + FunctionVersionURN: "sn:cn:yrk:c53626012ba84727b938ca8bf03108ef:function:0@default@aliasfunc:1", + Weight: 0, + }}, + } + return fakeAliasEle +} +func ClearAliasRoute() { + aliases = &Aliases{ + AliasMap: &sync.Map{}, + } +} + +func TestOptAlias(t *testing.T) { + ClearAliasRoute() + defer ClearAliasRoute() + convey.Convey("AddAlias success", t, func() { + fakeAliasEle := GetFakeAliasEle() + aliases.AddAlias(fakeAliasEle) + ele, ok := aliases.AliasMap.Load(fakeAliasEle.AliasURN) + convey.So(ok, convey.ShouldBeTrue) + convey.So(ele, convey.ShouldNotBeNil) + }) + convey.Convey("update Alias success", t, func() { + fakeAliasEle := GetFakeAliasEle() + fakeAliasEle.RoutingConfigs = []*routingConfig{ + { + FunctionVersionURN: "sn:cn:yrk:12345678901234561234567890123456:function:helloworld:$latest", + Weight: 50, + }, + { + FunctionVersionURN: "sn:cn:yrk:12345678901234561234567890123456:function:helloworld:v1", + Weight: 50, + }, + } + aliases.AddAlias(fakeAliasEle) + ele, ok := aliases.AliasMap.Load(fakeAliasEle.AliasURN) + convey.So(ok, convey.ShouldBeTrue) + convey.So(ele.(*AliasElement).RoutingConfigs[0].Weight, convey.ShouldEqual, 50) + convey.So(ele.(*AliasElement).RoutingConfigs[1].Weight, convey.ShouldEqual, 50) + }) + convey.Convey("remove Alias success", t, func() { + fakeAliasEle := GetFakeAliasEle() + aliases.AddAlias(fakeAliasEle) + aliases.RemoveAlias(fakeAliasEle.AliasURN) + ele, ok := aliases.AliasMap.Load(fakeAliasEle.AliasURN) + convey.So(ok, convey.ShouldBeFalse) + convey.So(ele, convey.ShouldBeNil) + }) +} + +func TestGetFuncURNFromAlias(t *testing.T) { + ClearAliasRoute() + defer ClearAliasRoute() + convey.Convey("alias does not exist", t, func() { + urn := aliases.GetFuncURNFromAlias(aliasURN) + convey.So(urn, convey.ShouldEqual, aliasURN) + }) + + convey.Convey("alias get error", t, func() { + aliases.AliasMap.Store(aliasURN, "456") + urn := aliases.GetFuncURNFromAlias(aliasURN) + aliases.AliasMap.Delete(aliasURN) + convey.So(urn, convey.ShouldEqual, "") + }) + convey.Convey("alias get error", t, func() { + aliases.AddAlias(GetFakeAliasEle()) + urn := aliases.GetFuncURNFromAlias(aliasURN) + convey.So(urn, convey.ShouldNotEqual, aliasURN) + convey.So(urn, convey.ShouldNotEqual, "") + convey.So(urn, convey.ShouldNotContainSubstring, "myaliasv1") + }) + +} + +func TestFetchInfoFromAliasKey(t *testing.T) { + path := "/sn/aliases/business/yrk/tenant/12345678901234561234567890123456/function/helloworld/myalias" + aliasKey := FetchInfoFromAliasKey(path) + + assert.Equal(t, aliasKey.FunctionID, "helloworld") + assert.Equal(t, aliasKey.AliasName, "myalias") + + path = "/sn/aliases/business/yrk/tenant/12345678901234561234567890123456/function/helloworld" + aliasKey = FetchInfoFromAliasKey(path) + assert.Empty(t, aliasKey) +} + +func TestBuildURNFromAliasKey(t *testing.T) { + path := "/sn/aliases/business/yrk/tenant/12345678901234561234567890123456/function/helloworld/myalias" + urn := BuildURNFromAliasKey(path) + assert.Contains(t, urn, "myalias") +} + +func TestGetFuncVersionURNWithParamsMatch(t *testing.T) { + ClearAliasRoute() + defer ClearAliasRoute() + fakeAliasEle := GetFakeRuleAliasEle() + aliases.AddAlias(fakeAliasEle) + params := map[string]string{} + params["userType"] = "VIP" + params["age"] = "10" + params["devType"] = "P40" + + aliasUrn := "sn:cn:yrk:12345678901234561234567890123456:function:helloworld:myaliasrulev1" + wantFuncVer := "sn:cn:yrk:172120022620195843:function:0@default@test_func:3" + got := GetAliases().GetFuncVersionURNWithParams(aliasUrn, params) + assert.Equal(t, wantFuncVer, got) +} + +func TestGetFuncVersionURNWithParamsNotMatch(t *testing.T) { + ClearAliasRoute() + defer ClearAliasRoute() + fakeAliasEle := GetFakeRuleAliasEle() + aliases.AddAlias(fakeAliasEle) + params := map[string]string{} + params["userType"] = "VIP" + params["age"] = "50" + params["devType"] = "P40" + + aliasUrn := "sn:cn:yrk:12345678901234561234567890123456:function:helloworld:myaliasrulev1" + wantFuncVer := "sn:cn:yrk:12345678901234561234567890123456:function:helloworld:$latest" + got := GetAliases().GetFuncVersionURNWithParams(aliasUrn, params) + assert.Equal(t, wantFuncVer, got) +} + +func TestMarshalTenantAliasList(t *testing.T) { + ClearAliasRoute() + defer ClearAliasRoute() + fakeAliasEle := GetFakeRuleAliasEle() + aliases.AddAlias(fakeAliasEle) + params := map[string]string{} + params["userType"] = "VIP" + params["age"] = "10" + params["devType"] = "P40" + + type args struct { + tenantID string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {"case1", args{tenantID: "12345678901234561234567890123456"}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := MarshalTenantAliasList(tt.args.tenantID) + if (err != nil) != tt.wantErr { + t.Errorf("MarshalTenantAliasList() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} + +func TestCheckUrnWithParamsMatchRules(t *testing.T) { + convey.Convey("CheckAliasRoutingChange", t, func() { + aliases.AddAlias(GetFakeRuleAliasEle()) + + aliasRuleURN := "sn:cn:yrk:12345678901234561234567890123456:function:helloworld:myaliasrulev1" + urnWithParam_old := "sn:cn:yrk:172120022620195843:function:0@default@test_func:latest" + convey.So(aliases.CheckAliasRoutingChange(aliasRuleURN, urnWithParam_old, make(map[string]string)), + convey.ShouldEqual, true) + + convey.So(aliases.CheckAliasRoutingChange(aliasRuleURN, urnWithParam_old, make(map[string]string)), + convey.ShouldEqual, true) + + aliases.AddAlias(GetFakeWeightAliasEle()) + aliasWeight := "sn:cn:yrk:12345678901234561234567890123456:function:0@default@aliasfunc:myaliasrulev1" + aliasURN_old := "sn:cn:yrk:c53626012ba84727b938ca8bf03108ef:function:0@default@aliasfunc:1" + convey.So(aliases.CheckAliasRoutingChange(aliasWeight, aliasURN_old, make(map[string]string)), + convey.ShouldEqual, true) + + convey.So(aliases.CheckAliasRoutingChange(aliasWeight, "old alias urn needed update session", + make(map[string]string)), convey.ShouldEqual, true) + }) +} + +func TestAliasWeightLoadBalancer(t *testing.T) { + convey.Convey("AliasWeightLoadBalancer", t, func() { + fakeAliasEle := &AliasElement{ + AliasURN: aliasURN, + FunctionURN: "sn:cn:yrk:12345678901234561234567890123456:function:helloworld", + FunctionVersionURN: "sn:cn:yrk:12345678901234561234567890123456:function:helloworld:1", + Name: "myaliasv1", + FunctionVersion: "1", + RevisionID: "20210617023315921", + Description: "", + RoutingConfigs: []*routingConfig{ + { + FunctionVersionURN: "sn:cn:yrk:12345678901234561234567890123456:function:helloworld:2", + Weight: 80, + }, + { + FunctionVersionURN: "sn:cn:yrk:12345678901234561234567890123456:function:helloworld:1", + Weight: 20, + }, + }, + } + ClearAliasRoute() + aliases.AddAlias(fakeAliasEle) + + aliasElementIf, _ := aliases.AliasMap.Load(fakeAliasEle.AliasURN) + aliasElement := aliasElementIf.(*AliasElement) + urnMap1 := []string{} + urnMap2 := make([]string, 50) + for i := 0; i < 50; i++ { + urn := aliasElement.getFuncVersionURN() + urnMap1 = append(urnMap1, urn) + } + var count int + for index, urn := range urnMap1 { + if urn == "sn:cn:yrk:12345678901234561234567890123456:function:helloworld:2" { + count++ + urnMap2[index] = urn + } + } + convey.So(count, convey.ShouldEqual, 40) + + for i := 0; i < 50; i++ { + if urnMap2[i] != "" { + newUrn := aliasElement.getFuncVersionURN() + if newUrn != urnMap2[i] { + fmt.Printf("index:%d oldUrn:%s, newUrn:%s \n", i, urnMap2[i], newUrn) + } + urnMap2[i] = newUrn + } + } + count = 0 + for _, urn := range urnMap2 { + if urn == "sn:cn:yrk:12345678901234561234567890123456:function:helloworld:2" { + count++ + } + } + convey.So(count, convey.ShouldEqual, 32) + }) +} + +func Test_ifAliasRoutingChanged(t *testing.T) { + convey.Convey("ifAliasRoutingChanged", t, func() { + convey.Convey("same type weight UpdateAllURN", func() { + origin := &AliasElement{RoutingType: "weight"} + newAlias := &AliasElement{RoutingType: "weight"} + mapEvent := ifAliasRoutingChanged(origin, newAlias) + convey.So(mapEvent[""], convey.ShouldEqual, UpdateAllURN) + }) + + convey.Convey("same type weight UpdateWeightGreyURN UpdateMainURN", func() { + origin := &AliasElement{RoutingType: "weight", RoutingConfigs: []*routingConfig{ + { + FunctionVersionURN: "function/latest", + Weight: 100, + }, + }, + FunctionVersion: "0", + } + newAlias := &AliasElement{RoutingType: "weight", RoutingConfigs: []*routingConfig{ + { + FunctionVersionURN: "function/1", + Weight: 80, + }, + { + FunctionVersionURN: "function/2", + Weight: 20, + }, + { + FunctionVersionURN: "function/3", + Weight: 0, + }, + }, + FunctionVersion: "1", + } + mapEvent := ifAliasRoutingChanged(origin, newAlias) + convey.So(mapEvent["function/3"], convey.ShouldEqual, UpdateWeightGreyURN) + convey.So(mapEvent["function/latest"], convey.ShouldEqual, UpdateWeightGreyURN) + convey.So(mapEvent["0"], convey.ShouldEqual, UpdateMainURN) + }) + + convey.Convey("same type rule", func() { + origin := &AliasElement{RoutingType: routingTypeRule, RoutingRules: routingRules{ + RuleLogic: "and", + Rules: nil, + GrayVersion: "0", + }, + FunctionVersionURN: "function/0", + } + newAlias := &AliasElement{RoutingType: routingTypeRule, RoutingRules: routingRules{ + RuleLogic: "or", + Rules: nil, + GrayVersion: "1", + }, + FunctionVersionURN: "function/1", + } + mapEvent := ifAliasRoutingChanged(origin, newAlias) + convey.So(mapEvent[origin.RoutingRules.GrayVersion], convey.ShouldEqual, UpdateAllURN) + convey.So(mapEvent[origin.FunctionVersionURN], convey.ShouldEqual, UpdateMainURN) + }) + + convey.Convey("different type rule weight", func() { + newAlias := &AliasElement{RoutingType: routingTypeRule, RoutingRules: routingRules{ + RuleLogic: "and", + Rules: nil, + GrayVersion: "0", + }, + FunctionVersionURN: "function/0", + } + origin := &AliasElement{RoutingType: "weight", RoutingConfigs: []*routingConfig{ + { + FunctionVersionURN: "function/latest", + Weight: 100, + }, + }, + FunctionVersion: "0", + FunctionVersionURN: "function/1", + } + mapEvent := ifAliasRoutingChanged(origin, newAlias) + convey.So(mapEvent[""], convey.ShouldEqual, UpdateAllURN) + }) + + convey.Convey("different type weight rule", func() { + origin := &AliasElement{RoutingType: routingTypeRule, RoutingRules: routingRules{ + RuleLogic: "and", + Rules: nil, + GrayVersion: "0", + }, + FunctionVersionURN: "function/0", + } + newAlias := &AliasElement{RoutingType: "weight", RoutingConfigs: []*routingConfig{ + { + FunctionVersionURN: "function/latest", + Weight: 0, + }, + }, + FunctionVersion: "0", + FunctionVersionURN: "function/1", + } + mapEvent := ifAliasRoutingChanged(origin, newAlias) + convey.So(mapEvent[""], convey.ShouldEqual, UpdateAllURN) + + newAlias1 := &AliasElement{RoutingType: "weight", RoutingConfigs: []*routingConfig{ + { + FunctionVersionURN: "function/latest", + Weight: 0, + }, + }, + FunctionVersion: "0", + FunctionVersionURN: "function/1", + } + mapEvent1 := ifAliasRoutingChanged(origin, newAlias1) + convey.So(mapEvent1[""], convey.ShouldEqual, UpdateAllURN) + + newAlias2 := &AliasElement{RoutingType: "weight", RoutingConfigs: []*routingConfig{ + { + FunctionVersionURN: "function/latest", + Weight: 100, + }, + }, + FunctionVersion: "0", + FunctionVersionURN: "function/1", + } + mapEvent2 := ifAliasRoutingChanged(origin, newAlias2) + convey.So(mapEvent2[""], convey.ShouldEqual, NoneUpdate) + }) + }) +} + +func TestResolveAliasedFunctionNameToURN(t *testing.T) { + patches := gomonkey.NewPatches() + defer patches.Reset() + + patches.ApplyFunc(GetAliases().GetFuncVersionURNWithParams, func(aliasUrn string, params map[string]string) string { + return "resolved_" + aliasUrn + }) + + testCases := []struct { + name string + functionNameWithAlias string + tenantID string + params map[string]string + expectedURN string + }{ + { + name: "Simple function name without alias", + functionNameWithAlias: "myFunction", + tenantID: "tenant1", + params: nil, + expectedURN: "sn:cn:yrk:tenant1:function:0@default@myFunction:latest", + }, + { + name: "Function name with version number", + functionNameWithAlias: "myFunction:2", + tenantID: "tenant1", + params: nil, + expectedURN: "sn:cn:yrk:tenant1:function:0@default@myFunction:2", + }, + { + name: "Function name with alias", + functionNameWithAlias: "myFunction:prod", + tenantID: "tenant1", + params: map[string]string{"key": "value"}, + expectedURN: "sn:cn:yrk:tenant1:function:0@default@myFunction:prod", + }, + { + name: "Invalid function name (too many splits)", + functionNameWithAlias: "myFunction:prod:extra", + tenantID: "tenant1", + params: nil, + expectedURN: "", + }, + { + name: "Empty function name", + functionNameWithAlias: "", + tenantID: "tenant1", + params: nil, + expectedURN: "sn:cn:yrk:tenant1:function:0@default@:latest", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := ResolveAliasedFunctionNameToURN(tc.functionNameWithAlias, tc.tenantID, tc.params) + assert.Equal(t, tc.expectedURN, result, "URN resolution should match expected output") + }) + } +} diff --git a/frontend/pkg/common/faas_common/aliasroute/event.go b/frontend/pkg/common/faas_common/aliasroute/event.go new file mode 100644 index 0000000000000000000000000000000000000000..e853d20d3182d1eeac47e525e6c2a68d89a75897 --- /dev/null +++ b/frontend/pkg/common/faas_common/aliasroute/event.go @@ -0,0 +1,38 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package aliasroute event +package aliasroute + +const ( + // NoneUpdate - + NoneUpdate = iota + // UpdateAllURN event + UpdateAllURN + // UpdateMainURN alias change its main functionURN event + UpdateMainURN + // UpdateWeightGreyURN alias[type weight] change its grey functionURN event + UpdateWeightGreyURN + // Delete event + Delete +) + +// AliasEvent - +type AliasEvent struct { + Type int + AliasURN string + FunctionVersionURN string +} diff --git a/frontend/pkg/common/faas_common/aliasroute/expression.go b/frontend/pkg/common/faas_common/aliasroute/expression.go new file mode 100644 index 0000000000000000000000000000000000000000..c05c11c26b5cb449fc9ddeca72e0690a512d4f63 --- /dev/null +++ b/frontend/pkg/common/faas_common/aliasroute/expression.go @@ -0,0 +1,101 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package aliasroute alias routing in busclient +package aliasroute + +import ( + "strconv" + "strings" + + "frontend/pkg/common/faas_common/logger/log" +) + +const ( + expressionSize = 3 +) + +// Expression rule expression struct +type Expression struct { + leftVal string + operator string + rightVal string +} + +func compareIntegerStrings(a, b string) (int, error) { + numA, err := strconv.Atoi(a) + if err != nil { + return 0, err + } + + numB, err := strconv.Atoi(b) + if err != nil { + return 0, err + } + + if numA < numB { + return -1, nil + } else if numA > numB { + return 1, nil + } else { + return 0, nil + } +} + +// Execute the rule expression +func (exp *Expression) Execute(params map[string]string) bool { + log.GetLogger().Debugf("params %v, exp.leftVal %v,exp.rightVal %v", params, exp.leftVal, exp.rightVal) + val, exist := params[exp.leftVal] + if !exist { + log.GetLogger().Warnf("cannot find val for %s in params", exp.leftVal) + return false + } + + switch exp.operator { + case "=": + return strings.TrimSpace(val) == strings.TrimSpace(exp.rightVal) + case "!=": + return strings.TrimSpace(val) != strings.TrimSpace(exp.rightVal) + case ">": + ret, err := compareIntegerStrings(val, exp.rightVal) + return err == nil && ret == 1 + case "<": + ret, err := compareIntegerStrings(val, exp.rightVal) + return err == nil && ret == -1 + case ">=": + ret, err := compareIntegerStrings(val, exp.rightVal) + return err == nil && (ret == 1 || ret == 0) + case "<=": + ret, err := compareIntegerStrings(val, exp.rightVal) + return err == nil && (ret == -1 || ret == 0) + case "in": + return matchStr(val, exp.rightVal) + default: + log.GetLogger().Warnf("unknown operator(%s), return false", val, exp.operator) + return false + } +} + +func matchStr(str string, targetStr string) bool { + tars := strings.Split(targetStr, ",") + for _, tar := range tars { + // The rvalue of the 'in' operator ignores "" + if tar != "" && strings.TrimSpace(str) == strings.TrimSpace(tar) { + return true + } + } + return false +} diff --git a/frontend/pkg/common/faas_common/aliasroute/expression_test.go b/frontend/pkg/common/faas_common/aliasroute/expression_test.go new file mode 100644 index 0000000000000000000000000000000000000000..ad46b6ea1abf4368e006398fe2c5a7cd3359474a --- /dev/null +++ b/frontend/pkg/common/faas_common/aliasroute/expression_test.go @@ -0,0 +1,188 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aliasroute + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +type ExpressionTestSuite struct { + alias AliasElement +} + +func (suite *ExpressionTestSuite) SetupTest() { + +} + +func (suite *ExpressionTestSuite) TearDownTest() { + +} + +func (suite *ExpressionTestSuite) TestEquel() { + +} + +func genExpression(str string) (Expression, error) { + partition := strings.Split(str, ":") + if len(partition) != expressionSize { + return Expression{}, fmt.Errorf("express(#{str}) string format is error") + } + return Expression{ + leftVal: partition[0], + operator: partition[1], + rightVal: partition[2], + }, nil +} + +func ExecuteExp(t *testing.T, expStr string, params map[string]string) bool { + exp, err := genExpression(expStr) + if err != nil { + t.Error("gen expression fail: ", expStr) + return false + } + return exp.Execute(params) +} + +func TestExpEq(t *testing.T) { + params := map[string]string{} + params["id"] = "123" + + got := ExecuteExp(t, "id:=:123", params) + assert.True(t, got) + + got = ExecuteExp(t, "id:=:444", params) + assert.False(t, got) +} + +func TestExpNotEq(t *testing.T) { + params := map[string]string{} + params["id"] = "123" + + got := ExecuteExp(t, "id:!=:200", params) + assert.True(t, got) + + got = ExecuteExp(t, "id:!=:123", params) + assert.False(t, got) +} + +func TestExpLt(t *testing.T) { + params := map[string]string{} + params["id"] = "123" + params["type"] = "p40" + + got := ExecuteExp(t, "id:<:200", params) + assert.True(t, got) + + got = ExecuteExp(t, "id:<:100", params) + assert.False(t, got) + + got = ExecuteExp(t, "type:<:100", params) + assert.False(t, got) +} + +func TestExpLtEq(t *testing.T) { + params := map[string]string{} + params["id"] = "123" + params["type"] = "p40" + + got := ExecuteExp(t, "id:<=:200", params) + assert.True(t, got) + + got = ExecuteExp(t, "id:<=:100", params) + assert.False(t, got) + + got = ExecuteExp(t, "id:<=:123", params) + assert.True(t, got) + + got = ExecuteExp(t, "type:<=:100", params) + assert.False(t, got) +} + +func TestExpGt(t *testing.T) { + params := map[string]string{} + params["id"] = "123" + params["type"] = "p40" + + got := ExecuteExp(t, "id:>:200", params) + assert.False(t, got) + + got = ExecuteExp(t, "id:>:100", params) + assert.True(t, got) + + got = ExecuteExp(t, "type:>:100", params) + assert.False(t, got) +} + +func TestExpGtEq(t *testing.T) { + params := map[string]string{} + params["id"] = "123" + params["type"] = "p40" + + got := ExecuteExp(t, "id:>=:200", params) + assert.False(t, got) + + got = ExecuteExp(t, "id:>=:100", params) + assert.True(t, got) + + got = ExecuteExp(t, "id:>=:123", params) + assert.True(t, got) + + got = ExecuteExp(t, "type:>=:1", params) + assert.False(t, got) +} + +func TestExpIn(t *testing.T) { + params := map[string]string{} + params["type"] = "p40" + + got := ExecuteExp(t, "type:in:p40,mate40", params) + assert.True(t, got) + + got = ExecuteExp(t, "type:in:mate40, p40", params) + assert.True(t, got) + + got = ExecuteExp(t, "type:in:mate40, p40 , p30", params) + assert.True(t, got) + + got = ExecuteExp(t, "type:in:mate40,p30", params) + assert.False(t, got) + + got = ExecuteExp(t, "type:in:", params) + assert.False(t, got) +} + +func TestExpExcept(t *testing.T) { + params := map[string]string{} + params["id"] = "123" + params["type"] = "p40" + + got := ExecuteExp(t, "age:<:30", params) + assert.False(t, got) + + got = ExecuteExp(t, "id:<:", params) + assert.False(t, got) + + got = ExecuteExp(t, "id:<:abc", params) + assert.False(t, got) + + got = ExecuteExp(t, "id:||:123", params) + assert.False(t, got) +} diff --git a/frontend/pkg/common/faas_common/autogc/algorithm.go b/frontend/pkg/common/faas_common/autogc/algorithm.go new file mode 100644 index 0000000000000000000000000000000000000000..abfa6f060b04322458de709dea4550b12e34acb7 --- /dev/null +++ b/frontend/pkg/common/faas_common/autogc/algorithm.go @@ -0,0 +1,60 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package autogc + +// Algorithm is algorithm for adjusting GOGC +type Algorithm interface { + Init(totalMemory, threshold uint64) + NextGOGC(currentMemory uint64, preGOGC int) int +} + +const ( + defaultMaxGOGC = 500 + defaultMinGCStep = 50 * MB + percent = 100 +) + +func min(x, y int) int { + if x < y { + return x + } + return y +} + +// DefaultAlg defines default algorithm to adjust GOGC +// when current memory <= threshold, it will adjust GOGC to match threshold, but not above defaultMaxGOGC (500) +// when current memory > threshold, it will adjust GOGC so that GC will trigger every defaultMinGCStep (50MB) heap alloc +type DefaultAlg struct { + total uint64 + threshold uint64 + maxGOGC int +} + +// Init initializes alg with total memory and memory threshold +func (da *DefaultAlg) Init(total, threshold uint64) { + da.total = total + da.threshold = threshold + da.maxGOGC = defaultMaxGOGC +} + +// NextGOGC calculates appropriated GOGC with current memory and previous GOGC +func (da *DefaultAlg) NextGOGC(currentMemory uint64, preGOGC int) int { + if da.threshold >= currentMemory+defaultMinGCStep { + return min(da.maxGOGC, int(percent*(float64(da.threshold)/float64(currentMemory)-1.0))) + } + return int(percent * defaultMinGCStep / currentMemory) +} diff --git a/frontend/pkg/common/faas_common/autogc/algorithm_test.go b/frontend/pkg/common/faas_common/autogc/algorithm_test.go new file mode 100644 index 0000000000000000000000000000000000000000..efe5b25af8ab3f9f32d54cbbf7907eeb472118a7 --- /dev/null +++ b/frontend/pkg/common/faas_common/autogc/algorithm_test.go @@ -0,0 +1,60 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package autogc + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDefaultAlg(t *testing.T) { + assert.Equal(t, 4*GB, 4294967296) + + alg := DefaultAlg{} + alg.Init(4*GB, 3200*MB) + + tests := []struct { + current uint64 + excepted int + }{ + { + current: 40 * MB, + excepted: defaultMaxGOGC, + }, + { + current: 3200 * MB, + excepted: 1, + }, + { + current: 3201 * MB, + excepted: 1, + }, + { + current: 3100 * MB, + excepted: 3, + }, + { + current: 2000 * MB, + excepted: 60, + }, + } + + for _, test := range tests { + assert.Equal(t, test.excepted, alg.NextGOGC(test.current, 0)) + } +} diff --git a/frontend/pkg/common/faas_common/autogc/autogc.go b/frontend/pkg/common/faas_common/autogc/autogc.go new file mode 100644 index 0000000000000000000000000000000000000000..c2f1c3b8b158997f217d9d0d67e6d1a17fdba65e --- /dev/null +++ b/frontend/pkg/common/faas_common/autogc/autogc.go @@ -0,0 +1,108 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package autogc adjusts GOGC automatically inspired by +// https://eng.uber.com/how-we-saved-70k-cores-across-30-mission-critical-services/ +package autogc + +import ( + "os" + "runtime" + "runtime/debug" + "strconv" + + "frontend/pkg/common/faas_common/logger/log" +) + +var ( + gcChannel = make(chan struct{}, 1) + gcAlg Algorithm + previousGOGC = 100 +) + +const ( + defaultMemoryThreshold = 80 +) + +// InitAutoGOGC starts to adjust GOGC automatically +func InitAutoGOGC() { + currentThreshold, err := strconv.Atoi(os.Getenv("AUTO_GC_MEMORY_THRESHOLD")) + if err != nil { + currentThreshold = defaultMemoryThreshold + log.GetLogger().Warnf("failed to get AUTO_GC_MEMORY_THRESHOLD, use default threshold, %s", err.Error()) + } else if currentThreshold <= 0 || currentThreshold > percent { + currentThreshold = defaultMemoryThreshold + } + log.GetLogger().Infof("current auto gc memory threshold: %d", currentThreshold) + limit, err := parseCGroupMemoryLimit() + if err != nil { + log.GetLogger().Errorf("failed to read cgroup memory limit, err: %s", err.Error()) + return + } + log.GetLogger().Infof("cgroup memory limit is %d, memory %d", limit, uint64(currentThreshold)*limit/percent) + + gcAlg = &DefaultAlg{} + if percent == 0 { + return + } + gcAlg.Init(limit, uint64(currentThreshold)*limit/percent) + + newCycleRefObj() + + go runAutoGOGC() +} + +func runAutoGOGC() { + file, err := os.Open(memPath) + if err != nil { + log.GetLogger().Errorf("failed to open statm file") + return + } + defer file.Close() + buffer := make([]byte, KB) + for range gcChannel { + rss, err := parseRSS(file, buffer) + if err != nil { + log.GetLogger().Errorf("failed to parse RSS, err: %s", err.Error()) + return + } + previousGOGC = debug.SetGCPercent(gcAlg.NextGOGC(rss, previousGOGC)) + } +} + +type finalizer struct { + ref *finalizerRef +} + +type finalizerRef struct { + parent *finalizer +} + +func finalizerHandler(f *finalizerRef) { + select { + case gcChannel <- struct{}{}: + default: + } + runtime.SetFinalizer(f, finalizerHandler) +} + +func newCycleRefObj() *finalizer { + f := &finalizer{} + f.ref = &finalizerRef{parent: f} + runtime.SetFinalizer(f.ref, finalizerHandler) + f.ref = nil + return f +} diff --git a/frontend/pkg/common/faas_common/autogc/autogc_test.go b/frontend/pkg/common/faas_common/autogc/autogc_test.go new file mode 100644 index 0000000000000000000000000000000000000000..0c7b0f33d951f83fcefc3af5824cac18c46c91ca --- /dev/null +++ b/frontend/pkg/common/faas_common/autogc/autogc_test.go @@ -0,0 +1,49 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package autogc + +import ( + "os" + "runtime" + "runtime/debug" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/stretchr/testify/assert" + + "frontend/pkg/common/faas_common/utils" +) + +func TestInitAutoGOGC(t *testing.T) { + InitAutoGOGC() + runtime.GC() + assert.Equal(t, 100, previousGOGC) +} + +func TestInitAutoGOGC2(t *testing.T) { + patches := utils.InitPatchSlice() + patches.Append(utils.PatchSlice{ + gomonkey.ApplyFunc(debug.SetGCPercent, + func(percent int) int { + return 100 + })}) + defer patches.ResetAll() + os.Setenv("AUTO_GC_MEMORY_THRESHOLD", "120") + InitAutoGOGC() + runtime.GC() + assert.Equal(t, 100, previousGOGC) +} diff --git a/frontend/pkg/common/faas_common/autogc/util.go b/frontend/pkg/common/faas_common/autogc/util.go new file mode 100644 index 0000000000000000000000000000000000000000..cc100a95397a7e5567e8addac16fe65a843e640a --- /dev/null +++ b/frontend/pkg/common/faas_common/autogc/util.go @@ -0,0 +1,75 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package autogc + +import ( + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "strconv" + "strings" +) + +const ( + cgroupMemLimitPath = "/sys/fs/cgroup/memory/memory.limit_in_bytes" + rssValueFieldIndex = 1 + base = 10 + bitSize = 64 +) + +// constants of memory unit +const ( + B = 1 << (10 * iota) + KB + MB + GB +) + +var ( + pageSize = uint64(os.Getpagesize()) + memPath = fmt.Sprintf("/proc/%d/statm", os.Getpid()) +) + +func parseCGroupMemoryLimit() (uint64, error) { + v, err := ioutil.ReadFile(cgroupMemLimitPath) + if err != nil { + return 0, err + } + return strconv.ParseUint(strings.TrimSpace(string(v)), base, bitSize) +} + +func parseRSS(f io.ReadSeeker, buffer []byte) (uint64, error) { + _, err := f.Seek(0, io.SeekStart) + if err != nil { + return 0, err + } + _, err = f.Read(buffer) + if err != nil && err != io.EOF { + return 0, err + } + fields := strings.Split(string(buffer), " ") + if len(fields) < (rssValueFieldIndex + 1) { + return 0, errors.New("invalid statm fields") + } + rss, err := strconv.ParseUint(fields[rssValueFieldIndex], base, bitSize) + if err != nil { + return 0, err + } + return rss * pageSize, nil +} diff --git a/frontend/pkg/common/faas_common/autogc/util_test.go b/frontend/pkg/common/faas_common/autogc/util_test.go new file mode 100644 index 0000000000000000000000000000000000000000..d8d52b7086b7fed74f7f5f4a305d914c2183f941 --- /dev/null +++ b/frontend/pkg/common/faas_common/autogc/util_test.go @@ -0,0 +1,44 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package autogc + +import ( + "bytes" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseRSS(t *testing.T) { + r := bytes.NewReader([]byte("367597 12113 4058 3810 0 47257 0\n")) + buffer := make([]byte, KB) + + for i := 0; i < 10; i++ { + rss, err := parseRSS(r, buffer) + assert.Nil(t, err, "parseRSS should return no error") + assert.Equal(t, uint64(12113*os.Getpagesize()), rss) + } + + r = bytes.NewReader([]byte("123")) + _, err := parseRSS(r, make([]byte, KB)) + assert.Error(t, err, "parseRSS should failed") + + r = bytes.NewReader([]byte("123 abcde 132")) + _, err = parseRSS(r, make([]byte, KB)) + assert.Error(t, err, "parseRSS should failed") +} diff --git a/frontend/pkg/common/faas_common/config/config.go b/frontend/pkg/common/faas_common/config/config.go new file mode 100644 index 0000000000000000000000000000000000000000..b7269b03e46f114660c95e13aebe13577733c4b8 --- /dev/null +++ b/frontend/pkg/common/faas_common/config/config.go @@ -0,0 +1,25 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package config - +package config + +// TLSConfig certificate configuration +type TLSConfig struct { + CaContent string `json:"caContent"` + KeyContent string `json:"keyContent"` + CertContent string `json:"certContent"` +} diff --git a/frontend/pkg/common/faas_common/constant/app.go b/frontend/pkg/common/faas_common/constant/app.go new file mode 100644 index 0000000000000000000000000000000000000000..152dfe95763dcf7eff8bcc15cd47d9e3a03192e1 --- /dev/null +++ b/frontend/pkg/common/faas_common/constant/app.go @@ -0,0 +1,72 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package constant - +package constant + +// used for app-job +const ( + // UserMetadataKey key used for the app createOpts + UserMetadataKey = "USER_PROVIDED_METADATA" + // EntryPointKey entrypoint for starting app + EntryPointKey = "ENTRYPOINT" + // AppFN - + FunctionNameApp = "app" + // AppFuncId - + AppFuncId = "12345678901234561234567890123456/0-system-faasExecutorPosixCustom/$latest" + // AppType type for invoking create-app + AppType = "SUBMISSION" + // AppStatusPending - + AppStatusPending = "PENDING" + // AppStatusRunning - + AppStatusRunning = "RUNNING" + // AppStatusSucceeded - + AppStatusSucceeded = "SUCCEEDED" + // AppStatusFailed - + AppStatusFailed = "FAILED" + // AppStatusStopped - + AppStatusStopped = "STOPPED" + + // AppInvokeTimeout 30min + AppInvokeTimeout = 1800 +) + +// AppInfo - Ray job JobDetails +type AppInfo struct { + Key string `json:"key"` + // Enum: "SUBMISSION" "DRIVER" + Type string `json:"type"` + Entrypoint string `json:"entrypoint"` + SubmissionID string `json:"submission_id"` + DriverInfo DriverInfo `json:"driver_info" valid:",optional"` + // Status Enum: "PENDING" "RUNNING" "STOPPED" "SUCCEEDED" "FAILED" + Status string `json:"status" valid:",optional"` + StartTime string `json:"start_time" valid:",optional"` + EndTime string `json:"end_time" valid:",optional"` + Metadata map[string]string `json:"metadata" valid:",optional"` + RuntimeEnv map[string]interface{} `json:"runtime_env" valid:",optional"` + DriverAgentHttpAddress string `json:"driver_agent_http_address" valid:",optional"` + DriverNodeID string `json:"driver_node_id" valid:",optional"` + DriverExitCode int32 `json:"driver_exit_code" valid:",optional"` + ErrorType string `json:"error_type" valid:",optional"` +} + +// DriverInfo - +type DriverInfo struct { + ID string `json:"id"` + NodeIPAddress string `json:"node_ip_address"` + PID string `json:"pid"` +} diff --git a/frontend/pkg/common/faas_common/constant/constant.go b/frontend/pkg/common/faas_common/constant/constant.go new file mode 100644 index 0000000000000000000000000000000000000000..5f35680e01141c38e50c771a734c636496716132 --- /dev/null +++ b/frontend/pkg/common/faas_common/constant/constant.go @@ -0,0 +1,521 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package constant - +package constant + +import "time" + +const ( + // LibruntimeHeaderSize is the header length of libruntime package + LibruntimeHeaderSize = 16 +) + +const ( + // BusinessTypeFG is the business type of FunctionGraph + BusinessTypeFG = iota + // BusinessTypeWiseCloud is the business type of WiseCloud + BusinessTypeWiseCloud +) + +const ( + // BackendTypeKernel - + BackendTypeKernel = iota + // BackendTypeFG - + BackendTypeFG +) + +const ( + // DeployModeContainer - + DeployModeContainer = "Container" + // DeployModeProcesses - + DeployModeProcesses = "Processes" +) + +const ( + // HeaderRequestID - + HeaderRequestID = "X-Request-Id" + // HeaderTraceID - + HeaderTraceID = "X-Trace-Id" + // HeaderTraceParent - + HeaderTraceParent = "Traceparent" +) + +const ( + // KernelResourceNotEnoughErrCode is the error code of kernel resource not enough + KernelResourceNotEnoughErrCode = 1002 + // KernelInnerSystemErrCode is the error code of kernel inner system error + KernelInnerSystemErrCode = 3003 + // KernelRequestErrBetweenRuntimeAndBus is the error code of bus communicate with runtime + KernelRequestErrBetweenRuntimeAndBus = 3001 + // KernelUserCodeLoadErrCode is the error code if kernel user code load error + KernelUserCodeLoadErrCode = 2001 + // KernelUserFunctionExceptionErrCode is the error code of kernel when user function exception + KernelUserFunctionExceptionErrCode = 2002 + // KernelCreateLimitErrCode is the error code of kernel when create limited + KernelCreateLimitErrCode = 1012 + // KernelWriteEtcdCircuitErrCode is the error code of kernel when write etcd failed or circuit + KernelWriteEtcdCircuitErrCode = 3005 + // KernelDataSystemUnavailable is the error code of kernel when data system is unavailable + KernelDataSystemUnavailable = 3015 + // KernelNPUFAULTErrCode is the error code of kernel when user exit with npu card is fault + KernelNPUFAULTErrCode = 3016 +) + +const ( + // InsReqSuccessCode is the return code when instance request succeeds + InsReqSuccessCode = 6030 + // InsReqSuccessMessage is the return message when instance request succeeds + InsReqSuccessMessage = "instance request successfully" + // UnsupportedOperationErrorCode is the return code when operation is not supported + UnsupportedOperationErrorCode = 6031 + // UnsupportedOperationErrorMessage is the return message when operation is not supported + UnsupportedOperationErrorMessage = "operation not supported" + // FuncNotExistErrorCode is the return code when function does not exist + FuncNotExistErrorCode = 6032 + // FuncNotExistErrorMessage is the return message when function does not exist + FuncNotExistErrorMessage = "function not exist" + // InsNotExistErrorCode is the return code when instance does not exist + InsNotExistErrorCode = 6033 + // InsNotExistErrorMessage is the return message when instance does not exist + InsNotExistErrorMessage = "instance not exist" + // InsAcquireFailedErrorCode is the return code when acquire instance fails + InsAcquireFailedErrorCode = 6034 + // InsAcquireLeaseExistErrorCode - is the return code when acquire repeated lease + InsAcquireLeaseExistErrorCode = 6035 + // InsAcquireFailedErrorMessage is the return message when acquire instance fails + InsAcquireFailedErrorMessage = "failed to acquire instance" + // LeaseExpireOrDeletedErrorCode is the return code when lease expires or be deleted + LeaseExpireOrDeletedErrorCode = 6036 + // LeaseExpireOrDeletedErrorMessage is the return message when lease expires or be deleted + LeaseExpireOrDeletedErrorMessage = "lease expires or deleted" + // AcquireLeaseTrafficLimitErrorCode - + AcquireLeaseTrafficLimitErrorCode = 6037 + // AcquireLeaseTrafficLimitErrorMessage is reach max limit of acquiring lease concurrently + AcquireLeaseTrafficLimitErrorMessage = "reach max limit of acquiring lease concurrently" + // LeaseErrorInstanceIsAbnormalMessage - lease op failed, instance is abnormal + LeaseErrorInstanceIsAbnormalMessage = "lease op failed, instance is abnormal" + // InsAcquireTimeOutErrorCode is the return code when acquire instance timout + InsAcquireTimeOutErrorCode = 6038 + // AcquireLeaseVPCConflictErrorCode The called function instance has a VPC conflict + AcquireLeaseVPCConflictErrorCode = 6039 + // InstancesConfigEtcdPrefix - + InstancesConfigEtcdPrefix = "/instances" + // InstancePathPrefix is the etcd path where the instance info will be placed + InstancePathPrefix = "/sn/instance" + // ModuleSchedulerPrefix is the etcd path where the module scheduler info will be placed + ModuleSchedulerPrefix = "/sn/faas-scheduler/instances" + // SchedulerRolloutPrefix - + SchedulerRolloutPrefix = "/sn/faas-scheduler/rollout" + // RolloutConfigPrefix - + RolloutConfigPrefix = "/sn/faas-scheduler/rolloutConfig" + // HTTPTriggerPrefix - + HTTPTriggerPrefix = "/sn/triggers/triggerType/HTTP/business/" + // FunctionPrefix - + FunctionPrefix = "/sn/functions" + // AliasPrefix - + AliasPrefix = "/sn/aliases" + // LeasePrefix - + LeasePrefix = "/sn/lease" + // FunctionAvailClusterPrefix Used to identify whether the called function vpc conflicts with the cluster network + FunctionAvailClusterPrefix = "/sn/function/available/clusters/" + // FrontendInstancePrefix frontend instance information recorded in meta etcd + FrontendInstancePrefix = "/sn/frontend/instances" + // TenantQuotaPrefix define the key prefix of etcd for tenant metadata + TenantQuotaPrefix = "/sn/quota/cluster" + + // ETCDEventKeySeparator is the separator of ETCD event key + ETCDEventKeySeparator = "/" + + // DefaultMaxRequestBodySize frontend maximum request body size + DefaultMaxRequestBodySize = 100 * 1024 * 1024 + + // DefaultMapSize default map size + DefaultMapSize = 3 + // DefaultHostAliasesSliceSize default host aliases slice size + DefaultHostAliasesSliceSize = 4 + // MinCustomResourcesSize is min custom resource size of invoke + MinCustomResourcesSize = 0 + + // SchedulerExclusivityKey is the key for tenant exclusivity scheduler + SchedulerExclusivityKey = "exclusivity" + // SchedulerRecoverTime - + SchedulerRecoverTime = 30 * time.Second + // DefaultServerWriteTimeOut 1300s + DefaultServerWriteTimeOut = 1300 * time.Second + // SchedulerKeyTypeFunction - + SchedulerKeyTypeFunction = "function" + // SchedulerKeyTypeModule - + SchedulerKeyTypeModule = "module" + // StaticInstanceApplier mark the instance is created by static function + StaticInstanceApplier = "static_function" +) + +const ( + // KeySeparator is the separator in an ETCD key + KeySeparator = "/" + // ValidEtcdKeyLenForInstance is the valid len of an instance ETCD key + ValidEtcdKeyLenForInstance = 14 + // SysFunctionTenantID is the tenantID of a system function + SysFunctionTenantID = "0" + // FaasFrontendMark is a part of the function name of a faasfrontend system function + FaasFrontendMark = "system-faasfrontend" + // FaasSchedulerMark is a part of the function name of a faasscheduler system function + FaasSchedulerMark = "system-faasscheduler" + // FunctionsIndexForInstance is the functions index of an valid instance ETCD key + FunctionsIndexForInstance = 2 + // TenantIndexForInstance is the tenant index of an valid instance ETCD key + TenantIndexForInstance = 5 + // TenantIDIndexForInstance is the tenantID index of an valid instance ETCD key + TenantIDIndexForInstance = 6 + // FunctionIndexForInstance is the functon index of an valid instance ETCD key + FunctionIndexForInstance = 7 + // FunctionNameIndexForInstance is the functon name index of an valid instance ETCD key + FunctionNameIndexForInstance = 8 + // InstanceIDIndexForInstance is the instanceID index of an valid instance ETCD key + InstanceIDIndexForInstance = 13 + // FaasSchedulerName is function name of a faasscheduler system function + FaasSchedulerName = "0-system-faasscheduler" +) + +// InstanceStatus is stauts of instance_status object +type InstanceStatus int + +const ( + // KernelInstanceStatusExited instance is exited + KernelInstanceStatusExited InstanceStatus = -1 + // KernelInstanceStatusNew instance is not created + KernelInstanceStatusNew InstanceStatus = 0 + // KernelInstanceStatusScheduling instance is scheduling + KernelInstanceStatusScheduling InstanceStatus = 1 + // KernelInstanceStatusCreating instance is creating + KernelInstanceStatusCreating InstanceStatus = 2 + // KernelInstanceStatusRunning instance is running + KernelInstanceStatusRunning InstanceStatus = 3 + // KernelInstanceStatusFailed instance is failed + KernelInstanceStatusFailed InstanceStatus = 4 + // KernelInstanceStatusExiting instance is exiting + KernelInstanceStatusExiting InstanceStatus = 5 + // KernelInstanceStatusFatal instance abnormal exits + KernelInstanceStatusFatal InstanceStatus = 6 + // KernelInstanceStatusScheduleFailed instance is schedule failed + KernelInstanceStatusScheduleFailed InstanceStatus = 7 + // KernelInstanceStatusEvicting instance is evicting + KernelInstanceStatusEvicting InstanceStatus = 9 + // KernelInstanceStatusEvicted instance is evicted + KernelInstanceStatusEvicted InstanceStatus = 10 + // KernelInstanceStatusSubHealth instance is sub health + KernelInstanceStatusSubHealth InstanceStatus = 11 +) + +// InstanceStatusType is EXIT_TYPE of instance_status object +type InstanceStatusType int + +const ( + // KernelInstanceStatusTypeNoneExit - + KernelInstanceStatusTypeNoneExit InstanceStatusType = 0 + // KernelInstanceStatusTypeReturn - + KernelInstanceStatusTypeReturn InstanceStatusType = 1 + // KernelInstanceStatusTypeExceptionInfo - + KernelInstanceStatusTypeExceptionInfo InstanceStatusType = 2 + // KernelInstanceStatusTypeOomInfo - + KernelInstanceStatusTypeOomInfo InstanceStatusType = 3 + // KernelInstanceStatusTypeStandardInfo - + KernelInstanceStatusTypeStandardInfo InstanceStatusType = 4 + // KernelInstanceStatusTypeUnknownError - + KernelInstanceStatusTypeUnknownError InstanceStatusType = 5 + // KernelInstanceStatusTypeUserKillInfo - + KernelInstanceStatusTypeUserKillInfo InstanceStatusType = 6 +) + +const ( + // RuntimeTypeCpp - + RuntimeTypeCpp = "cpp" + // RuntimeTypeCppBin - + RuntimeTypeCppBin = "cppbin" + // RuntimeTypeJava - + RuntimeTypeJava = "java" + // RuntimeTypeNodejs - + RuntimeTypeNodejs = "nodejs" + // RuntimeTypePython - + RuntimeTypePython = "python" + // RuntimeTypeCustom - + RuntimeTypeCustom = "custom" + // RuntimeTypeFusion - + RuntimeTypeFusion = "fusion" + // RuntimeTypeHTTP - + RuntimeTypeHTTP = "http" +) + +const ( + // ExtendedCallHandler used as kernel metadata extendedMetaData.extended_handler.handler field + ExtendedCallHandler = "handler" + // ExtendedInitHandler used as kernel metadata extendedMetaData.extended_handler.initializer field + ExtendedInitHandler = "initializer" + // CallHandler - + CallHandler = "call" + // InitHandler - + InitHandler = "init" + // CheckPointHandler - + CheckPointHandler = "checkpoint" + // RecoverHandler - + RecoverHandler = "recover" + // ShutdownHandler - + ShutdownHandler = "shutdown" + // SignalHandler - + SignalHandler = "signal" +) + +const ( + // PythonCallExecutor - + PythonCallExecutor = "faas_executor.faasCallHandler" + // PythonInitExecutor - + PythonInitExecutor = "faas_executor.faasInitHandler" + // PythonCheckPointExecutor - + PythonCheckPointExecutor = "faas_executor.faasCheckPointHandler" + // PythonRecoverExecutor - + PythonRecoverExecutor = "faas_executor.faasRecoverHandler" + // PythonShutDownExecutor - + PythonShutDownExecutor = "faas_executor.faasShutDownHandler" + // PythonSignalExecutor - + PythonSignalExecutor = "faas_executor.faasSignalHandler" +) + +const ( + // MaxTraceIDLength is the max length of traceID + MaxTraceIDLength = 128 +) + +const ( + // DefaultListenIP - + DefaultListenIP = "127.0.0.1" + // BusProxyHTTPPort - + BusProxyHTTPPort = "22423" +) + +const ( + // TraceIDRuntimeCallCtx Key value of the traceID parameter in the context input parameter of CallHandler + TraceIDRuntimeCallCtx = "traceID" +) + +const ( + // DefaultURNVersion is the default version of a URN + DefaultURNVersion = "latest" + // DefaultNameSpace is the default namespace + DefaultNameSpace = "default" +) + +const ( + // ClusterNameEnvKey defines env key for cluster name + ClusterNameEnvKey = "CLUSTER_NAME" + // PodIPEnvKey define pod ip env key + PodIPEnvKey = "POD_IP" + // HostNameEnvKey defines the hostname env key + HostNameEnvKey = "HOSTNAME" + // HostIPEnvKey defines the host ip env key + HostIPEnvKey = "HOST_IP" + // ResourceCPUName - + ResourceCPUName = "CPU" + // ResourceMemoryName - + ResourceMemoryName = "Memory" + // ResourceEphemeralStorage - + ResourceEphemeralStorage = "ephemeral-storage" + // CustomContainerRuntimeType is the runtime type for http function + CustomContainerRuntimeType = "custom image" + // CustomImageExtraTimeout is the timeout to offset non-pool start of custom image + CustomImageExtraTimeout = 300 + // PosixCustomRuntimeType is the runtime type for posix custom + PosixCustomRuntimeType = "posix-custom-runtime" + // CommonExtraTimeout - + CommonExtraTimeout = 2 + // TrafficRedundantRate limit redundancy rate for traffic limitation + TrafficRedundantRate = 1.1 + // SystemExtraTimeout - + SystemExtraTimeout = 5 + // KernelScheduleTimeout is the timeout set in kernel to avoid instance schedule timeout + KernelScheduleTimeout = 5 + // ModuleScheduler - + ModuleScheduler = "ModuleScheduler" + // AffinityPoolIDKey - + AffinityPoolIDKey = "AFFINITY_POOL_ID" + // UnUseAntiOtherLabelsKey - + UnUseAntiOtherLabelsKey = "unUseAntiOtherLabels" + + // BusinessTypeServe - + BusinessTypeServe = "serve" + // URLSeparator is the separator of http url + URLSeparator = "/" + // ApplicationIndex - + ApplicationIndex = 0 +) + +const ( + // TrueStr - + TrueStr = "true" +) + +const ( + // HeaderInvokeURN - + HeaderInvokeURN = "X-Tag-VersionUrn" + // HeaderStateKey - + HeaderStateKey = "X-State-Key" + // HeaderNodeLabel is node label + HeaderNodeLabel = "X-Node-Label" + // HeaderCPUSize is cpu size specified by invoke + HeaderCPUSize = "X-Instance-Cpu" + // HeaderMemorySize is cpu memory specified by invoke + HeaderMemorySize = "X-Instance-Memory" + // HeaderCustomResource is customResource specified by invoke + HeaderCustomResource = "X-Instance-CustomResource" + // HeaderCustomResourceNew is customResource specified by invoke + HeaderCustomResourceNew = "X-Instance-Custom-Resource" + // HeaderContentType - + HeaderContentType = "Content-Type" + // HeaderContentLength - + HeaderContentLength = "Content-Length" + // HeaderBillingDuration - + HeaderBillingDuration = "X-Billing-Duration" + // HeaderInnerCode - + HeaderInnerCode = "X-Inner-Code" + // HeaderInvokeSummary - + HeaderInvokeSummary = "X-Invoke-Summary" + // HeaderLogResult - + HeaderLogResult = "X-Log-Result" + // HeaderLogType - + HeaderLogType = "X-Log-Type" + // DefaultLogFlag is the default flag for log + DefaultLogFlag = "None" + // HeaderAuthTimestamp is the timestamp for authorization + HeaderAuthTimestamp = "X-Timestamp-Auth" + // HeaderAuthorization is authorization + HeaderAuthorization = "Authorization" + // HeaderInvokeAlias indicates alias of current invocation + HeaderInvokeAlias = "x-invoke-alias" + // HeaderRetryFlag - + HeaderRetryFlag = "X-Retry-Flag" + // HeaderInstanceID - + HeaderInstanceID = "X-Instance-Id" + // HeaderInstanceIP - + HeaderInstanceIP = "X-Instance-Ip" + // HeaderWorkerCost - + HeaderWorkerCost = "X-Worker-Cost" + // HeaderCallInstance - + HeaderCallInstance = "X-Call-Instance" + // HeaderCallNode - + HeaderCallNode = "X-Call-Node" + + // HeaderEventSourceID - + HeaderEventSourceID = "X-Event-Source-Id" + // HeaderCallType is the request type + HeaderCallType = "X-Call-Type" + // HeaderForceDeploy is Force Deploy + HeaderForceDeploy = "X-Force-Deploy" + // HeaderStreamAPIGEvent - + HeaderStreamAPIGEvent = "X-Stream-Apig-Event" + // HeaderRequestStreamName - + HeaderRequestStreamName = "X-Request-Stream-Name" + // HeaderResponseStreamName - + HeaderResponseStreamName = "X-Response-Stream-Name" + // HeaderFrontendResponseStreamName - + HeaderFrontendResponseStreamName = "X-Frontend-Response-Stream-Name" + // HeaderRemoteClientId - + HeaderRemoteClientId = "X-Remote-Client-Id" +) + +const ( + // NewLease for add a lease of client + NewLease = "NewLease" + // KeepAlive for keep client alive + KeepAlive = "KeepAlive" + // DelLease for del a lease of client + DelLease = "DelLease" +) +const ( + // MetaFuncKey key used to match functions within ETCD + MetaFuncKey = "/sn/functions/business/yrk/tenant/%s/function/%s/version/%s" + // SilentFuncKey key used to match silent functions within ETCD + SilentFuncKey = "/silent/sn/functions/business/yrk/tenant/%s/function/%s/version/%s" +) + +const ( + // RuntimeInstanceName is instance name specified by user + RuntimeInstanceName = "instanceName" + // InstanceCreateEvent key of instance create event + InstanceCreateEvent = "instanceCreateEvent" + // InstanceRequirementResourcesKey key of FunctionSystemClient.Invoke args[1] + InstanceRequirementResourcesKey = "resourcesData" + // InstanceRequirementInsIDKey key of FunctionSystemClient.Invoke args[1] + InstanceRequirementInsIDKey = "designateInstanceID" + // InstanceCallerPodName name of Instance Caller.Invoke args[1] + InstanceCallerPodName = "instanceCallerPodName" + // InstanceTrafficLimited - name of instance traffic limit key args[1] + InstanceTrafficLimited = "instanceTrafficLimited" + // InstanceRequirementPoolLabel - key of poolLabel + InstanceRequirementPoolLabel = "poolLabel" + // InstanceSessionConfig is the key of instance session config in instance acquiring + InstanceSessionConfig = "instanceSessionConfig" + // InstanceRequirementInvokeLabel - name of instance label args[1] + InstanceRequirementInvokeLabel = "instanceInvokeLabel" +) + +const ( + // HeaderTenantID - + HeaderTenantID = "X-Tenant-Id" + // HeaderFunctionName - + HeaderFunctionName = "X-Function-Name" + // HeaderDataSystemPayloadInfo - + HeaderDataSystemPayloadInfo = "X-Data-System-Payload-Info" + // HeaderClientID - + HeaderClientID = "X-Client-Id" + // HeaderTargetServiceID - + HeaderTargetServiceID = "X-Target-Service-Id" +) + +const ( + // PipInstallPrefix - + PipInstallPrefix = "pip3.9 install" + // WorkingDirType - + WorkingDirType = "working_dir" + // PipCheckSuffix - + PipCheckSuffix = "pip3.9 check" +) + +const ( + // KillSignalVal - + KillSignalVal = 1 + // StopAppSignalVal used for stop-app + StopAppSignalVal = 7 + + // KillSignalAliasUpdate is signal for alias update + KillSignalAliasUpdate = 64 + // KillSignalFaaSSchedulerUpdate is signal for faasscheduler update + KillSignalFaaSSchedulerUpdate = 72 +) + +const ( + // InstanceNameNote notes instance name + InstanceNameNote = "INSTANCE_NAME_NOTE" + // FunctionKeyNote - is used to describe the function + FunctionKeyNote = "FUNCTION_KEY_NOTE" + // ResourceSpecNote - is used to describe the resource + ResourceSpecNote = "RESOURCE_SPEC_NOTE" + // SchedulerIDNote - is used to decribe the schedulerID + SchedulerIDNote = "SCHEDULER_ID_NOTE" + // InstanceTypeNote - is used to decribe the instance type: "scaled", "reserved", "state" + InstanceTypeNote = "INSTANCE_TYPE_NOTE" + // InstanceLabelNode - + InstanceLabelNode = "INSTANCE_LABEL_NOTE" +) diff --git a/frontend/pkg/common/faas_common/constant/delegate.go b/frontend/pkg/common/faas_common/constant/delegate.go new file mode 100644 index 0000000000000000000000000000000000000000..86057ff33e88a1b30a3404a84b915d6a2c284304 --- /dev/null +++ b/frontend/pkg/common/faas_common/constant/delegate.go @@ -0,0 +1,96 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package constant + +const ( + // DelegateVolumeMountKey is the key for DELEGATE_VOLUME_MOUNTS in CreateOption + DelegateVolumeMountKey = "DELEGATE_VOLUME_MOUNTS" + // DelegateInitVolumeMountKey is the key for DELEGATE_INIT_VOLUME_MOUNTS in CreateOption + DelegateInitVolumeMountKey = "DELEGATE_INIT_VOLUME_MOUNTS" + // DelegateAgentVolumeMountKey is the key for DELEGATE_AGENT_VOLUME_MOUNTS i + DelegateAgentVolumeMountKey = "DELEGATE_AGENT_VOLUME_MOUNTS" + // DelegateVolumesKey is the key for DELEGATE_VOLUMES in CreateOption + DelegateVolumesKey = "DELEGATE_VOLUMES" + // DelegateHostAliases is the key for DELEGATE_HOST_ALIASES in CreateOption + DelegateHostAliases = "DELEGATE_HOST_ALIASES" + // DelegateDownloadKey is the key for DelegateDownload in CreateOption + DelegateDownloadKey = "DELEGATE_DOWNLOAD" + // DelegateBootstrapKey is the key for DelegateStart in CreateOption + DelegateBootstrapKey = "DELEGATE_BOOTSTRAP" + // DelegateLayerDownloadKey is the key for DelegateLayerDownload in CreateOption + DelegateLayerDownloadKey = "DELEGATE_LAYER_DOWNLOAD" + // DelegateMountKey is the key for DELEGATE_MOUNT in CreateOption + DelegateMountKey = "DELEGATE_MOUNT" + // DelegateEncryptKey is the key for DELEGATE_ENCRYPT in CreateOption + DelegateEncryptKey = "DELEGATE_ENCRYPT" + // DelegateContainerKey is the key for DELEGATE_CONTAINER in CreateOption + DelegateContainerKey = "DELEGATE_CONTAINER" + // DelegateContainerSideCars is the key for DELEGATE_SIDECARS in CreateOption + DelegateContainerSideCars = "DELEGATE_SIDECARS" + // DelegateInitContainers is the key for DELEGATE_INIT_CONTAINERS in CreateOption + DelegateInitContainers = "DELEGATE_INIT_CONTAINERS" + // DelegatePodAnnotations is used to transfer pod annotations to the kernel during instance creation + DelegatePodAnnotations = "DELEGATE_POD_ANNOTATIONS" + // DelegatePodLabels is used to transfer pod labels to the kernel during instance creation + DelegatePodLabels = "DELEGATE_POD_LABELS" + // DelegatePodInitLabels - + DelegatePodInitLabels = "DELEGATE_POD_INIT_LABELS" + // DelegatePodSeccompProfile is key for DELEGATE_POD_SECCOMP_PROFILE in CreateOption + DelegatePodSeccompProfile = "DELEGATE_POD_SECCOMP_PROFILE" + // DelegateInitVolumeMounts is key for DELEGATE_INIT_VOLUME_MOUNTS in CreateOption + DelegateInitVolumeMounts = "DELEGATE_INIT_VOLUME_MOUNTS" + // DelegateNuwaRuntimeInfo is key for DELEGATE_NUWA_RUNTIME_INFO in CreateOption + DelegateNuwaRuntimeInfo = "DELEGATE_NUWA_RUNTIME_INFO" + // DelegateInitEnv is key for DelegateInitEnv in CreateOption + DelegateInitEnv = "DELEGATE_INIT_ENV" + // EnvDelegateEncrypt - + EnvDelegateEncrypt = "DELEGATE_ENCRYPT" + // DelegateTolerations is the key for DELEGATE_TOLERATIONS in CreateOption + DelegateTolerations = "DELEGATE_TOLERATIONS" + + // DelegateRuntimeManagerTag the key of runtime-manager's image tag + DelegateRuntimeManagerTag = "DELEGATE_RUNTIME_MANAGER" + // DelegateNodeAffinity is the key for DELEGATE_NODE_AFFINITY in CreateOption + DelegateNodeAffinity = "DELEGATE_NODE_AFFINITY" + + // DelegateNodeAffinityPolicy - + DelegateNodeAffinityPolicy = "DELEGATE_NODE_AFFINITY_POLICY" + // DelegateAffinity - + DelegateAffinity = "DELEGATE_AFFINITY" + // DelegateNodeAffinityPolicyCoverage - + DelegateNodeAffinityPolicyCoverage = "coverage" + // DelegateNodeAffinityPolicyAggregation - + DelegateNodeAffinityPolicyAggregation = "aggregation" + + // InstanceLifeCycle - + InstanceLifeCycle = "lifecycle" + // InstanceLifeCycleDetached - + InstanceLifeCycleDetached = "detached" + + // DelegateDirectoryInfo is the path that will be monitored its disk usage + DelegateDirectoryInfo = "DELEGATE_DIRECTORY_INFO" + // DelegateDirectoryQuota is the quota of the path + DelegateDirectoryQuota = "DELEGATE_DIRECTORY_QUOTA" + // PostStartExec - + PostStartExec = "POST_START_EXEC" + // DelegateEnvVar - + DelegateEnvVar = "DELEGATE_ENV_VAR" + // BusinessTypeTypeNote - is used to decribe the instance business type: "Serve", "FaaS", "Actor" + BusinessTypeTypeNote = "BUSINESS_TYPE_NOTE" + // FaasInvokeTimeout is function exec timeout + FaasInvokeTimeout = "INVOKE_TIMEOUT" +) diff --git a/frontend/pkg/common/faas_common/constant/functiongraph.go b/frontend/pkg/common/faas_common/constant/functiongraph.go new file mode 100644 index 0000000000000000000000000000000000000000..03c7ed019b5733f5068134212c627714c3282b9b --- /dev/null +++ b/frontend/pkg/common/faas_common/constant/functiongraph.go @@ -0,0 +1,156 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package constant - +package constant + +const ( + // BusinessTypeWebSocket websocket business type + BusinessTypeWebSocket = "WEBSOCKET" + // BusinessTypeCAE cae business type + BusinessTypeCAE = "CAE" + // BusinessTypeFaaS FaaS business type + BusinessTypeFaaS = "FaaS" + // BusinessType business type key + BusinessType = "BUSINESS_TYPE" + // LanguageJava8 language java8 + LanguageJava8 = "java8" +) + +const ( + // ZoneKey zone key + ZoneKey = "KUBERNETES_IO_AVAILABLEZONE" + // ZoneNameLen define zone length + ZoneNameLen = 255 + // DefaultAZ default az + DefaultAZ = "defaultaz" + + // PodNamespaceEnvKey define pod namespace env key + PodNamespaceEnvKey = "POD_NAMESPACE" + + // FunctionLoadTimeoutEnvKey load Function timeout time + FunctionLoadTimeoutEnvKey = "LOAD_FUNCTION_TIMEOUT" + + // ResourceLimitsMemory Memory limit, in bytes + ResourceLimitsMemory = "MEMORY_LIMIT_BYTES" + + // FuncBranchEnvKey is branch env key + FuncBranchEnvKey = "FUNC_BRANCH" + + // DataSystemBranchEnvKey is branch env key + DataSystemBranchEnvKey = "DATASYSTEM_CAPABILITY" + + // HTTPort busproxy httpserver listen port + HTTPort = "22423" + // TCPort busproxy tcpserver listen port + TCPort = "32568" + // BusWorkerServerTCPort bus worker server listen port + BusWorkerServerTCPort = "32569" + // BusRuntimeServerPort bus listen port for + BusRuntimeServerPort = "32570" + // DefaultCachePort indicates the default port of a cache-manager server + DefaultCachePort = "9993" + + // PlatformTenantID is tenant ID of platform function + PlatformTenantID = "0" + + // RuntimeLogOptTail - + RuntimeLogOptTail = "Tail" + // RuntimeLayerDirName - + RuntimeLayerDirName = "layer" + // RuntimeFuncDirName - + RuntimeFuncDirName = "func" + + // FunctionTaskAppID - + FunctionTaskAppID = "function-task" + + // BackpressureCode indicate that frontend should choose another proxy/worker and retry + BackpressureCode = 211429 + // HeaderBackpressure indicate that proxy can backpressure this request + HeaderBackpressure = "X-Backpressure" + // HeaderBackpressureNums Backpressure numbers counter + HeaderBackpressureNums = "X-Backpressure-Nums" + // MonitorFileName monitor file name + MonitorFileName = "monitor-disk" + + // DefaultFuncLogIndex default function log's index + DefaultFuncLogIndex = -2 + + // IsClusterUpgrading indicate that the cluster is in upgrading phase + IsClusterUpgrading = "FAAS_CLUSTER_IS_UPGRADING" +) + +const ( + // WorkerManagerApplier mark the instance is created by minInstance + WorkerManagerApplier = "worker-manager" + // ASBResApplier mark the instance is created by ASBRes + ASBResApplier = "ASBRes" + // FunctionTaskApplier mark the instance is created by minInstance + FunctionTaskApplier = "functiontask" + // PredictionApplier mark the instance is created by smart warmer predict + PredictionApplier = "prediction" + // FaasSchedulerApplier the instance is created by faas scheduler + FaasSchedulerApplier = "faas-scheduler" + // PoolInfoPrefix pool info prefix in redis + PoolInfoPrefix = "ClusterState_Pool" + // PoolInfoSep pool info separator in redis + PoolInfoSep = "_" + // ClusterIDKey cluster id key in system env + ClusterIDKey = "CLUSTER_ID" + // DefaultRecordingInterval default pool info recording interval, unit is second + DefaultRecordingInterval = 5 + // DefaultRecordExpiredTime default pool info record expired time, unit is second + DefaultRecordExpiredTime = 900 +) + +const ( + // FunctionAccessor - defines the microservice component name. + FunctionAccessor = "FunctionAccessor" + // FunctionTask - + FunctionTask = "FunctionTask" + // InstanceManager - + InstanceManager = "FunctionInstanceManager" + // StateManager - + StateManager = "StateManager" + // CacheManager - + CacheManager = "CacheManager" + // CacheServiceName indicates the header of the cache service + CacheServiceName = "cache-manager" + // FaaSScheduler - + FaaSScheduler = "faas-scheduler" + // SnapshotManager - + SnapshotManager = "SnapshotManager" + // Autoscaler define the alarm type + Autoscaler = "Autoscaler" +) + +// header constant key for FG +const ( + // FGHeaderRequestID - + FGHeaderRequestID = "X-Request-Id" + // FGHeaderAccessKey - + FGHeaderAccessKey = "X-Access-Key" + // FGHeaderSecretKey - + FGHeaderSecretKey = "X-Secret-Key" + // FGHeaderSecurityAccessKey - + FGHeaderSecurityAccessKey = "X-Security-Access-Key" + // FGHeaderSecuritySecretKey - + FGHeaderSecuritySecretKey = "X-Security-Secret-Key" + // FGHeaderAuthToken - + FGHeaderAuthToken = "X-Auth-Token" + // FGHeaderSecurityToken - + FGHeaderSecurityToken = "X-Security-Token" +) diff --git a/frontend/pkg/common/faas_common/constant/wisecloud.go b/frontend/pkg/common/faas_common/constant/wisecloud.go new file mode 100644 index 0000000000000000000000000000000000000000..4630d45438058e2ba74ba685b51317f3b1c1e8f1 --- /dev/null +++ b/frontend/pkg/common/faas_common/constant/wisecloud.go @@ -0,0 +1,42 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package constant - +package constant + +// header constant key for caas +const ( + CaaSHeaderTraceID = "X-Caas-Trace-Id" + CaaSHeaderRequestID = "X-Cff-Request-Id" +) + +// CaaS alarm +const ( + // WiseCloudSite site + WiseCloudSite = "WISECLOUD_SITE" + // TenantID WiseCloud tenantID + TenantID = "WISECLOUD_TENANTID" + // ApplicationID WiseCloud applicationId + ApplicationID = "WISECLOUD_APPLICATIONID" + // ServiceID WiseCloud serviceId + ServiceID = "WISECLOUD_SERVICEID" + // ClusterName define cluster env key + ClusterName = "CLUSTER_NAME" + // PodNameEnvKey define pod name env key + PodNameEnvKey = "POD_NAME" + // CloudMapId define in wiseCloud about cloudMap id + CloudMapId = "X_WISECLOUD_CLOUDMAP_ID" +) diff --git a/frontend/pkg/common/faas_common/crypto/cryptoapi_mock.go b/frontend/pkg/common/faas_common/crypto/cryptoapi_mock.go new file mode 100644 index 0000000000000000000000000000000000000000..173bfb19fa6ea18d7139e518ae1a4690198069bc --- /dev/null +++ b/frontend/pkg/common/faas_common/crypto/cryptoapi_mock.go @@ -0,0 +1,42 @@ +//go:build !scc + +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package crypto for auth +package crypto + +// SccConfig - +type SccConfig struct { + Enable bool `json:"enable" valid:"optional"` + SecretName string `json:"secretName" valid:"optional"` + Algorithm string `json:"algorithm" valid:"optional"` +} + +// SCCInitialized - +func SCCInitialized() bool { + return false +} + +// InitializeSCC - +func InitializeSCC(config SccConfig) error { + return nil +} + +// SCCDecrypt - +func SCCDecrypt(cipher []byte) (string, error) { + return "", nil +} diff --git a/frontend/pkg/common/faas_common/crypto/scc_crypto.go b/frontend/pkg/common/faas_common/crypto/scc_crypto.go new file mode 100644 index 0000000000000000000000000000000000000000..cdbd84c7b4fa72e01861597eb7bc916d584d707c --- /dev/null +++ b/frontend/pkg/common/faas_common/crypto/scc_crypto.go @@ -0,0 +1,166 @@ +//go:build scc + +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package crypto for auth +package crypto + +import ( + "cryptoapi" + "encoding/json" + "fmt" + "path" + "sync" + + corev1 "k8s.io/api/core/v1" + + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/utils" +) + +var ( + sccInitialized bool = false + m sync.RWMutex +) + +const ( + // Aes256Cbc - + Aes256Cbc = "AES256_CBC" + // Aes128Gcm - + Aes128Gcm = "AES128_GCM" + // Aes256Gcm - + Aes256Gcm = "AES256_GCM" + // Sm4Cbc - + Sm4Cbc = "SM4_CBC" + // Sm4Ctr - + Sm4Ctr = "SM4_CTR" + + sccConfigDefaultPath = "/home/snuser/secret/scc" +) + +// SccConfig - +type SccConfig struct { + Enable bool `json:"enable" valid:"optional"` + SecretName string `json:"secretName" valid:"optional"` + Algorithm string `json:"algorithm" valid:"optional"` + SccConfigPath string `json:"sccConfigPath" valid:"optional"` +} + +// SCCInitialized - +func SCCInitialized() bool { + m.RLock() + defer m.RUnlock() + return sccInitialized +} + +// GetSCCAlgorithm - +func GetSCCAlgorithm(algorithm string) int { + switch algorithm { + case Aes256Cbc: + return cryptoapi.ALG_AES256_CBC + case Aes128Gcm: + return cryptoapi.ALG_AES128_GCM + case Aes256Gcm: + return cryptoapi.ALG_AES256_GCM + case Sm4Cbc: + return cryptoapi.ALG_SM4_CBC + case Sm4Ctr: + return cryptoapi.ALG_SM4_CTR + default: + return cryptoapi.ALG_AES256_GCM + } +} + +// InitializeSCC - +func InitializeSCC(config SccConfig) error { + m.Lock() + defer m.Unlock() + + if !config.Enable { + return nil + } + options := cryptoapi.NewSccOptions() + sccConfigPath := config.SccConfigPath + if sccConfigPath == "" { + sccConfigPath = sccConfigDefaultPath + } + options.PrimaryKeyFile = path.Join(sccConfigPath, "primary.ks") + options.StandbyKeyFile = path.Join(sccConfigPath, "standby.ks") + options.LogPath = "/tmp/log/" + options.LogFile = "scc" + options.DefaultAlgorithm = GetSCCAlgorithm(config.Algorithm) + options.RandomDevice = "/dev/random" + options.EnableChangeFilePermission = 1 + cryptoapi.Finalize() + err := cryptoapi.InitializeWithConfig(options) + if err != nil { + log.GetLogger().Errorf("Initialize SCC Error = [%s]", err.Error()) + return err + } + sccInitialized = true + return nil +} + +// FinalizeSCC - +func FinalizeSCC() { + m.Lock() + defer m.Unlock() + sccInitialized = false + cryptoapi.Finalize() +} + +// SCCEncrypt - +func SCCEncrypt(plainInput string) ([]byte, error) { + cipher, err := cryptoapi.Encrypt(plainInput) + if err != nil { + log.GetLogger().Errorf("SCC Encrypt Error = [%s]", err.Error()) + return nil, err + } + + return []byte(cipher), nil +} + +// SCCDecrypt - +func SCCDecrypt(cipher []byte) (string, error) { + plain, err := cryptoapi.Decrypt(string(cipher)) + if err != nil { + log.GetLogger().Errorf("SCC Decrypt Error = [%s]", err.Error()) + return "", err + } + + return plain, nil +} + +// GenerateSCCVolumesAndMounts - +func GenerateSCCVolumesAndMounts(secretName string, builder *utils.VolumeBuilder) (string, string, error) { + if builder == nil { + return "", "", fmt.Errorf("volume builder is nil") + } + builder.AddVolume(corev1.Volume{Name: "scc-ks", + VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: secretName}}}) + builder.AddVolumeMount(utils.ContainerRuntimeManager, + corev1.VolumeMount{Name: "scc-ks", MountPath: "/home/snuser/resource/scc"}) + volumesData, err := json.Marshal(builder.Volumes) + if err != nil { + return "", "", err + } + volumesMountData, err := json.Marshal(builder.Mounts[utils.ContainerRuntimeManager]) + if err != nil { + return "", "", err + } + return string(volumesData), string(volumesMountData), nil +} diff --git a/frontend/pkg/common/faas_common/crypto/scc_crypto_test.go b/frontend/pkg/common/faas_common/crypto/scc_crypto_test.go new file mode 100644 index 0000000000000000000000000000000000000000..f271f254015a51bec35a07a97fdc5012b401f962 --- /dev/null +++ b/frontend/pkg/common/faas_common/crypto/scc_crypto_test.go @@ -0,0 +1,85 @@ +//go:build scc + +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This test file can also be used as a tool to create, encrypt and decrypt our secrets and cipher texts +package crypto + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSCCEncryptDecryptInitialized(t *testing.T) { + var c = SccConfig{ + Enable: true, + Algorithm: "AES256_GCM", + } + ret := InitializeSCC(c) + assert.Nil(t, ret) + input := "text to encrypt" + encrypted, err := SCCEncrypt(input) + fmt.Printf("encrypted : %s\n", string(encrypted)) + assert.Nil(t, err) + decrypt, err := SCCDecrypt(encrypted) + fmt.Printf("decrypt : %s\n", decrypt) + assert.Nil(t, err) + assert.Equal(t, input, decrypt) + assert.NotEqual(t, encrypted, input) + FinalizeSCC() +} + +func TestSCCEncryptDecryptNotInitialized(t *testing.T) { + var c = SccConfig{ + Enable: false, + Algorithm: "AES256_GCM", + } + ret := InitializeSCC(c) + assert.Nil(t, ret) + input := "text to encrypt" + encrypted, _ := SCCEncrypt(input) + fmt.Printf("encrypted : %s\n", string(encrypted)) + decrypt, _ := SCCDecrypt(encrypted) + fmt.Printf("decrypt : %s\n", decrypt) + FinalizeSCC() +} + +func TestSCCEncryptDecryptAlgorithms(t *testing.T) { + var c = SccConfig{ + Enable: true, + Algorithm: "AES256_GCM", + } + + algorithms := []string{"AES256_CBC", "AES128_GCM", "AES256_GCM", "SM4_CBC", "SM4_CTR", "DEFAULT"} + for _, algo := range algorithms { + FinalizeSCC() + c.Algorithm = algo + ret := InitializeSCC(c) + assert.Nil(t, ret) + input := "text to encrypt" + encrypted, err := SCCEncrypt(input) + fmt.Printf("encrypted : %s\n", string(encrypted)) + assert.Nil(t, err) + decrypt, err := SCCDecrypt(encrypted) + fmt.Printf("decrypt : %s\n", decrypt) + assert.Nil(t, err) + assert.Equal(t, input, decrypt) + assert.NotEqual(t, encrypted, input) + } +} diff --git a/frontend/pkg/common/faas_common/datasystemclient/api.go b/frontend/pkg/common/faas_common/datasystemclient/api.go new file mode 100644 index 0000000000000000000000000000000000000000..e08e547072c71b8a4455d722ec1a46812a90ba9c --- /dev/null +++ b/frontend/pkg/common/faas_common/datasystemclient/api.go @@ -0,0 +1,903 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package datasystemclient is data system client used for communicating with data system worker. +// To use data system, you should export the data system lib path. Please refer to the Dockerfile of the frontend. +// The lib should copied to home/sn/bin/datasystem/lib. Please refer to +// functioncore/build/common/common_compile.sh and the Dockerfile of the frontend. +// NOTE: To change the version of data system, must revise the version in the common_compile.sh, test.sh and the go.mod +package datasystemclient + +import ( + "errors" + "fmt" + "io" + "net/http" + "reflect" + "runtime" + "strings" + "time" + "unsafe" + + "go.uber.org/zap" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/grpc/pb/data" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/types" +) + +// Config - required parameter +type Config struct { + TenantID string + NodeIP string + NoNeedGenKey bool + NeedEncrypt bool + KeyPrefix string + Cluster string + Limit uint64 + DataKey []byte + + invalidIP []string + useLastUsedNode bool +} + +const ( + errKeyNotFound = 3 + errOutOfMemory = 6 + errDsWorkerNotReady = 8 + errTryAgain = 19 + errRPCCancelled = 1000 + errRPCUnavailable = 1002 + errAsyncQueueFull = 2003 + + errDsClientNil = 11001 // 数据系统错误目前到6000，errDsClientNil表示frontend中析构了该client，为frontend封装，因此从11000开始计数，避开数据系统节点返回的错误码区间 +) + +const ( + defaultUploadTTLSecond = 24 * 60 * 60 + defaultExecuteTTLSecond = 30 * 60 + defaultDataSystemTimeoutMs = 60 * 1000 + defaultDownloadLimit = 32 * 1024 * 1024 + defaultSubscribeTimeoutMs = 100 + // DefaultDataSystemPort - + DefaultDataSystemPort = 31501 +) + +const ( + // StreamEndElement - + StreamEndElement = `\xE0\xFF\xE0\xFF` + // StreamEndElementSize - + StreamEndElementSize = 16 +) + +var ( + // ErrKeyNotFound - the key on data system is not found + ErrKeyNotFound = errors.New("key not found on data system") + // ErrValueSizeExceeded - the value size exceeded the limit + ErrValueSizeExceeded = errors.New("value size exceeded the limit") + // UploadTTLSecond - upload data to data system timeout in uploadFile function + UploadTTLSecond uint32 + // ExecuteTTLSecond - upload data to data system timeout in execute function + // should greater than the time of algorithm execute + ExecuteTTLSecond uint32 + // UploadWriteMode - L2Cache policy used during uploading in uploadFile function + UploadWriteMode api.WriteModeEnum + // ExecuteWriteMode - L2Cache policy used during uploading in execute function + ExecuteWriteMode api.WriteModeEnum + + nodeEtcdKeyPrefix = "/datasystem/cluster/" + + timeoutMs int + port int + + clientMap = concurrentMap{mp: make(map[string]*nodeIP2ClientMap)} + localClientLibruntime api.LibruntimeAPI +) + +// SubscribeParam - +type SubscribeParam struct { + StreamName string + TimeoutMs uint32 + ExpectReceiveNum int32 + Callback func() + TraceId string +} + +func initDataSystemCommon(cfg *types.DataSystemConfig, stopCh <-chan struct{}) { + port = DefaultDataSystemPort + timeoutMs = cfg.TimeoutMs + if timeoutMs <= 0 { + timeoutMs = defaultDataSystemTimeoutMs + } + var dataSystemKeyPrefixList []string + if len(cfg.Clusters) != 0 { + for _, cluster := range cfg.Clusters { + dataSystemKeyPrefix := "/" + cluster + nodeEtcdKeyPrefix + dataSystemKeyPrefixList = append(dataSystemKeyPrefixList, dataSystemKeyPrefix) + } + } else { + dataSystemKeyPrefixList = append(dataSystemKeyPrefixList, nodeEtcdKeyPrefix) + } + log.GetLogger().Infof("init data system success,timeoutMs: %d", timeoutMs) + go StartWatch(dataSystemKeyPrefixList, stopCh) +} + +func setLocalClient(rt api.LibruntimeAPI) { + localClientLibruntime = rt +} + +// InitDataSystemLibruntime - init data system before using api +func InitDataSystemLibruntime(cfg *types.DataSystemConfig, rt api.LibruntimeAPI, stopCh <-chan struct{}) { + setLocalClient(rt) + initDataSystemCommon(cfg, stopCh) +} + +func getClient(cfg *Config, traceId string) (DsClientImpl, bool, error) { + logger := log.GetLogger().With(zap.Any("traceId", traceId)) + cache, err := getDataSystemCacheByCluster(cfg.Cluster) + if err != nil { + logger.Warnf("getDataSystemCacheByCluster failed, err: %s", err.Error()) + return DsClientImpl{}, false, err + } + if cfg.NodeIP == "" || !cache.ifNodeExist(cfg.NodeIP) { + // if node ip is empty, get a random dataSystem node + if cfg.useLastUsedNode { + cfg.NodeIP, err = cache.getLastUsedNodeWithInvalidNode(cfg.invalidIP) + } else { + cfg.NodeIP, err = cache.getRandomNodeWithInvalidNode(cfg.invalidIP) + } + if err != nil { + return DsClientImpl{}, false, err + } + logger.Infof("get a random node ip: %s", cfg.NodeIP) + + } + + // double check + if client, existed := clientMap.get(cfg.TenantID, cfg.NodeIP); existed { + return client, false, nil + } + + client, err := clientMap.getOrCreate(cfg.TenantID, cfg.NodeIP) + if err != nil { + cache.invalidateNode(cfg.NodeIP) + clientMap.deleteClient(cfg.TenantID, cfg.NodeIP) + logger.Warnf("Failed to create the client. replace the node and try again.: %s, err: %s", + cfg.NodeIP, err.Error()) + return DsClientImpl{}, true, err + } + return client, false, nil +} + +func getDataSystemCacheByCluster(cluster string) (*Cache, error) { + if cluster == "" { + cluster = noCluster + } + cacheData, ok := dataSystemCache.Load(cluster) + if !ok { + log.GetLogger().Errorf("no datasystem node in cluster %s", cluster) + return nil, errors.New("no data system node is available") + } + cache, ok := cacheData.(*Cache) + if !ok { + return nil, errors.New("dataSystem cache is invalid") + } + return cache, nil +} + +// shouldRetry - 数据系统相关错误码处理逻辑：1.需要重试的错误码以白名单方式处理，2.不在白名单中的错误码直接返回失败 +func shouldRetry(code int) bool { + retryCode := map[int]struct{}{ + errOutOfMemory: {}, + errAsyncQueueFull: {}, + errRPCCancelled: {}, + errTryAgain: {}, + errDsClientNil: {}, + errRPCUnavailable: {}, + errDsWorkerNotReady: {}, + } + _, ok := retryCode[code] + return ok +} + +// UploadWithoutKeyRetry - the key is returned by data system +func UploadWithoutKeyRetry(value []byte, config *Config, param api.SetParam, traceID string) (string, error) { + for { + genKey, needRetry, err := uploadWithoutKey(value, config, param, traceID) + if err == nil { + return genKey, nil + } + if needRetry { + log.GetLogger().Debugf("upload without key will retry, failed ip: %s", config.NodeIP) + config.invalidIP = append(config.invalidIP, config.NodeIP) + config.NodeIP = "" + continue + } + return "", err + } +} + +func uploadWithoutKey(value []byte, config *Config, param api.SetParam, traceID string) (string, bool, error) { + dsClient, retry, err := getClient(config, traceID) + if err != nil { + return "", retry, err + } + if dsClient.kvClient == nil { + return "", false, fmt.Errorf("dsclient is nil") + } + runtime.LockOSThread() + dsClient.kvClient.SetTraceID(traceID) + key, status := dsClient.kvClient.KVSetWithoutKey(value, param) + runtime.UnlockOSThread() + if status.IsError() { + if shouldRetry(status.Code) { + log.GetLogger().Warnf("uploadWithoutKey dsClient(nodeIP: %s) is unavailable, code: %d, err: %s,"+ + " retry other clients, traceID: %s", config.NodeIP, status.Code, status.Err, traceID) + return "", true, status.Err + } + log.GetLogger().Warnf("dsClient(nodeIP: %s) is unavailable, code: %d, can't retry err: %s, traceID: %s", + config.NodeIP, status.Code, status.Err, traceID) + return "", false, status.Err + } + return key, false, nil +} + +// UploadWithKeyRetry - the key is returned by data system 返回的是数据系统生成的key，不带前缀拼接 +func UploadWithKeyRetry(value []byte, config *Config, param api.SetParam, traceID string) (string, error) { + for { + genKey, retry, err := uploadWithKey(value, config, param, traceID) + if err == nil { + return genKey, nil + } + if retry { + log.GetLogger().Debugf("upload with key will retry, failed ip: %s,traceID: %s", + config.NodeIP, traceID) + config.invalidIP = append(config.invalidIP, config.NodeIP) + config.NodeIP = "" + continue + } + log.GetLogger().Errorf("upload with key failed err: %s,NodeIP: %s,traceID: %s", err.Error(), + config.NodeIP, traceID) + return "", err + } +} + +func uploadWithKey(value []byte, config *Config, param api.SetParam, traceID string) (string, bool, error) { + dsClient, retry, err := getClient(config, traceID) + if err != nil { + return "", retry, err + } + if dsClient.kvClient == nil { + return "", false, fmt.Errorf("dsclient is nil") + } + var key string + var genKey string + key, genKey, err = getDataSystemKey(config, dsClient, traceID) + if err != nil { + return "", true, err + } + if config.NeedEncrypt { + value, err = encryptData(config, value) + if err != nil { + return "", false, fmt.Errorf("failed to encrypt value: %v", err) + } + } + runtime.LockOSThread() + dsClient.kvClient.SetTraceID(traceID) + if err = localClientLibruntime.SetTenantID(config.TenantID); err != nil { + runtime.UnlockOSThread() + return "", false, err + } + status := dsClient.kvClient.KVSet(key, value, param) + runtime.UnlockOSThread() + if status.IsError() { + if shouldRetry(status.Code) { + log.GetLogger().Warnf("uploadWithKey dsClient(nodeIP: %s) is unavailable, code: %d, err: %s,"+ + " retry other clients, traceID: %s", config.NodeIP, status.Code, status.Err, traceID) + return "", true, status.Err + } + log.GetLogger().Warnf("dsClient(nodeIP: %s) is unavailable, code: %d, can't retry err: %s, traceID: %s", + config.NodeIP, status.Code, status.Err, traceID) + return "", false, status.Err + } + return genKey, false, nil +} + +func getDataSystemKey(config *Config, dsClient DsClientImpl, + traceID string) (string, string, error) { + if config.NoNeedGenKey { + return config.KeyPrefix, "", nil + } + + genKey := dsClient.kvClient.GenerateKey() + + if genKey == "" { + log.GetLogger().Warnf("failed to generate key, retry other clients, nodeIP: %s, traceId: %s", + config.NodeIP, traceID) + return "", "", fmt.Errorf("failed to generate key") + } + + key := config.KeyPrefix + genKey + + return key, genKey, nil +} + +// DownloadArrayRetry - download multiple keys at once +func DownloadArrayRetry(keys []string, config *Config, traceID string) ([][]byte, error) { + if config.Limit == 0 { + config.Limit = defaultDownloadLimit + } + for { + byteValues, retry, err := downloadArray(keys, config, traceID) + if err == nil { + return byteValues, nil + } + if retry { + config.invalidIP = append(config.invalidIP, config.NodeIP) + config.NodeIP = "" + continue + } + return nil, err + } +} + +func downloadArray(keys []string, config *Config, traceID string) ([][]byte, bool, error) { + dsClient, retry, err := getClient(config, traceID) + if err != nil { + return nil, retry, err + } + if dsClient.kvClient == nil { + return nil, false, fmt.Errorf("dsclient is nil") + } + runtime.LockOSThread() + dsClient.kvClient.SetTraceID(traceID) + if err = localClientLibruntime.SetTenantID(config.TenantID); err != nil { + runtime.UnlockOSThread() + return nil, false, err + } + sizes, queryErr := dsClient.kvClient.KVQuerySize(keys) + retry, err = checkStatus(queryErr, config, traceID) + if err != nil { + runtime.UnlockOSThread() + return nil, retry, err + } + var totalSize uint64 + for _, val := range sizes { + totalSize += val + } + if totalSize > config.Limit { + runtime.UnlockOSThread() + log.GetLogger().Errorf("query size: %d exceeded the limit: %d", totalSize, config.Limit) + return nil, false, ErrValueSizeExceeded + } + byteValues, status := dsClient.kvClient.KVGetMulti(keys) + runtime.UnlockOSThread() + retry, err = checkStatus(status, config, traceID) + if err != nil { + return nil, retry, err + } + for i, v := range byteValues { + if config.NeedEncrypt { + val, err := decryptData(config, v) + if err != nil { + return nil, false, fmt.Errorf("failed to decrypt value: %v", err) + } + byteValues[i] = val + continue + } + byteValues[i] = v + } + + return byteValues, false, nil +} + +func checkStatus(status api.ErrorInfo, config *Config, traceID string) (bool, error) { + if status.IsError() { + if status.Code == errKeyNotFound { + return false, ErrKeyNotFound + } + if shouldRetry(status.Code) { + log.GetLogger().Warnf("dsClient(nodeIP: %s) is unavailable, code: %d, err: %s,"+ + " retry other clients, traceID: %s", config.NodeIP, status.Code, status.Err, traceID) + return true, status.Err + } + log.GetLogger().Warnf("dsClient(nodeIP: %s) is unavailable, code: %d, can't retry err: %s, traceID: %s", + config.NodeIP, status.Code, status.Err, traceID) + return false, status.Err + } + return false, nil +} + +// DeleteArrayRetry - delete multiple keys at once; the first output parameter is the keys failed to delete +func DeleteArrayRetry(keys []string, config *Config, traceID string) ([]string, error) { + for { + failedKeys, retry, err := deleteArray(keys, config, traceID) + if err == nil { + return nil, nil + } + if retry { + config.invalidIP = append(config.invalidIP, config.NodeIP) + config.NodeIP = "" + continue + } + return failedKeys, err + } +} + +func deleteArray(keys []string, config *Config, traceID string) ([]string, bool, error) { + dsClient, retry, err := getClient(config, traceID) + if err != nil { + return keys, retry, err + } + if dsClient.kvClient == nil { + return nil, false, fmt.Errorf("dsclient is nil") + } + runtime.LockOSThread() + dsClient.kvClient.SetTraceID(traceID) + if err = localClientLibruntime.SetTenantID(config.TenantID); err != nil { + runtime.UnlockOSThread() + return nil, false, err + } + failedKeys, status := dsClient.kvClient.KVDelMulti(keys) + runtime.UnlockOSThread() + if status.IsError() { + if shouldRetry(status.Code) { + log.GetLogger().Warnf("dsClient(nodeIP: %s) is unavailable, code: %d, err: %s,"+ + " retry other clients, traceID: %s", config.NodeIP, status.Code, status.Err, traceID) + return keys, true, status.Err + } + log.GetLogger().Warnf("dsClient(nodeIP: %s) is unavailable, code: %d, can't retry err: %s, traceID: %s", + config.NodeIP, status.Code, status.Err, traceID) + return keys, false, status.Err + } else if len(failedKeys) > 0 { + return failedKeys, false, fmt.Errorf("some keys failed to delete") + } + return nil, false, nil +} + +// ObjPut - put objects to datasystem +func ObjPut(req *data.PutRequest, config *Config, traceID string) api.ErrorInfo { + if localClientLibruntime == nil { + return api.ErrorInfo{Code: errRPCUnavailable, Err: fmt.Errorf("dsclient is nil")} + } + paramLibruntime := api.PutParam{ + WriteMode: api.WriteModeEnum(req.WriteMode), + ConsistencyType: api.ConsistencyTypeEnum(req.ConsistencyType), + CacheType: api.CacheTypeEnum(req.CacheType), + } + runtime.LockOSThread() + var status api.ErrorInfo + err := putRaw(req, config.TenantID, paramLibruntime) + if err != nil { + status.Code = err.(api.ErrorInfo).Code + status.Err = err + } + runtime.UnlockOSThread() + if status.IsError() { + log.GetLogger().Warnf("ObjPut dsClient(nodeIP: %s) is unavailable, code: %d, err: %s, traceID: %s", + config.NodeIP, status.Code, status.Err, traceID) + return status + } + return status +} + +func putRaw(req *data.PutRequest, tenantID string, paramLibruntime api.PutParam) error { + if err := localClientLibruntime.SetTenantID(tenantID); err != nil { + return err + } + if len(req.NestedObjectIds) == 0 { + return localClientLibruntime.PutRaw(req.ObjectId, req.ObjectData, paramLibruntime) + } + return localClientLibruntime.PutRaw(req.ObjectId, req.ObjectData, paramLibruntime, req.NestedObjectIds...) +} + +// ObjGet - get objects to datasystem +func ObjGet(req *data.GetRequest, config *Config, traceID string) ([][]byte, api.ErrorInfo) { + if localClientLibruntime == nil { + return nil, api.ErrorInfo{Code: errRPCUnavailable, Err: fmt.Errorf("dsclient is nil")} + } + + var byteVals [][]byte + var status api.ErrorInfo + + runtime.LockOSThread() + var errInfo error + byteVals, errInfo = getRaw(req, config.TenantID) + log.GetLogger().Debugf("libruntime api get values size:%d", len(byteVals)) + if errInfo != nil { + status.Code = errInfo.(api.ErrorInfo).Code + status.Err = errInfo + } + runtime.UnlockOSThread() + if status.IsError() { + log.GetLogger().Warnf("ObjGet dsClient(nodeIP: %s) is unavailable, code: %d, err: %s, traceID: %s", + config.NodeIP, status.Code, status.Err, traceID) + return byteVals, status + } + return byteVals, status +} + +func getRaw(req *data.GetRequest, tenantID string) ([][]byte, error) { + if err := localClientLibruntime.SetTenantID(tenantID); err != nil { + return nil, err + } + return localClientLibruntime.GetRaw(req.ObjectIds, int(req.TimeoutMs)) +} + +// GIncreaseRef - increase global ref +func GIncreaseRef(req *data.IncreaseRefRequest, config *Config, traceID string) ([]string, api.ErrorInfo) { + if localClientLibruntime == nil { + return nil, api.ErrorInfo{Code: errRPCUnavailable, Err: fmt.Errorf("dsclient is nil")} + } + + var status api.ErrorInfo + var values []string + + runtime.LockOSThread() + var errorInfo error + values, errorInfo = gIncreaseRefRaw(req, config.TenantID) + if errorInfo != nil { + status.Code = errorInfo.(api.ErrorInfo).Code + status.Err = errorInfo + } + + runtime.UnlockOSThread() + if status.IsError() { + log.GetLogger().Warnf("dsClient(nodeIP: %s) is unavailable, code: %d, err: %s, traceID: %s", + config.NodeIP, status.Code, status.Err, traceID) + return values, status + } + return values, status +} + +func gIncreaseRefRaw(req *data.IncreaseRefRequest, tenantID string) ([]string, error) { + if err := localClientLibruntime.SetTenantID(tenantID); err != nil { + return nil, err + } + return localClientLibruntime.GIncreaseRefRaw(req.ObjectIds, req.RemoteClientId) +} + +// GDecreaseRef - decrease global ref +func GDecreaseRef(req *data.DecreaseRefRequest, config *Config, traceID string) ([]string, api.ErrorInfo) { + if localClientLibruntime == nil { + return nil, api.ErrorInfo{Code: errRPCUnavailable, Err: fmt.Errorf("dsclient is nil")} + } + + var status api.ErrorInfo + var values []string + + runtime.LockOSThread() + var err error + values, err = gDecreaseRefRaw(req, config.TenantID) + if err != nil { + status.Code = err.(api.ErrorInfo).Code + status.Err = err + } + runtime.UnlockOSThread() + if status.IsError() { + log.GetLogger().Warnf("GDecreaseRef dsClient(nodeIP: %s) is unavailable, code: %d, err: %s, traceID: %s", + config.NodeIP, status.Code, status.Err, traceID) + return values, status + } + return values, status +} + +func gDecreaseRefRaw(req *data.DecreaseRefRequest, tenantID string) ([]string, error) { + if err := localClientLibruntime.SetTenantID(tenantID); err != nil { + return nil, err + } + return localClientLibruntime.GDecreaseRefRaw(req.ObjectIds, req.RemoteClientId) +} + +// Set - set kv to datasystem +func Set(req *data.KvSetRequest, config *Config, traceID string) api.ErrorInfo { + if localClientLibruntime == nil { + return api.ErrorInfo{Code: errRPCUnavailable, Err: fmt.Errorf("dsclient is nil")} + } + paramLibruntime := api.SetParam{ + WriteMode: api.WriteModeEnum(req.WriteMode), + TTLSecond: req.TtlSecond, + Existence: req.Existence, + CacheType: api.CacheTypeEnum(req.CacheType), + } + log.GetLogger().Debugf("set kv to datasystem, key:%s, param:%v", req.Key, paramLibruntime) + var status api.ErrorInfo + runtime.LockOSThread() + if err := kvSet(req, config.TenantID, paramLibruntime); err != nil { + status.Code = err.(api.ErrorInfo).Code + status.Err = err + } + runtime.UnlockOSThread() + if status.IsError() { + log.GetLogger().Warnf("SetKV dsClient(nodeIP: %s) is unavailable, code: %d, err: %s, traceID: %s", + config.NodeIP, status.Code, status.Err, traceID) + return status + } + return status +} + +func kvSet(req *data.KvSetRequest, tenantID string, paramLibruntime api.SetParam) error { + if err := localClientLibruntime.SetTenantID(tenantID); err != nil { + return err + } + return localClientLibruntime.KVSet(req.Key, req.Value, paramLibruntime) +} + +// MSetTx - set multi kvs to datasystem transactionally +func MSetTx(req *data.KvMSetTxRequest, config *Config, traceID string) api.ErrorInfo { + if localClientLibruntime == nil { + return api.ErrorInfo{Code: errRPCUnavailable, Err: fmt.Errorf("dsclient is nil")} + } + paramLibruntime := api.MSetParam{ + WriteMode: api.WriteModeEnum(req.WriteMode), + TTLSecond: req.TtlSecond, + Existence: req.Existence, + CacheType: api.CacheTypeEnum(req.CacheType), + } + log.GetLogger().Debugf("set multi kvs to datasystem, param:%v", paramLibruntime) + var status api.ErrorInfo + runtime.LockOSThread() + if err := kvMSetTx(req, config.TenantID, paramLibruntime); err != nil { + status.Code = err.(api.ErrorInfo).Code + status.Err = err + } + runtime.UnlockOSThread() + if status.IsError() { + log.GetLogger().Warnf("MSetTx dsClient(nodeIP: %s) is unavailable, code: %d, err: %s, traceID: %s", + config.NodeIP, status.Code, status.Err, traceID) + return status + } + return status +} + +func kvMSetTx(req *data.KvMSetTxRequest, tenantID string, paramLibruntime api.MSetParam) error { + if err := localClientLibruntime.SetTenantID(tenantID); err != nil { + return err + } + return localClientLibruntime.KVMSetTx(req.Keys, req.Values, paramLibruntime) +} + +// Get - get kv to datasystem +func Get(req *data.KvGetRequest, config *Config, traceID string) ([][]byte, api.ErrorInfo) { + if localClientLibruntime == nil { + return nil, api.ErrorInfo{Code: errRPCUnavailable, Err: fmt.Errorf("dsclient is nil")} + } + var status api.ErrorInfo + var byteValues [][]byte + runtime.LockOSThread() + var err error + byteValues, err = kvGetMulti(req, config.TenantID) + if err != nil { + status.Code = err.(api.ErrorInfo).Code + status.Err = err + } + runtime.UnlockOSThread() + if status.IsError() { + log.GetLogger().Warnf("Get dsClient(nodeIP: %s) is unavailable, code: %d, err: %s, traceID: %s", + config.NodeIP, status.Code, status.Err, traceID) + return byteValues, status + } + return byteValues, status +} + +func kvGetMulti(req *data.KvGetRequest, tenantID string) ([][]byte, error) { + if err := localClientLibruntime.SetTenantID(tenantID); err != nil { + return nil, err + } + return localClientLibruntime.KVGetMulti(req.Keys, uint(req.TimeoutMs)) +} + +// Del - del kv to datasystem +func Del(req *data.KvDelRequest, config *Config, traceID string) ([]string, api.ErrorInfo) { + if localClientLibruntime == nil { + return nil, api.ErrorInfo{Code: errRPCUnavailable, Err: fmt.Errorf("dsclient is nil")} + } + var values []string + var status api.ErrorInfo + + runtime.LockOSThread() + var err error + values, err = kvDelMulti(req, config.TenantID) + if err != nil { + status.Code = err.(api.ErrorInfo).Code + status.Err = err + } + runtime.UnlockOSThread() + if status.IsError() { + log.GetLogger().Warnf("Del dsClient(nodeIP: %s) is unavailable, code: %d, err: %s, traceID: %s", + config.NodeIP, status.Code, status.Err, traceID) + return values, status + } + return values, status +} + +func kvDelMulti(req *data.KvDelRequest, tenantID string) ([]string, error) { + if err := localClientLibruntime.SetTenantID(tenantID); err != nil { + return nil, err + } + return localClientLibruntime.KVDelMulti(req.Keys) +} + +// GReleaseGRefs - release global ref +func GReleaseGRefs(remoteClientID string, config *Config, traceID string) api.ErrorInfo { + if localClientLibruntime == nil { + return api.ErrorInfo{Code: errRPCUnavailable, Err: fmt.Errorf("dsclient is nil")} + } + var status api.ErrorInfo + runtime.LockOSThread() + localClientLibruntime.SetTraceID(traceID) + err := localClientLibruntime.ReleaseGRefs(remoteClientID) + if err != nil { + status.Code = err.(api.ErrorInfo).Code + status.Err = err + } + runtime.UnlockOSThread() + if status.IsError() { + log.GetLogger().Warnf("dsClient(nodeIP: %s) is unavailable, code: %d, err: %s, traceID: %s", + config.NodeIP, status.Code, status.Err, traceID) + return status + } + return status +} + +// SubscribeStream - +func SubscribeStream(param SubscribeParam, ctx StreamCtx) error { + var subscriptionConfig api.SubscriptionConfig + subscriptionConfig.SubscriptionName = param.StreamName + subscriptionConfig.SubscriptionType = api.Stream + subscriptionConfig.TraceId = param.TraceId + runtime.LockOSThread() + if err := localClientLibruntime.SetTenantID(ctx.GetRequestHeader(constant.HeaderTenantID)); err != nil { + runtime.UnlockOSThread() + return err + } + consumer, errorInfo := localClientLibruntime.Subscribe(param.StreamName, subscriptionConfig) + runtime.UnlockOSThread() + if errorInfo == nil { + receiveStream(param, consumer, ctx) + return nil + + } + log.GetLogger().Warnf("create stream consumer failed, streamName: "+ + "%s, message: %s", param.StreamName, errorInfo.Error()) + return errorInfo +} + +func receiveStream(param SubscribeParam, consumer api.StreamConsumer, ctx StreamCtx) { + ctx.SetResponseHeader("Content-Type", "text/event-stream") + ctx.SetResponseHeader("Cache-Control", "no-cache") + ctx.SetResponseHeader("Connection", "keep-alive") + logger := log.GetLogger().With(zap.Any("streamName", param.StreamName)) + ctx.Stream(func(w io.Writer) bool { + defer streamFinishedHandler(param, consumer) + cancelCh := ctx.Done() + closeCh := make(<-chan bool) + notify, ok := w.(http.CloseNotifier) + err := ctx.FlushResult(w, []byte("")) + if err != nil { + logger.Warnf("flush stream result failed, error: %s", err.Error()) + return false + } + if ok { + closeCh = notify.CloseNotify() + } + startTime := time.Now() + for { + select { + case _, ok = <-cancelCh: + if !ok { + logger.Warnf("cancel channel is closed") + } + logger.Warnf("subscribe request of stream client is canceled") + return false + // This case takes effect only when ginCtx is used. + // When the client is disconnected, return and close consumer. + case _, ok = <-closeCh: + if !ok { + logger.Warnf("close channel is closed") + } + logger.Warnf("http connection of stream client is closed") + return false + default: + } + elements, errorInfo := receiveElements(consumer, defaultSubscribeTimeoutMs, param.ExpectReceiveNum) + if errorInfo != nil { + logger.Warnf("receive stream error: %s,element size: %d", errorInfo.Error(), len(elements)) + break + } + if len(elements) == 0 && time.Now().Sub(startTime).Milliseconds() <= int64(param.TimeoutMs) { + continue + } + if len(elements) == 0 { + logger.Warnf("receive stream failed,element size is zero") + break + } + startTime = time.Now() + result, streamEnd := processElements(elements, consumer, logger) + err := ctx.FlushResult(w, result) + if err != nil { + logger.Warnf("flush stream result failed, error: %s,bytes: %s", err.Error(), result) + if strings.Contains(err.Error(), "connection closed") || + strings.Contains(err.Error(), "connection was forcibly closed by the remote host") { + return true + } + } + if streamEnd { + logger.Infof("receive end element,close consumer") + return false + } + } + return false + }) +} + +func streamFinishedHandler(param SubscribeParam, consumer api.StreamConsumer) { + logger := log.GetLogger().With(zap.Any("streamName", param.StreamName)) + if err := consumer.Close(); err != nil { + logger.Errorf("failed to close consumer %s", err.Error()) + } + logger.Infof("consumer close success") + if param.Callback != nil { + param.Callback() + } +} + +func processElements(elements []api.Element, consumer api.StreamConsumer, logger api.FormatLogger) ([]byte, bool) { + var ( + result []byte + streamEnd bool + ) + for i := range elements { + sh := reflect.SliceHeader{ + Data: uintptr(unsafe.Pointer(elements[i].Ptr)), + Len: int(elements[i].Size), + Cap: int(elements[i].Size), + } + bytes := *(*[]byte)(unsafe.Pointer(&sh)) + logger.Debugf("receive stream with element: %s, size, %d", string(bytes), elements[i].Size) + consumer.Ack(elements[i].Id) + if elements[i].Size == StreamEndElementSize && + string(bytes) == StreamEndElement { + streamEnd = true + break + } + result = append(result, bytes...) + } + result = append(result, '\n') + return result, streamEnd +} + +func receiveElements(consumer api.StreamConsumer, timeoutMs uint32, expectReceiveNum int32) ([]api.Element, error) { + if expectReceiveNum <= 0 { + return consumer.Receive(timeoutMs) + } else { + expectNum := uint32(expectReceiveNum) + return consumer.ReceiveExpectNum(expectNum, timeoutMs) + } +} + +func encryptData(cfg *Config, data []byte) ([]byte, error) { + var res []byte + return res, nil +} + +func decryptData(cfg *Config, data []byte) ([]byte, error) { + var res []byte + return res, nil +} diff --git a/frontend/pkg/common/faas_common/datasystemclient/api_test.go b/frontend/pkg/common/faas_common/datasystemclient/api_test.go new file mode 100644 index 0000000000000000000000000000000000000000..44e7630f54fb09e16f1cc353663d68467ff29763 --- /dev/null +++ b/frontend/pkg/common/faas_common/datasystemclient/api_test.go @@ -0,0 +1,1471 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package datasystemclient is data system client used for communicating with data system worker. +// To use data system, you should export the data system lib path. Please refer to the Dockerfile of the frontend. +// The lib should copied to home/sn/bin/datasystem/lib. Please refer to +// functioncore/build/common/common_compile.sh and the Dockerfile of the frontend. +// NOTE: To change the version of data system, must revise the version in the common_compile.sh, test.sh and the go.mod +package datasystemclient + +import ( + "encoding/base64" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "reflect" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/gin-gonic/gin" + "github.com/smartystreets/goconvey/convey" + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/grpc/pb/data" + "frontend/pkg/common/faas_common/logger/log" + mockUtils "frontend/pkg/common/faas_common/utils" +) + +type FakeKvClient struct { + num int +} + +func (f *FakeKvClient) KVSet(key string, value []byte, param api.SetParam) api.ErrorInfo { + switch key { + case "key1": + return api.ErrorInfo{Code: 0, Err: nil} + case "key2": + return api.ErrorInfo{Code: errOutOfMemory, Err: errors.New("err2")} + } + return api.ErrorInfo{Code: 0, Err: nil} +} + +func (f *FakeKvClient) KVSetWithoutKey(value []byte, param api.SetParam) (string, api.ErrorInfo) { + return "", api.ErrorInfo{} +} + +func (f *FakeKvClient) KVGet(key string, timeoutms ...uint32) ([]byte, api.ErrorInfo) { + switch key { + case "key1": + return []byte("value1"), api.ErrorInfo{Code: 0, Err: nil} + case "key2": + return []byte("value1"), api.ErrorInfo{Code: errOutOfMemory, Err: errors.New("err2")} + } + return []byte(""), api.ErrorInfo{} +} + +func (f *FakeKvClient) KVGetMulti(keys []string, timeoutms ...uint32) ([][]byte, api.ErrorInfo) { + switch len(keys) { + case 3: + return nil, api.ErrorInfo{Code: errKeyNotFound} + case 2: + return [][]byte{[]byte("value1"), []byte("value2")}, api.ErrorInfo{Code: 0, Err: nil} + case 1: + return [][]byte{[]byte("value3")}, api.ErrorInfo{Code: errOutOfMemory, Err: errors.New("err2")} + } + return [][]byte{}, api.ErrorInfo{} +} + +func (f *FakeKvClient) KVQuerySize(keys []string) ([]uint64, api.ErrorInfo) { + switch len(keys) { + case 1: + return []uint64{10}, api.ErrorInfo{} + case 2: + return []uint64{10, 10}, api.ErrorInfo{} + case 3: + return []uint64{100, 100, 100}, api.ErrorInfo{} + case 4: + return []uint64{}, api.ErrorInfo{Code: errOutOfMemory, Err: errors.New("err2")} + case 5: + return []uint64{100000, 100000, 100000, 100000, 100000}, api.ErrorInfo{} + } + return []uint64{}, api.ErrorInfo{} +} + +func (f *FakeKvClient) KVDel(key string) api.ErrorInfo { + switch key { + case "key1": + return api.ErrorInfo{Code: 0, Err: nil} + case "key2": + return api.ErrorInfo{Code: errOutOfMemory, Err: errors.New("err2")} + } + return api.ErrorInfo{} +} + +func (f *FakeKvClient) KVDelMulti(keys []string) ([]string, api.ErrorInfo) { + switch len(keys) { + case 2: + return nil, api.ErrorInfo{Code: 0, Err: nil} + case 1: + return []string{"key3"}, api.ErrorInfo{Code: errOutOfMemory, Err: errors.New("err2")} + } + return []string{}, api.ErrorInfo{} +} + +func (f *FakeKvClient) GenerateKey() string { + return "1" +} + +func (f *FakeKvClient) SetTraceID(traceID string) { +} + +func (f *FakeKvClient) DestroyClient() {} + +func TestUploadWithKeyRetryKvClient(t *testing.T) { + cache := &Cache{ + nodeList: []string{}, + invalidMap: make(map[string]struct{}, 1), + } + cache.addNode("127.2.2.101", log.GetLogger()) + cache.addNode("127.2.2.102", log.GetLogger()) + dataSystemCache.Store(noCluster, cache) + type args struct { + deviceID string + value []byte + config *Config + param api.SetParam + sourceClient DsClientImpl + } + tests := []struct { + name string + args args + wantErr bool + patchesFunc mockUtils.PatchesFunc + }{ + {"case1 succeed to upload key", args{deviceID: "key1", value: []byte("value1"), + config: &Config{TenantID: "tenant1", NodeIP: "127.2.2.101"}, + param: api.SetParam{WriteMode: api.NoneL2Cache, TTLSecond: 60 * 1000}, + sourceClient: DsClientImpl{kvClient: &FakeKvClient{num: 1}}}, false, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(NewClient, + func(tenantID string, nodeIP string) (DsClientImpl, error) { + return DsClientImpl{kvClient: &FakeKvClient{num: 1}}, nil + }), + gomonkey.ApplyFunc((*Cache).healthCheckProcess, + func(_ *Cache, node string) { + return + }), + }) + return patches + }}, + + {"case2 succeed to upload key when node ip is null", args{deviceID: "key1", value: []byte("value1"), + config: &Config{TenantID: "tenant1", NodeIP: ""}, + param: api.SetParam{WriteMode: api.NoneL2Cache, TTLSecond: 60 * 1000}, + sourceClient: DsClientImpl{kvClient: &FakeKvClient{num: 1}}}, false, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(NewClient, + func(tenantID string, nodeIP string) (DsClientImpl, error) { + return DsClientImpl{kvClient: &FakeKvClient{num: 1}}, nil + }), + gomonkey.ApplyFunc((*Cache).healthCheckProcess, + func(_ *Cache, node string) { + return + }), + }) + return patches + }}, + {"case3 failed to upload key when node ip and tenantID is null", args{deviceID: "key1", value: []byte("value1"), + config: &Config{TenantID: "", NodeIP: ""}, + param: api.SetParam{WriteMode: api.NoneL2Cache, TTLSecond: 60 * 1000}, + sourceClient: DsClientImpl{kvClient: &FakeKvClient{num: 1}}}, true, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(NewClient, + func(tenantID string, nodeIP string) (DsClientImpl, error) { + return DsClientImpl{}, errors.New("failed to upload") + }), + gomonkey.ApplyFunc((*Cache).healthCheckProcess, + func(_ *Cache, node string) { + return + }), + }) + return patches + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + patches := tt.patchesFunc() + localClientLibruntime = &mockUtils.FakeLibruntimeSdkClient{} + if _, err := UploadWithKeyRetry(tt.args.value, tt.args.config, tt.args.param, + "traceID"); (err != nil) != tt.wantErr { + t.Errorf("UploadWithKeyRetry() error = %v, wantErr %v", err, tt.wantErr) + } + patches.ResetAll() + }) + } + clientMap = concurrentMap{mp: make(map[string]*nodeIP2ClientMap)} + dataSystemCache.Delete(noCluster) +} + +func TestUploadWithKeyRetry(t *testing.T) { + cache := &Cache{ + nodeList: []string{}, + invalidMap: make(map[string]struct{}, 1), + } + cache.addNode("127.2.2.101", log.GetLogger()) + cache.addNode("127.2.2.102", log.GetLogger()) + dataSystemCache.Store(noCluster, cache) + type args struct { + deviceID string + value []byte + config *Config + param api.SetParam + localClientLibruntime api.LibruntimeAPI + } + tests := []struct { + name string + args args + wantErr bool + patchesFunc mockUtils.PatchesFunc + }{ + {"case1 succeed to upload key", args{deviceID: "key1", value: []byte("value1"), + config: &Config{TenantID: "tenant1", NodeIP: "127.2.2.101"}, + param: api.SetParam{WriteMode: api.NoneL2Cache, TTLSecond: 60 * 1000}, + localClientLibruntime: &mockUtils.FakeLibruntimeSdkClient{}}, + false, + func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(NewClient, + func(tenantID string, nodeIP string) (DsClientImpl, error) { + return DsClientImpl{kvClient: &FakeKvClient{}}, nil + }), + gomonkey.ApplyFunc((*Cache).healthCheckProcess, + func(_ *Cache, node string) { + return + }), + }) + return patches + }}, + + {"case2 succeed to upload key when node ip is null", args{deviceID: "key1", value: []byte("value1"), + config: &Config{TenantID: "tenant1", NodeIP: ""}, + param: api.SetParam{WriteMode: api.NoneL2Cache, TTLSecond: 60 * 1000}, + localClientLibruntime: &mockUtils.FakeLibruntimeSdkClient{}}, + false, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(NewClient, + func(tenantID string, nodeIP string) (DsClientImpl, error) { + return DsClientImpl{kvClient: &FakeKvClient{}}, nil + }), + gomonkey.ApplyFunc((*Cache).healthCheckProcess, + func(_ *Cache, node string) { + return + }), + }) + return patches + }}, + {"case3 failed to upload key when node ip and tenantID is null", args{deviceID: "key1", value: []byte("value1"), + config: &Config{TenantID: "", NodeIP: ""}, + param: api.SetParam{WriteMode: api.NoneL2Cache, TTLSecond: 60 * 1000}, + localClientLibruntime: &mockUtils.FakeLibruntimeSdkClient{}}, true, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(NewClient, + func(tenantID string, nodeIP string) (DsClientImpl, error) { + return DsClientImpl{}, errors.New("failed to upload") + }), + gomonkey.ApplyFunc((*Cache).healthCheckProcess, + func(_ *Cache, node string) { + return + }), + }) + return patches + }}, + {"case4 failed to set tenant id", args{deviceID: "key1", value: []byte("value1"), + config: &Config{TenantID: "tenant1", NodeIP: "127.2.2.101"}, + param: api.SetParam{WriteMode: api.NoneL2Cache, TTLSecond: 60 * 1000}, + localClientLibruntime: &mockUtils.FakeLibruntimeSdkClient{}}, + true, + func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(NewClient, + func(tenantID string, nodeIP string) (DsClientImpl, error) { + return DsClientImpl{kvClient: &FakeKvClient{}}, nil + }), + gomonkey.ApplyFunc((*Cache).healthCheckProcess, + func(_ *Cache, node string) { + return + }), + gomonkey.ApplyFunc((*mockUtils.FakeLibruntimeSdkClient).SetTenantID, + func(_ *mockUtils.FakeLibruntimeSdkClient, tenantID string) error { + return errors.New("set tenant failed") + }), + }) + return patches + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + patches := tt.patchesFunc() + localClientLibruntime = tt.args.localClientLibruntime + tt.args.config.KeyPrefix = tt.args.deviceID + if _, err := UploadWithKeyRetry(tt.args.value, tt.args.config, tt.args.param, + "traceID"); (err != nil) != tt.wantErr { + t.Errorf("UploadWithKeyRetry() error = %v, wantErr %v", err, tt.wantErr) + } + patches.ResetAll() + }) + } + clientMap = concurrentMap{mp: make(map[string]*nodeIP2ClientMap)} + dataSystemCache.Delete(noCluster) +} + +func TestDownloadArrayRetryKvClient(t *testing.T) { + cache := &Cache{ + nodeList: []string{}, + invalidMap: make(map[string]struct{}, 1), + } + cache.addNode("127.2.2.101", log.GetLogger()) + cache.addNode("127.2.2.102", log.GetLogger()) + dataSystemCache.Store(noCluster, cache) + type args struct { + keys []string + config *Config + sourceClient DsClientImpl + } + tests := []struct { + name string + args args + want [][]byte + wantErr bool + patchesFunc mockUtils.PatchesFunc + }{ + {"case1 succeed to download array", args{keys: []string{"key1", "key2"}, + config: &Config{TenantID: "tenant1", NodeIP: "127.2.2.101"}, + sourceClient: DsClientImpl{kvClient: &FakeKvClient{1}}}, [][]byte{[]byte("value1"), []byte("value2")}, + false, + func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(NewClient, + func(tenantID string, nodeIP string) (DsClientImpl, error) { + return DsClientImpl{kvClient: &FakeKvClient{1}}, nil + }), + gomonkey.ApplyFunc((*Cache).healthCheckProcess, + func(_ *Cache, node string) { + return + }), + }) + return patches + }}, + {"case2 failed to query size", args{keys: []string{"key1", "key2", "key3", "key4"}, + config: &Config{TenantID: "tenant1", NodeIP: "127.2.2.101"}, + sourceClient: DsClientImpl{kvClient: &FakeKvClient{1}}}, nil, true, + func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(NewClient, + func(tenantID string, nodeIP string) (DsClientImpl, error) { + return DsClientImpl{kvClient: &FakeKvClient{1}}, nil + }), + gomonkey.ApplyFunc((*Cache).healthCheckProcess, + func(_ *Cache, node string) { + return + }), + }) + return patches + }}, + {"case3 failed to download array node ip and tenantID is null", args{keys: []string{"key3"}, + config: &Config{TenantID: "", NodeIP: ""}, + sourceClient: DsClientImpl{kvClient: &FakeKvClient{1}}}, nil, true, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(NewClient, + func(tenantID string, nodeIP string) (DsClientImpl, error) { + return DsClientImpl{}, errors.New("failed to download") + }), + gomonkey.ApplyFunc((*Cache).healthCheckProcess, + func(_ *Cache, node string) { + return + }), + }) + return patches + }}, + {"case4 errKeyNotFound", args{keys: []string{"key1", "key2", "key3"}, + config: &Config{TenantID: "tenant1", NodeIP: "127.2.2.101"}, + sourceClient: DsClientImpl{kvClient: &FakeKvClient{1}}}, nil, true, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(NewClient, + func(tenantID string, nodeIP string) (DsClientImpl, error) { + return DsClientImpl{}, errors.New("failed to download") + }), + gomonkey.ApplyFunc((*Cache).healthCheckProcess, + func(_ *Cache, node string) { + return + }), + }) + return patches + }}, + {"case5 failed to query in limit", args{keys: []string{"key1", "key2", "key3", "key4", "key5"}, + config: &Config{TenantID: "tenant1", NodeIP: "127.2.2.101", Limit: 2}, + sourceClient: DsClientImpl{kvClient: &FakeKvClient{1}}}, nil, true, + func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(NewClient, + func(tenantID string, nodeIP string) (DsClientImpl, error) { + return DsClientImpl{kvClient: &FakeKvClient{1}}, nil + }), + gomonkey.ApplyFunc((*Cache).healthCheckProcess, + func(_ *Cache, node string) { + return + }), + }) + return patches + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + patches := tt.patchesFunc() + localClientLibruntime = &mockUtils.FakeLibruntimeSdkClient{} + got, err := DownloadArrayRetry(tt.args.keys, tt.args.config, "traceID") + if (err != nil) != tt.wantErr { + t.Errorf("DownloadArrayRetry() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("DownloadArrayRetry() got = %v, want %v", got, tt.want) + } + patches.ResetAll() + }) + } + clientMap = concurrentMap{mp: make(map[string]*nodeIP2ClientMap)} + dataSystemCache.Delete(noCluster) +} + +func Test_getDataSystemKey(t *testing.T) { + convey.Convey("getDataSystemKey ok", t, func() { + config := &Config{} + config.NoNeedGenKey = true + config.KeyPrefix = "/dt" + key, genKey, err := getDataSystemKey(config, DsClientImpl{}, "aaaa") + convey.So(key, convey.ShouldNotBeNil) + convey.So(genKey, convey.ShouldEqual, "") + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("getDataSystemKey ok 1", t, func() { + p := gomonkey.ApplyFunc(getClient, func(cfg *Config, _ string) (DsClientImpl, bool, error) { + return DsClientImpl{kvClient: &FakeKvClient{num: 1}}, false, nil + }) + defer p.Reset() + config := &Config{} + config.NoNeedGenKey = false + config.KeyPrefix = "aaa" + dsClient, _, _ := getClient(config, "") + _, _, err := getDataSystemKey(config, dsClient, "aaaa") + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("getDataSystemKey ok 2", t, func() { + p := gomonkey.ApplyFunc(getClient, func(cfg *Config) (DsClientImpl, bool, error) { + return DsClientImpl{kvClient: &FakeKvClient{num: 1}}, false, nil + }) + defer p.Reset() + config := &Config{} + config.NoNeedGenKey = false + config.KeyPrefix = "aaa" + config.useLastUsedNode = true + dsClient, _, _ := getClient(config, "") + _, _, err := getDataSystemKey(config, dsClient, "aaaa") + convey.So(err, convey.ShouldBeNil) + }) +} + +func TestUploadWithoutKeyRetry(t *testing.T) { + cache := &Cache{ + nodeList: []string{}, + invalidMap: make(map[string]struct{}, 1), + } + cache.addNode("127.2.2.101", log.GetLogger()) + cache.addNode("127.2.2.102", log.GetLogger()) + dataSystemCache.Store(noCluster, cache) + type args struct { + value []byte + config *Config + param api.SetParam + localClientLibruntime api.LibruntimeAPI + } + tests := []struct { + name string + args args + want string + wantErr bool + patchesFunc mockUtils.PatchesFunc + }{ + {"case1 succeed to upload without key", args{value: []byte("value1"), + config: &Config{TenantID: "tenant1", NodeIP: "127.2.2.101"}, + param: api.SetParam{WriteMode: api.NoneL2Cache, TTLSecond: 60 * 1000}, + localClientLibruntime: &mockUtils.FakeLibruntimeSdkClient{}}, + "", + false, + func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(NewClient, + func(tenantID string, nodeIP string) (DsClientImpl, error) { + return DsClientImpl{kvClient: &FakeKvClient{}}, nil + }), + gomonkey.ApplyFunc((*Cache).healthCheckProcess, + func(_ *Cache, node string) { + return + }), + }) + return patches + }}, + {"case2 failed to upload without key node ip and tenantID is null", args{value: []byte("value2"), + config: &Config{TenantID: "", NodeIP: ""}, + param: api.SetParam{WriteMode: api.NoneL2Cache, TTLSecond: 60 * 1000}, + localClientLibruntime: &mockUtils.FakeLibruntimeSdkClient{}}, "", true, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(NewClient, + func(tenantID string, nodeIP string) (DsClientImpl, error) { + return DsClientImpl{}, errors.New("failed to upload") + }), + gomonkey.ApplyFunc((*Cache).healthCheckProcess, + func(_ *Cache, node string) { + return + }), + }) + return patches + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + patches := tt.patchesFunc() + localClientLibruntime = tt.args.localClientLibruntime + got, err := UploadWithoutKeyRetry(tt.args.value, tt.args.config, tt.args.param, "traceID") + if (err != nil) != tt.wantErr { + t.Errorf("UploadWithoutKeyRetry() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("UploadWithoutKeyRetry() got = %v, want %v", got, tt.want) + } + patches.ResetAll() + }) + } + clientMap = concurrentMap{mp: make(map[string]*nodeIP2ClientMap)} + dataSystemCache.Delete(noCluster) +} + +func TestDeleteArrayRetryKvClient(t *testing.T) { + cache := &Cache{ + nodeList: []string{}, + invalidMap: make(map[string]struct{}, 1), + } + cache.addNode("127.2.2.101", log.GetLogger()) + cache.addNode("127.2.2.102", log.GetLogger()) + dataSystemCache.Store(noCluster, cache) + type args struct { + keys []string + config *Config + sourceClient DsClientImpl + } + tests := []struct { + name string + args args + want []string + wantErr bool + patchesFunc mockUtils.PatchesFunc + }{ + {"case1 succeed to download array", args{keys: []string{"key1", "key2"}, + config: &Config{TenantID: "tenant1", NodeIP: "127.2.2.101"}, + sourceClient: DsClientImpl{kvClient: &FakeKvClient{1}}}, nil, false, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(NewClient, + func(tenantID string, nodeIP string) (DsClientImpl, error) { + return DsClientImpl{kvClient: &FakeKvClient{num: 1}}, nil + }), + gomonkey.ApplyFunc((*Cache).healthCheckProcess, + func(_ *Cache, node string) { + return + }), + }) + return patches + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + patches := tt.patchesFunc() + got, err := DeleteArrayRetry(tt.args.keys, tt.args.config, "traceID") + if (err != nil) != tt.wantErr { + t.Errorf("DeleteArrayRetry() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("DeleteArrayRetry() got = %v, want %v", got, tt.want) + } + patches.ResetAll() + }) + } + clientMap = concurrentMap{mp: make(map[string]*nodeIP2ClientMap)} + dataSystemCache.Delete(noCluster) +} + +func TestDeleteArrayRetry(t *testing.T) { + cache := &Cache{ + nodeList: []string{}, + invalidMap: make(map[string]struct{}, 1), + } + cache.addNode("127.2.2.101", log.GetLogger()) + cache.addNode("127.2.2.102", log.GetLogger()) + dataSystemCache.Store(noCluster, cache) + type args struct { + keys []string + config *Config + sourceClient DsClientImpl + } + tests := []struct { + name string + args args + want []string + wantErr bool + patchesFunc mockUtils.PatchesFunc + }{ + {"case1 succeed to download array", args{keys: []string{"key1", "key2"}, + config: &Config{TenantID: "tenant1", NodeIP: "127.2.2.101"}, + sourceClient: DsClientImpl{kvClient: &FakeKvClient{}}}, nil, false, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(NewClient, + func(tenantID string, nodeIP string) (DsClientImpl, error) { + return DsClientImpl{kvClient: &FakeKvClient{num: 1}}, nil + }), + gomonkey.ApplyFunc((*Cache).healthCheckProcess, + func(_ *Cache, node string) { + return + }), + }) + return patches + }}, + {"case2 failed to download array after retry", args{keys: []string{"key3"}, + config: &Config{TenantID: "tenant1", NodeIP: "127.2.2.102"}, + sourceClient: DsClientImpl{kvClient: &FakeKvClient{}}}, []string{"key3"}, true, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(NewClient, + func(tenantID string, nodeIP string) (DsClientImpl, error) { + return DsClientImpl{}, errors.New("failed to delete") + }), + gomonkey.ApplyFunc((*Cache).healthCheckProcess, + func(_ *Cache, node string) { + return + }), + }) + return patches + }}, + {"case3 failed to download node ip and tenantID is null", args{keys: []string{"key3"}, + config: &Config{TenantID: "", NodeIP: ""}, + sourceClient: DsClientImpl{kvClient: &FakeKvClient{}}}, []string{"key3"}, true, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(NewClient, + func(tenantID string, nodeIP string) (DsClientImpl, error) { + return DsClientImpl{}, errors.New("failed to delete") + }), + gomonkey.ApplyFunc((*Cache).healthCheckProcess, + func(_ *Cache, node string) { + return + }), + }) + return patches + }}, + { + "case4 len key is 0", args{keys: []string{}, + config: &Config{TenantID: "", NodeIP: ""}, + sourceClient: DsClientImpl{kvClient: &FakeKvClient{}}}, []string{}, true, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(NewClient, + func(tenantID string, nodeIP string) (DsClientImpl, error) { + return DsClientImpl{}, errors.New("failed to delete") + }), + gomonkey.ApplyFunc((*Cache).healthCheckProcess, + func(_ *Cache, node string) { + return + }), + }) + return patches + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + patches := tt.patchesFunc() + got, err := DeleteArrayRetry(tt.args.keys, tt.args.config, "traceID") + if (err != nil) != tt.wantErr { + t.Errorf("DeleteArrayRetry() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("DeleteArrayRetry() got = %v, want %v", got, tt.want) + } + patches.ResetAll() + }) + } + clientMap = concurrentMap{mp: make(map[string]*nodeIP2ClientMap)} + dataSystemCache.Delete(noCluster) +} + +func TestDeleteClient(t *testing.T) { + cm := concurrentMap{mp: make(map[string]*nodeIP2ClientMap)} + var sourceClient = &FakeKvClient{} + cm.mp["tenant1"] = &nodeIP2ClientMap{ + clientMap: map[string]DsClientImpl{ + "127.2.2.101": {kvClient: sourceClient}, + "127.2.2.102": {kvClient: sourceClient}, + }, + RWMutex: sync.RWMutex{}, + logger: log.GetLogger(), + } + convey.Convey("delete tenantID & nodeIP", t, func() { + cm.deleteClient("tenant1", "127.2.2.101") + client, ok := cm.get("tenant1", "127.2.2.101") + convey.So(client.kvClient, convey.ShouldBeNil) + convey.So(ok, convey.ShouldBeFalse) + }) + convey.Convey("delete tenantID", t, func() { + cm.deleteTenant("tenant1") + client := cm.getOneClient("tenant1") + convey.So(client, convey.ShouldBeNil) + }) + convey.Convey("delete nodeIp", t, func() { + cm.mp["tenant1"] = &nodeIP2ClientMap{ + clientMap: map[string]DsClientImpl{ + "127.2.2.101": {kvClient: sourceClient}, + "127.2.2.102": {kvClient: sourceClient}, + }, + RWMutex: sync.RWMutex{}, + logger: log.GetLogger(), + } + cm.mp["tenant2"] = &nodeIP2ClientMap{ + clientMap: map[string]DsClientImpl{ + "127.2.2.101": {kvClient: sourceClient}, + "127.2.2.102": {kvClient: sourceClient}, + }, + RWMutex: sync.RWMutex{}, + logger: log.GetLogger(), + } + cm.mp["tenant3"] = &nodeIP2ClientMap{ + clientMap: map[string]DsClientImpl{ + "127.2.2.101": {kvClient: sourceClient}, + }, + RWMutex: sync.RWMutex{}, + logger: log.GetLogger(), + } + cm.deleteNodeIp("127.2.2.101", log.GetLogger()) + convey.So(cm.mp["tenant1"].size(), convey.ShouldEqual, 1) + convey.So(cm.mp["tenant2"].size(), convey.ShouldEqual, 1) + _, ok := cm.mp["tenant3"] + convey.So(ok, convey.ShouldEqual, false) + }) +} + +func Test_uploadWithKeyKvClient(t *testing.T) { + convey.Convey("uploadWithKey test", t, func() { + convey.Convey("test encrypt", func() { + p := gomonkey.ApplyFunc(getClient, func(cfg *Config, _ string) (DsClientImpl, bool, error) { + return DsClientImpl{kvClient: &FakeKvClient{num: 1}}, false, nil + }) + defer p.Reset() + key, b, err := uploadWithKey([]byte("value"), &Config{NeedEncrypt: true, KeyPrefix: "aaa"}, api.SetParam{}, + "") + convey.So(key, convey.ShouldEqual, "") + convey.So(b, convey.ShouldBeFalse) + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("test set tenantID fail", func() { + patches := []*gomonkey.Patches{ + gomonkey.ApplyFunc(getClient, func(cfg *Config, _ string) (DsClientImpl, bool, error) { + return DsClientImpl{kvClient: &FakeKvClient{num: 1}}, false, nil + }), + gomonkey.ApplyFunc((*mockUtils.FakeLibruntimeSdkClient).SetTenantID, + func(_ *mockUtils.FakeLibruntimeSdkClient, tenantID string) error { + return errors.New("set tenant failed") + }), + } + for _, patch := range patches { + patch.Reset() + } + + key, b, err := uploadWithKey([]byte("value"), &Config{NeedEncrypt: false}, api.SetParam{}, "") + convey.So(key, convey.ShouldEqual, "") + convey.So(b, convey.ShouldBeFalse) + convey.So(err, convey.ShouldNotBeNil) + }) + }) +} + +func Test_uploadWithKey(t *testing.T) { + type args struct { + deviceID string + value []byte + config *Config + param api.SetParam + traceID string + } + tests := []struct { + name string + args args + want1 bool + wantErr bool + patchesFunc mockUtils.PatchesFunc + }{ + {"case1 dsClient is nil", args{config: &Config{}}, true, true, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(getClient, + func(cfg *Config, _ string) (DsClientImpl, bool, error) { return DsClientImpl{}, true, errors.New("e") }), + }) + return patches + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + patches := tt.patchesFunc() + _, got1, err := uploadWithKey(tt.args.value, tt.args.config, tt.args.param, tt.args.traceID) + if (err != nil) != tt.wantErr { + t.Errorf("uploadWithKey() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got1 != tt.want1 { + t.Errorf("uploadWithKey() got1 = %v, want %v", got1, tt.want1) + } + patches.ResetAll() + }) + } +} + +type invokerLibruntimeMock struct { + setTenantIDSuccessfully bool +} + +func (c *invokerLibruntimeMock) CreateInstance(funcMeta api.FunctionMeta, args []api.Arg, + invokeOpt api.InvokeOptions) (instanceID string, err error) { + return "", nil +} + +func (c *invokerLibruntimeMock) InvokeByInstanceId(funcMeta api.FunctionMeta, instanceID string, args []api.Arg, + invokeOpt api.InvokeOptions) (returnObjectID string, err error) { + return "", nil +} + +func (c *invokerLibruntimeMock) InvokeByFunctionName(funcMeta api.FunctionMeta, args []api.Arg, + invokeOpt api.InvokeOptions) (returnObjectID string, err error) { + return "", nil +} + +func (c *invokerLibruntimeMock) AcquireInstance(state string, funcMeta api.FunctionMeta, acquireOpt api.InvokeOptions) (api.InstanceAllocation, error) { + return api.InstanceAllocation{}, nil +} + +func (c *invokerLibruntimeMock) ReleaseInstance(allocation api.InstanceAllocation, stateID string, abnormal bool, option api.InvokeOptions) { + +} + +func (c *invokerLibruntimeMock) Kill(instanceID string, signal int, payload []byte) (err error) { + return nil +} + +func (c *invokerLibruntimeMock) CreateInstanceRaw(createReqRaw []byte) (createRespRaw []byte, err error) { + return []byte{}, nil +} + +func (c *invokerLibruntimeMock) InvokeByInstanceIdRaw(invokeReqRaw []byte) (resultRaw []byte, err error) { + return []byte{}, nil +} + +func (f *invokerLibruntimeMock) KillRaw(killReqRaw []byte) (killRespRaw []byte, err error) { + return []byte{}, nil +} + +func (f *invokerLibruntimeMock) SaveState(state []byte) (stateID string, err error) { + return "", nil +} + +func (f *invokerLibruntimeMock) LoadState(checkpointID string) (state []byte, err error) { + return []byte{}, nil +} + +func (f *invokerLibruntimeMock) Exit(code int, message string) { + return +} + +func (f *invokerLibruntimeMock) Finalize() { + return +} + +func (f *invokerLibruntimeMock) KVSet(key string, value []byte, param api.SetParam) (err error) { + return nil +} + +func (f *invokerLibruntimeMock) KVSetWithoutKey(value []byte, param api.SetParam) (key string, err error) { + return "", nil +} + +func (f *invokerLibruntimeMock) KVMSetTx(keys []string, values [][]byte, param api.MSetParam) error { + return nil +} + +func (f *invokerLibruntimeMock) KVGet(key string, timeoutms uint) (value []byte, err error) { + return []byte{}, nil +} + +func (f *invokerLibruntimeMock) KVGetMulti(keys []string, timeoutms uint) (values [][]byte, err error) { + return [][]byte{}, nil +} + +func (f *invokerLibruntimeMock) KVDel(key string) (err error) { + return nil +} + +func (f *invokerLibruntimeMock) KVDelMulti(keys []string) (failedKeys []string, err error) { + return []string{}, nil +} + +func (f *invokerLibruntimeMock) QueryGlobalProducersNum(streamName string) (uint64, error) { + return 0, nil +} + +func (f *invokerLibruntimeMock) QueryGlobalConsumersNum(streamName string) (uint64, error) { + return 0, nil +} + +func (f *invokerLibruntimeMock) Wait(objectIDs []string, waitNum uint64, timeoutMs int) (readyIDs, unReadyIDs []string, errors map[string]error) { + return []string{}, []string{}, make(map[string]error) +} + +func (f *invokerLibruntimeMock) SetTraceID(traceID string) { + return +} + +func (f *invokerLibruntimeMock) SetTenantID(tenantID string) error { + if f.setTenantIDSuccessfully { + return nil + } + return api.ErrorInfo{Code: 1001, Err: errors.New("failed to set tenant id")} +} + +func (f *invokerLibruntimeMock) Put(objectID string, value []byte, param api.PutParam, + nestedObjectIDs ...string) (err error) { + return nil +} + +func (f *invokerLibruntimeMock) Get(objectIDs []string, timeoutMs int) (data [][]byte, err error) { + return [][]byte{}, nil +} + +func (f *invokerLibruntimeMock) GIncreaseRef(objectIDs []string, remoteClientID ...string) (failedIDs []string, err error) { + return []string{}, nil +} + +func (f *invokerLibruntimeMock) GDecreaseRef(objectIDs []string, remoteClientID ...string) (failedIDs []string, err error) { + return []string{}, nil +} + +func (f *invokerLibruntimeMock) ReleaseGRefs(remoteClientID string) error { + return nil +} + +func (f *invokerLibruntimeMock) GetAsync(objectID string, cb api.GetAsyncCallback) { + return +} + +func (f *invokerLibruntimeMock) GetFormatLogger() api.FormatLogger { + return nil +} + +func (c *invokerLibruntimeMock) CreateProducer(streamName string, producerConf api.ProducerConf) (producer api.StreamProducer, err error) { + return producer, err +} + +func (c *invokerLibruntimeMock) Subscribe(streamName string, config api.SubscriptionConfig) (consumer api.StreamConsumer, err error) { + return consumer, nil +} + +func (c *invokerLibruntimeMock) DeleteStream(streamName string) (err error) { + return nil +} + +func (c *invokerLibruntimeMock) CreateClient(config api.ConnectArguments) (api.KvClient, error) { + return &FakeKvClient{1}, nil +} + +func (l *invokerLibruntimeMock) GIncreaseRefRaw(objectIDs []string, remoteClientID ...string) ([]string, error) { + return []string{}, nil +} + +func (l *invokerLibruntimeMock) PutRaw(objectID string, value []byte, param api.PutParam, nestedObjectIDs ...string) error { + return nil +} + +func (l *invokerLibruntimeMock) GetRaw(objectIDs []string, timeoutMs int) ([][]byte, error) { + return [][]byte{}, nil +} + +func (l *invokerLibruntimeMock) GDecreaseRefRaw(objectIDs []string, remoteClientID ...string) ([]string, error) { + return []string{}, nil +} +func (l *invokerLibruntimeMock) GetCredential() api.Credential { + return api.Credential{} +} + +func (l *invokerLibruntimeMock) UpdateSchdulerInfo(schedulerName string, schedulerId string, option string) { + return +} + +func (f *invokerLibruntimeMock) IsHealth() bool { + return true +} + +func (f *invokerLibruntimeMock) IsDsHealth() bool { + return true +} + +func TestKVGet(t *testing.T) { + convey.Convey("TestKVGetLibruntime", t, func() { + keyNotFound := 1 + mock := &invokerLibruntimeMock{setTenantIDSuccessfully: true} + kvStore := map[string][]byte{} + testKey := "test_key" + testValue := []byte{'1', '1', '1'} + patches := []*gomonkey.Patches{ + gomonkey.ApplyMethod(reflect.TypeOf(mock), "KVGetMulti", func(_ *invokerLibruntimeMock, keys []string, + timeoutms uint) ([][]byte, error) { + values := [][]byte{} + for _, key := range keys { + if value, ok := kvStore[key]; ok { + values = append(values, value) + continue + } + return values, api.ErrorInfo{Code: keyNotFound, Err: fmt.Errorf("key %s not found", key)} + } + return values, nil + }), + gomonkey.ApplyMethod(reflect.TypeOf(mock), "KVSet", func(_ *invokerLibruntimeMock, key string, + value []byte, param api.SetParam) error { + kvStore[key] = value + return nil + }), + } + defer func() { + for _, patch := range patches { + patch.Reset() + } + }() + + setLocalClient(mock) + setReq := data.KvSetRequest{ + Key: testKey, + Value: testValue, + } + err := Set(&setReq, &Config{}, "") + convey.So(err.Code, convey.ShouldEqual, 0) + convey.So(err.Err, convey.ShouldBeNil) + + getReq := data.KvGetRequest{ + Keys: []string{testKey}, + } + values, status := Get(&getReq, &Config{}, "") + convey.So(status.Code, convey.ShouldEqual, 0) + convey.So(status.Err, convey.ShouldBeNil) + convey.So(string(values[0]), convey.ShouldEqual, string(testValue)) + }) +} + +func TestObjPut(t *testing.T) { + convey.Convey("ObjPut", t, func() { + convey.Convey("dsclient is nil", func() { + localClientLibruntime = nil + put := ObjPut(&data.PutRequest{}, &Config{}, "test-trace-ID") + convey.So(put.Code, convey.ShouldEqual, errRPCUnavailable) + }) + convey.Convey("success", func() { + mock := &invokerLibruntimeMock{setTenantIDSuccessfully: true} + setLocalClient(mock) + put := ObjPut(&data.PutRequest{ + ObjectId: "", + ObjectData: nil, + WriteMode: int32(os.O_WRONLY), + ConsistencyType: 1, + NestedObjectIds: nil, + }, &Config{}, "test-trace-ID") + convey.So(put.Err, convey.ShouldBeNil) + }) + + convey.Convey("put failed", func() { + failedMock := &invokerLibruntimeMock{setTenantIDSuccessfully: false} + setLocalClient(failedMock) + put := ObjPut(&data.PutRequest{}, &Config{}, "test-trace-ID") + convey.So(put.Code, convey.ShouldEqual, 1001) + }) + }) +} + +func TestObjGet(t *testing.T) { + convey.Convey("ObjGet", t, func() { + convey.Convey("dsclient is nil", func() { + localClientLibruntime = nil + _, put := ObjGet(&data.GetRequest{}, &Config{}, "test-trace-ID") + convey.So(put.Code, convey.ShouldEqual, errRPCUnavailable) + }) + convey.Convey("success", func() { + localClientLibruntime = &invokerLibruntimeMock{setTenantIDSuccessfully: true} + values, put := ObjGet(&data.GetRequest{ + ObjectIds: nil, + TimeoutMs: 0, + }, &Config{}, "test-trace-ID") + convey.So(put.Err, convey.ShouldBeNil) + convey.So(len(values), convey.ShouldEqual, 0) + }) + }) +} + +func TestGIncreaseRef(t *testing.T) { + convey.Convey("GIncreaseRef", t, func() { + convey.Convey("dsclient is nil", func() { + localClientLibruntime = nil + _, put := GIncreaseRef(&data.IncreaseRefRequest{}, &Config{}, "test-trace-ID") + convey.So(put.Code, convey.ShouldEqual, errRPCUnavailable) + }) + convey.Convey("success", func() { + localClientLibruntime = &invokerLibruntimeMock{setTenantIDSuccessfully: true} + _, status := GIncreaseRef(&data.IncreaseRefRequest{}, &Config{}, "test-trace-ID") + convey.So(status.Code, convey.ShouldEqual, 0) + }) + convey.Convey("failed", func() { + localClientLibruntime = &invokerLibruntimeMock{setTenantIDSuccessfully: false} + _, status := GIncreaseRef(&data.IncreaseRefRequest{}, &Config{}, "test-trace-ID") + convey.So(status.Code, convey.ShouldNotEqual, 0) + }) + }) +} + +func TestGDecreaseRef(t *testing.T) { + convey.Convey("GDecreaseRef", t, func() { + convey.Convey("dsclient is nil", func() { + localClientLibruntime = nil + _, put := GDecreaseRef(&data.DecreaseRefRequest{}, &Config{}, "test-trace-ID") + convey.So(put.Code, convey.ShouldEqual, errRPCUnavailable) + }) + convey.Convey("success", func() { + localClientLibruntime = &invokerLibruntimeMock{setTenantIDSuccessfully: true} + _, status := GDecreaseRef(&data.DecreaseRefRequest{}, &Config{}, "test-trace-ID") + convey.So(status.Code, convey.ShouldEqual, 0) + }) + convey.Convey("failed", func() { + localClientLibruntime = &invokerLibruntimeMock{setTenantIDSuccessfully: false} + _, status := GDecreaseRef(&data.DecreaseRefRequest{}, &Config{}, "test-trace-ID") + convey.So(status.Code, convey.ShouldNotEqual, 0) + }) + }) +} + +func TestSet(t *testing.T) { + convey.Convey("Set", t, func() { + convey.Convey("dsclient is nil", func() { + localClientLibruntime = nil + put := Set(&data.KvSetRequest{}, &Config{}, "test-trace-ID") + convey.So(put.Code, convey.ShouldEqual, errRPCUnavailable) + }) + convey.Convey("success", func() { + localClientLibruntime = &invokerLibruntimeMock{setTenantIDSuccessfully: true} + status := Set(&data.KvSetRequest{}, &Config{}, "test-trace-ID") + convey.So(status.Code, convey.ShouldEqual, 0) + }) + convey.Convey("failed", func() { + localClientLibruntime = &invokerLibruntimeMock{setTenantIDSuccessfully: false} + status := Set(&data.KvSetRequest{}, &Config{}, "test-trace-ID") + convey.So(status.Code, convey.ShouldNotEqual, 0) + }) + }) +} + +func TestMSetTx(t *testing.T) { + convey.Convey("MSetTx", t, func() { + convey.Convey("dsclient is nil", func() { + localClientLibruntime = nil + put := MSetTx(&data.KvMSetTxRequest{}, &Config{}, "test-trace-ID") + convey.So(put.Code, convey.ShouldEqual, errRPCUnavailable) + }) + convey.Convey("success", func() { + localClientLibruntime = &invokerLibruntimeMock{setTenantIDSuccessfully: true} + status := MSetTx(&data.KvMSetTxRequest{}, &Config{}, "test-trace-ID") + convey.So(status.Code, convey.ShouldEqual, 0) + }) + convey.Convey("failed", func() { + localClientLibruntime = &invokerLibruntimeMock{setTenantIDSuccessfully: false} + status := MSetTx(&data.KvMSetTxRequest{}, &Config{}, "test-trace-ID") + convey.So(status.Code, convey.ShouldNotEqual, 0) + }) + }) +} + +func TestGet(t *testing.T) { + convey.Convey("Get", t, func() { + convey.Convey("dsclient is nil", func() { + localClientLibruntime = nil + _, put := Get(&data.KvGetRequest{}, &Config{}, "test-trace-ID") + convey.So(put.Code, convey.ShouldEqual, errRPCUnavailable) + }) + convey.Convey("success", func() { + localClientLibruntime = &invokerLibruntimeMock{setTenantIDSuccessfully: true} + _, status := Get(&data.KvGetRequest{}, &Config{}, "test-trace-ID") + convey.So(status.Code, convey.ShouldEqual, 0) + }) + convey.Convey("failed", func() { + localClientLibruntime = &invokerLibruntimeMock{setTenantIDSuccessfully: false} + _, status := Get(&data.KvGetRequest{}, &Config{}, "test-trace-ID") + convey.So(status.Code, convey.ShouldNotEqual, 0) + }) + }) +} + +func TestDel(t *testing.T) { + convey.Convey("Del", t, func() { + convey.Convey("dsclient is nil", func() { + localClientLibruntime = nil + _, put := Del(&data.KvDelRequest{}, &Config{}, "test-trace-ID") + convey.So(put.Code, convey.ShouldEqual, errRPCUnavailable) + }) + + convey.Convey("success", func() { + localClientLibruntime = &invokerLibruntimeMock{setTenantIDSuccessfully: true} + _, status := Del(&data.KvDelRequest{}, &Config{}, "test-trace-ID") + convey.So(status.Code, convey.ShouldEqual, 0) + }) + + convey.Convey("failed", func() { + localClientLibruntime = &invokerLibruntimeMock{setTenantIDSuccessfully: false} + _, status := Del(&data.KvDelRequest{}, &Config{}, "test-trace-ID") + convey.So(status.Code, convey.ShouldNotEqual, 0) + }) + }) + +} + +func TestGReleaseGRefs(t *testing.T) { + convey.Convey("GReleaseGRefs", t, func() { + convey.Convey("dsclient is nil", func() { + localClientLibruntime = nil + put := GReleaseGRefs("test-ID", &Config{}, "test-trace-ID") + convey.So(put.Code, convey.ShouldEqual, errRPCUnavailable) + }) + convey.Convey("success", func() { + localClientLibruntime = &invokerLibruntimeMock{setTenantIDSuccessfully: true} + status := GReleaseGRefs("test-ID", &Config{}, "test-trace-ID") + convey.So(status.Code, convey.ShouldEqual, 0) + }) + }) +} + +func Test_downloadArray(t *testing.T) { + convey.Convey("download array test", t, func() { + localClientLibruntime = &mockUtils.FakeLibruntimeSdkClient{} + convey.Convey("decrypt failed", func() { + p := gomonkey.ApplyFunc(getClient, func(cfg *Config, _ string) (DsClientImpl, bool, error) { + return DsClientImpl{kvClient: &FakeKvClient{}}, false, nil + }) + defer p.Reset() + key, b, err := downloadArray([]string{"aaa", "bbb"}, &Config{NeedEncrypt: true, TenantID: "aaaa"}, "") + convey.So(key, convey.ShouldBeNil) + convey.So(b, convey.ShouldBeFalse) + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("decrypt failed with tenantID dataKey", func() { + p := gomonkey.ApplyFunc(getClient, func(cfg *Config, _ string) (DsClientImpl, bool, error) { + return DsClientImpl{kvClient: &FakeKvClient{}}, false, nil + }) + defer p.Reset() + key, b, err := downloadArray([]string{"aaa", "bbb"}, &Config{NeedEncrypt: true, TenantID: "aaaa", DataKey: []byte("test")}, "") + convey.So(key, convey.ShouldBeNil) + convey.So(b, convey.ShouldBeFalse) + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("decrypt ok", func() { + p := gomonkey.ApplyFunc(getClient, func(cfg *Config, _ string) (DsClientImpl, bool, error) { + return DsClientImpl{kvClient: &FakeKvClient{}}, false, nil + }) + defer p.Reset() + p1 := gomonkey.ApplyFunc(decryptData, func(cfg *Config, data []byte) ([]byte, error) { + return []byte("hello"), nil + }) + defer p1.Reset() + key, b, err := downloadArray([]string{"aaa", "bbb"}, &Config{NeedEncrypt: true, TenantID: "aaaa", Limit: 1000000, DataKey: []byte("test")}, "") + convey.So(key, convey.ShouldNotBeNil) + convey.So(b, convey.ShouldBeFalse) + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("retry", func() { + kvclient := &FakeKvClient{} + p := gomonkey.ApplyFunc(getClient, func(cfg *Config, _ string) (DsClientImpl, bool, error) { + return DsClientImpl{kvClient: kvclient}, false, nil + }) + defer p.Reset() + p1 := gomonkey.ApplyFunc(decryptData, func(cfg *Config, data []byte) ([]byte, error) { + return []byte("hello"), nil + }) + defer p1.Reset() + + errCode := 0 + p2 := gomonkey.ApplyMethod(reflect.TypeOf(kvclient), "KVQuerySize", func(_ *FakeKvClient, _ []string) ([]uint64, api.ErrorInfo) { + return nil, api.ErrorInfo{Code: errCode, Err: fmt.Errorf("do KVQuerySize func failed")} + }) + defer p2.Reset() + /* + errOutOfMemory = 6 + errDsWorkerNotReady = 8 + errTryAgain = 19 + errRPCCancelled = 1000 + errRPCUnavailable = 1002 + errAsyncQueueFull = 2003 + + errDsClientNil = 11001 + */ + testCodeMap := []int{6, 8, 19, 1000, 1002, 2003, 11001} + for _, code := range testCodeMap { + errCode = code + _, b, err := downloadArray([]string{"aaa", "bbb"}, &Config{NeedEncrypt: true, TenantID: "aaaa", Limit: 1000000, DataKey: []byte("test")}, "") + convey.So(b, convey.ShouldBeTrue) + convey.So(err, convey.ShouldNotBeNil) + } + + }) + }) +} + +func Test_SubscribeStream(t *testing.T) { + convey.Convey("test SubscribeStream simple failed", t, func() { + localClientLibruntime = &invokerLibruntimeMock{setTenantIDSuccessfully: true} + p := gomonkey.ApplyMethod(reflect.TypeOf(localClientLibruntime), "Subscribe", + func(_ *invokerLibruntimeMock, streamName string, + config api.SubscriptionConfig) (consumer api.StreamConsumer, err error) { + return nil, fmt.Errorf("just for test") + }) + defer p.Reset() + errInfo := SubscribeStream(SubscribeParam{ + StreamName: "", + TimeoutMs: 0, + ExpectReceiveNum: 2, + TraceId: "traceId", + }, &GinCtxAdapter{&gin.Context{Request: &http.Request{}}}) + convey.So(errInfo.Error(), convey.ShouldEqual, "just for test") + }) +} + +type MockCloseNotifier struct { + flushFlag bool +} + +// CloseNotify +func (m *MockCloseNotifier) CloseNotify() <-chan bool { + notify := make(chan bool, 1) + return notify +} +func (m *MockCloseNotifier) Flush() { + m.flushFlag = true +} + +// MockResponseWriter +type MockResponseWriter struct { + http.ResponseWriter + *MockCloseNotifier +} + +func Test_receiveStream(t *testing.T) { + convey.Convey("test SubscribeStream simple failed", t, func() { + var testData uint8 = 10 + count := 0 + mockConsumer := &mockUtils.FakeStreamConsumer{} + p := gomonkey.ApplyMethod(reflect.TypeOf(mockConsumer), "ReceiveExpectNum", + func(_ *mockUtils.FakeStreamConsumer, expectNum uint32, timeoutMs uint32) ([]api.Element, error) { + if count == 0 { + count = count + 1 + return []api.Element{{ + Ptr: &testData, + Size: 8, + }}, nil + } else { + return []api.Element{}, errors.New("Producer not found") + } + }) + defer p.Reset() + q := gomonkey.ApplyMethod(reflect.TypeOf(&GinCtxAdapter{}), "Done", + func(_ *GinCtxAdapter) <-chan struct{} { + done := make(<-chan struct{}, 2) + return done + }) + defer q.Reset() + streamName := "test-Stream" + timeoutMs := 100 + expectReceiveNum := 2 + consumer := &mockUtils.FakeStreamConsumer{} + rw := MockResponseWriter{ + ResponseWriter: httptest.NewRecorder(), + MockCloseNotifier: &MockCloseNotifier{}, + } + ctx, _ := gin.CreateTestContext(rw) + ginCtx := &GinCtxAdapter{ + Context: ctx, + } + var called int32 + callback := func() { + atomic.AddInt32(&called, 1) + } + + receiveStream(SubscribeParam{ + StreamName: streamName, + TimeoutMs: uint32(timeoutMs), + ExpectReceiveNum: int32(expectReceiveNum), + Callback: callback, + }, consumer, ginCtx) + convey.So(rw.flushFlag, convey.ShouldBeTrue) + convey.So(called, convey.ShouldEqual, 1) + }) +} + +func Test_createAesCryptor(t *testing.T) { + convey.Convey(" test createAesCryptor", t, func() { + // secBase64 := "cANhJ2LdE93A/d/tRJH1Lf32GzMEriQTVS91SMZ1Qp8=" + pa := gomonkey.ApplyFunc(StartWatch, func(dataSystemKeyPrefixList []string, stopCh <-chan struct{}) {}) + defer func() { + time.Sleep(200 * time.Millisecond) + pa.Reset() + }() + convey.Convey("01. error, data key is nil", func() { + cpt, err := createAesCryptor("tenantA", nil) + convey.So(err, convey.ShouldNotBeNil) + convey.So(cpt, convey.ShouldBeNil) + }) + convey.Convey("02. ok, data key is not nil", func() { + cpt, err := createAesCryptor("tenantA", []byte("abcdefg")) + convey.So(err, convey.ShouldBeNil) + convey.So(cpt, convey.ShouldNotBeNil) + }) + }) +} \ No newline at end of file diff --git a/frontend/pkg/common/faas_common/datasystemclient/cache.go b/frontend/pkg/common/faas_common/datasystemclient/cache.go new file mode 100644 index 0000000000000000000000000000000000000000..cf5a3feba532e2e7091f5f59114b77b9c99925ce --- /dev/null +++ b/frontend/pkg/common/faas_common/datasystemclient/cache.go @@ -0,0 +1,350 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package datasystemclient is data system client used for communicating with data system worker. +// To use data system, you should export the data system lib path. Please refer to the Dockerfile of the frontend. +// The lib should copied to home/sn/bin/datasystem/lib. Please refer to +// functioncore/build/common/common_compile.sh and the Dockerfile of the frontend. +// NOTE: To change the version of data system, must revise the version in the common_compile.sh, test.sh and the go.mod +package datasystemclient + +import ( + "errors" + "math/rand" + "strings" + "sync" + "time" + + "go.uber.org/zap" + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/utils" +) + +const ( + dataSystemKeyWithAZLen = 5 + dataSystemKeyWithoutAZLen = 4 + dataSystemEndpointsLen = 2 + noCluster = "noCluster" +) + +var dataSystemCache = sync.Map{} + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +// Cache - +type Cache struct { + nodeList []string + invalidMap map[string]struct{} + lastUsedNode string + lock sync.RWMutex +} + +func (c *Cache) addNode(node string, logger api.FormatLogger) { + c.lock.Lock() + defer c.lock.Unlock() + for _, v := range c.nodeList { + if node == v { + logger.Warnf("the node is already existed, no need add") + return + } + } + c.nodeList = append(c.nodeList, node) + logger.Infof("add dataSystem node successfully") +} + +func (c *Cache) deleteNode(node string, logger api.FormatLogger) { + c.lock.Lock() + defer c.lock.Unlock() + if !c.delete(node) { + logger.Warnf("the node is not exist") + return + } + // delete invalid node + delete(c.invalidMap, node) + logger.Infof("delete dataSystem node from cache successfully") +} + +func (c *Cache) isEmpty() bool { + c.lock.Lock() + defer c.lock.Unlock() + return len(c.nodeList) == 0 && len(c.invalidMap) == 0 +} + +// must be used in a method with a lock +func (c *Cache) delete(node string) bool { + l := len(c.nodeList) + for i, v := range c.nodeList { + if node == v { + // no need for order + // replace the last digit to the index need to delete, and then delete the last digit + c.nodeList[i] = c.nodeList[l-1] + c.nodeList = c.nodeList[:l-1] + return true + } + } + return false +} + +func (c *Cache) ifNodeExist(node string) bool { + c.lock.RLock() + defer c.lock.RUnlock() + for _, v := range c.nodeList { + if v == node { + return true + } + } + return false +} + +func (c *Cache) getRandomNode() (string, error) { + c.lock.RLock() + defer c.lock.RUnlock() + if len(c.nodeList) == 0 { + log.GetLogger().Warnf("no data system node is available") + return "", errors.New("no data system node is available") + } + node := c.nodeList[rand.Intn(len(c.nodeList))] + return node, nil +} + +func (c *Cache) getLastUsedNodeWithInvalidNode(invalidNodes []string) (string, error) { + if c.lastUsedNode != "" && !utils.IsStringInArray(c.lastUsedNode, invalidNodes) { + return c.lastUsedNode, nil + } + node, err := c.getRandomNodeWithInvalidNode(invalidNodes) + if err == nil { + c.lastUsedNode = node + } + return node, err +} + +func (c *Cache) getRandomNodeWithInvalidNode(invalidNodes []string) (string, error) { + if len(invalidNodes) == 0 { + return c.getRandomNode() + } + c.lock.RLock() + defer c.lock.RUnlock() + var nodeList []string + for _, node := range c.nodeList { + if !utils.IsStringInArray(node, invalidNodes) { + nodeList = append(nodeList, node) + } + } + if len(nodeList) == 0 { + log.GetLogger().Warnf("no data system node is available") + return "", errors.New("no data system node is available") + } + node := nodeList[rand.Intn(len(nodeList))] + return node, nil +} + +func (c *Cache) invalidateNode(node string) { + c.lock.Lock() + defer c.lock.Unlock() + if !c.delete(node) { + log.GetLogger().Warnf("invalid node is already deleted") + return + } + c.invalidMap[node] = struct{}{} + // the number of failed nodes is not too large, and too many coroutines are not started + // if used a single-process to traversal map, high-latency operations need add read lock + go c.healthCheckProcess(node) +} + +func (c *Cache) checkInvalidNode(node string) bool { + c.lock.RLock() + if _, ok := c.invalidMap[node]; !ok { + c.lock.RUnlock() + return false + } + c.lock.RUnlock() + return true +} + +func (c *Cache) restoreInvalidNode(node string) { + c.lock.Lock() + delete(c.invalidMap, node) + c.nodeList = append(c.nodeList, node) + c.lock.Unlock() +} + +func (c *Cache) healthCheckProcess(node string) { + log.GetLogger().Infof("start the health check process, nodeIP: %s", node) + trigger := time.NewTicker(5 * time.Second) // healthCheck interval + defer trigger.Stop() + for { + <-trigger.C + if !c.checkInvalidNode(node) { + log.GetLogger().Warnf("invalid node has been deleted, stop health check process, node: %s", node) + return + } + + if !healthCheck(node) { + continue + } + c.restoreInvalidNode(node) + return + } +} + +func healthCheck(ip string) bool { + _, err := NewClient("", ip) + if err != nil { + return false + } + return true +} + +func processDataSystemEvent(event *etcd3.Event) { + logger := log.GetLogger().With(zap.Any("etcdType", event.Type), zap.Any("revisionId", event.Rev)) + logger.Infof("process dataSystem etcd event type") + switch event.Type { + case etcd3.PUT: + err := processAddEvent(event, logger) + if err != nil { + logger.Warnf("process data system put event error: %s", err.Error()) + } + case etcd3.DELETE: + err := processDeleteEvent(event, logger) + if err != nil { + logger.Warnf("process data system delete event error: %s", err.Error()) + } + case etcd3.ERROR: + logger.Warnf("etcd error event: %s", event.Value) + default: + logger.Warnf("unsupported event: %s", event.Value) + } +} + +func processAddEvent(event *etcd3.Event, logger api.FormatLogger) error { + logger = logger.With(zap.Any("etcdValue", string(event.Value))) + ip, az, err := parseDsKey(event.Key) + if err != nil { + logger.Errorf("failed to parse dataSystem Key, err: %s", err.Error()) + return err + } + cacheData, _ := dataSystemCache.LoadOrStore(az, &Cache{ + nodeList: []string{}, + invalidMap: make(map[string]struct{}, 1), + }) + cache, ok := cacheData.(*Cache) + if !ok { + return errors.New("dataSystem load from cache is invalid") + } + + _, status, err := parseDsValue(string(event.Value)) + if err != nil { + log.GetLogger().Warnf("failed to parse dataSystemValue, err: %s", err.Error()) + } + localDataSystemStatusCache.SetLocalDataSystemStatus(ip, status) + readyStatus := map[string]struct{}{ + dataSystemStatusReady: struct{}{}, // only ready status can add + } + _, ready := readyStatus[status] + if err != nil || !ready { + cache.deleteNode(ip, logger) + clientMap.deleteNodeIp(ip, logger) + logger.Warnf("but node is not ready, don't add") + if isShutdownFronted() { + destroy() + } + return nil + } + + cache.addNode(ip, logger) + logger.Warnf("add node to cache") + return nil +} + +func processDeleteEvent(event *etcd3.Event, logger api.FormatLogger) error { + ip, az, err := parseDsKey(event.Key) + if err != nil { + logger.Errorf("failed to parse dataSystem Key, err: %s", err.Error()) + return err + } + cacheData, ok := dataSystemCache.Load(az) + if !ok { + logger.Warnf("no datasystem node in az %s,no need to delete", az) + return nil + } + cache, ok := cacheData.(*Cache) + if !ok { + return errors.New("dataSystem cache is invalid") + } + localDataSystemStatusCache.SetLocalDataSystemStatus(ip, "") + cache.deleteNode(ip, logger) + if cache.isEmpty() { + dataSystemCache.Delete(az) + } + clientMap.deleteNodeIp(ip, logger) + if isShutdownFronted() { + destroy() + } + return nil +} + +// get ip and az form dataSystem key +// dataSystem key format: /[AZ]/datasystem/cluster/[ip:port] +func parseDsKey(key string) (string, string, error) { + keys := strings.Split(key, "/") + if len(keys) != dataSystemKeyWithAZLen && len(keys) != dataSystemKeyWithoutAZLen { // length of dataSystem key + return "", "", errors.New("invalid length of dataSystem key") + } + az := noCluster + endpoints := strings.Split(keys[len(keys)-1], ":") // index of endpoints in dataSystem key + if len(endpoints) != dataSystemEndpointsLen { // length of endpoints key + return "", "", errors.New("invalid length of endpoints in dataSystem key") + } + if len(keys) == dataSystemKeyWithAZLen { + az = keys[1] + } + return endpoints[0], az, nil +} + +// get timestamp and status form dataSystem value +// dataSystem value format: 1748573798753243935;ready +func parseDsValue(value string) (string, string, error) { + splits := strings.Split(value, ";") + if len(splits) != 2 { // magic number + return "", "", errors.New("invalid format of dataSystem key") + } + timeStamp, status := splits[0], splits[1] // magic number + return timeStamp, status, nil +} + +// dataSystemKeyFilter no need filter +func dataSystemKeyFilter(event *etcd3.Event) bool { + return false +} + +// StartWatch - +func StartWatch(dataSystemKeyPrefixList []string, stopCh <-chan struct{}) { + etcdClient := etcd3.GetDataSystemEtcdClient() + if etcdClient == nil { + etcdClient = etcd3.GetMetaEtcdClient() + log.GetLogger().Infof("watch dataSystem from meta etcd") + } + for _, dataSystemKeyPrefix := range dataSystemKeyPrefixList { + watcher := etcd3.NewEtcdWatcher(dataSystemKeyPrefix, dataSystemKeyFilter, + processDataSystemEvent, stopCh, etcdClient) + watcher.StartWatch() + } +} diff --git a/frontend/pkg/common/faas_common/datasystemclient/cache_test.go b/frontend/pkg/common/faas_common/datasystemclient/cache_test.go new file mode 100644 index 0000000000000000000000000000000000000000..ab426e01b24c19f284da5368d7f9b7b560b8cf8c --- /dev/null +++ b/frontend/pkg/common/faas_common/datasystemclient/cache_test.go @@ -0,0 +1,200 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package datasystemclient is data system client used for communicating with data system worker. +// To use data system, you should export the data system lib path. Please refer to the Dockerfile of the frontend. +// The lib should copied to home/sn/bin/datasystem/lib. Please refer to +// functioncore/build/common/common_compile.sh and the Dockerfile of the frontend. +// NOTE: To change the version of data system, must revise the version in the common_compile.sh, test.sh and the go.mod +package datasystemclient + +import ( + "sync" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/common/faas_common/etcd3" + mockUtils "frontend/pkg/common/faas_common/utils" + "github.com/stretchr/testify/assert" + clientv3 "go.etcd.io/etcd/client/v3" +) + +func Test_processDataSystemEvent(t *testing.T) { + dataSystemCache = sync.Map{} + defer gomonkey.ApplyFunc(destroy, func() { + return + }).Reset() + convey.Convey("add simple event", t, func() { + // 添加ready的数据系统节点 + event := &etcd3.Event{ + Type: etcd3.PUT, + Key: "/AZ1/datasystem/cluster/8.8.8.8:31501", + Value: []byte("1748573798753243935;ready"), + } + + processDataSystemEvent(event) + cacheRaw, ok := dataSystemCache.Load("AZ1") + convey.So(ok, convey.ShouldBeTrue) + cache, ok := cacheRaw.(*Cache) + convey.So(ok, convey.ShouldBeTrue) + + convey.So(len(cache.nodeList), convey.ShouldEqual, 1) + convey.So(cache.nodeList[0], convey.ShouldEqual, "8.8.8.8") + _, ok = cache.invalidMap["8.8.8.8"] + convey.So(ok, convey.ShouldBeFalse) + + // 添加状态节点异常的数据系统节点 + for _, status := range []string{"start", "restart", "exiting", "sfafdfafd"} { + event.Value = []byte("1748573798753243935;" + status) + processDataSystemEvent(event) + convey.So(len(cache.nodeList), convey.ShouldEqual, 0) + } + + // 添加状态节点中etcd value格式异常的数据系统节点 + event.Value = []byte("1748573798753243935ready") + processDataSystemEvent(event) + convey.So(len(cache.nodeList), convey.ShouldEqual, 0) + }) + convey.Convey("simple delete event", t, func() { + event := &etcd3.Event{ + Type: etcd3.PUT, + Key: "/AZ1/datasystem/cluster/8.8.8.8:31501", + Value: []byte("1748573798753243935;ready"), + } + + processDataSystemEvent(event) + event = &etcd3.Event{ + Type: etcd3.DELETE, + Key: "/AZ1/datasystem/cluster/8.8.8.8:31501", + Value: []byte("1748573798753243935;ready"), + } + processDataSystemEvent(event) + _, ok := dataSystemCache.Load("AZ1") + convey.So(ok, convey.ShouldBeFalse) + + event.Type = etcd3.PUT + processDataSystemEvent(event) + + event.Type = etcd3.DELETE + event.Key = "/AZ1/datasystem/cluster/8.8.8.9:31501" + processDataSystemEvent(event) + + cacheRaw, ok := dataSystemCache.Load("AZ1") + convey.So(ok, convey.ShouldBeTrue) + cache, ok := cacheRaw.(*Cache) + convey.So(ok, convey.ShouldBeTrue) + convey.So(len(cache.nodeList), convey.ShouldEqual, 1) + + event.Key = "/AZ1/datasystem/cluster/8.8.8.8:31501" + processDataSystemEvent(event) + _, ok = dataSystemCache.Load("AZ1") + convey.So(ok, convey.ShouldBeFalse) + }) +} + +func TestCache_healthCheckProcess(t *testing.T) { + type fields struct { + nodeList []string + invalidMap map[string]struct{} + lock sync.RWMutex + } + type args struct { + node string + } + tests := []struct { + name string + fields fields + args args + patchesFunc mockUtils.PatchesFunc + }{ + {"case1 node in invalidMap", fields{ + nodeList: []string{}, + invalidMap: map[string]struct{}{"8.8.8.8": struct{}{}}, + lock: sync.RWMutex{}, + }, args{node: "8.8.8.8"}, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(NewClient, func(tenantID string, nodeIP string) (DsClientImpl, error) { return DsClientImpl{}, nil }), + }) + return patches + }}, + {"case2 node not in invalidMap", fields{ + nodeList: []string{}, + invalidMap: map[string]struct{}{}, + lock: sync.RWMutex{}, + }, args{node: "8.8.8.8"}, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + return patches + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + patches := tt.patchesFunc() + c := &Cache{ + nodeList: tt.fields.nodeList, + invalidMap: tt.fields.invalidMap, + lock: tt.fields.lock, + } + c.healthCheckProcess(tt.args.node) + patches.ResetAll() + }) + } +} + +func Test_StartWatch(t *testing.T) { + etcdClient := &etcd3.EtcdClient{ + Client: &clientv3.Client{}, + } + watchFlag := false + defer gomonkey.ApplyFunc(etcd3.GetMetaEtcdClient, func() *etcd3.EtcdClient { + return etcdClient + }).Reset() + defer gomonkey.ApplyFunc((*etcd3.EtcdWatcher).StartWatch, func(ew *etcd3.EtcdWatcher) { + watchFlag = true + }).Reset() + stop := make(chan struct{}) + StartWatch([]string{"/datasystem/cluster/"}, stop) + close(stop) + assert.True(t, watchFlag) +} + +func Test_ParseDsKey(t *testing.T) { + key := "/cluster001/datasystem/cluster/127.0.0.1:8080" + ip, az, err := parseDsKey(key) + assert.Nil(t, err) + assert.Equal(t, ip, "127.0.0.1") + assert.Equal(t, az, "cluster001") + + key = "/datasystem/cluster/127.0.0.1:8080" + ip, az, err = parseDsKey(key) + assert.Nil(t, err) + assert.Equal(t, ip, "127.0.0.1") + assert.Equal(t, az, noCluster) + + key = "/cluster001/datasystem/cluster" + ip, az, err = parseDsKey(key) + assert.NotNil(t, err) + assert.Equal(t, ip, "") + assert.Equal(t, az, "") + + key = "/cluster001/datasystem/cluster/127.0.0.1" + ip, az, err = parseDsKey(key) + assert.NotNil(t, err) + assert.Equal(t, ip, "") + assert.Equal(t, az, "") +} diff --git a/frontend/pkg/common/faas_common/datasystemclient/concurrent_map.go b/frontend/pkg/common/faas_common/datasystemclient/concurrent_map.go new file mode 100644 index 0000000000000000000000000000000000000000..c4ce75e525b152ad136864bc2f51a54c727a0e99 --- /dev/null +++ b/frontend/pkg/common/faas_common/datasystemclient/concurrent_map.go @@ -0,0 +1,223 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package datasystemclient is data system client used for communicating with data system worker. +// To use data system, you should export the data system lib path. Please refer to the Dockerfile of the frontend. +// The lib should copied to home/sn/bin/datasystem/lib. Please refer to +// functioncore/build/common/common_compile.sh and the Dockerfile of the frontend. +// NOTE: To change the version of data system, must revise the version in the common_compile.sh, test.sh and the go.mod +package datasystemclient + +import ( + "errors" + "sync" + + "go.uber.org/zap" + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/logger/log" +) + +// clients - key: node IP; value: data system worker instance, *api.DataSystemClient +type nodeIP2ClientMap struct { + clientMap map[string]DsClientImpl + sync.RWMutex + logger api.FormatLogger +} + +// DsClientImpl - +type DsClientImpl struct { + kvClient api.KvClient +} + +func (n *nodeIP2ClientMap) deleteAll() { + n.Lock() + defer n.Unlock() + for _, client := range n.clientMap { + client.kvClient.DestroyClient() + } + n.clientMap = make(map[string]DsClientImpl) + n.logger.Infof("delete all the client") +} + +func (n *nodeIP2ClientMap) delete(nodeIp string) { + n.Lock() + defer n.Unlock() + client, ok := n.clientMap[nodeIp] + if !ok { + return + } + delete(n.clientMap, nodeIp) + client.kvClient.DestroyClient() + n.logger.Infof("delete %s client", nodeIp) +} + +func (n *nodeIP2ClientMap) get(nodeIp string) (DsClientImpl, bool) { + n.RLock() + defer n.RUnlock() + client, ok := n.clientMap[nodeIp] + return client, ok +} + +func (n *nodeIP2ClientMap) getRandomOne() (DsClientImpl, bool) { + n.RLock() + defer n.RUnlock() + + for _, client := range n.clientMap { + return client, true + } + return DsClientImpl{}, false +} + +func (n *nodeIP2ClientMap) add(nodeIp string, client DsClientImpl) { + n.Lock() + defer n.Unlock() + if c, ok := n.clientMap[nodeIp]; ok { + c.kvClient.DestroyClient() + } + n.clientMap[nodeIp] = client + n.logger.Infof("add %s client", nodeIp) +} + +func (n *nodeIP2ClientMap) size() int { + n.RLock() + defer n.RUnlock() + return len(n.clientMap) +} + +type concurrentMap struct { + // clients - key: tenantID; value: map + mp map[string]*nodeIP2ClientMap + sync.RWMutex +} + +func (m *concurrentMap) get(tenantID string, nodeIP string) (DsClientImpl, bool) { + m.RLock() + defer m.RUnlock() + tenantMap, existed := m.mp[tenantID] + if !existed { + return DsClientImpl{}, false + } + client, existed := tenantMap.get(nodeIP) + return client, existed +} + +func (m *concurrentMap) getOneClient(tenantID string) api.KvClient { + m.RLock() + defer m.RUnlock() + tenantMap, existed := m.mp[tenantID] + if !existed { + return nil + } + client, ok := tenantMap.getRandomOne() + if ok { + return client.kvClient + } + return nil +} + +func (m *concurrentMap) getOrCreate(tenantID string, nodeIP string) (DsClientImpl, error) { + m.Lock() + defer m.Unlock() + // double check Before creating a thread, perform the get operation again to check whether other threads have been + // created. If no, continue to create threads to prevent repeated creation. + tenantMap, existed := m.mp[tenantID] + if existed { + if client, existed := tenantMap.get(nodeIP); existed { + return client, nil + } + } else { + m.mp[tenantID] = &nodeIP2ClientMap{ + clientMap: make(map[string]DsClientImpl), + RWMutex: sync.RWMutex{}, + logger: log.GetLogger().With(zap.Any("tenantId", tenantID)), + } + } + newClient, err := NewClient(tenantID, nodeIP) + if err != nil { + return DsClientImpl{}, err + } + m.mp[tenantID].add(nodeIP, newClient) + return newClient, nil +} + +// NewClient - +func NewClient(tenantID string, nodeIP string) (DsClientImpl, error) { + if localClientLibruntime == nil { + log.GetLogger().Errorf("local dataSystem client is nil") + return DsClientImpl{}, errors.New("local dataSystem client is nil") + } + credential := localClientLibruntime.GetCredential() + // create + var dsClient DsClientImpl + createConfigLibruntime := api.ConnectArguments{ + Host: nodeIP, + Port: port, + TimeoutMs: timeoutMs, + TenantID: tenantID, + AccessKey: credential.AccessKey, + SecretKey: credential.SecretKey, + } + newClient, err := localClientLibruntime.CreateClient(createConfigLibruntime) + if err != nil { + log.GetLogger().Errorf("failed to create dataSystem client: %s", err.Error()) + return dsClient, err + } + dsClient.kvClient = newClient + log.GetLogger().Infof("create new datasystem client nodeIP is: %s,tenantID: %s", nodeIP, tenantID) + return dsClient, nil +} + +func (m *concurrentMap) deleteNodeIp(nodeIp string, logger api.FormatLogger) { + m.Lock() + defer m.Unlock() + deleteEmptyList := make([]string, 0) + for tenantId, tenantMap := range m.mp { + tenantMap.delete(nodeIp) + if tenantMap.size() == 0 { + deleteEmptyList = append(deleteEmptyList, tenantId) + } + } + for _, k := range deleteEmptyList { + delete(m.mp, k) + } + logger.Infof("delete nodeIp from clientMap ok") +} + +func (m *concurrentMap) deleteClient(tenantID string, nodeIP string) { + m.Lock() + defer m.Unlock() + tenantMap, existed := m.mp[tenantID] + if !existed { + return + } + tenantMap.delete(nodeIP) + if tenantMap.size() == 0 { + delete(m.mp, tenantID) + } + log.GetLogger().Infof("delete nodeIp: %s, tenantId: %s from clientMap ok", nodeIP, tenantID) +} + +func (m *concurrentMap) deleteTenant(tenantID string) { + m.Lock() + defer m.Unlock() + tenantMap, ok := m.mp[tenantID] + if ok { + tenantMap.deleteAll() + } + delete(m.mp, tenantID) + log.GetLogger().Infof("delete tenantId: %s from clientMap ok", tenantID) +} diff --git a/frontend/pkg/common/faas_common/datasystemclient/concurrent_map_test.go b/frontend/pkg/common/faas_common/datasystemclient/concurrent_map_test.go new file mode 100644 index 0000000000000000000000000000000000000000..a13cba7499147bb6a7f6c3d9c4c4353be30d3d1c --- /dev/null +++ b/frontend/pkg/common/faas_common/datasystemclient/concurrent_map_test.go @@ -0,0 +1,104 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package datasystemclient is data system client used for communicating with data system worker. +// To use data system, you should export the data system lib path. Please refer to the Dockerfile of the frontend. +// The lib should copied to home/sn/bin/datasystem/lib. Please refer to +// functioncore/build/common/common_compile.sh and the Dockerfile of the frontend. +// NOTE: To change the version of data system, must revise the version in the common_compile.sh, test.sh and the go.mod +package datasystemclient + +import ( + "sync" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/types" + mockUtils "frontend/pkg/common/faas_common/utils" +) + +func TestNewKvClient(t *testing.T) { + dsConfig := &types.DataSystemConfig{ + TimeoutMs: 60000, + Clusters: []string{"AZ1"}, + } + lease := gomonkey.ApplyFunc(StartWatch, + func(dataSystemKeyPrefixList []string, stopCh <-chan struct{}) { return }) + InitDataSystemLibruntime(dsConfig, &mockUtils.FakeLibruntimeSdkClient{}, make(chan struct{})) + type args struct { + tenantID string + nodeIP string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {"case1 succeed to new a client", args{ + tenantID: "t1", + nodeIP: "127.0.0.2", + }, false}} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := NewClient(tt.args.tenantID, tt.args.nodeIP) + if (err != nil) != tt.wantErr { + t.Errorf("NewClient() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } + time.Sleep(100 * time.Millisecond) + lease.Reset() +} + +func TestNodeIP2ClientMap(t *testing.T) { + convey.Convey("simple", t, func() { + n := &nodeIP2ClientMap{ + clientMap: make(map[string]DsClientImpl), + RWMutex: sync.RWMutex{}, + logger: log.GetLogger(), + } + + n.add("1.1.1.1", DsClientImpl{kvClient: &FakeKvClient{}}) + convey.So(n.size(), convey.ShouldEqual, 1) + + n.add("1.1.1.1", DsClientImpl{kvClient: &FakeKvClient{}}) + convey.So(n.size(), convey.ShouldEqual, 1) + + _, ok := n.get("1.1.1.1") + convey.So(ok, convey.ShouldBeTrue) + + _, ok = n.get("2.2.2.2") + convey.So(ok, convey.ShouldBeFalse) + + n.delete("1.1.1.1") + convey.So(n.size(), convey.ShouldEqual, 0) + + n.add("1.1.1.1", DsClientImpl{kvClient: &FakeKvClient{}}) + n.add("2.1.1.1", DsClientImpl{kvClient: &FakeKvClient{}}) + n.add("3.1.1.1", DsClientImpl{kvClient: &FakeKvClient{}}) + n.add("4.1.1.1", DsClientImpl{kvClient: &FakeKvClient{}}) + _, ok = n.getRandomOne() + convey.So(ok, convey.ShouldBeTrue) + + n.deleteAll() + convey.So(n.size(), convey.ShouldEqual, 0) + }) +} diff --git a/frontend/pkg/common/faas_common/datasystemclient/fault_manager.go b/frontend/pkg/common/faas_common/datasystemclient/fault_manager.go new file mode 100644 index 0000000000000000000000000000000000000000..6bec129ec75e3ddde71c15c718c534a54e316417 --- /dev/null +++ b/frontend/pkg/common/faas_common/datasystemclient/fault_manager.go @@ -0,0 +1,145 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package datasystemclient + +import ( + "os" + "sync" + "sync/atomic" + "syscall" + "time" + + "frontend/pkg/common/faas_common/logger/log" +) + +const ( + // 节点已准备好对外服务 + dataSystemStatusReady = "ready" + // 节点启动 + dataSystemStatusStart = "start" + // 节点重启 + dataSystemStatusRestart = "restart" + // 节点对账恢复中 + dataSystemStatusRecover = "recover" + // etcd故障期间重启的状态 + dataSystemStatusDRst = "d_rst" + // 节点退出（主动缩容） + dataSystemStatusExiting = "exiting" +) + +const readinessDuration = 15 * time.Second + +var ( + localDataSystemStatusCache = &LocalDataSystemStatusCache{} + shutdownFlag = atomic.Bool{} + streamEnable = atomic.Bool{} +) + +// LocalDataSystemStatusCache 本地数据系统状态缓存结构体 +type LocalDataSystemStatusCache struct { + status string + lock sync.RWMutex +} + +// IsStatusReady - +func (d *LocalDataSystemStatusCache) IsStatusReady() bool { + d.lock.RLock() + defer d.lock.RUnlock() + if d.status != dataSystemStatusReady { + log.GetLogger().Debugf("data system status is not ready, status: %s", d.status) + return false + } + return true +} + +// SetLocalDataSystemStatus - +func (d *LocalDataSystemStatusCache) SetLocalDataSystemStatus(ip, status string) { + d.lock.Lock() + defer d.lock.Unlock() + localNode := os.Getenv("NODE_IP") + if localNode == "" { + log.GetLogger().Debugf("get local node is empty") + return + } + if ip != localNode { + log.GetLogger().Debugf("node[%s] is not local data system node[%s]", ip, localNode) + return + } + log.GetLogger().Infof("save local data system node[%s] status[%s]", ip, status) + d.status = status +} + +// GetLocalDataSystemStatus - +func (d *LocalDataSystemStatusCache) GetLocalDataSystemStatus() string { + d.lock.RLock() + defer d.lock.RUnlock() + return d.status +} + +// IsLocalDataSystemStatusReady - +func IsLocalDataSystemStatusReady() bool { + return localDataSystemStatusCache.IsStatusReady() +} + +// SetStreamEnable - +func SetStreamEnable(streamEnableConfig bool) { + streamEnable.Store(streamEnableConfig) +} + +func isShutdownFronted() bool { + if !streamEnable.Load() { + log.GetLogger().Infof("it's not stream scenario, skip shutdown frontend") + return false + } + skipShutdownStatusMap := map[string]struct{}{ + dataSystemStatusReady: {}, + dataSystemStatusStart: {}, + dataSystemStatusRestart: {}, + dataSystemStatusRecover: {}, + dataSystemStatusDRst: {}, + } + status := localDataSystemStatusCache.GetLocalDataSystemStatus() + if _, ok := skipShutdownStatusMap[status]; ok { + log.GetLogger().Debugf("status is [%s], skip shutdown frontend", status) + return false + } + return true +} + +func destroy() { + time.Sleep(readinessDuration) + if shutdownFlag.Swap(true) { + log.GetLogger().Infof("shutdown frontend has been triggered, skip this operation") + return + } + defer func() { + shutdownFlag.Store(false) + }() + log.GetLogger().Infof("local dataSystem status is not ready, prepare shutdown frontend") + pid := os.Getpid() + proc, err := os.FindProcess(pid) + if err != nil { + log.GetLogger().Errorf("get process pid failed, pid: %d, err: %v", pid, err) + return + } + err = proc.Signal(syscall.SIGTERM) + if err != nil { + log.GetLogger().Errorf("send SIGTERM signal to the process failed, pid: %d, err: %v", pid, err) + return + } + log.GetLogger().Infof("send SIGTERM signal to the process success, pid: %d", pid) +} diff --git a/frontend/pkg/common/faas_common/datasystemclient/fault_manager_test.go b/frontend/pkg/common/faas_common/datasystemclient/fault_manager_test.go new file mode 100644 index 0000000000000000000000000000000000000000..844b2ddee721cbafd8079af310ff110bcd27e924 --- /dev/null +++ b/frontend/pkg/common/faas_common/datasystemclient/fault_manager_test.go @@ -0,0 +1,151 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package datasystemclient + +import ( + "os" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" +) + +func TestIsStatusReady(t *testing.T) { + convey.Convey("test *LocalDataSystemStatusCache IsStatusReady()", t, func() { + var dataSystemStatusCache LocalDataSystemStatusCache + convey.Convey("test dataSystem status is ready", func() { + dataSystemStatusCache.status = dataSystemStatusReady + result := dataSystemStatusCache.IsStatusReady() + convey.So(result, convey.ShouldBeTrue) + }) + + convey.Convey("test dataSystem status is not ready", func() { + dataSystemStatusCache.status = dataSystemStatusExiting + result := dataSystemStatusCache.IsStatusReady() + convey.So(result, convey.ShouldBeFalse) + }) + }) +} + +func TestLocalDataSystemStatusCacheGetLocalDataSystemStatus(t *testing.T) { + convey.Convey("test *LocalDataSystemStatusCache GetLocalDataSystemStatus", t, func() { + convey.Convey("test get dataSystem status", func() { + var dataSystemStatusCache LocalDataSystemStatusCache + dataSystemStatusCache.status = dataSystemStatusReady + convey.So(dataSystemStatusCache.GetLocalDataSystemStatus(), convey.ShouldEqual, dataSystemStatusReady) + }) + }) +} + +func TestLocalDataSystemStatusCacheSetLocalDataSystemStatus(t *testing.T) { + convey.Convey("test *LocalDataSystemStatusCache SetLocalDataSystemStatus", t, func() { + var dataSystemStatusCache LocalDataSystemStatusCache + convey.Convey("test set dataSystem status, when NODE_IP is empty", func() { + dataSystemStatusCache.SetLocalDataSystemStatus("", dataSystemStatusReady) + convey.So(dataSystemStatusCache.GetLocalDataSystemStatus(), convey.ShouldEqual, "") + }) + + convey.Convey("test set dataSystem status, when NODE_IP is not equal dataSystemStatusCache", func() { + defer gomonkey.ApplyFunc(os.Getenv, func(key string) string { + return "0.0.0.0" + }).Reset() + dataSystemStatusCache.SetLocalDataSystemStatus("0.0.0.1", dataSystemStatusReady) + convey.So(dataSystemStatusCache.GetLocalDataSystemStatus(), convey.ShouldNotEqual, dataSystemStatusReady) + }) + + convey.Convey("test set dataSystem status, when NODE_IP is equal dataSystemStatusCache", func() { + defer gomonkey.ApplyFunc(os.Getenv, func(key string) string { + return "0.0.0.0" + }).Reset() + dataSystemStatusCache.SetLocalDataSystemStatus("0.0.0.0", dataSystemStatusReady) + convey.So(dataSystemStatusCache.GetLocalDataSystemStatus(), convey.ShouldEqual, dataSystemStatusReady) + }) + }) +} + +func TestIsLocalDataSystemStatusReady(t *testing.T) { + convey.Convey("test IsLocalDataSystemStatusReady", t, func() { + original := localDataSystemStatusCache.status + defer func() { + localDataSystemStatusCache.status = original + }() + convey.Convey("local dataSystem status is ready", func() { + localDataSystemStatusCache.status = dataSystemStatusReady + result := IsLocalDataSystemStatusReady() + convey.So(result, convey.ShouldBeTrue) + }) + + convey.Convey("local dataSystem status is not ready", func() { + localDataSystemStatusCache.status = dataSystemStatusExiting + result := IsLocalDataSystemStatusReady() + convey.So(result, convey.ShouldBeFalse) + }) + }) +} + +func TestSetStreamEnable(t *testing.T) { + convey.Convey("test SetStreamEnable", t, func() { + convey.Convey("test set streamEnable", func() { + SetStreamEnable(false) + convey.So(streamEnable.Load(), convey.ShouldBeFalse) + }) + }) +} + +func TestIsShutdownFronted(t *testing.T) { + convey.Convey("test is shout down frontend", t, func() { + originalStreamEnable := streamEnable.Load() + defer func() { + streamEnable.Store(originalStreamEnable) + }() + streamEnable.Store(true) + + convey.Convey("when streamEnable is false, skip shutdown", func() { + streamEnable.Store(false) + result := isShutdownFronted() + convey.So(result, convey.ShouldBeFalse) + }) + + convey.Convey("when dataSystem status is ready, skip shutdown", func() { + defer gomonkey.ApplyMethodFunc(&LocalDataSystemStatusCache{}, "GetLocalDataSystemStatus", func() string { + return dataSystemStatusReady + }).Reset() + result := isShutdownFronted() + convey.So(result, convey.ShouldBeFalse) + }) + + convey.Convey("when dataSystem status is exiting, skip shutdown", func() { + defer gomonkey.ApplyMethodFunc(&LocalDataSystemStatusCache{}, "GetLocalDataSystemStatus", func() string { + return dataSystemStatusExiting + }).Reset() + result := isShutdownFronted() + convey.So(result, convey.ShouldBeTrue) + }) + }) +} + +func TestDestroy(t *testing.T) { + convey.Convey("test destroy frontend, when watch dataSystem", t, func() { + convey.Convey("destroy success", func() { + defer gomonkey.ApplyMethodFunc(&os.Process{}, "Signal", func(sig os.Signal) error { + return nil + }).Reset() + destroy() + convey.So("", convey.ShouldBeEmpty) + }) + }) +} diff --git a/frontend/pkg/common/faas_common/datasystemclient/kvclient.go b/frontend/pkg/common/faas_common/datasystemclient/kvclient.go new file mode 100644 index 0000000000000000000000000000000000000000..ff1627362d35f2417900006bfa089b01c3725aae --- /dev/null +++ b/frontend/pkg/common/faas_common/datasystemclient/kvclient.go @@ -0,0 +1,207 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package datasystemclient is data system kv client. +package datasystemclient + +import ( + "fmt" + "runtime" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/logger/log" +) + +// Option options for kv client +type Option struct { + TenantID string + NodeIP string + Cluster string + WriteMode api.WriteModeEnum + TTLSecond uint32 +} + +// KVPutWithRetry put kv to ds with retry +func KVPutWithRetry(key string, value []byte, option *Option, traceID string) error { + log.GetLogger().Debugf("datasystem kv put %s, %s, traceID: %s", key, string(value), traceID) + config := &Config{ + invalidIP: []string{}, + useLastUsedNode: true, + + TenantID: option.TenantID, + NodeIP: option.NodeIP, + Cluster: option.Cluster, + KeyPrefix: key, + NoNeedGenKey: true, + } + setParam := api.SetParam{ + WriteMode: option.WriteMode, + TTLSecond: option.TTLSecond, + } + for { + retry, err := kvPut(value, setParam, config, traceID) + if err == nil { + return nil + } + if retry { + log.GetLogger().Debugf("put with key will retry, failed ip: %s,traceID: %s, err: %s", + config.NodeIP, traceID, err.Error()) + config.invalidIP = append(config.invalidIP, config.NodeIP) + config.NodeIP = "" + continue + } + log.GetLogger().Errorf("get with key failed err: %s,NodeIP: %s,traceID: %s", err.Error(), + config.NodeIP, traceID) + return err + } +} + +// KVGetWithRetry get kv to ds with retry +func KVGetWithRetry(key string, option *Option, traceID string) ([]byte, error) { + log.GetLogger().Debugf("datasystem kv get %s, traceID: %s", key, traceID) + config := &Config{ + invalidIP: []string{}, + useLastUsedNode: true, + + TenantID: option.TenantID, + NodeIP: option.NodeIP, + Cluster: option.Cluster, + } + for { + resp, retry, err := kvGet(key, config, traceID) + if err == nil { + log.GetLogger().Debugf("datasystem kv get %s, %s, traceID: %s", key, string(resp), traceID) + return resp, nil + } + if retry { + log.GetLogger().Debugf("get with key will retry, failed ip: %s,traceID: %s, err: %s", + config.NodeIP, traceID, err.Error()) + config.invalidIP = append(config.invalidIP, config.NodeIP) + config.NodeIP = "" + continue + } + log.GetLogger().Errorf("get with key failed err: %s,NodeIP: %s,traceID: %s", err.Error(), + config.NodeIP, traceID) + return nil, err + } +} + +// KVDelWithRetry del kv to ds with retry +func KVDelWithRetry(key string, option *Option, traceID string) error { + log.GetLogger().Debugf("datasystem kv del %s, traceID: %s", key, traceID) + config := &Config{ + invalidIP: []string{}, + useLastUsedNode: true, + + TenantID: option.TenantID, + NodeIP: option.NodeIP, + Cluster: option.Cluster, + } + for { + retry, err := kvDel(key, config, traceID) + if err == nil { + return nil + } + if retry { + log.GetLogger().Debugf("del with key will retry, failed ip: %s,traceID: %s, err: %s", + config.NodeIP, traceID, err.Error()) + config.invalidIP = append(config.invalidIP, config.NodeIP) + config.NodeIP = "" + continue + } + log.GetLogger().Errorf("get with key failed err: %s,NodeIP: %s,traceID: %s", err.Error(), + config.NodeIP, traceID) + return err + } +} + +func kvPut(value []byte, param api.SetParam, config *Config, traceID string) (bool, error) { + dsClient, retry, err := getClient(config, traceID) + if err != nil { + return retry, err + } + if dsClient.kvClient == nil { + return false, fmt.Errorf("dsclient is nil") + } + key, _, err := getDataSystemKey(config, dsClient, traceID) + if err != nil { + return false, err + } + runtime.LockOSThread() + dsClient.kvClient.SetTraceID(traceID) + if err = localClientLibruntime.SetTenantID(config.TenantID); err != nil { + runtime.UnlockOSThread() + return false, err + } + errInfo := dsClient.kvClient.KVSet(key, value, param) + runtime.UnlockOSThread() + if errInfo.IsError() { + if shouldRetry(errInfo.Code) { + return true, errInfo.Err + } + return false, errInfo.Err + } + return false, nil +} + +func kvGet(key string, config *Config, traceID string) ([]byte, bool, error) { + dsClient, retry, err := getClient(config, traceID) + if err != nil { + return nil, retry, err + } + if dsClient.kvClient == nil { + return nil, false, fmt.Errorf("dsclient is nil") + } + runtime.LockOSThread() + dsClient.kvClient.SetTraceID(traceID) + if err = localClientLibruntime.SetTenantID(config.TenantID); err != nil { + runtime.UnlockOSThread() + return nil, false, err + } + resp, errInfo := dsClient.kvClient.KVGet(key) + runtime.UnlockOSThread() + retry, err = checkStatus(errInfo, config, traceID) + if err != nil { + return nil, retry, err + } + return resp, false, nil +} + +func kvDel(key string, config *Config, traceID string) (bool, error) { + dsClient, retry, err := getClient(config, traceID) + if err != nil { + return retry, err + } + if dsClient.kvClient == nil { + return false, fmt.Errorf("dsclient is nil") + } + runtime.LockOSThread() + dsClient.kvClient.SetTraceID(traceID) + if err = localClientLibruntime.SetTenantID(config.TenantID); err != nil { + runtime.UnlockOSThread() + return false, err + } + errInfo := dsClient.kvClient.KVDel(key) + runtime.UnlockOSThread() + if errInfo.IsError() { + if shouldRetry(errInfo.Code) { + return true, errInfo.Err + } + return false, errInfo.Err + } + return false, nil +} diff --git a/frontend/pkg/common/faas_common/datasystemclient/kvclient_test.go b/frontend/pkg/common/faas_common/datasystemclient/kvclient_test.go new file mode 100644 index 0000000000000000000000000000000000000000..03d1aa53d6e8a1e334f192f8b7aab5ec0ab7e943 --- /dev/null +++ b/frontend/pkg/common/faas_common/datasystemclient/kvclient_test.go @@ -0,0 +1,355 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package datasystemclient is data system client used for communicating with data system worker. +package datasystemclient + +import ( + "fmt" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/stretchr/testify/assert" + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/logger/log" + mockUtils "frontend/pkg/common/faas_common/utils" +) + +func localInit() { + cache := &Cache{ + nodeList: []string{}, + invalidMap: make(map[string]struct{}, 1), + } + localClientLibruntime = &mockUtils.FakeLibruntimeSdkClient{} + cache.addNode("127.2.2.101", log.GetLogger()) + cache.addNode("127.2.2.102", log.GetLogger()) + dataSystemCache.Store(noCluster, cache) +} + +func localRecover() { + dataSystemCache.Delete(noCluster) +} + +func TestKVDelWithRetry(t *testing.T) { + defer gomonkey.ApplyFunc(NewClient, + func(tenantID string, nodeIP string) (DsClientImpl, error) { + return DsClientImpl{kvClient: &FakeKvClient{num: 1}}, nil + }).Reset() + defer gomonkey.ApplyFunc((*Cache).healthCheckProcess, + func(_ *Cache, node string) { + return + }) + localInit() + defer localRecover() + type args struct { + key string + option *Option + traceID string + } + tests := []struct { + name string + args args + wantErr assert.ErrorAssertionFunc + }{ + { + name: "del success", + args: args{ + key: "aaaa", + option: &Option{ + TenantID: "tenant1", + NodeIP: "127.2.2.101", + Cluster: noCluster, + WriteMode: 0, + TTLSecond: 60, + }, + traceID: "aaaaa", + }, + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + if err != nil { + t.Errorf("del failed, err %s", err.Error()) + return false + } + return true + }, + }, + { + name: "del failed", + args: args{ + key: "key2", + option: &Option{ + TenantID: "tenant1", + NodeIP: "127.2.2.101", + Cluster: noCluster, + WriteMode: 0, + TTLSecond: 60, + }, + traceID: "aaaaa", + }, + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + return assert.Equal(t, "no data system node is available", err.Error()) + }, + }, + { + name: "find other node success", + args: args{ + key: "aaaa", + option: &Option{ + TenantID: "tenant1", + NodeIP: "127.2.2.1", + Cluster: noCluster, + WriteMode: 0, + TTLSecond: 60, + }, + traceID: "aaaaa", + }, + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + if err != nil { + t.Errorf("del failed, err %s", err.Error()) + return false + } + return true + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.wantErr(t, KVDelWithRetry(tt.args.key, tt.args.option, tt.args.traceID), fmt.Sprintf("KVDelWithRetry(%v, %v, %v)", tt.args.key, tt.args.option, tt.args.traceID)) + }) + } +} + +func TestKVGetWithRetry(t *testing.T) { + defer gomonkey.ApplyFunc(NewClient, + func(tenantID string, nodeIP string) (DsClientImpl, error) { + return DsClientImpl{kvClient: &FakeKvClient{num: 1}}, nil + }).Reset() + defer gomonkey.ApplyFunc((*Cache).healthCheckProcess, + func(_ *Cache, node string) { + return + }) + localInit() + defer localRecover() + type args struct { + key string + option *Option + traceID string + } + tests := []struct { + name string + args args + want []byte + wantErr assert.ErrorAssertionFunc + }{ + { + name: "get success", + args: args{ + key: "key1", + option: &Option{ + TenantID: "tenant1", + NodeIP: "127.2.2.101", + Cluster: noCluster, + WriteMode: 0, + TTLSecond: 60, + }, + traceID: "aaaaa", + }, + want: []byte("value1"), + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + if err != nil { + t.Errorf("get failed, err %s", err.Error()) + return false + } + return true + }, + }, + { + name: "get failed", + args: args{ + key: "key2", + option: &Option{ + TenantID: "tenant1", + NodeIP: "127.2.2.101", + Cluster: noCluster, + WriteMode: 0, + TTLSecond: 60, + }, + traceID: "aaaaa", + }, + want: nil, + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + return assert.Equal(t, "no data system node is available", err.Error()) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := KVGetWithRetry(tt.args.key, tt.args.option, tt.args.traceID) + if !tt.wantErr(t, err, fmt.Sprintf("KVGetWithRetry(%v, %v, %v)", tt.args.key, tt.args.option, tt.args.traceID)) { + return + } + assert.Equalf(t, tt.want, got, "KVGetWithRetry(%v, %v, %v)", tt.args.key, tt.args.option, tt.args.traceID) + }) + } +} + +func TestKVPutWithRetry(t *testing.T) { + defer gomonkey.ApplyFunc(NewClient, + func(tenantID string, nodeIP string) (DsClientImpl, error) { + return DsClientImpl{kvClient: &FakeKvClient{num: 1}}, nil + }).Reset() + defer gomonkey.ApplyFunc((*Cache).healthCheckProcess, + func(_ *Cache, node string) { + return + }) + localInit() + defer localRecover() + type args struct { + key string + value []byte + option *Option + traceID string + } + tests := []struct { + name string + args args + wantErr assert.ErrorAssertionFunc + }{ + { + name: "put success", + args: args{ + key: "key1", + value: []byte("value1"), + option: &Option{ + TenantID: "tenant1", + NodeIP: "127.2.2.101", + Cluster: noCluster, + WriteMode: 0, + TTLSecond: 60, + }, + traceID: "aaaaa", + }, + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + if err != nil { + t.Errorf("del failed, err %s", err.Error()) + return false + } + return true + }, + }, + { + name: "put failed", + args: args{ + key: "key2", + value: []byte("value1"), + option: &Option{ + TenantID: "tenant1", + NodeIP: "127.2.2.101", + Cluster: noCluster, + WriteMode: 0, + TTLSecond: 60, + }, + traceID: "aaaaa", + }, + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + return assert.Equal(t, "no data system node is available", err.Error()) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.wantErr(t, KVPutWithRetry(tt.args.key, tt.args.value, tt.args.option, tt.args.traceID), fmt.Sprintf("KVPutWithRetry(%v, %v, %v, %v)", tt.args.key, tt.args.value, tt.args.option, tt.args.traceID)) + }) + } +} + +func Test_kvDel(t *testing.T) { + type args struct { + key string + config *Config + traceID string + } + tests := []struct { + name string + args args + want bool + wantErr assert.ErrorAssertionFunc + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := kvDel(tt.args.key, tt.args.config, tt.args.traceID) + if !tt.wantErr(t, err, fmt.Sprintf("kvDel(%v, %v, %v)", tt.args.key, tt.args.config, tt.args.traceID)) { + return + } + assert.Equalf(t, tt.want, got, "kvDel(%v, %v, %v)", tt.args.key, tt.args.config, tt.args.traceID) + }) + } +} + +func Test_kvGet(t *testing.T) { + type args struct { + key string + config *Config + traceID string + } + tests := []struct { + name string + args args + want []byte + want1 bool + wantErr assert.ErrorAssertionFunc + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1, err := kvGet(tt.args.key, tt.args.config, tt.args.traceID) + if !tt.wantErr(t, err, fmt.Sprintf("kvGet(%v, %v, %v)", tt.args.key, tt.args.config, tt.args.traceID)) { + return + } + assert.Equalf(t, tt.want, got, "kvGet(%v, %v, %v)", tt.args.key, tt.args.config, tt.args.traceID) + assert.Equalf(t, tt.want1, got1, "kvGet(%v, %v, %v)", tt.args.key, tt.args.config, tt.args.traceID) + }) + } +} + +func Test_kvPut(t *testing.T) { + type args struct { + value []byte + param api.SetParam + config *Config + traceID string + } + tests := []struct { + name string + args args + want bool + wantErr assert.ErrorAssertionFunc + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := kvPut(tt.args.value, tt.args.param, tt.args.config, tt.args.traceID) + if !tt.wantErr(t, err, fmt.Sprintf("kvPut(%v, %v, %v, %v)", tt.args.value, tt.args.param, tt.args.config, tt.args.traceID)) { + return + } + assert.Equalf(t, tt.want, got, "kvPut(%v, %v, %v, %v)", tt.args.value, tt.args.param, tt.args.config, tt.args.traceID) + }) + } +} diff --git a/frontend/pkg/common/faas_common/datasystemclient/streamctx.go b/frontend/pkg/common/faas_common/datasystemclient/streamctx.go new file mode 100644 index 0000000000000000000000000000000000000000..914ffa4327f4ab5be9ffb7970544a5ce614b5ac3 --- /dev/null +++ b/frontend/pkg/common/faas_common/datasystemclient/streamctx.go @@ -0,0 +1,113 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package datasystemclient + +import ( + "bufio" + "io" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/valyala/fasthttp" +) + +// StreamCtx - +type StreamCtx interface { + GetRequestHeader(key string) string + SetResponseHeader(key, value string) + Stream(writer func(w io.Writer) bool) + FlushResult(w io.Writer, result []byte) error + Done() <-chan struct{} +} + +// GinCtxAdapter - +type GinCtxAdapter struct { + *gin.Context +} + +// GetRequestHeader - +func (gt *GinCtxAdapter) GetRequestHeader(key string) string { + return gt.Request.Header.Get(key) +} + +// SetResponseHeader - +func (gt *GinCtxAdapter) SetResponseHeader(key, value string) { + gt.Writer.Header().Set(key, value) +} + +// Stream - +func (gt *GinCtxAdapter) Stream(writer func(w io.Writer) bool) { + gt.Context.Stream(writer) +} + +// FlushResult - +func (gt *GinCtxAdapter) FlushResult(w io.Writer, result []byte) error { + _, err := w.Write(result) + if err != nil { + return err + } + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + return nil +} + +// Done - +func (gt *GinCtxAdapter) Done() <-chan struct{} { + return gt.Request.Context().Done() +} + +// FastHttpCtxAdapter - +type FastHttpCtxAdapter struct { + *fasthttp.RequestCtx +} + +// GetRequestHeader - +func (ft *FastHttpCtxAdapter) GetRequestHeader(key string) string { + return string(ft.Request.Header.Peek(key)) +} + +// SetResponseHeader - +func (ft *FastHttpCtxAdapter) SetResponseHeader(key, value string) { + ft.Response.Header.Set(key, value) +} + +// Stream - +func (ft *FastHttpCtxAdapter) Stream(writer func(w io.Writer) bool) { + ft.SetBodyStreamWriter(func(w *bufio.Writer) { + writer(w) + }) +} + +func (ft *FastHttpCtxAdapter) FlushResult(w io.Writer, result []byte) error { + _, err := w.Write(result) + if err != nil { + return err + } + if f, ok := w.(*bufio.Writer); ok { + // When the client is disconnected, return and close consumer. + if err = f.Flush(); err != nil { + return err + } + } + return nil +} + +// Done - +func (ft *FastHttpCtxAdapter) Done() <-chan struct{} { + return ft.RequestCtx.Done() +} diff --git a/frontend/pkg/common/faas_common/etcd3/cache.go b/frontend/pkg/common/faas_common/etcd3/cache.go new file mode 100644 index 0000000000000000000000000000000000000000..d717f4f1ff68a505f421aa90bd2515fc1e1c6e00 --- /dev/null +++ b/frontend/pkg/common/faas_common/etcd3/cache.go @@ -0,0 +1,408 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd3 - +package etcd3 + +import ( + "bufio" + "bytes" + "encoding/json" + "errors" + "fmt" + "os" + "sort" + "strconv" + "strings" + "time" + + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/utils" +) + +const ( + cacheMetaFilePrefix = "etcdCacheMeta_" + cacheDataFilePrefix = "etcdCacheData_" + backupFileSuffix = "_backup" + cacheDataSplitNum = 3 +) + +var ( + // ErrInvalidCacheMeta - + ErrInvalidCacheMeta = errors.New("invalid cache meta") + // ErrCacheDataNotExist - + ErrCacheDataNotExist = errors.New("cache data not exist") + // ErrCacheDataMD5Mismatch - + ErrCacheDataMD5Mismatch = errors.New("cache data md5 mismatch") + cacheDataSeparator = "|" + cacheDataLineFeed = []byte("\n") +) + +// ETCDCacheMeta - +type ETCDCacheMeta struct { + Revision int64 `json:"revision"` + CacheMD5 string `json:"cacheMD5"` +} + +func (ew *EtcdWatcher) setCacheFilePath() { + cacheMetaFileName := fmt.Sprintf("%s%s", cacheMetaFilePrefix, strings.ReplaceAll(ew.key, "/", "#")) + cacheDataFileName := fmt.Sprintf("%s%s", cacheDataFilePrefix, strings.ReplaceAll(ew.key, "/", "#")) + ew.cacheConfig.MetaFilePath = fmt.Sprintf("%s/%s", ew.cacheConfig.PersistPath, cacheMetaFileName) + ew.cacheConfig.DataFilePath = fmt.Sprintf("%s/%s", ew.cacheConfig.PersistPath, cacheDataFileName) + ew.cacheConfig.BackupFilePath = fmt.Sprintf("%s%s", ew.cacheConfig.DataFilePath, backupFileSuffix) +} + +func (ew *EtcdWatcher) processETCDCache() { + log.GetLogger().Infof("start processing ETCD cache") + ew.setCacheFilePath() + persistInterval := ew.cacheConfig.FlushInterval + ticker := time.NewTicker(time.Minute * time.Duration(persistInterval)) + defer ticker.Stop() + // only record event with latest revision which is easier for flushCacheFile + eventCache := make(map[string]*Event, ew.cacheConfig.FlushThreshold) + for { + select { + case <-ticker.C: + log.GetLogger().Infof("ticker triggers, flushing cache now") + if err := ew.flushCacheToFile(eventCache); err == nil { + eventCache = make(map[string]*Event, ew.cacheConfig.FlushThreshold) + } + case event := <-ew.CacheChan: + log.GetLogger().Infof("threshold triggers, flushing cache now") + preEvent, exist := eventCache[event.Key] + if !exist || (exist && preEvent.Rev < event.Rev) { + eventCache[event.Key] = event + } + if len(eventCache) > ew.cacheConfig.FlushThreshold { + if err := ew.flushCacheToFile(eventCache); err == nil { + eventCache = make(map[string]*Event, ew.cacheConfig.FlushThreshold) + } + } + case <-ew.configCh: + log.GetLogger().Infof("cache config changed, new config is %+v", ew.cacheConfig) + if !ew.cacheConfig.EnableCache { + log.GetLogger().Warnf("etcd cache disabled, stop processing cache") + return + } + if ew.cacheConfig.FlushInterval != persistInterval { + persistInterval = ew.cacheConfig.FlushInterval + ticker.Reset(time.Minute * time.Duration(persistInterval)) + } + case <-ew.stopCh: + log.GetLogger().Warnf("etcd watcher stopped, stop processing cache") + return + } + } +} + +func (ew *EtcdWatcher) getCacheMeta() *ETCDCacheMeta { + var cacheMeta *ETCDCacheMeta + _, statErr := os.Stat(ew.cacheConfig.MetaFilePath) + if os.IsNotExist(statErr) { + return &ETCDCacheMeta{} + } + if statErr == nil { + cacheMetaData, err := os.ReadFile(ew.cacheConfig.MetaFilePath) + if err != nil { + log.GetLogger().Errorf("failed to read cache meta file %s error %s", ew.cacheConfig.MetaFilePath, + err.Error()) + return nil + } + cacheMeta = &ETCDCacheMeta{} + if err = json.Unmarshal(cacheMetaData, cacheMeta); err != nil { + log.GetLogger().Errorf("failed to unmarshal cache meta file %s error %s", ew.cacheConfig.MetaFilePath, + err.Error()) + return nil + } + return cacheMeta + } + return nil +} + +func (ew *EtcdWatcher) cleanCacheFile(cleanMeta, cleanData, cleanBackup bool) { + if cleanMeta { + if err := os.Remove(ew.cacheConfig.MetaFilePath); err != nil { + log.GetLogger().Errorf("failed to remove cache meta file %s error %s", ew.cacheConfig.MetaFilePath, + err.Error()) + } + } + if cleanData { + if err := os.Remove(ew.cacheConfig.DataFilePath); err != nil { + log.GetLogger().Errorf("failed to remove cache data file %s error %s", ew.cacheConfig.DataFilePath, + err.Error()) + } + } + if cleanBackup { + if err := os.Remove(ew.cacheConfig.BackupFilePath); err != nil { + log.GetLogger().Errorf("failed to remove cache backup file %s error %s", ew.cacheConfig.BackupFilePath, + err.Error()) + } + } +} + +// processDataBackup turns dataFile to backFile +func (ew *EtcdWatcher) processDataBackup(cacheMeta *ETCDCacheMeta) error { + _, statDataFileErr := os.Stat(ew.cacheConfig.DataFilePath) + _, statBackupFileErr := os.Stat(ew.cacheConfig.BackupFilePath) + // need to handle backupFife if either dataFile or backupFile exists + if statDataFileErr == nil || statBackupFileErr == nil { + // if backupFile doesn't exist, it's the normal case, rename dataFile to backupFile if it exists. + // if backupFile exists, it's the fault case where flush is interrupted, remove dataFile if it exists. + if statDataFileErr == nil && os.IsNotExist(statBackupFileErr) { + if err := os.Rename(ew.cacheConfig.DataFilePath, ew.cacheConfig.BackupFilePath); err != nil { + log.GetLogger().Errorf("failed to rename cache file to %s error %s", ew.cacheConfig.BackupFilePath, + err.Error()) + ew.cleanCacheFile(true, true, true) + return err + } + } else if statDataFileErr == nil && statBackupFileErr == nil { + if err := os.Remove(ew.cacheConfig.DataFilePath); err != nil { + log.GetLogger().Errorf("failed to remove dirty cache file %s error %s", ew.cacheConfig.DataFilePath, + err.Error()) + return err + } + } + if utils.CalcFileMD5(ew.cacheConfig.BackupFilePath) != cacheMeta.CacheMD5 { + log.GetLogger().Errorf("md5 mismatch for cache backup file %s", ew.cacheConfig.BackupFilePath) + ew.cleanCacheFile(true, true, true) + return ErrCacheDataMD5Mismatch + } + } + return nil +} + +// flushCacheToFile will modify the given eventCache during processing +func (ew *EtcdWatcher) flushCacheToFile(eventCache map[string]*Event) error { + ew.Lock() + if ew.cacheFlushing { + ew.Unlock() + return nil + } + ew.cacheFlushing = true + defer func() { + ew.Lock() + ew.cacheFlushing = false + ew.Unlock() + }() + ew.Unlock() + cacheMeta := ew.getCacheMeta() + if cacheMeta == nil { + ew.cleanCacheFile(true, true, true) + return ErrInvalidCacheMeta + } + // backup dataFile if it exists, will generate new dataFile from backupFile and eventCache + if err := ew.processDataBackup(cacheMeta); err != nil { + return err + } + var scanner *bufio.Scanner + _, statBackupFileErr := os.Stat(ew.cacheConfig.BackupFilePath) + if statBackupFileErr == nil { + backupFile, err := os.OpenFile(ew.cacheConfig.BackupFilePath, os.O_RDONLY, 0600) + if err != nil { + log.GetLogger().Errorf("failed to open cache backup file %s error %s", ew.cacheConfig.BackupFilePath, + err.Error()) + return err + } + defer func() { + if err := backupFile.Close(); err != nil { + log.GetLogger().Errorf("failed to close backup file %s error %s", ew.cacheConfig.BackupFilePath, + err.Error()) + } + if err := os.Remove(ew.cacheConfig.BackupFilePath); err != nil { + log.GetLogger().Errorf("failed to remove backup file %s error %s", ew.cacheConfig.BackupFilePath, + err.Error()) + } + }() + scanner = bufio.NewScanner(backupFile) + } + dataFile, err := os.OpenFile(ew.cacheConfig.DataFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC|os.O_SYNC, 0600) + if err != nil { + log.GetLogger().Errorf("failed to open cache file %s error %s", ew.cacheConfig.DataFilePath, err.Error()) + return err + } + eventList := generateSortedCacheList(eventCache) + offset := int64(0) + for scanner != nil && scanner.Scan() { + line := scanner.Text() + items := strings.SplitN(line, cacheDataSeparator, cacheDataSplitNum) + if len(items) != cacheDataSplitNum { + log.GetLogger().Warnf("skip invalid data %s in cache file %s", line, ew.cacheConfig.BackupFilePath) + continue + } + scanKey, scanValue := items[0], []byte(items[2]) + scanRevision, err := strconv.ParseInt(items[1], 10, 64) + if err != nil { + log.GetLogger().Errorf("invalid revision format of %s in line %s cache file %s", items[1], line, + ew.cacheConfig.BackupFilePath) + continue + } + if scanRevision > cacheMeta.Revision { + cacheMeta.Revision = scanRevision + } + index := -1 + for i, event := range eventList { + if event.Rev > cacheMeta.Revision { + cacheMeta.Revision = event.Rev + } + // eventList keeps keys in lexicographical order which is also the order we set in cache data file, this + // loop only handles eventKey <= scanKey scenario which contains two types of keys : 1. eventKey which goes + // before scanKey with PUT type 2. eventKey equals to scanKey which will update or delete scanKey if it has + // a newer revision + if event.Key < scanKey { + if event.Type == PUT { + offset = flushEventToFile(dataFile, offset, []byte(event.Key), event.Value, event.Rev) + } + index = i + continue + } + if event.Key == scanKey { + // should not update or delete if event revision is older than cacheMeta + if event.Rev > scanRevision && event.Type == PUT { + scanValue = event.Value + scanRevision = event.Rev + } else if event.Rev > scanRevision && event.Type == DELETE { + scanValue = nil + } + index = i + } + // here eventKey >= scanKey no need to go further + break + } + if index != -1 { + eventList = eventList[index+1:] + } + if scanValue != nil { + offset = flushEventToFile(dataFile, offset, []byte(scanKey), scanValue, scanRevision) + } + } + for _, event := range eventList { + if event.Rev > cacheMeta.Revision { + cacheMeta.Revision = event.Rev + } + if event.Type == PUT { + offset = flushEventToFile(dataFile, offset, []byte(event.Key), event.Value, event.Rev) + } + } + if err = dataFile.Close(); err != nil { + log.GetLogger().Errorf("failed to close cache data file %s error %s", ew.cacheConfig.DataFilePath, + err.Error()) + } + if offset == 0 { + log.GetLogger().Errorf("failed to write data file %s", ew.cacheConfig.DataFilePath) + ew.cleanCacheFile(false, true, false) + return errors.New("failed to write data file") + } + cacheMeta.CacheMD5 = utils.CalcFileMD5(ew.cacheConfig.DataFilePath) + cacheMetaData, err := json.Marshal(cacheMeta) + if err != nil { + log.GetLogger().Errorf("failed to marshal cache meta error %s", err.Error()) + return err + } + if err = os.WriteFile(ew.cacheConfig.MetaFilePath, cacheMetaData, 0600); err != nil { + log.GetLogger().Errorf("failed to write cache meta file %s error %s", ew.cacheConfig.MetaFilePath, + err.Error()) + ew.cleanCacheFile(false, true, false) + return err + } + log.GetLogger().Infof("succeed to flush cache") + return nil +} + +func (ew *EtcdWatcher) restoreCacheFromFile() error { + ew.setCacheFilePath() + _, statBackupFileErr := os.Stat(ew.cacheConfig.BackupFilePath) + if statBackupFileErr == nil { + // backupFile exists, it's the fault scenario, flushCacheToFile with nil to restore dataFile from backupFile + ew.flushCacheToFile(nil) + } + _, statDataFileErr := os.Stat(ew.cacheConfig.DataFilePath) + if os.IsNotExist(statDataFileErr) { + return ErrCacheDataNotExist + } + cacheMeta := ew.getCacheMeta() + if cacheMeta == nil { + ew.cleanCacheFile(true, true, true) + return ErrInvalidCacheMeta + } + if utils.CalcFileMD5(ew.cacheConfig.DataFilePath) != cacheMeta.CacheMD5 { + log.GetLogger().Errorf("md5 mismatch for cache data file %s", ew.cacheConfig.DataFilePath) + ew.cleanCacheFile(true, true, true) + return ErrCacheDataMD5Mismatch + } + dataFile, err := os.OpenFile(ew.cacheConfig.DataFilePath, os.O_RDONLY, 0600) + if err != nil { + log.GetLogger().Errorf("failed to open cache backup file %s error %s", ew.cacheConfig.DataFilePath, + err.Error()) + ew.cleanCacheFile(true, true, true) + return err + } + scanner := bufio.NewScanner(dataFile) + for scanner.Scan() { + line := scanner.Text() + items := strings.SplitN(line, cacheDataSeparator, cacheDataSplitNum) + if len(items) != cacheDataSplitNum { + log.GetLogger().Warnf("skip invalid data %s in cache file %s", line, ew.cacheConfig.DataFilePath) + continue + } + scanKey, scanValue := items[0], []byte(items[2]) + scanRevision, err := strconv.ParseInt(items[1], 10, 64) + if err != nil { + log.GetLogger().Errorf("invalid revision format of %s in line %s file %s", items[1], line, + ew.cacheConfig.DataFilePath) + continue + } + ew.sendEvent(&Event{ + Type: PUT, + Key: scanKey, + Value: scanValue, + Rev: scanRevision, + }) + } + if err = dataFile.Close(); err != nil { + log.GetLogger().Errorf("failed to close cache backup file %s error %s", ew.cacheConfig.DataFilePath, + err.Error()) + } + ew.initialRev = cacheMeta.Revision + log.GetLogger().Infof("succeed to restore etcd cache to revision %d", cacheMeta.Revision) + return nil +} + +func flushEventToFile(f *os.File, offset int64, key, value []byte, revision int64) int64 { + buffer := new(bytes.Buffer) + buffer.Write(key) + buffer.Write([]byte(cacheDataSeparator)) + buffer.Write([]byte(strconv.FormatInt(revision, 10))) + buffer.Write([]byte(cacheDataSeparator)) + buffer.Write(value) + buffer.Write(cacheDataLineFeed) + _, err := f.WriteAt(buffer.Bytes(), offset) + if err != nil { + log.GetLogger().Errorf("failed to write content to cache file error %s", err.Error()) + return offset + } + return offset + int64(buffer.Len()) +} + +func generateSortedCacheList(cache map[string]*Event) []*Event { + cacheList := make([]*Event, 0, len(cache)) + for _, v := range cache { + cacheList = append(cacheList, v) + } + sort.Slice(cacheList, func(i, j int) bool { + return cacheList[i].Key < cacheList[j].Key + }) + return cacheList +} diff --git a/frontend/pkg/common/faas_common/etcd3/cache_test.go b/frontend/pkg/common/faas_common/etcd3/cache_test.go new file mode 100644 index 0000000000000000000000000000000000000000..e7b8e8da78bda474e5d74abf913d5a7237fcd22d --- /dev/null +++ b/frontend/pkg/common/faas_common/etcd3/cache_test.go @@ -0,0 +1,362 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd3 - +package etcd3 + +import ( + "encoding/json" + "errors" + "os" + "reflect" + "testing" + "time" + + "frontend/pkg/common/faas_common/utils" + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" +) + +func TestProcessETCDCache(t *testing.T) { + hackTicker := time.NewTicker(50 * time.Millisecond) + resetDuration := time.Duration(0) + patches := []*gomonkey.Patches{ + gomonkey.ApplyMethod(reflect.TypeOf(&time.Ticker{}), "Reset", func(_ *time.Ticker, d time.Duration) { + resetDuration = d + }), + } + defer func() { + for _, p := range patches { + p.Reset() + } + }() + convey.Convey("ticker case", t, func() { + patch := gomonkey.ApplyFunc(time.NewTicker, func(d time.Duration) *time.Ticker { + hackTicker.Reset(50 * time.Millisecond) + return hackTicker + }) + os.Remove("etcdCacheMeta_#sn#function") + os.Remove("etcdCacheData_#sn#function") + os.Remove("etcdCacheData_#sn#function_backup") + stopCh := make(chan struct{}, 1) + ew := newEtcdWatcher() + ew.stopCh = stopCh + go ew.processETCDCache() + time.Sleep(500 * time.Millisecond) + ew.CacheChan <- &Event{ + Rev: 100, + Type: PUT, + Key: "/sn/function/123/hello/latest", + Value: []byte(`{"name":"hello","version":"latest"}`), + } + time.Sleep(500 * time.Millisecond) + stopCh <- struct{}{} + time.Sleep(500 * time.Millisecond) + data, err := os.ReadFile("etcdCacheData_#sn#function") + convey.So(err, convey.ShouldBeNil) + convey.So(string(data), convey.ShouldEqual, "/sn/function/123/hello/latest|100|{\"name\":\"hello\",\"version\":\"latest\"}\n") + data, err = os.ReadFile("etcdCacheMeta_#sn#function") + convey.So(err, convey.ShouldBeNil) + convey.So(string(data), convey.ShouldEqual, "{\"revision\":100,\"cacheMD5\":\"4fca8f1c736ca30135ed16538f4aebfc\"}") + patch.Reset() + }) + convey.Convey("threshold case", t, func() { + os.Remove("etcdCacheMeta_#sn#function") + os.Remove("etcdCacheData_#sn#function") + os.Remove("etcdCacheData_#sn#function_backup") + stopCh := make(chan struct{}, 1) + ew := newEtcdWatcher() + ew.stopCh = stopCh + ew.cacheConfig.FlushThreshold = 0 + go ew.processETCDCache() + time.Sleep(500 * time.Millisecond) + ew.CacheChan <- &Event{ + Rev: 100, + Type: PUT, + Key: "/sn/function/123/hello/latest", + Value: []byte(`{"name":"hello","version":"latest"}`), + } + time.Sleep(500 * time.Millisecond) + stopCh <- struct{}{} + time.Sleep(500 * time.Millisecond) + data, err := os.ReadFile("etcdCacheData_#sn#function") + convey.So(err, convey.ShouldBeNil) + convey.So(string(data), convey.ShouldEqual, "/sn/function/123/hello/latest|100|{\"name\":\"hello\",\"version\":\"latest\"}\n") + data, err = os.ReadFile("etcdCacheMeta_#sn#function") + convey.So(err, convey.ShouldBeNil) + convey.So(string(data), convey.ShouldEqual, "{\"revision\":100,\"cacheMD5\":\"4fca8f1c736ca30135ed16538f4aebfc\"}") + }) + convey.Convey("config update case", t, func() { + os.Remove("etcdCacheMeta_#sn#function") + os.Remove("etcdCacheData_#sn#function") + os.Remove("etcdCacheData_#sn#function_backup") + ew := newEtcdWatcher() + go ew.processETCDCache() + time.Sleep(500 * time.Millisecond) + ew.cacheConfig.FlushInterval = 20 + ew.configCh <- struct{}{} + time.Sleep(500 * time.Millisecond) + convey.So(resetDuration, convey.ShouldEqual, 20*time.Minute) + ew.cacheConfig.EnableCache = false + ew.configCh <- struct{}{} + }) + os.Remove("etcdCacheMeta_#sn#function") + os.Remove("etcdCacheData_#sn#function") + os.Remove("etcdCacheData_#sn#function_backup") +} + +func TestFlushCacheToFile(t *testing.T) { + stopCh := make(chan struct{}, 1) + ew := &EtcdWatcher{ + key: "/sn/function", + CacheChan: make(chan *Event, 10), + configCh: make(chan struct{}, 1), + stopCh: stopCh, + cacheConfig: EtcdCacheConfig{ + EnableCache: true, + PersistPath: "./", + FlushInterval: 10, + FlushThreshold: 10, + }, + } + ew.setCacheFilePath() + convey.Convey("no dataFile and no backupFile", t, func() { + os.Remove("etcdCacheMeta_#sn#function") + os.Remove("etcdCacheData_#sn#function") + os.Remove("etcdCacheData_#sn#function_backup") + eventBuffer := map[string]*Event{ + "/sn/function/123/hello/latest": &Event{ + Rev: 100, + Key: "/sn/function/123/hello/latest", + Value: []byte(`{"name":"hello","version":"latest"}`), + }, + } + err := ew.flushCacheToFile(eventBuffer) + convey.So(err, convey.ShouldBeNil) + _, stateMetaFileErr := os.Stat("./etcdCacheMeta_#sn#function") + _, stateDataFileErr := os.Stat("./etcdCacheData_#sn#function") + _, stateBackupFileErr := os.Stat("./etcdCacheData_#sn#function_backup") + convey.So(stateMetaFileErr, convey.ShouldBeNil) + convey.So(stateDataFileErr, convey.ShouldBeNil) + convey.So(os.IsNotExist(stateBackupFileErr), convey.ShouldEqual, true) + }) + convey.Convey("no backupFile", t, func() { + os.Remove("etcdCacheMeta_#sn#function") + os.Remove("etcdCacheData_#sn#function") + os.Remove("etcdCacheData_#sn#function_backup") + // dataFile exists and no metaFile + os.WriteFile("./etcdCacheData_#sn#function", []byte("/sn/function/123/hello/latest|100|{\"name\":\"hello\",\"version\":\"latest\"}\n"), 0600) + err := ew.flushCacheToFile(nil) + convey.So(err, convey.ShouldNotBeNil) + _, errStatMeta := os.Stat("./etcdCacheMeta_#sn#function") + _, errStatData := os.Stat("./etcdCacheData_#sn#function") + _, errStatBackup := os.Stat("./etcdCacheData_#sn#function_backup") + convey.So(os.IsNotExist(errStatMeta), convey.ShouldEqual, true) + convey.So(os.IsNotExist(errStatData), convey.ShouldEqual, true) + convey.So(os.IsNotExist(errStatBackup), convey.ShouldEqual, true) + // dataFile exists and metaFile exists + os.WriteFile("./etcdCacheMeta_#sn#function", []byte(`{"revision":101,"cacheMD5":"726eb6f3140438ac1cbe334777e1a272"}`), 0600) + os.WriteFile("./etcdCacheData_#sn#function", []byte("/sn/function/123/goodbye/latest|101|{\"name\":\"goodbye\",\"version\":\"latest\"}\n/sn/function/123/hello/latest|100|{\"name\":\"hello\",\"version\":\"latest\"}\n/sn/function/123/invalid/latest|xxx|{\"name\":\"invalid\",\"version\":\"latest\"}\nThisIsInvalidData\n"), 0600) + eventBuffer := map[string]*Event{ + "/sn/function/123/goodbye/latest": &Event{ + Rev: 102, + Type: PUT, + Key: "/sn/function/123/goodbye/latest", + Value: []byte(`{"name":"goodbye","version":"v1"}`), + }, + "/sn/function/123/hello/latest": &Event{ + Rev: 103, + Type: DELETE, + Key: "/sn/function/123/hello/latest", + Value: []byte(`{"name":"hello","version":"latest"}`), + }, + "/sn/function/123/echo/latest": &Event{ + Rev: 104, + Type: PUT, + Key: "/sn/function/123/echo/latest", + Value: []byte(`{"name":"echo","version":"latest"}`), + }, + } + err = ew.flushCacheToFile(eventBuffer) + convey.So(err, convey.ShouldBeNil) + data, err := os.ReadFile("./etcdCacheMeta_#sn#function") + convey.So(err, convey.ShouldBeNil) + meta := &ETCDCacheMeta{} + err = json.Unmarshal(data, meta) + convey.So(err, convey.ShouldBeNil) + convey.So(meta.Revision, convey.ShouldEqual, 104) + convey.So(meta.CacheMD5, convey.ShouldEqual, "006731eddc832c067f9814b64ae12833") + convey.So(utils.CalcFileMD5("./etcdCacheData_#sn#function"), convey.ShouldEqual, "006731eddc832c067f9814b64ae12833") + _, stateBackupFileErr := os.Stat("./etcdCacheData_#sn#function_backup") + convey.So(os.IsNotExist(stateBackupFileErr), convey.ShouldEqual, true) + }) + convey.Convey("backupFile exists", t, func() { + os.Remove("etcdCacheMeta_#sn#function") + os.Remove("etcdCacheData_#sn#function") + os.Remove("etcdCacheData_#sn#function_backup") + // backupFile mismatch with metaFile + os.WriteFile("./etcdCacheMeta_#sn#function", []byte(`{"revision":100,"cacheMD5":"4f4449c598ec58854d7104c4a64e979f"}`), 0600) + os.WriteFile("./etcdCacheData_#sn#function_backup", []byte("/sn/function/123/hello/latest|100|{\"name\":\"hello\",\"version\":\"latest\"}\n"), 0600) + err := ew.flushCacheToFile(nil) + convey.So(err, convey.ShouldNotBeNil) + _, errStatMeta := os.Stat("./etcdCacheMeta_#sn#function") + _, errStatData := os.Stat("./etcdCacheData_#sn#function") + _, errStatBackup := os.Stat("./etcdCacheData_#sn#function_backup") + convey.So(os.IsNotExist(errStatMeta), convey.ShouldEqual, true) + convey.So(os.IsNotExist(errStatData), convey.ShouldEqual, true) + convey.So(os.IsNotExist(errStatBackup), convey.ShouldEqual, true) + // backupFile exists and dataFile exists + os.WriteFile("./etcdCacheMeta_#sn#function", []byte(`{"revision":100,"cacheMD5":"4fca8f1c736ca30135ed16538f4aebfc"}`), 0600) + os.WriteFile("./etcdCacheData_#sn#function", []byte("/sn/function/123/goodbye/latest|101|{\"name\":\"goodbye\",\"version\":\"latest\"}\n"), 0600) + os.WriteFile("./etcdCacheData_#sn#function_backup", []byte("/sn/function/123/hello/latest|100|{\"name\":\"hello\",\"version\":\"latest\"}\n"), 0600) + err = ew.flushCacheToFile(nil) + convey.So(err, convey.ShouldBeNil) + data, err := os.ReadFile("./etcdCacheMeta_#sn#function") + convey.So(err, convey.ShouldBeNil) + meta := &ETCDCacheMeta{} + err = json.Unmarshal(data, meta) + convey.So(err, convey.ShouldBeNil) + convey.So(meta.Revision, convey.ShouldEqual, 100) + convey.So(meta.CacheMD5, convey.ShouldEqual, "4fca8f1c736ca30135ed16538f4aebfc") + convey.So(utils.CalcFileMD5("./etcdCacheData_#sn#function"), convey.ShouldEqual, "4fca8f1c736ca30135ed16538f4aebfc") + _, stateBackupFileErr := os.Stat("./etcdCacheData_#sn#function_backup") + convey.So(os.IsNotExist(stateBackupFileErr), convey.ShouldEqual, true) + }) + convey.Convey("file close fail", t, func() { + patch1 := gomonkey.ApplyMethod(reflect.TypeOf(&os.File{}), "Close", func(f *os.File) error { + return errors.New("some error") + }) + fileHack, _ := os.OpenFile("./xxx", os.O_WRONLY|os.O_CREATE|os.O_TRUNC|os.O_SYNC, 0600) + patch2 := gomonkey.ApplyFunc(os.OpenFile, func(name string, flag int, perm os.FileMode) (*os.File, error) { + return fileHack, nil + }) + os.Remove("etcdCacheMeta_#sn#function") + os.Remove("etcdCacheData_#sn#function") + os.Remove("etcdCacheData_#sn#function_backup") + os.WriteFile("./etcdCacheMeta_#sn#function", []byte(`{"revision":100,"cacheMD5":"4fca8f1c736ca30135ed16538f4aebfc"}`), 0600) + os.WriteFile("./etcdCacheData_#sn#function", []byte("/sn/function/123/goodbye/latest|101|{\"name\":\"goodbye\",\"version\":\"latest\"}\n"), 0600) + os.WriteFile("./etcdCacheData_#sn#function_backup", []byte("/sn/function/123/hello/latest|100|{\"name\":\"hello\",\"version\":\"latest\"}\n"), 0600) + err := ew.flushCacheToFile(nil) + convey.So(err, convey.ShouldNotBeNil) + os.Remove("etcdCacheMeta_#sn#function") + err = ew.flushCacheToFile(nil) + convey.So(err, convey.ShouldNotBeNil) + patch1.Reset() + patch2.Reset() + fileHack.Close() + os.Remove("./xxx") + }) + os.Remove("etcdCacheMeta_#sn#function") + os.Remove("etcdCacheData_#sn#function") + os.Remove("etcdCacheData_#sn#function_backup") +} + +func TestRestoreCacheFromFile(t *testing.T) { + stopCh := make(chan struct{}, 1) + ew := &EtcdWatcher{ + key: "/sn/function", + ResultChan: make(chan *Event, 10), + CacheChan: make(chan *Event, 10), + configCh: make(chan struct{}, 1), + stopCh: stopCh, + cacheConfig: EtcdCacheConfig{ + EnableCache: true, + PersistPath: "./", + FlushInterval: 10, + FlushThreshold: 10, + }, + } + convey.Convey("no dataFile", t, func() { + err := ew.restoreCacheFromFile() + convey.So(err, convey.ShouldNotBeNil) + convey.So(len(ew.ResultChan), convey.ShouldEqual, 0) + }) + convey.Convey("backupFile exists", t, func() { + os.Remove("etcdCacheMeta_#sn#function") + os.Remove("etcdCacheData_#sn#function") + os.Remove("etcdCacheData_#sn#function_backup") + // invalid metaFile + os.WriteFile("./etcdCacheMeta_#sn#function", []byte(`this is a invalid json`), 0600) + os.WriteFile("./etcdCacheData_#sn#function_backup", []byte("/sn/function/123/hello/latest|100|{\"name\":\"hello\",\"version\":\"latest\"}\n"), 0600) + err := ew.restoreCacheFromFile() + convey.So(err, convey.ShouldNotBeNil) + convey.So(len(ew.ResultChan), convey.ShouldEqual, 0) + _, errStatMeta := os.Stat("./etcdCacheMeta_#sn#function") + _, errStatData := os.Stat("./etcdCacheData_#sn#function") + _, errStatBackup := os.Stat("./etcdCacheData_#sn#function_backup") + convey.So(os.IsNotExist(errStatMeta), convey.ShouldEqual, true) + convey.So(os.IsNotExist(errStatData), convey.ShouldEqual, true) + convey.So(os.IsNotExist(errStatBackup), convey.ShouldEqual, true) + // backupFile mismatches with metaFile + os.WriteFile("./etcdCacheMeta_#sn#function", []byte(`{"revision":100,"cacheMD5":"4f4449c598ec58854d7104c4a64e979f"}`), 0600) + os.WriteFile("./etcdCacheData_#sn#function_backup", []byte("/sn/function/123/hello/latest|100|{\"name\":\"hello\",\"version\":\"latest\"}\n"), 0600) + err = ew.restoreCacheFromFile() + convey.So(err, convey.ShouldNotBeNil) + convey.So(len(ew.ResultChan), convey.ShouldEqual, 0) + _, errStatMeta = os.Stat("./etcdCacheMeta_#sn#function") + _, errStatData = os.Stat("./etcdCacheData_#sn#function") + _, errStatBackup = os.Stat("./etcdCacheData_#sn#function_backup") + convey.So(os.IsNotExist(errStatMeta), convey.ShouldEqual, true) + convey.So(os.IsNotExist(errStatData), convey.ShouldEqual, true) + convey.So(os.IsNotExist(errStatBackup), convey.ShouldEqual, true) + }) + convey.Convey("dataFile exist and no backupFile", t, func() { + os.Remove("etcdCacheMeta_#sn#function") + os.Remove("etcdCacheData_#sn#function") + os.Remove("etcdCacheData_#sn#function_backup") + // dataFile mismatches with metaFile + os.WriteFile("./etcdCacheMeta_#sn#function", []byte(`{"revision":100,"cacheMD5":"4f4449c598ec58854d7104c4a64e979f"}`), 0600) + os.WriteFile("./etcdCacheData_#sn#function", []byte("/sn/function/123/hello/latest|100|{\"name\":\"hello\",\"version\":\"latest\"}\n"), 0600) + err := ew.restoreCacheFromFile() + convey.So(err, convey.ShouldNotBeNil) + convey.So(len(ew.ResultChan), convey.ShouldEqual, 0) + _, errStatMeta := os.Stat("./etcdCacheMeta_#sn#function") + _, errStatData := os.Stat("./etcdCacheData_#sn#function") + _, errStatBackup := os.Stat("./etcdCacheData_#sn#function_backup") + convey.So(os.IsNotExist(errStatMeta), convey.ShouldEqual, true) + convey.So(os.IsNotExist(errStatData), convey.ShouldEqual, true) + convey.So(os.IsNotExist(errStatBackup), convey.ShouldEqual, true) + // dataFile matches with metaFile + os.WriteFile("./etcdCacheMeta_#sn#function", []byte(`{"revision":100,"cacheMD5":"03d9ff29f229e0123e427a1c84ad5afb"}`), 0600) + os.WriteFile("./etcdCacheData_#sn#function", []byte("/sn/function/123/hello/latest|100|{\"name\":\"hello\",\"version\":\"latest\"}\nThisIsInvalidData\n"), 0600) + err = ew.restoreCacheFromFile() + convey.So(err, convey.ShouldBeNil) + convey.So(len(ew.ResultChan), convey.ShouldEqual, 1) + event := <-ew.ResultChan + convey.So(event, convey.ShouldResemble, &Event{ + Rev: 100, + Key: "/sn/function/123/hello/latest", + Value: []byte(`{"name":"hello","version":"latest"}`), + }) + }) + os.Remove("etcdCacheMeta_#sn#function") + os.Remove("etcdCacheData_#sn#function") + os.Remove("etcdCacheData_#sn#function_backup") +} + +func newEtcdWatcher() *EtcdWatcher { + return &EtcdWatcher{ + key: "/sn/function", + CacheChan: make(chan *Event, 10), + configCh: make(chan struct{}, 1), + cacheConfig: EtcdCacheConfig{ + EnableCache: true, + PersistPath: "./", + FlushInterval: 10, + FlushThreshold: 10, + }, + } +} diff --git a/frontend/pkg/common/faas_common/etcd3/client.go b/frontend/pkg/common/faas_common/etcd3/client.go new file mode 100644 index 0000000000000000000000000000000000000000..70fbabe6fef7b72aa7ccd9f89e52c6292330ffd3 --- /dev/null +++ b/frontend/pkg/common/faas_common/etcd3/client.go @@ -0,0 +1,485 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd3 client +package etcd3 + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "sync" + "time" + + "go.etcd.io/etcd/client/v3" + + "frontend/pkg/common/faas_common/alarm" + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/log" +) + +var ( + routerEtcdClient *EtcdClient + metaEtcdClient *EtcdClient + caeMetaEtcdClient *EtcdClient + dataSystemEtcdClient *EtcdClient +) + +const ( + // Router router etcd type + Router = "route" + + // Meta meta etcd type + Meta = "meta" + + // CAEMeta cae meta etcd type + CAEMeta = "CAEMeta" + + // DataSystem cae meta etcd type + DataSystem = "DataSystem" + + defaultEtcdLostContactTime = 5 * time.Minute +) + +var ( + errInitRouterEtcd = errors.New("failed to init router etcd client") + errInitMetadataEtcd = errors.New("failed to init metadata etcd client") + errInitCAEMetadataEtcd = errors.New("failed to init CAE metadata etcd client") + errInitDataSystemEtcd = errors.New("failed to init dataSystem etcd client") +) + +// GetRouterEtcdClient - +func GetRouterEtcdClient() *EtcdClient { + return routerEtcdClient +} + +// GetMetaEtcdClient - +func GetMetaEtcdClient() *EtcdClient { + return metaEtcdClient +} + +// GetCAEMetaEtcdClient - +func GetCAEMetaEtcdClient() *EtcdClient { + return caeMetaEtcdClient +} + +// GetDataSystemEtcdClient - +func GetDataSystemEtcdClient() *EtcdClient { + return dataSystemEtcdClient +} + +// GetEtcdStatusLostContact - +func (e *EtcdClient) GetEtcdStatusLostContact() bool { + return e.etcdStatusAfterLostContact +} + +// GetEtcdStatusNow - +func (e *EtcdClient) GetEtcdStatusNow() bool { + return e.etcdStatusNow +} + +// GetEtcdType - +func (e *EtcdClient) GetEtcdType() string { + return e.etcdType +} + +// InitParam - +func InitParam() *EtcdInitParam { + return new(EtcdInitParam) +} + +// WithRouteEtcdConfig - +func (e *EtcdInitParam) WithRouteEtcdConfig(config EtcdConfig) *EtcdInitParam { + e.routeEtcdConfig = &config + return e +} + +// WithMetaEtcdConfig - +func (e *EtcdInitParam) WithMetaEtcdConfig(config EtcdConfig) *EtcdInitParam { + e.metaEtcdConfig = &config + return e +} + +// WithCAEMetaEtcdConfig - +func (e *EtcdInitParam) WithCAEMetaEtcdConfig(config EtcdConfig) *EtcdInitParam { + e.CAEMetaEtcdConfig = &config + return e +} + +// WithDataSystemEtcdConfig - +func (e *EtcdInitParam) WithDataSystemEtcdConfig(config EtcdConfig) *EtcdInitParam { + e.DataSystemEtcdConfig = &config + return e +} + +// WithStopCh - +func (e *EtcdInitParam) WithStopCh(ch <-chan struct{}) *EtcdInitParam { + e.stopCh = ch + return e +} + +// WithAlarmSwitch - +func (e *EtcdInitParam) WithAlarmSwitch(enableAlarm bool) *EtcdInitParam { + e.enableAlarm = enableAlarm + return e +} + +// InitRouterEtcdClient - +func InitRouterEtcdClient(etcdConfig EtcdConfig, alarmConfig alarm.Config, stopCh <-chan struct{}) error { + if err := InitParam(). + WithRouteEtcdConfig(etcdConfig). + WithStopCh(stopCh). + WithAlarmSwitch(alarmConfig.EnableAlarm). + InitClient(); err != nil { + return err + } + if routerClient := GetRouterEtcdClient(); routerClient != nil { + if err := routerClient.EtcdHeatBeat(); err != nil { + errInfo := fmt.Sprintf("failed to check etcd client conn, err: %s", err.Error()) + log.GetLogger().Errorf(errInfo) + routerClient.reportOrClearAlarm(alarm.GenerateAlarmLog, errInfo, alarm.Level2) + time.Sleep(DurationContextTimeout) + return err + } + } + return nil +} + +// InitMetaEtcdClient - +func InitMetaEtcdClient(etcdConfig EtcdConfig, alarmConfig alarm.Config, stopCh <-chan struct{}) error { + if err := InitParam(). + WithMetaEtcdConfig(etcdConfig). + WithAlarmSwitch(alarmConfig.EnableAlarm). + WithStopCh(stopCh). + InitClient(); err != nil { + return err + } + if metaClient := GetMetaEtcdClient(); metaClient != nil { + if err := metaClient.EtcdHeatBeat(); err != nil { + errInfo := fmt.Sprintf("failed to check etcd client conn, err: %s", err.Error()) + log.GetLogger().Errorf(errInfo) + metaClient.reportOrClearAlarm(alarm.GenerateAlarmLog, errInfo, alarm.Level2) + time.Sleep(DurationContextTimeout) + return err + } + } + return nil +} + +// InitCAEMetaEtcdClient - +func InitCAEMetaEtcdClient(etcdConfig EtcdConfig, alarmConfig alarm.Config, stopCh <-chan struct{}) error { + if err := InitParam(). + WithCAEMetaEtcdConfig(etcdConfig). + WithAlarmSwitch(alarmConfig.EnableAlarm). + WithStopCh(stopCh). + InitClient(); err != nil { + return err + } + if metaClient := GetCAEMetaEtcdClient(); metaClient != nil { + if err := metaClient.EtcdHeatBeat(); err != nil { + errInfo := fmt.Sprintf("failed to check etcd client conn, err: %s", err.Error()) + log.GetLogger().Errorf(errInfo) + metaClient.reportOrClearAlarm(alarm.GenerateAlarmLog, errInfo, alarm.Level2) + time.Sleep(DurationContextTimeout) + return err + } + } + return nil +} + +// InitDataSystemEtcdClient - +func InitDataSystemEtcdClient(etcdConfig EtcdConfig, alarmConfig alarm.Config, stopCh <-chan struct{}) error { + if err := InitParam(). + WithDataSystemEtcdConfig(etcdConfig). + WithAlarmSwitch(alarmConfig.EnableAlarm). + WithStopCh(stopCh). + InitClient(); err != nil { + return err + } + if etcdClient := GetDataSystemEtcdClient(); etcdClient != nil { + if err := etcdClient.EtcdHeatBeat(); err != nil { + errInfo := fmt.Sprintf("failed to check etcd client conn, err: %s", err.Error()) + log.GetLogger().Errorf(errInfo) + etcdClient.reportOrClearAlarm(alarm.GenerateAlarmLog, errInfo, alarm.Level2) + time.Sleep(DurationContextTimeout) + return err + } + } + return nil +} + +// InitClient initialize etcdClient based on initialization parameters. +func (e *EtcdInitParam) InitClient() error { + if e.routeEtcdConfig != nil && e.initRouteEtcdClient() != nil { + return errInitRouterEtcd + } + if e.metaEtcdConfig != nil && e.initMetadataEtcdClient() != nil { + return errInitMetadataEtcd + } + if e.CAEMetaEtcdConfig != nil && e.initCAEMetadataEtcdClient() != nil { + return errInitCAEMetadataEtcd + } + if e.DataSystemEtcdConfig != nil && e.initDataSystemEtcdClient() != nil { + return errInitDataSystemEtcd + } + return nil +} + +func (e *EtcdInitParam) initRouteEtcdClient() error { + if routerEtcdClient != nil { + return nil + } + var err error + if routerEtcdClient, err = newClient(e.routeEtcdConfig, e.stopCh, e.enableAlarm, Router); err != nil { + log.GetLogger().Errorf("failed to new router etcd client with error: %s", err.Error()) + return err + } + return nil +} + +func (e *EtcdInitParam) initMetadataEtcdClient() error { + if metaEtcdClient != nil { + return nil + } + var err error + log.GetLogger().Infof("new meta etcd client") + if metaEtcdClient, err = newClient(e.metaEtcdConfig, e.stopCh, e.enableAlarm, Meta); err != nil { + log.GetLogger().Errorf("failed to new metadata etcd client with error: %s", err.Error()) + return err + } + return nil +} + +func (e *EtcdInitParam) initCAEMetadataEtcdClient() error { + if caeMetaEtcdClient != nil { + return nil + } + var err error + log.GetLogger().Infof("new CAE meta etcd client") + if caeMetaEtcdClient, err = newClient(e.CAEMetaEtcdConfig, e.stopCh, e.enableAlarm, CAEMeta); err != nil { + log.GetLogger().Errorf("failed to new CAE metadata etcd client with error: %s", err.Error()) + return err + } + return nil +} + +func (e *EtcdInitParam) initDataSystemEtcdClient() error { + if dataSystemEtcdClient != nil { + return nil + } + var err error + log.GetLogger().Infof("new DataSystem etcd client") + if dataSystemEtcdClient, err = newClient(e.DataSystemEtcdConfig, e.stopCh, e.enableAlarm, DataSystem); err != nil { + log.GetLogger().Errorf("failed to new DataSystem etcd client with error: %s", err.Error()) + return err + } + return nil +} + +func newClient(config *EtcdConfig, stopCh <-chan struct{}, enableAlarm bool, + etcdType string) (*EtcdClient, error) { + if stopCh == nil { + return nil, errors.New("etcd stopCh should not be nil") + } + client, err := buildClient(config) + if err != nil { + log.GetLogger().Errorf("failed to new %s etcd client, %s", etcdType, err.Error()) + return nil, err + } + client.stopCh = stopCh + client.config = config + client.etcdType = etcdType + client.isAlarmEnable = enableAlarm + client.etcdStatusAfterLostContact = true + client.etcdStatusNow = true + + go client.keepConnAlive() + return client, nil +} + +func buildClient(config *EtcdConfig) (*EtcdClient, error) { + cfg, err := GetEtcdAuthType(*config).GetEtcdConfig() + if err != nil { + log.GetLogger().Errorf("failed to create shared etcd client error %s", err.Error()) + return nil, err + } + cfg.DialTimeout = etcdDialTimeout + cfg.DialKeepAliveTime = etcdKeepaliveTime + cfg.DialKeepAliveTimeout = etcdKeepaliveTimeout + cfg.Endpoints = config.Servers + etcdClient, err := clientv3.New(*cfg) + if err != nil { + log.GetLogger().Errorf("failed to create shared etcd client error %s", err.Error()) + return nil, err + } + return &EtcdClient{ + Client: etcdClient, + clientExitCh: make(chan struct{}), + cond: sync.NewCond(&sync.Mutex{}), + }, nil +} + +func (e *EtcdClient) keepConnAlive() { + timer := time.NewTimer(keepConnAliveTTL) + for { + select { + case <-timer.C: + e.checkConnState() + timer.Reset(keepConnAliveTTL) + case _, ok := <-e.stopCh: + if !ok { + log.GetLogger().Warnf("stop channel is closed and quits keep %s etcd conn alive task", e.etcdType) + } + e.cond.Broadcast() + timer.Stop() + return + } + } +} + +// EtcdHeatBeat - +func (e *EtcdClient) EtcdHeatBeat() error { + ctx, cancel := context.WithTimeout(context.Background(), keepConnAliveTTL) + defer cancel() + _, err := e.Client.Get(ctx, "alive", clientv3.WithKeysOnly()) + return err +} + +func (e *EtcdClient) checkConnState() { + e.rwMutex.RLock() + err := e.EtcdHeatBeat() + e.rwMutex.RUnlock() + + if err != nil { + if e.etcdTimer == nil { + e.abnormalContinuouslyTimes++ + e.etcdTimer = time.AfterFunc(defaultEtcdLostContactTime, func() { + e.etcdStatusAfterLostContact = false + errInfo := fmt.Sprintf("etcd %s lost contact over %v, etcdStatusAfterLostContact is %v", + e.etcdType, defaultEtcdLostContactTime, e.etcdStatusAfterLostContact) + e.reportOrClearAlarm(alarm.GenerateAlarmLog, errInfo, alarm.Level3) + log.GetLogger().Warnf(errInfo) + }) + } + e.etcdStatusNow = false + e.exitOnce.Do(func() { + close(e.clientExitCh) + }) + errInfo := fmt.Sprintf("failed to check etcd client conn, err: %s", err.Error()) + log.GetLogger().Errorf(errInfo) + e.reportOrClearAlarm(alarm.GenerateAlarmLog, errInfo, alarm.Level2) + if err = e.restart(); err != nil { + log.GetLogger().Errorf("failed to restart etcd client, %s", err.Error()) + } + return + } + if e.etcdStatusAfterLostContact == false { + e.reportOrClearAlarm(alarm.ClearAlarmLog, "Clear critical alarm, "+ + "The connection to etcd has been restored", alarm.Level3) + } + if e.abnormalContinuouslyTimes > 0 { + e.reportOrClearAlarm(alarm.ClearAlarmLog, "Clear major alarm, "+ + "The connection to etcd has been restored", alarm.Level2) + e.abnormalContinuouslyTimes = 0 + } + + if e.etcdTimer != nil { + e.etcdTimer.Stop() + e.etcdTimer = nil + e.etcdStatusAfterLostContact = true + log.GetLogger().Infof("reconnect to %s etcd", e.etcdType) + } + if !e.etcdStatusNow { + e.clientExitCh = make(chan struct{}) + e.exitOnce = sync.Once{} + e.cond.Broadcast() + } + e.etcdStatusNow = true +} + +func (e *EtcdClient) reportOrClearAlarm(opType string, detail string, alarmLevel string) { + if e.isAlarmEnable { + alarmDetail := &alarm.Detail{ + SourceTag: os.Getenv(constant.PodNameEnvKey) + "|" + os.Getenv(constant.PodIPEnvKey) + + "|" + os.Getenv(constant.ClusterName) + "|MetadataEtcdConnection", + OpType: opType, + Details: detail, + StartTimestamp: 0, + EndTimestamp: 0, + } + if alarmDetail.OpType == alarm.GenerateAlarmLog { + alarmDetail.StartTimestamp = int(time.Now().Unix()) + } else { + alarmDetail.EndTimestamp = int(time.Now().Unix()) + } + alarmInfo := &alarm.LogAlarmInfo{ + AlarmID: alarm.MetadataEtcdConnection00001, + AlarmName: "MetadataEtcdConnection", + AlarmLevel: alarmLevel, + } + if e.etcdType == Router { + alarmDetail.SourceTag = os.Getenv(constant.PodNameEnvKey) + "|" + os.Getenv(constant.PodIPEnvKey) + + "|" + os.Getenv(constant.ClusterName) + "|RouterEtcdConnection" + alarmInfo.AlarmID = alarm.RouterEtcdConnection00001 + alarmInfo.AlarmName = "RouterEtcdConnection" + } + if e.etcdType == CAEMeta { + alarmDetail.SourceTag = os.Getenv(constant.PodNameEnvKey) + "|" + os.Getenv(constant.PodIPEnvKey) + + "|" + os.Getenv(constant.ClusterName) + "|CAEMetadataEtcdConnection" + alarmInfo.AlarmID = alarm.RouterEtcdConnection00001 + alarmInfo.AlarmName = "CAEMetadataEtcdConnection" + } + alarm.ReportOrClearAlarm(alarmInfo, alarmDetail) + } +} + +func (e *EtcdClient) restart() error { + log.GetLogger().Infof("start to rebuild %s etcd client", e.etcdType) + recreatedClient, err := buildClient(e.config) + if err != nil { + log.GetLogger().Errorf("failed to recreate %s etcd client, %s", e.etcdType, err.Error()) + return err + } + e.rwMutex.Lock() + e.stop() + e.Client = recreatedClient.Client + e.rwMutex.Unlock() + return nil +} + +func (e *EtcdClient) stop() { + if err := e.Client.Close(); err != nil { + log.GetLogger().Errorf("failed to close %s etcd client, %s", e.etcdType, err.Error()) + } +} + +// AttachAZPrefix - +func (e *EtcdClient) AttachAZPrefix(key string) string { + if e.config != nil && len(e.config.AZPrefix) != 0 { + return fmt.Sprintf("/%s%s", e.config.AZPrefix, key) + } + return key +} + +// DetachAZPrefix - +func (e *EtcdClient) DetachAZPrefix(key string) string { + if e.config != nil && len(e.config.AZPrefix) != 0 { + return strings.TrimPrefix(key, fmt.Sprintf("/%s", e.config.AZPrefix)) + } + return key +} diff --git a/frontend/pkg/common/faas_common/etcd3/client_test.go b/frontend/pkg/common/faas_common/etcd3/client_test.go new file mode 100644 index 0000000000000000000000000000000000000000..dacf3ad2ed010cf8b6195eda92ee449027eda88c --- /dev/null +++ b/frontend/pkg/common/faas_common/etcd3/client_test.go @@ -0,0 +1,363 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd3 client +package etcd3 + +import ( + "context" + "errors" + "reflect" + "sync" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + "go.etcd.io/etcd/client/v3" + + "frontend/pkg/common/faas_common/alarm" +) + +type KvMock struct { +} + +func (k *KvMock) Put(ctx context.Context, key, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error) { + //TODO implement me + panic("implement me") +} + +func (k *KvMock) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + return nil, nil +} + +func (k *KvMock) Delete(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.DeleteResponse, error) { + //TODO implement me + panic("implement me") +} + +func (k *KvMock) Compact(ctx context.Context, rev int64, opts ...clientv3.CompactOption) (*clientv3.CompactResponse, error) { + //TODO implement me + panic("implement me") +} + +func (k *KvMock) Do(ctx context.Context, op clientv3.Op) (clientv3.OpResponse, error) { + //TODO implement me + panic("implement me") +} + +func (k *KvMock) Txn(ctx context.Context) clientv3.Txn { + //TODO implement me + panic("implement me") +} + +func TestInitEtcdClientOK(t *testing.T) { + stopCh := make(chan struct{}) + patches := []*gomonkey.Patches{ + gomonkey.ApplyFunc(buildClient, func(config *EtcdConfig) (*EtcdClient, error) { + return &EtcdClient{clientExitCh: make(chan struct{}), cond: sync.NewCond(&sync.Mutex{})}, nil + }), + } + defer func() { + close(stopCh) + for _, patch := range patches { + patch.Reset() + } + }() + + convey.Convey("new RouteClient", t, func() { + err := InitParam(). + WithRouteEtcdConfig(EtcdConfig{}). + WithStopCh(stopCh).InitClient() + convey.So(err, convey.ShouldBeNil) + convey.So(GetRouterEtcdClient(), convey.ShouldNotBeNil) + }) + + convey.Convey("new MetadataClient", t, func() { + err := InitParam(). + WithMetaEtcdConfig(EtcdConfig{}). + WithStopCh(stopCh).InitClient() + convey.So(err, convey.ShouldBeNil) + convey.So(GetMetaEtcdClient(), convey.ShouldNotBeNil) + }) + + convey.Convey("new CAEMetadataClient", t, func() { + err := InitParam(). + WithCAEMetaEtcdConfig(EtcdConfig{}). + WithStopCh(stopCh).InitClient() + convey.So(err, convey.ShouldBeNil) + convey.So(GetCAEMetaEtcdClient(), convey.ShouldNotBeNil) + }) + + convey.Convey("new DataSystemEtcdClient", t, func() { + err := InitParam(). + WithDataSystemEtcdConfig(EtcdConfig{}). + WithStopCh(stopCh).InitClient() + convey.So(err, convey.ShouldBeNil) + convey.So(GetDataSystemEtcdClient(), convey.ShouldNotBeNil) + }) +} + +func TestInitEtcdClientFail(t *testing.T) { + var stopCh chan struct{} + routerEtcdClient = nil + metaEtcdClient = nil + caeMetaEtcdClient = nil + convey.Convey("new RouteClient", t, func() { + err := InitParam(). + WithRouteEtcdConfig(EtcdConfig{}). + WithStopCh(stopCh).InitClient() + convey.So(err, convey.ShouldNotBeNil) + }) + + convey.Convey("new MetadataClient", t, func() { + err := InitParam(). + WithMetaEtcdConfig(EtcdConfig{}). + WithStopCh(stopCh).InitClient() + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("new RouteClient", t, func() { + stopCh = make(chan struct{}) + err := InitParam(). + WithRouteEtcdConfig(EtcdConfig{}). + WithStopCh(stopCh).InitClient() + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("new CAE MetadataClient", t, func() { + err := InitParam(). + WithCAEMetaEtcdConfig(EtcdConfig{}). + WithStopCh(stopCh).InitClient() + convey.So(err, convey.ShouldNotBeNil) + }) + close(stopCh) +} + +func TestInitEtcdClientKeepAliveOK(t *testing.T) { + stopCh := make(chan struct{}) + kv := &KvMock{} + client := &clientv3.Client{KV: kv} + patches := []*gomonkey.Patches{ + gomonkey.ApplyFunc(clientv3.New, func(cfg clientv3.Config) (*clientv3.Client, error) { + return client, nil + }), + gomonkey.ApplyMethod(reflect.TypeOf(kv), "Get", func(_ *KvMock, ctx context.Context, key string, + opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + return nil, nil + }), + gomonkey.ApplyGlobalVar(&keepConnAliveTTL, time.Duration(100)*time.Millisecond), + } + defer func() { + close(stopCh) + for _, patch := range patches { + patch.Reset() + } + }() + + convey.Convey("etcd client alive", t, func() { + err := InitParam(). + WithRouteEtcdConfig(EtcdConfig{}). + WithStopCh(stopCh).InitClient() + time.Sleep(200 * time.Millisecond) + convey.So(err, convey.ShouldBeNil) + convey.So(GetRouterEtcdClient().GetEtcdStatusNow(), convey.ShouldEqual, true) + }) +} + +func TestInitEtcdClientKeepAliveReconnect(t *testing.T) { + routerEtcdClient = nil + metaEtcdClient = nil + stopCh := make(chan struct{}) + kv := &KvMock{} + client := &clientv3.Client{KV: kv} + patches := []*gomonkey.Patches{ + gomonkey.ApplyFunc(clientv3.New, func(cfg clientv3.Config) (*clientv3.Client, error) { + return client, nil + }), + gomonkey.ApplyMethod(reflect.TypeOf(kv), "Get", + func(_ *KvMock, ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + return nil, errors.New("lost connection") + }), + gomonkey.ApplyMethod(reflect.TypeOf(client), "Close", func(_ *clientv3.Client) error { + return nil + }), + gomonkey.ApplyGlobalVar(&keepConnAliveTTL, time.Duration(100)*time.Millisecond), + } + defer func() { + close(stopCh) + for _, patch := range patches { + patch.Reset() + } + }() + + convey.Convey("lost etcd client and reconnect", t, func() { + err := InitParam(). + WithRouteEtcdConfig(EtcdConfig{}). + WithStopCh(stopCh).InitClient() + time.Sleep(200 * time.Millisecond) + convey.So(err, convey.ShouldBeNil) + convey.So(GetRouterEtcdClient().GetEtcdStatusNow(), convey.ShouldEqual, false) + + patches = append(patches, gomonkey.ApplyMethod(reflect.TypeOf(kv), "Get", + func(_ *KvMock, ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + return nil, nil + })) + time.Sleep(200 * time.Millisecond) + convey.So(GetRouterEtcdClient().GetEtcdStatusNow(), convey.ShouldEqual, true) + convey.So(GetRouterEtcdClient().GetEtcdStatusLostContact(), convey.ShouldEqual, true) + }) +} + +func TestInitMetaEtcdClient(t *testing.T) { + convey.Convey("InitMetaEtcdClient", t, func() { + convey.Convey("failed to init", func() { + defer gomonkey.ApplyMethod(reflect.TypeOf(&EtcdInitParam{}), "InitClient", + func(e *EtcdInitParam) error { + return errors.New("failed to init") + }).Reset() + stop := make(chan struct{}) + err := InitMetaEtcdClient(EtcdConfig{}, alarm.Config{}, stop) + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("failed to heat beat", func() { + defer gomonkey.ApplyMethod(reflect.TypeOf(&EtcdInitParam{}), "InitClient", + func(e *EtcdInitParam) error { + metaEtcdClient = &EtcdClient{} + return nil + }).Reset() + defer gomonkey.ApplyMethod(reflect.TypeOf(metaEtcdClient), "EtcdHeatBeat", func(e *EtcdClient) error { + return errors.New("failed to heart beat") + }).Reset() + stop := make(chan struct{}) + err := InitMetaEtcdClient(EtcdConfig{}, alarm.Config{}, stop) + convey.So(err, convey.ShouldNotBeNil) + }) + }) +} + +func TestInitCAEMetaEtcdClient(t *testing.T) { + convey.Convey("InitCAEMetaEtcdClient", t, func() { + convey.Convey("failed to init", func() { + defer gomonkey.ApplyMethod(reflect.TypeOf(&EtcdInitParam{}), "InitClient", + func(e *EtcdInitParam) error { + return errors.New("failed to init") + }).Reset() + stop := make(chan struct{}) + err := InitMetaEtcdClient(EtcdConfig{}, alarm.Config{}, stop) + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("failed to heat beat", func() { + defer gomonkey.ApplyMethod(reflect.TypeOf(&EtcdInitParam{}), "InitClient", + func(e *EtcdInitParam) error { + caeMetaEtcdClient = &EtcdClient{} + return nil + }).Reset() + defer gomonkey.ApplyMethod(reflect.TypeOf(caeMetaEtcdClient), "EtcdHeatBeat", func(e *EtcdClient) error { + return errors.New("failed to heart beat") + }).Reset() + stop := make(chan struct{}) + err := InitCAEMetaEtcdClient(EtcdConfig{}, alarm.Config{}, stop) + convey.So(err, convey.ShouldNotBeNil) + }) + }) +} + +func TestInitDataSystemEtcdClient(t *testing.T) { + convey.Convey("InitDataSystemEtcdClient", t, func() { + convey.Convey("failed to init", func() { + defer gomonkey.ApplyMethod(reflect.TypeOf(&EtcdInitParam{}), "InitClient", + func(e *EtcdInitParam) error { + return errors.New("failed to init") + }).Reset() + stop := make(chan struct{}) + err := InitDataSystemEtcdClient(EtcdConfig{}, alarm.Config{}, stop) + convey.So(err.Error(), convey.ShouldContainSubstring, "failed to init") + }) + convey.Convey("failed to heat beat", func() { + defer gomonkey.ApplyMethod(reflect.TypeOf(&EtcdInitParam{}), "InitClient", + func(e *EtcdInitParam) error { + dataSystemEtcdClient = &EtcdClient{} + return nil + }).Reset() + defer gomonkey.ApplyMethod(reflect.TypeOf(dataSystemEtcdClient), "EtcdHeatBeat", func(e *EtcdClient) error { + return errors.New("failed to heart beat") + }).Reset() + stop := make(chan struct{}) + err := InitDataSystemEtcdClient(EtcdConfig{}, alarm.Config{}, stop) + convey.So(err.Error(), convey.ShouldContainSubstring, "failed to heart beat") + }) + }) +} + +func TestInitRouterEtcdClient(t *testing.T) { + convey.Convey("InitMetaEtcdClient", t, func() { + convey.Convey("failed to init", func() { + defer gomonkey.ApplyMethod(reflect.TypeOf(&EtcdInitParam{}), "InitClient", + func(e *EtcdInitParam) error { + return errors.New("failed to init") + }).Reset() + stop := make(chan struct{}) + err := InitRouterEtcdClient(EtcdConfig{}, alarm.Config{}, stop) + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("failed to heat beat", func() { + defer gomonkey.ApplyMethod(reflect.TypeOf(&EtcdInitParam{}), "InitClient", + func(e *EtcdInitParam) error { + routerEtcdClient = &EtcdClient{} + return nil + }).Reset() + defer gomonkey.ApplyMethod(reflect.TypeOf(routerEtcdClient), "EtcdHeatBeat", func(e *EtcdClient) error { + return errors.New("failed to heart beat") + }).Reset() + stop := make(chan struct{}) + err := InitRouterEtcdClient(EtcdConfig{}, alarm.Config{}, stop) + convey.So(err, convey.ShouldNotBeNil) + }) + }) +} + +func Test_reportOrClearAlarm(t *testing.T) { + convey.Convey("reportOrClearAlarm", t, func() { + convey.Convey("no test assertion", func() { + e := EtcdClient{isAlarmEnable: true, etcdType: Router} + e.reportOrClearAlarm(alarm.GenerateAlarmLog, "告警", "INFO") + }) + }) +} + +func Test_AZPrefixProcess(t *testing.T) { + convey.Convey("test AZPrefix", t, func() { + convey.Convey("AttachAZPrefix", func() { + e := EtcdClient{config: &EtcdConfig{ + AZPrefix: "az1", + }} + key := e.AttachAZPrefix("/sn/instance/xxx") + convey.So(key, convey.ShouldEqual, "/az1/sn/instance/xxx") + e.config.AZPrefix = "" + key = e.AttachAZPrefix("/sn/instance/xxx") + convey.So(key, convey.ShouldEqual, "/sn/instance/xxx") + }) + convey.Convey("DetachAZPrefix", func() { + e := EtcdClient{config: &EtcdConfig{ + AZPrefix: "az1", + }} + key := e.DetachAZPrefix("/az1/sn/instance/xxx") + convey.So(key, convey.ShouldEqual, "/sn/instance/xxx") + e.config.AZPrefix = "" + key = e.DetachAZPrefix("/sn/instance/xxx") + convey.So(key, convey.ShouldEqual, "/sn/instance/xxx") + }) + }) +} diff --git a/frontend/pkg/common/faas_common/etcd3/config.go b/frontend/pkg/common/faas_common/etcd3/config.go new file mode 100644 index 0000000000000000000000000000000000000000..52371d29c50c631c91deb903ce636fa5f9dcdee4 --- /dev/null +++ b/frontend/pkg/common/faas_common/etcd3/config.go @@ -0,0 +1,278 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd3 implements crud and watch operations based etcd clientv3 +package etcd3 + +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "io/ioutil" + "os" + + "go.etcd.io/etcd/client/v3" + + commonCrypto "frontend/pkg/common/crypto" + "frontend/pkg/common/faas_common/crypto" + "frontend/pkg/common/faas_common/localauth" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/sts" + "frontend/pkg/common/faas_common/sts/cert" + commontls "frontend/pkg/common/faas_common/tls" + "frontend/pkg/common/faas_common/utils" +) + +// EtcdAuth etcd authentication interface +type EtcdAuth interface { + GetEtcdConfig() (*clientv3.Config, error) +} + +type noAuth struct { +} + +type tlsAuth struct { + caFile string + certFile string + keyFile string + user string + password string +} + +type pwdAuth struct { + user string + password string +} + +type clientTLSAuth struct { + cerfile []byte + keyfile []byte + cafile []byte + passphrasefile []byte +} + +// GetEtcdAuthType etcd authentication type +func GetEtcdAuthType(etcdConfig EtcdConfig) EtcdAuth { + if etcdConfig.AuthType == "TLS" { + return &clientTLSAuth{ + cerfile: []byte(etcdConfig.CertFile), + keyfile: []byte(etcdConfig.KeyFile), + cafile: []byte(etcdConfig.CaFile), + passphrasefile: []byte(etcdConfig.PassphraseFile), + } + } + if etcdConfig.SslEnable { + if os.Getenv(sts.EnvSTSEnable) == "true" { + return &tlsAuth{} + } + return &tlsAuth{ + certFile: etcdConfig.CertFile, + keyFile: etcdConfig.KeyFile, + caFile: etcdConfig.CaFile, + user: etcdConfig.User, + password: etcdConfig.Password, + } + } + if etcdConfig.Password == "" { + return &noAuth{} + } + if len(etcdConfig.User) != 0 || len(etcdConfig.Password) != 0 { + return &pwdAuth{ + user: etcdConfig.User, + password: etcdConfig.Password, + } + } + return &noAuth{} +} + +func (n *noAuth) GetEtcdConfig() (*clientv3.Config, error) { + return &clientv3.Config{}, nil +} + +func (t *tlsAuth) GetEtcdConfig() (*clientv3.Config, error) { + if os.Getenv(sts.EnvSTSEnable) == "true" { + return BuildStsCfg() + } + pool, err := commontls.GetX509CACertPool(t.caFile) + if err != nil { + log.GetLogger().Errorf("failed to getX509CACertPool: %s", err.Error()) + return nil, err + } + + var certs []tls.Certificate + if certs, err = commontls.LoadServerTLSCertificate(t.certFile, t.keyFile, "", "LOCAL", false); err != nil { + log.GetLogger().Errorf("failed to loadServerTLSCertificate: %s", err.Error()) + return nil, err + } + + clientAuthMode := tls.NoClientCert + cfg := &clientv3.Config{ + TLS: &tls.Config{ + RootCAs: pool, + Certificates: certs, + ClientAuth: clientAuthMode, + }, + } + if len(t.user) != 0 && len(t.password) != 0 { + pwd, err := localauth.Decrypt(t.password) + if err != nil { + log.GetLogger().Errorf("failed to decrypt etcd config with error %s", err) + return nil, err + } + cfg.Username = t.user + cfg.Password = string(pwd) + utils.ClearStringMemory(t.password) + } + return cfg, nil +} + +func (p *pwdAuth) GetEtcdConfig() (*clientv3.Config, error) { + if len(p.user) == 0 || len(p.password) == 0 { + return nil, errors.New("etcd user or password is empty") + } + pwd, err := localauth.Decrypt(p.password) + if err != nil { + log.GetLogger().Errorf("failed to decrypt etcd config with error %s", err) + return nil, err + } + cfg := &clientv3.Config{ + Username: p.user, + Password: string(pwd), + } + utils.ClearStringMemory(p.password) + return cfg, nil +} + +func (c *clientTLSAuth) getPassphrase() ([]byte, error) { + // check whether the passphrasefile file exists. If the file exists, the client key is encrypted using a password. + // If the file does not exist, the client key is not encrypted and can be directly read. + var keyPwd []byte + var err error + if _, err = os.Stat(string(c.passphrasefile)); err == nil { + keyPwd, err = ioutil.ReadFile(string(c.passphrasefile)) + if err != nil { + log.GetLogger().Errorf("failed to read passphrasefile, err: %s", err.Error()) + return nil, err + } + if crypto.SCCInitialized() { + pwd, err := crypto.SCCDecrypt(keyPwd) + if err != nil { + log.GetLogger().Errorf("failed to decrypt passphrasefile, err: %s", err.Error()) + return nil, err + } + keyPwd = []byte(pwd) + } + } + + return keyPwd, nil +} + +func (c *clientTLSAuth) getTLSConfig(encryptedKeyPEM []byte, keyPwd []byte, + certPEM []byte, caCertPEM []byte) (*tls.Config, error) { + // Decode will find the next PEM formatted block (certificate, private key etc) in the input. + // It returns that block and the remainder of the input. + // If no PEM data is found, keyBlock is nil and the whole of the input is returned in rest. + // When keyBlock is nil, an error is reported. + // You do not need to pay attention to the content of the second return value. + keyBlock, _ := pem.Decode(encryptedKeyPEM) + if keyBlock == nil { + log.GetLogger().Errorf("failed to decode key PEM block") + return nil, fmt.Errorf("failed to decode key PEM block") + } + keyDER, err := commonCrypto.DecryptPEMBlock(keyBlock, keyPwd) + if err != nil { + log.GetLogger().Errorf("failed to decrypt key: err: %s", err.Error()) + return nil, err + } + + key, err := x509.ParsePKCS1PrivateKey(keyDER) + if err != nil { + log.GetLogger().Errorf("failed to parse private key: err: %s", err.Error()) + return nil, err + } + + certPool := x509.NewCertPool() + if !certPool.AppendCertsFromPEM(caCertPEM) { + log.GetLogger().Errorf("failed to append CA certificate") + return nil, fmt.Errorf("failed to append CA certificate") + } + + clientCert, err := tls.X509KeyPair(certPEM, pem.EncodeToMemory( + &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})) + if err != nil { + log.GetLogger().Errorf("failed to create client certificate: %s", err.Error()) + return nil, err + } + + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{clientCert}, + RootCAs: certPool, + } + return tlsConfig, nil +} + +func (c *clientTLSAuth) GetEtcdConfig() (*clientv3.Config, error) { + keyPwd, err := c.getPassphrase() + if err != nil { + return nil, err + } + + certPEM, err := ioutil.ReadFile(string(c.cerfile)) + if err != nil { + log.GetLogger().Errorf("failed to read cert file: %s", err.Error()) + return nil, err + } + + caCertPEM, err := ioutil.ReadFile(string(c.cafile)) + if err != nil { + log.GetLogger().Errorf("failed to read ca file: %s", err.Error()) + return nil, err + } + + encryptedKeyPEM, err := ioutil.ReadFile(string(c.keyfile)) + if err != nil { + log.GetLogger().Errorf("failed to read key file: %s", err.Error()) + return nil, err + } + tlsConfig, err := c.getTLSConfig(encryptedKeyPEM, keyPwd, certPEM, caCertPEM) + if err != nil { + return nil, err + } + return &clientv3.Config{ + TLS: tlsConfig, + }, nil +} + +// BuildStsCfg - Construct tlsConfig from sts p12 +func BuildStsCfg() (*clientv3.Config, error) { + caCertsPool, tlsCert, err := cert.LoadCerts() + if err != nil { + log.GetLogger().Errorf("failed to get X509CACertPool and TLSCertificate: %s", err.Error()) + return nil, err + } + + clientAuthMode := tls.NoClientCert + tlsConfig := &clientv3.Config{ + TLS: &tls.Config{ + RootCAs: caCertsPool, + Certificates: []tls.Certificate{*tlsCert}, + ClientAuth: clientAuthMode, + }, + } + return tlsConfig, nil +} diff --git a/frontend/pkg/common/faas_common/etcd3/config_test.go b/frontend/pkg/common/faas_common/etcd3/config_test.go new file mode 100644 index 0000000000000000000000000000000000000000..98574a79267fa1835d31598c878dff63c158df95 --- /dev/null +++ b/frontend/pkg/common/faas_common/etcd3/config_test.go @@ -0,0 +1,469 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd3 implements crud and watch operations based etcd clientv3 +package etcd3 + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "io/ioutil" + "os" + "strings" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + . "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + clientv3 "go.etcd.io/etcd/client/v3" + + commonCrypto "frontend/pkg/common/crypto" + "frontend/pkg/common/faas_common/crypto" + "frontend/pkg/common/faas_common/localauth" + "frontend/pkg/common/faas_common/sts/cert" + commontls "frontend/pkg/common/faas_common/tls" + mockUtils "frontend/pkg/common/faas_common/utils" +) + +func TestGetEtcdAuthType(t *testing.T) { + convey.Convey("tlsAuth", t, func() { + etcdConfig := EtcdConfig{ + SslEnable: true, + } + etcdAuth := GetEtcdAuthType(etcdConfig) + convey.So(etcdAuth, convey.ShouldResemble, &tlsAuth{ + certFile: etcdConfig.CertFile, + keyFile: etcdConfig.KeyFile, + caFile: etcdConfig.CaFile, + }) + }) + convey.Convey("tlsAuth", t, func() { + etcdConfig := EtcdConfig{ + SslEnable: false, + Password: "", + } + etcdAuth := GetEtcdAuthType(etcdConfig) + convey.So(etcdAuth, convey.ShouldResemble, &noAuth{}) + }) + convey.Convey("tlsAuth", t, func() { + etcdConfig := EtcdConfig{ + SslEnable: false, + Password: "p123", + } + etcdAuth := GetEtcdAuthType(etcdConfig) + convey.So(etcdAuth, convey.ShouldResemble, &pwdAuth{ + user: etcdConfig.User, + password: etcdConfig.Password, + }) + }) + convey.Convey("clientTLSAuth", t, func() { + etcdConfig := EtcdConfig{ + AuthType: "TLS", + CaFile: "CaFile", + CertFile: "CertFile", + KeyFile: "KeyFile", + PassphraseFile: "PassphraseFile", + } + etcdAuth := GetEtcdAuthType(etcdConfig) + convey.So(etcdAuth, convey.ShouldResemble, &clientTLSAuth{ + cerfile: []byte("CertFile"), + keyfile: []byte("KeyFile"), + cafile: []byte("CaFile"), + passphrasefile: []byte("PassphraseFile"), + }) + }) +} + +func TestGetEtcdConfig(t *testing.T) { + defer gomonkey.ApplyFunc(localauth.Decrypt, func(src string) ([]byte, error) { + return []byte(strings.Clone(src)), nil + }).Reset() + convey.Convey("noAuth", t, func() { + noAuth := &noAuth{} + cfg, err := noAuth.GetEtcdConfig() + convey.So(cfg, convey.ShouldResemble, &clientv3.Config{}) + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("tlsAuth", t, func() { + defer gomonkey.ApplyFunc(tls.X509KeyPair, func(certPEMBlock []byte, keyPEMBlock []byte) (tls.Certificate, error) { + return tls.Certificate{}, nil + }).Reset() + defer gomonkey.ApplyFunc(ioutil.ReadFile, func(filename string) ([]byte, error) { + return []byte{}, nil + }).Reset() + defer gomonkey.ApplyFunc(commontls.LoadServerTLSCertificate, func(certFile, keyFile, passPhase, decryptTool string, + isHTTPS bool) ([]tls.Certificate, error) { + return nil, nil + }).Reset() + tlsAuth := &tlsAuth{ + user: "root", + password: string([]byte("123")), + } + cfg, err := tlsAuth.GetEtcdConfig() + convey.So(err, convey.ShouldBeNil) + convey.So(cfg, convey.ShouldNotBeNil) + }) + convey.Convey("tlsAuth error", t, func() { + defer gomonkey.ApplyFunc(localauth.Decrypt, func(src string) ([]byte, error) { + return nil, errors.New("some error") + }).Reset() + tlsAuth := &tlsAuth{} + _, err := tlsAuth.GetEtcdConfig() + convey.So(err, convey.ShouldNotBeNil) + tlsAuth.user, tlsAuth.password = "root", "123" + _, err = tlsAuth.GetEtcdConfig() + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("pwdAuth", t, func() { + pwdAuth := &pwdAuth{ + user: "root", + password: string([]byte("123")), + } + cfg, err := pwdAuth.GetEtcdConfig() + convey.So(cfg.Password, convey.ShouldEqual, "123") + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("pwdAuth error", t, func() { + defer gomonkey.ApplyFunc(localauth.Decrypt, func(src string) ([]byte, error) { + return nil, errors.New("some error") + }).Reset() + pwdAuth := &pwdAuth{ + password: string([]byte("123")), + } + _, err := pwdAuth.GetEtcdConfig() + convey.So(err, convey.ShouldNotBeNil) + pwdAuth.user = "root" + convey.So(err, convey.ShouldNotBeNil) + }) +} + +func TestGetPassphrase(t *testing.T) { + patches := []*Patches{ + ApplyFunc(os.Stat, func(name string) (os.FileInfo, error) { + return nil, nil + }), + ApplyFunc(os.ReadFile, func(string) ([]byte, error) { + return []byte("dummyPassphrase"), nil + }), + ApplyFunc(crypto.SCCInitialized, func() bool { + return false + }), + } + defer func() { + for _, patch := range patches { + time.Sleep(100 * time.Millisecond) + patch.Reset() + } + }() + + c := &clientTLSAuth{ + passphrasefile: []byte("path/to/passphrasefile"), + } + + passphrase, err := c.getPassphrase() + assert.Nil(t, err) + assert.Equal(t, []byte("dummyPassphrase"), passphrase) +} + +func TestBuildStsCfg(t *testing.T) { + tests := []struct { + name string + wantErr bool + patchesFunc mockUtils.PatchesFunc + }{ + {"case1", false, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(cert.LoadCerts, func() (*x509.CertPool, *tls.Certificate, + error) { + return &x509.CertPool{}, &tls.Certificate{}, nil + })}) + return patches + }}, + {"case2 LoadCerts error", true, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(cert.LoadCerts, func() (*x509.CertPool, *tls.Certificate, + error) { + return &x509.CertPool{}, &tls.Certificate{}, errors.New("error") + })}) + return patches + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + patches := tt.patchesFunc() + _, err := BuildStsCfg() + if (err != nil) != tt.wantErr { + t.Errorf("BuildStsCfg() error = %v, wantErr %v", err, tt.wantErr) + return + } + patches.ResetAll() + }) + } +} + +func TestGetTLSConfig(t *testing.T) { + // Create test data + testKey, _ := rsa.GenerateKey(rand.Reader, 2048) + keyDER := x509.MarshalPKCS1PrivateKey(testKey) + encryptedPEM := pem.EncodeToMemory(&pem.Block{ + Type: "ENCRYPTED PRIVATE KEY", + Bytes: keyDER, + }) + certPEM := []byte("test cert") + caCertPEM := []byte("test CA cert") + keyPwd := []byte("test password") + + tests := []struct { + name string + mockSetups func() []*gomonkey.Patches + expectedError bool + errorContains string + }{ + { + name: "decrypt PEM block failure", + mockSetups: func() []*gomonkey.Patches { + var patches []*gomonkey.Patches + + p1 := gomonkey.ApplyFunc(pem.Decode, func(data []byte) (*pem.Block, []byte) { + return &pem.Block{}, nil + }) + patches = append(patches, p1) + + p2 := gomonkey.ApplyFunc(commonCrypto.DecryptPEMBlock, func(block *pem.Block, password []byte) ([]byte, error) { + return nil, fmt.Errorf("decrypt error") + }) + patches = append(patches, p2) + + return patches + }, + expectedError: true, + errorContains: "decrypt error", + }, + { + name: "parse private key failure", + mockSetups: func() []*gomonkey.Patches { + var patches []*gomonkey.Patches + + p1 := gomonkey.ApplyFunc(pem.Decode, func(data []byte) (*pem.Block, []byte) { + return &pem.Block{}, nil + }) + patches = append(patches, p1) + + p2 := gomonkey.ApplyFunc(commonCrypto.DecryptPEMBlock, func(block *pem.Block, password []byte) ([]byte, error) { + return []byte("invalid key"), nil + }) + patches = append(patches, p2) + + p3 := gomonkey.ApplyFunc(x509.ParsePKCS1PrivateKey, func(der []byte) (*rsa.PrivateKey, error) { + return nil, fmt.Errorf("parse error") + }) + patches = append(patches, p3) + + return patches + }, + expectedError: true, + errorContains: "parse error", + }, + { + name: "append CA cert failure", + mockSetups: func() []*gomonkey.Patches { + var patches []*gomonkey.Patches + + p1 := gomonkey.ApplyFunc(pem.Decode, func(data []byte) (*pem.Block, []byte) { + return &pem.Block{}, nil + }) + patches = append(patches, p1) + + p2 := gomonkey.ApplyFunc(commonCrypto.DecryptPEMBlock, func(block *pem.Block, password []byte) ([]byte, error) { + return keyDER, nil + }) + patches = append(patches, p2) + + p3 := gomonkey.ApplyFunc(x509.ParsePKCS1PrivateKey, func(der []byte) (*rsa.PrivateKey, error) { + return testKey, nil + }) + patches = append(patches, p3) + + p4 := gomonkey.ApplyMethod((*x509.CertPool)(nil), "AppendCertsFromPEM", + func(_ *x509.CertPool, _ []byte) bool { + return false + }) + patches = append(patches, p4) + + return patches + }, + expectedError: true, + errorContains: "failed to append CA certificate", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + patches := tt.mockSetups() + defer func() { + for _, p := range patches { + p.Reset() + } + }() + + c := &clientTLSAuth{} + config, err := c.getTLSConfig(encryptedPEM, keyPwd, certPEM, caCertPEM) + + if tt.expectedError { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.errorContains) + assert.Nil(t, config) + } else { + assert.NoError(t, err) + assert.NotNil(t, config) + } + }) + } +} + +func mockTls(keyDER []byte, testKey *rsa.PrivateKey) []*Patches { + var patches []*gomonkey.Patches + + // Mock pem.Decode + p1 := gomonkey.ApplyFunc(pem.Decode, func(data []byte) (*pem.Block, []byte) { + return &pem.Block{ + Type: "ENCRYPTED PRIVATE KEY", + Bytes: keyDER, + }, nil + }) + patches = append(patches, p1) + + // Mock commonCrypto.DecryptPEMBlock + p2 := gomonkey.ApplyFunc(commonCrypto.DecryptPEMBlock, func(block *pem.Block, password []byte) ([]byte, error) { + return keyDER, nil + }) + patches = append(patches, p2) + + // Mock x509.ParsePKCS1PrivateKey + p3 := gomonkey.ApplyFunc(x509.ParsePKCS1PrivateKey, func(der []byte) (*rsa.PrivateKey, error) { + return testKey, nil + }) + patches = append(patches, p3) + + // Mock certPool.AppendCertsFromPEM + p4 := gomonkey.ApplyMethod((*x509.CertPool)(nil), "AppendCertsFromPEM", + func(_ *x509.CertPool, _ []byte) bool { + return true + }) + patches = append(patches, p4) + + // Mock tls.X509KeyPair + p5 := gomonkey.ApplyFunc(tls.X509KeyPair, func(certPEM, keyPEM []byte) (tls.Certificate, error) { + return tls.Certificate{}, nil + }) + patches = append(patches, p5) + + return patches +} + +func TestTlsAuthGetEtcdConfig(t *testing.T) { + patches := []*Patches{ + ApplyFunc(os.Stat, func(name string) (os.FileInfo, error) { + return nil, nil + }), + ApplyFunc(os.ReadFile, func(string) ([]byte, error) { + return []byte("dummyPassphrase"), nil + }), + ApplyFunc(crypto.SCCInitialized, func() bool { + return false + }), + } + defer func() { + for _, patch := range patches { + time.Sleep(100 * time.Millisecond) + patch.Reset() + } + }() + tests := []struct { + name string + mockSetups func() []*gomonkey.Patches + expectedError bool + errorContains string + }{ + { + name: "successful case", + mockSetups: func() []*gomonkey.Patches { + var patches []*gomonkey.Patches + + // Mock ioutil.ReadFile + p2 := gomonkey.ApplyFunc(ioutil.ReadFile, func(filename string) ([]byte, error) { + return []byte("test data"), nil + }) + patches = append(patches, p2) + + testKey, _ := rsa.GenerateKey(rand.Reader, 2048) + keyDER := x509.MarshalPKCS1PrivateKey(testKey) + tlsMocks := mockTls(keyDER, testKey) + patches = append(patches, tlsMocks...) + + return patches + }, + expectedError: false, + }, + { + name: "getPassphrase failure", + mockSetups: func() []*gomonkey.Patches { + p1 := gomonkey.ApplyFunc(pem.Decode, func(data []byte) (*pem.Block, []byte) { + return nil, nil + }) + return []*gomonkey.Patches{p1} + }, + expectedError: true, + errorContains: "failed to decode key PEM block", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + patches := tt.mockSetups() + defer func() { + for _, p := range patches { + p.Reset() + } + }() + + c := &clientTLSAuth{ + cerfile: []byte("cert.pem"), + cafile: []byte("ca.pem"), + keyfile: []byte("key.pem"), + } + config, err := c.GetEtcdConfig() + + if tt.expectedError { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.errorContains) + assert.Nil(t, config) + } else { + assert.NoError(t, err) + assert.NotNil(t, config) + } + }) + } +} diff --git a/frontend/pkg/common/faas_common/etcd3/event.go b/frontend/pkg/common/faas_common/etcd3/event.go new file mode 100644 index 0000000000000000000000000000000000000000..d5e5ab7c9a72de8aeb9456c9cf2e037e2ea76144 --- /dev/null +++ b/frontend/pkg/common/faas_common/etcd3/event.go @@ -0,0 +1,106 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd3 event +package etcd3 + +import ( + "go.etcd.io/etcd/api/v3/mvccpb" + "go.etcd.io/etcd/client/v3" +) + +const ( + // PUT event + PUT = iota + // DELETE event + DELETE + // HISTORYDELETE event + HISTORYDELETE + // HISTORYUPDATE event + HISTORYUPDATE + // ERROR unexpected event + ERROR + // SYNCED synced event + SYNCED +) + +// Event of databases +type Event struct { + Type int + Key string + Value []byte + PrevValue []byte + Rev int64 + ETCDType string +} + +// only type can be used +// notice watcher, ready to watch etcd kv. +func syncedEvent() *Event { + return &Event{ + Type: SYNCED, + Key: "", + Value: nil, + PrevValue: nil, + Rev: 0, + ETCDType: "", + } +} + +// parseKV converts a KeyValue retrieved from an initial sync() listing to a synthetic isCreated event. +func parseKV(kv *mvccpb.KeyValue, etcdType string) *Event { + return &Event{ + Type: PUT, + Key: string(kv.Key), + Value: kv.Value, + PrevValue: nil, + Rev: kv.ModRevision, + ETCDType: etcdType, + } +} + +func parseEvent(e *clientv3.Event, etcdType string) *Event { + eType := PUT + if e.Type == clientv3.EventTypeDelete { + eType = DELETE + } + ret := &Event{ + Type: eType, + Key: string(e.Kv.Key), + Value: e.Kv.Value, + Rev: e.Kv.ModRevision, + ETCDType: etcdType, + } + if e.PrevKv != nil { + ret.PrevValue = e.PrevKv.Value + } + return ret +} + +func parseHistoryEvent(e *clientv3.Event, etcdType string) *Event { + event := parseEvent(e, etcdType) + if event.Type == DELETE { + event.Type = HISTORYDELETE + } + if event.Type == PUT { + event.Type = HISTORYUPDATE + } + return event +} + +func parseErr(err error, source string) *Event { + return &Event{Type: ERROR, Value: []byte(err.Error()), ETCDType: source} +} diff --git a/frontend/pkg/common/faas_common/etcd3/event_test.go b/frontend/pkg/common/faas_common/etcd3/event_test.go new file mode 100644 index 0000000000000000000000000000000000000000..98e9393985c156db69b271333a15fb16e32af25d --- /dev/null +++ b/frontend/pkg/common/faas_common/etcd3/event_test.go @@ -0,0 +1,89 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package etcd3 + +import ( + "errors" + "reflect" + "testing" + + "github.com/smartystreets/goconvey/convey" + "go.etcd.io/etcd/api/v3/mvccpb" + "go.etcd.io/etcd/client/v3" +) + +func Test_syncedEvent(t *testing.T) { + convey.Convey("syncedEvent", t, func() { + event := syncedEvent() + convey.So(event.Type, convey.ShouldEqual, SYNCED) + }) +} + +func Test_parseKV(t *testing.T) { + convey.Convey("parseKV", t, func() { + kv := parseKV(&mvccpb.KeyValue{Key: []byte("key1"), Value: []byte("value1")}, Router) + convey.So(kv.Key, convey.ShouldEqual, "key1") + convey.So(string(kv.Value), convey.ShouldEqual, "value1") + convey.So(kv.Type, convey.ShouldEqual, PUT) + }) +} + +func Test_parseEvent(t *testing.T) { + convey.Convey("parseEvent", t, func() { + event := parseEvent(&clientv3.Event{ + Type: DELETE, + Kv: &mvccpb.KeyValue{Key: []byte("key1"), Value: []byte("value1")}, + PrevKv: &mvccpb.KeyValue{Key: []byte("key2"), Value: []byte("value2")}, + }, Router) + convey.So(event.Type, convey.ShouldEqual, DELETE) + convey.So(event.Key, convey.ShouldEqual, "key1") + convey.So(string(event.Value), convey.ShouldEqual, "value1") + convey.So(string(event.PrevValue), convey.ShouldEqual, "value2") + }) +} + +func Test_parseErr(t *testing.T) { + convey.Convey("parseErr", t, func() { + err := parseErr(errors.New("parseErr"), Router) + convey.So(err.Type, convey.ShouldEqual, ERROR) + convey.So(string(err.Value), convey.ShouldEqual, "parseErr") + }) +} + +func Test_parseHistoryEvent(t *testing.T) { + type args struct { + e *clientv3.Event + } + tests := []struct { + name string + args args + wantType int + }{ + {"case1", args{e: &clientv3.Event{ + Type: DELETE, + Kv: &mvccpb.KeyValue{Key: []byte("key1"), Value: []byte("value1")}, + PrevKv: &mvccpb.KeyValue{Key: []byte("key2"), Value: []byte("value2")}, + }}, HISTORYDELETE}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := parseHistoryEvent(tt.args.e, Router); !reflect.DeepEqual(got.Type, tt.wantType) { + t.Errorf("parseHistoryEvent() = %v, want %v", got.Type, tt.wantType) + } + }) + } +} diff --git a/frontend/pkg/common/faas_common/etcd3/instance_register.go b/frontend/pkg/common/faas_common/etcd3/instance_register.go new file mode 100644 index 0000000000000000000000000000000000000000..ea6b6930fd6435663f67e836753b9e105faa6c1b --- /dev/null +++ b/frontend/pkg/common/faas_common/etcd3/instance_register.go @@ -0,0 +1,147 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd3 - +package etcd3 + +import ( + "context" + "errors" + "time" + + "go.etcd.io/etcd/client/v3" + + "frontend/pkg/common/faas_common/logger/log" +) + +const ( + instanceEtcdKeyTTL = 30 + defaultRefreshInterval = 15 * time.Second +) + +var ( + refreshInterval = defaultRefreshInterval +) + +// EtcdRegister - register to specified ETCD +type EtcdRegister struct { + EtcdClient *EtcdClient + InstanceKey string + Value string + leaseID clientv3.LeaseID + StopCh <-chan struct{} +} + +// Register - register instance to meta etcd or router etcd +func (r *EtcdRegister) Register() error { + if r.EtcdClient != GetMetaEtcdClient() && r.EtcdClient != GetRouterEtcdClient() { + log.GetLogger().Errorf("etcdClient is not meta or route etcd") + return errors.New("etcdClient is not meta or route etcd") + } + var err error + err = r.putInstanceInfoToEtcd() + if err != nil { + log.GetLogger().Errorf("failed to register instance to %s etcd when start, error:%s", + r.EtcdClient.GetEtcdType(), err.Error()) + return err + } + go r.startRefreshLeaseJob() + return nil +} + +func (r *EtcdRegister) startRefreshLeaseJob() { + if r.StopCh == nil { + log.GetLogger().Errorf("StopCh is nil, lease in %s etcd will not be refreshed", + r.EtcdClient.GetEtcdType()) + return + } + refreshTicker := time.NewTicker(refreshInterval) + defer refreshTicker.Stop() + for { + select { + case <-refreshTicker.C: + r.refreshLease() + case <-r.StopCh: + log.GetLogger().Warnf("stopping refresh lease job") + refreshTicker.Stop() + r.stopLease() + return + } + } +} + +func (r *EtcdRegister) stopLease() { + revokeCtx := CreateEtcdCtxInfoWithTimeout(context.Background(), DurationContextTimeout) + err := r.EtcdClient.Revoke(revokeCtx, r.leaseID) + if err != nil { + log.GetLogger().Warnf("revoke lease in %s etcd failed, err:%s", + r.EtcdClient.GetEtcdType(), err.Error()) + } + ctx := CreateEtcdCtxInfoWithTimeout(context.Background(), DurationContextTimeout) + err = r.EtcdClient.Delete(ctx, r.InstanceKey) + if err != nil { + log.GetLogger().Errorf("delete key: %s,from %s etcd failed, err:%s", + r.InstanceKey, r.EtcdClient.GetEtcdType(), err.Error()) + } +} + +func (r *EtcdRegister) refreshLease() { + if !r.isKeyExist() { + if err := r.putInstanceInfoToEtcd(); err != nil { + return + } + } + keepAliveOnceCtx := CreateEtcdCtxInfoWithTimeout(context.Background(), DurationContextTimeout) + err := r.EtcdClient.KeepAliveOnce(keepAliveOnceCtx, r.leaseID) + if err != nil { + log.GetLogger().Errorf("unable to refresh lease in %s etcd:%s", + r.EtcdClient.GetEtcdType(), err.Error()) + } +} + +func (r *EtcdRegister) isKeyExist() bool { + ctx := CreateEtcdCtxInfoWithTimeout(context.Background(), DurationContextTimeout) + resp, err := r.EtcdClient.GetResponse(ctx, r.InstanceKey, + clientv3.WithKeysOnly(), clientv3.WithSerializable()) + if err != nil { + log.GetLogger().Errorf("failed to get new key:%s from %s etcd, err:%s", + r.InstanceKey, r.EtcdClient.GetEtcdType(), err.Error()) + return false + } + return len(resp.Kvs) > 0 +} + +func (r *EtcdRegister) putInstanceInfoToEtcd() error { + grantCtx := CreateEtcdCtxInfoWithTimeout(context.Background(), DurationContextTimeout) + id, err := r.EtcdClient.Grant(grantCtx, instanceEtcdKeyTTL) + if err != nil { + log.GetLogger().Errorf("failed to grant instance lease in %s etcd: %s", r.EtcdClient.GetEtcdType(), + err.Error()) + return err + } + + ctx := CreateEtcdCtxInfoWithTimeout(context.Background(), DurationContextTimeout) + err = r.EtcdClient.Put(ctx, r.InstanceKey, r.Value, clientv3.WithLease(id)) + if err != nil { + log.GetLogger().Errorf("unable to put new key:%s to %s etcd, err:%s", + r.InstanceKey, r.EtcdClient.GetEtcdType(), err.Error()) + return err + } + r.leaseID = id + log.GetLogger().Infof("register instance key:%s, value:%s to %s etcd successfully!", + r.InstanceKey, r.Value, r.EtcdClient.GetEtcdType()) + return nil +} diff --git a/frontend/pkg/common/faas_common/etcd3/instance_register_test.go b/frontend/pkg/common/faas_common/etcd3/instance_register_test.go new file mode 100644 index 0000000000000000000000000000000000000000..ec89d4bf9a9b3e0516be7fd2db0b6af3711f0e52 --- /dev/null +++ b/frontend/pkg/common/faas_common/etcd3/instance_register_test.go @@ -0,0 +1,267 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd3 - +package etcd3 + +import ( + "context" + "errors" + "fmt" + "os" + "reflect" + "sync/atomic" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + "go.etcd.io/etcd/api/v3/mvccpb" + "go.etcd.io/etcd/client/v3" +) + +type mockLease struct { +} + +func (m mockLease) Grant(ctx context.Context, ttl int64) (*clientv3.LeaseGrantResponse, error) { + return &clientv3.LeaseGrantResponse{ID: 1}, nil +} + +func (m mockLease) Revoke(ctx context.Context, id clientv3.LeaseID) (*clientv3.LeaseRevokeResponse, error) { + return nil, nil +} + +func (m mockLease) TimeToLive(ctx context.Context, id clientv3.LeaseID, opts ...clientv3.LeaseOption) (*clientv3.LeaseTimeToLiveResponse, error) { + return nil, nil +} + +func (m mockLease) Leases(ctx context.Context) (*clientv3.LeaseLeasesResponse, error) { + return nil, nil +} + +func (m mockLease) KeepAlive(ctx context.Context, id clientv3.LeaseID) (<-chan *clientv3.LeaseKeepAliveResponse, error) { + return nil, nil +} + +func (m mockLease) KeepAliveOnce(ctx context.Context, id clientv3.LeaseID) (*clientv3.LeaseKeepAliveResponse, error) { + return nil, nil +} + +func (m mockLease) Close() error { + panic("implement me") +} + +type mockKV struct { + put uint32 + get uint32 + delete uint32 + do uint32 +} + +func (fk *mockKV) Put(ctx context.Context, key, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error) { + atomic.AddUint32(&fk.put, 1) + return &clientv3.PutResponse{}, nil +} + +func (fk *mockKV) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + atomic.AddUint32(&fk.get, 1) + return &clientv3.GetResponse{Kvs: []*mvccpb.KeyValue{{}}}, nil +} + +func (fk *mockKV) Delete(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.DeleteResponse, error) { + atomic.AddUint32(&fk.delete, 1) + return &clientv3.DeleteResponse{}, nil +} + +func (mockKV) Compact(ctx context.Context, rev int64, opts ...clientv3.CompactOption) (*clientv3.CompactResponse, error) { + return &clientv3.CompactResponse{}, nil +} + +func (fk *mockKV) Do(ctx context.Context, op clientv3.Op) (clientv3.OpResponse, error) { + atomic.AddUint32(&fk.do, 1) + return clientv3.OpResponse{}, nil +} + +func (mockKV) Txn(ctx context.Context) clientv3.Txn { + return nil +} + +func TestRegisterInstance_PutInstanceToEtcd(t *testing.T) { + convey.Convey("test put instance info", t, func() { + patch := gomonkey.ApplyFunc(GetMetaEtcdClient, func() *EtcdClient { + return &EtcdClient{Client: &clientv3.Client{KV: &mockKV{}, Lease: &mockLease{}}} + }) + + defer func() { + patch.Reset() + }() + register := &EtcdRegister{ + EtcdClient: GetMetaEtcdClient(), + InstanceKey: "/sn/frontend/instances/CLUSTER_ID/HOST_IP/POD_NAME", + Value: "active", + } + convey.Convey("lease id not exist", func() { + var keyInput string + defer gomonkey.ApplyMethod(reflect.TypeOf(&EtcdClient{}), "Put", func(_ *EtcdClient, + ctxInfo EtcdCtxInfo, key string, value string, opts ...clientv3.OpOption) error { + keyInput = key + return nil + }).Reset() + defer gomonkey.ApplyFunc(os.Getenv, func(key string) string { + return key + }).Reset() + err := register.putInstanceInfoToEtcd() + convey.So(err, convey.ShouldBeNil) + convey.So(keyInput, convey.ShouldEqual, "/sn/frontend/instances/CLUSTER_ID/HOST_IP/POD_NAME") + }) + + convey.Convey("Grant error", func() { + defer gomonkey.ApplyMethod(reflect.TypeOf(&EtcdClient{}), "Grant", func(_ *EtcdClient, + ctxInfo EtcdCtxInfo, ttl int64) (clientv3.LeaseID, error) { + return 111, fmt.Errorf("grant failed") + }).Reset() + defer gomonkey.ApplyFunc(os.Getenv, func(key string) string { + return key + }).Reset() + err := register.putInstanceInfoToEtcd() + convey.So(err, convey.ShouldNotBeNil) + }) + + convey.Convey("put error", func() { + var keyInput string + defer gomonkey.ApplyMethod(reflect.TypeOf(&EtcdClient{}), "Put", func(_ *EtcdClient, + ctxInfo EtcdCtxInfo, key string, value string, opts ...clientv3.OpOption) error { + return fmt.Errorf("put failed") + }).Reset() + defer gomonkey.ApplyFunc(os.Getenv, func(key string) string { + return key + }).Reset() + err := register.putInstanceInfoToEtcd() + convey.So(err, convey.ShouldNotBeNil) + convey.So(keyInput, convey.ShouldBeBlank) + convey.So(err.Error(), convey.ShouldEqual, "put failed") + }) + }) +} + +func Test_registerInstance(t *testing.T) { + etcdClient := &EtcdClient{Client: &clientv3.Client{Lease: &mockLease{}}} + patch := gomonkey.ApplyFunc(GetMetaEtcdClient, func() *EtcdClient { + return etcdClient + }) + patch.ApplyMethod(reflect.TypeOf(&EtcdClient{}), "Put", func(_ *EtcdClient, + ctxInfo EtcdCtxInfo, key string, value string, opts ...clientv3.OpOption) error { + return errors.New("put etcd error") + }) + defer func() { + patch.Reset() + }() + + register := &EtcdRegister{ + EtcdClient: GetMetaEtcdClient(), + InstanceKey: "/sn/frontend/instances/CLUSTER_ID/HOST_IP/POD_NAME", + Value: "active", + } + err := register.Register() + assert.NotNil(t, err) +} + +func Test_isKeyExist(t *testing.T) { + convey.Convey("Test isKeyExist", t, func() { + patch := gomonkey.ApplyFunc(GetMetaEtcdClient, func() *EtcdClient { + return &EtcdClient{ + Client: &clientv3.Client{}, + } + }) + defer patch.Reset() + register := &EtcdRegister{ + EtcdClient: GetMetaEtcdClient(), + InstanceKey: "/sn/frontend/instances/CLUSTER_ID/HOST_IP/POD_NAME", + Value: "active", + } + convey.Convey("get etcd key return empty", func() { + patch := gomonkey.ApplyMethod(reflect.TypeOf(&EtcdClient{}), "GetResponse", func(_ *EtcdClient, + ctxInfo EtcdCtxInfo, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + return &clientv3.GetResponse{Kvs: []*mvccpb.KeyValue{}}, nil + }) + defer patch.Reset() + existed := register.isKeyExist() + convey.So(existed, convey.ShouldBeFalse) + }) + + convey.Convey("succeed to get etcd key", func() { + patch := gomonkey.ApplyMethod(reflect.TypeOf(&EtcdClient{}), "GetResponse", func(_ *EtcdClient, + ctxInfo EtcdCtxInfo, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + return &clientv3.GetResponse{Kvs: []*mvccpb.KeyValue{{Key: []byte("key")}}}, nil + }) + defer patch.Reset() + existed := register.isKeyExist() + convey.So(existed, convey.ShouldBeTrue) + }) + + convey.Convey("failed to get etcd key", func() { + patch := gomonkey.ApplyMethod(reflect.TypeOf(&EtcdClient{}), "GetResponse", func(_ *EtcdClient, + ctxInfo EtcdCtxInfo, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + return nil, errors.New("failed") + }) + defer patch.Reset() + existed := register.isKeyExist() + convey.So(existed, convey.ShouldBeFalse) + }) + }) +} + +func Test_startRefreshLeaseJob(t *testing.T) { + kv := &mockKV{} + patches := gomonkey.NewPatches() + patches.ApplyFunc((*EtcdClient).Put, func(_ *EtcdClient, ctxInfo EtcdCtxInfo, key string, value string, opts ...clientv3.OpOption) error { + return nil + }) + patches.ApplyFunc((*clientv3.Client).Ctx, func(_ *clientv3.Client) context.Context { return context.TODO() }) + patches.ApplyFunc((*clientv3.Client).Close, func(_ *clientv3.Client) error { return nil }) + patches.ApplyFunc(clientv3.NewKV, func(c *clientv3.Client) clientv3.KV { + return kv + }) + patches.ApplyFunc(GetMetaEtcdClient, func() *EtcdClient { + return &EtcdClient{Client: &clientv3.Client{KV: kv, Lease: &mockLease{}}} + }) + defer func() { + patches.Reset() + }() + refreshInterval = 1 * time.Millisecond + + register := &EtcdRegister{ + EtcdClient: GetMetaEtcdClient(), + InstanceKey: "/sn/frontend/instances/CLUSTER_ID/HOST_IP/POD_NAME", + Value: "active", + } + + // stop chan is nil, will not trigger refresh + go register.startRefreshLeaseJob() + time.Sleep(10 * time.Millisecond) + assert.Equal(t, uint32(0), atomic.LoadUint32(&kv.get)) + assert.Equal(t, uint32(0), atomic.LoadUint32(&kv.put)) + assert.Equal(t, uint32(0), atomic.LoadUint32(&kv.do)) + + stopCh := make(chan struct{}) + register.StopCh = stopCh + go register.startRefreshLeaseJob() + time.Sleep(100 * time.Millisecond) + assert.NotEqual(t, uint32(0), atomic.LoadUint32(&kv.get)) + close(stopCh) + time.Sleep(1 * time.Second) +} diff --git a/frontend/pkg/common/faas_common/etcd3/lease.go b/frontend/pkg/common/faas_common/etcd3/lease.go new file mode 100644 index 0000000000000000000000000000000000000000..bac561d3dc8171367efacf978e91350238e79b8d --- /dev/null +++ b/frontend/pkg/common/faas_common/etcd3/lease.go @@ -0,0 +1,55 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd3 implements crud and watch operations based etcd clientv3 +package etcd3 + +import ( + "go.etcd.io/etcd/client/v3" +) + +// Grant - +func (e *EtcdClient) Grant(ctxInfo EtcdCtxInfo, ttl int64) (clientv3.LeaseID, error) { + ctx, cancel := ctxInfo.Ctx, ctxInfo.Cancel + e.rwMutex.RLock() + resp, err := e.Client.Grant(ctx, ttl) + e.rwMutex.RUnlock() + cancel() + if err != nil { + return 0, err + } + return resp.ID, nil +} + +// KeepAliveOnce - +func (e *EtcdClient) KeepAliveOnce(ctxInfo EtcdCtxInfo, leaseID clientv3.LeaseID) error { + ctx, cancel := ctxInfo.Ctx, ctxInfo.Cancel + e.rwMutex.RLock() + _, err := e.Client.KeepAliveOnce(ctx, leaseID) + e.rwMutex.RUnlock() + cancel() + return err +} + +// Revoke - +func (e *EtcdClient) Revoke(ctxInfo EtcdCtxInfo, leaseID clientv3.LeaseID) error { + ctx, cancel := ctxInfo.Ctx, ctxInfo.Cancel + e.rwMutex.RLock() + _, err := e.Client.Revoke(ctx, leaseID) + e.rwMutex.RUnlock() + cancel() + return err +} diff --git a/frontend/pkg/common/faas_common/etcd3/lease_test.go b/frontend/pkg/common/faas_common/etcd3/lease_test.go new file mode 100644 index 0000000000000000000000000000000000000000..262e9a8eefec9d02c85c429ad98d5f89c760b6fb --- /dev/null +++ b/frontend/pkg/common/faas_common/etcd3/lease_test.go @@ -0,0 +1,69 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd3 implements crud and watch operations based etcd clientv3 +package etcd3 + +import ( + "context" + "testing" + "time" + + "github.com/smartystreets/goconvey/convey" + "go.etcd.io/etcd/client/v3" + + "frontend/pkg/common/faas_common/utils" +) + +// TestEtcdClient_Grant - +func TestEtcdClient_Grant(t *testing.T) { + convey.Convey("test: grant", t, func() { + client := &EtcdClient{ + Client: &clientv3.Client{ + Lease: utils.FakeEtcdLease{}, + }, + } + id, err := client.Grant(CreateEtcdCtxInfoWithTimeout(context.Background(), 100*time.Millisecond), 10) + convey.So(id, convey.ShouldEqual, 1) + convey.So(err, convey.ShouldBeNil) + }) +} + +// TestEtcdClient_KeepAliveOnce - +func TestEtcdClient_KeepAliveOnce(t *testing.T) { + convey.Convey("test: keepAliveOnce", t, func() { + client := &EtcdClient{ + Client: &clientv3.Client{ + Lease: utils.FakeEtcdLease{}, + }, + } + err := client.KeepAliveOnce(CreateEtcdCtxInfoWithTimeout(context.Background(), 100*time.Millisecond), 1) + convey.So(err, convey.ShouldBeNil) + }) +} + +// TestEtcdClient_KeepAliveOnce - +func TestEtcdClient_Revoke(t *testing.T) { + convey.Convey("test: revoke", t, func() { + client := &EtcdClient{ + Client: &clientv3.Client{ + Lease: utils.FakeEtcdLease{}, + }, + } + err := client.Revoke(CreateEtcdCtxInfoWithTimeout(context.Background(), 100*time.Millisecond), 1) + convey.So(err, convey.ShouldBeNil) + }) +} diff --git a/frontend/pkg/common/faas_common/etcd3/lock.go b/frontend/pkg/common/faas_common/etcd3/lock.go new file mode 100644 index 0000000000000000000000000000000000000000..0448b6e4190de98c5f209001c1a4ee8eed9330e1 --- /dev/null +++ b/frontend/pkg/common/faas_common/etcd3/lock.go @@ -0,0 +1,282 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd3 implements crud and watch operations based etcd clientv3 +package etcd3 + +import ( + "context" + "errors" + "fmt" + "sync/atomic" + "time" + + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/client/v3/concurrency" + + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/utils" +) + +const ( + defaultRequestTimeout = 30 * time.Second + refreshAheadTime = 1 * time.Second + lockedKeyHoldIndex = 1 +) + +var ( + // ErrEtcdResponseInvalid - + ErrEtcdResponseInvalid = errors.New("etcd response is invalid") + // ErrNoKeyCanBeFound - + ErrNoKeyCanBeFound = errors.New("no etcd key can be found") + // ErrNoKeyCanBeLocked - + ErrNoKeyCanBeLocked = errors.New("no etcd key can be locked") + lockFailCountLimit = 10 +) + +// EtcdLocker - +type EtcdLocker struct { + EtcdClient *EtcdClient + acquiredLock *concurrency.Mutex + LockedKey string + holderKey string + LeaseTTL int + leaseID clientv3.LeaseID + locked atomic.Uint32 + LockCallback func(locker *EtcdLocker) error + UnlockCallback func(locker *EtcdLocker) error + FailCallback func() + unlockCh chan struct{} + StopCh <-chan struct{} +} + +// GetLockedKey - +func (l *EtcdLocker) GetLockedKey() string { + return l.LockedKey +} + +// TryLockWithPrefix will get all identities(instanceID) distributed from control plane and try to lock one +func (l *EtcdLocker) TryLockWithPrefix(prefix string, filter func(k, v []byte) bool) error { + resp, err := l.EtcdClient.Get(CreateEtcdCtxInfoWithTimeout(context.TODO(), defaultRequestTimeout), prefix, + clientv3.WithPrefix()) + if err != nil { + log.GetLogger().Errorf("failed to get prefix %s from etcd error %s", prefix, err.Error()) + return err + } + if len(resp.Kvs) == 0 { + log.GetLogger().Warnf("no etcd key is found for prefix %s", prefix) + return ErrNoKeyCanBeLocked + } + var ( + locked bool + tryLockErr error + ) + for _, kv := range resp.Kvs { + if filter(kv.Key, kv.Value) { + tryLockErr = ErrNoKeyCanBeLocked + continue + } + tryLockErr = l.TryLock(string(kv.Key)) + if tryLockErr == nil { + locked = true + break + } + } + if !locked { + if tryLockErr != nil { + return tryLockErr + } else { + return ErrNoKeyCanBeLocked + } + } + return nil +} + +// TryLock - +func (l *EtcdLocker) TryLock(key string) error { + if err := l.tryLock(key); err != nil { + return err + } + go l.lockKeeperLoop() + return nil +} + +func (l *EtcdLocker) tryLock(key string) error { + grtCtx := CreateEtcdCtxInfoWithTimeout(context.Background(), DurationContextTimeout) + if l.leaseID == clientv3.NoLease { + leaseID, err := l.EtcdClient.Grant(grtCtx, int64(l.LeaseTTL)) + if err != nil { + log.GetLogger().Errorf("failed to grant lease for key in %s etcd error %s", l.EtcdClient.GetEtcdType(), + err.Error()) + return err + } + l.leaseID = leaseID + } + l.holderKey = fmt.Sprintf("%s/%x", key, l.leaseID) + log.GetLogger().Infof("generate holderKey %s", l.holderKey) + var lockErr error + defer func() { + if lockErr != nil { + log.GetLogger().Errorf("failed to lock key %s, delete holder key %s", key, l.holderKey) + rvkCtx := CreateEtcdCtxInfoWithTimeout(context.Background(), DurationContextTimeout) + if err := l.EtcdClient.Revoke(rvkCtx, l.leaseID); err != nil { + log.GetLogger().Errorf("failed to revoke lease %d error %d", l.leaseID, err.Error()) + } + l.leaseID = clientv3.NoLease + delCtx := CreateEtcdCtxInfoWithTimeout(context.Background(), DurationContextTimeout) + if err := l.EtcdClient.Delete(delCtx, l.holderKey); err != nil { + log.GetLogger().Errorf("failed to delete holder key %s error %d", l.holderKey, err.Error()) + } + } + }() + cmp := clientv3.Compare(clientv3.LeaseValue(l.holderKey), "=", clientv3.NoLease) + put := clientv3.OpPut(l.holderKey, "", clientv3.WithLease(l.leaseID)) + get := clientv3.OpGet(l.holderKey) + // key is already been put, we want to get the minimum holder key so use WithLimit(2) + getKeyHolder := clientv3.OpGet(key, []clientv3.OpOption{clientv3.WithPrefix(), clientv3.WithSort( + clientv3.SortByCreateRevision, clientv3.SortAscend), clientv3.WithLimit(lockedKeyHoldIndex + 1)}...) + txnCtx := CreateEtcdCtxInfoWithTimeout(context.Background(), DurationContextTimeout) + var resp *clientv3.TxnResponse + resp, lockErr = l.EtcdClient.Client.Txn(txnCtx.Ctx).If(cmp).Then(put, getKeyHolder).Else(get, getKeyHolder).Commit() + if lockErr != nil { + log.GetLogger().Errorf("failed to lock key %s, transaction error %s", key, lockErr.Error()) + return lockErr + } + if len(resp.Responses) != lockedKeyHoldIndex+1 { + log.GetLogger().Errorf("failed to lock key %s, transaction response size %s is invalid", key, + len(resp.Responses)) + lockErr = ErrEtcdResponseInvalid + return lockErr + } + var myRevision int64 + if resp.Succeeded { + myRevision = resp.Header.Revision + } else { + if len(resp.Responses[0].GetResponseRange().Kvs) == 0 { + log.GetLogger().Errorf("failed to lock key %s, transaction response[0] kvs size is 0", key) + lockErr = ErrEtcdResponseInvalid + return lockErr + } + myRevision = resp.Responses[0].GetResponseRange().Kvs[0].CreateRevision + } + log.GetLogger().Infof("get holderKey %s my revision %d", l.holderKey, myRevision) + // resp.Responses[1] contains info got from getKeyHolder, ideally looks like [originKey, holderKey] after sorting, + // because originKey is put by control plane and has lower revision than any holderKey attached with a lease + holderKvs := resp.Responses[1].GetResponseRange().Kvs + // holderKvs[0] is not the originKey means originKey is deleted + if len(holderKvs) == 0 || string(holderKvs[0].Key) != key { + log.GetLogger().Warnf("failed to find key %s, key may be deleted", l.holderKey) + lockErr = ErrNoKeyCanBeFound + return lockErr + } + // holderKvs[1] has different revision from myRevision means other one has locked this key before me + if len(holderKvs) > 1 && holderKvs[1].CreateRevision != myRevision { + log.GetLogger().Warnf("failed to lock key %s, key already locked, holder revision %d", l.holderKey, + holderKvs[1].CreateRevision) + lockErr = ErrNoKeyCanBeLocked + return lockErr + } + l.LockedKey = key + l.unlockCh = make(chan struct{}) + if l.LockCallback != nil { + if lockErr = l.LockCallback(l); lockErr != nil { + log.GetLogger().Warnf("failed to process lock callback of %s error %s", key, lockErr.Error()) + return lockErr + } + } + log.GetLogger().Infof("succeed to lock key %s", key) + return nil +} + +// Unlock - +func (l *EtcdLocker) Unlock() error { + delCtx := CreateEtcdCtxInfoWithTimeout(context.Background(), DurationContextTimeout) + err := l.EtcdClient.Delete(delCtx, l.holderKey) + if err != nil { + log.GetLogger().Errorf("failed to unlock key %s , delete holder %s error %s", l.LockedKey, l.holderKey, + err.Error()) + } + if l.UnlockCallback != nil { + if err = l.UnlockCallback(l); err != nil { + log.GetLogger().Errorf("failed to process unlock callback of %s error %s", l.LockedKey, err.Error()) + } + } + l.LockedKey = "" + l.holderKey = "" + l.leaseID = clientv3.NoLease + utils.SafeCloseChannel(l.unlockCh) + return err +} + +func (l *EtcdLocker) lockKeeperLoop() { + leaseTicker := time.NewTicker(time.Duration(l.LeaseTTL)*time.Second - refreshAheadTime) + defer leaseTicker.Stop() + failCount := 0 + for { + select { + case _, ok := <-l.unlockCh: + if !ok { + log.GetLogger().Warnf("unlock channel triggers for etcd lock of key %s", l.LockedKey) + } + return + case _, ok := <-l.StopCh: + if !ok { + log.GetLogger().Warnf("stop channel triggers for etcd lock of key %s", l.LockedKey) + } + l.Unlock() + return + case <-leaseTicker.C: + if l.leaseID == clientv3.NoLease { + // wait for multiple leaseTTL time to make sure lease is expired at server side + time.Sleep(time.Duration(l.LeaseTTL) * time.Second) + if err := l.tryLock(l.LockedKey); err == ErrNoKeyCanBeFound || err == ErrNoKeyCanBeLocked { + log.GetLogger().Errorf("cannot keep lock key %s, lock fail count %d error %s", l.LockedKey, + failCount, err) + if failCount >= lockFailCountLimit { + l.FailCallback() + return + } + failCount++ + } else { + failCount = 0 + } + } else { + getCtx := CreateEtcdCtxInfoWithTimeout(context.Background(), DurationContextTimeout) + resp, err := l.EtcdClient.Get(getCtx, l.LockedKey) + if err != nil { + log.GetLogger().Errorf("unable to get locked key %s in %s etcd error %s", l.LockedKey, + l.EtcdClient.GetEtcdType(), err.Error()) + l.leaseID = clientv3.NoLease + continue + } + if len(resp.Kvs) == 0 { + log.GetLogger().Warnf("locked key %s is deleted in %s etcd unlock now", l.LockedKey, + l.EtcdClient.GetEtcdType()) + l.Unlock() + l.FailCallback() + return + } + keepAliveOnceCtx := CreateEtcdCtxInfoWithTimeout(context.Background(), DurationContextTimeout) + err = l.EtcdClient.KeepAliveOnce(keepAliveOnceCtx, l.leaseID) + if err != nil { + log.GetLogger().Errorf("unable to refresh lease in %s etcd error %s", l.EtcdClient.GetEtcdType(), + err.Error()) + l.leaseID = clientv3.NoLease + } + } + } + } +} diff --git a/frontend/pkg/common/faas_common/etcd3/lock_test.go b/frontend/pkg/common/faas_common/etcd3/lock_test.go new file mode 100644 index 0000000000000000000000000000000000000000..df2ea05e13c88d7ba0ee74e8e520ec0fc2419162 --- /dev/null +++ b/frontend/pkg/common/faas_common/etcd3/lock_test.go @@ -0,0 +1,341 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd3 implements crud and watch operations based etcd clientv3 +package etcd3 + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" +) + +type fakeTxn struct { + response *clientv3.TxnResponse + err error +} + +func (t *fakeTxn) If(cs ...clientv3.Cmp) clientv3.Txn { + return t +} + +func (t *fakeTxn) Then(ops ...clientv3.Op) clientv3.Txn { + return t +} + +func (t *fakeTxn) Else(ops ...clientv3.Op) clientv3.Txn { + return t +} + +func (t *fakeTxn) Commit() (*clientv3.TxnResponse, error) { + return t.response, t.err +} + +type fakeKv struct { + txn *fakeTxn +} + +func (k *fakeKv) Put(ctx context.Context, key, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error) { + return nil, nil +} + +func (k *fakeKv) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + return nil, nil +} + +func (k *fakeKv) Delete(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.DeleteResponse, error) { + return nil, nil +} + +func (k *fakeKv) Compact(ctx context.Context, rev int64, opts ...clientv3.CompactOption) (*clientv3.CompactResponse, error) { + return nil, nil +} + +func (k *fakeKv) Do(ctx context.Context, op clientv3.Op) (clientv3.OpResponse, error) { + return clientv3.OpResponse{}, nil +} + +func (k *fakeKv) Txn(ctx context.Context) clientv3.Txn { + return k.txn +} + +func buildTxnResponse(success bool, revision int64, kvs1, kvs2 []*mvccpb.KeyValue) *clientv3.TxnResponse { + responses := []*etcdserverpb.ResponseOp{} + if kvs1 != nil { + responses = append(responses, &etcdserverpb.ResponseOp{ + Response: &etcdserverpb.ResponseOp_ResponseRange{ + ResponseRange: &etcdserverpb.RangeResponse{ + Kvs: kvs1, + }, + }, + }) + } + if kvs2 != nil { + responses = append(responses, &etcdserverpb.ResponseOp{ + Response: &etcdserverpb.ResponseOp_ResponseRange{ + ResponseRange: &etcdserverpb.RangeResponse{ + Kvs: kvs2, + }, + }, + }) + } + return &clientv3.TxnResponse{ + Succeeded: success, + Header: &etcdserverpb.ResponseHeader{Revision: revision}, + Responses: responses, + } +} + +func TestTryLock(t *testing.T) { + convey.Convey("test TryLock", t, func() { + ft := &fakeTxn{} + stopCh := make(chan struct{}) + lock := &EtcdLocker{EtcdClient: &EtcdClient{Client: &clientv3.Client{KV: &fakeKv{txn: ft}}}, LeaseTTL: 10, + StopCh: stopCh} + patches := []*gomonkey.Patches{ + gomonkey.ApplyFunc((*EtcdClient).Grant, func(_ *EtcdClient, ctxInfo EtcdCtxInfo, ttl int64) (clientv3.LeaseID, + error) { + return 123, nil + }), + gomonkey.ApplyFunc((*EtcdClient).Revoke, func(_ *EtcdClient, ctxInfo EtcdCtxInfo, leaseID clientv3.LeaseID) error { + return nil + }), + gomonkey.ApplyFunc((*EtcdClient).Delete, func(_ *EtcdClient, ctxInfo EtcdCtxInfo, etcdKey string, + opts ...clientv3.OpOption) error { + return nil + }), + } + defer func() { + close(stopCh) + time.Sleep(100 * time.Millisecond) + for _, p := range patches { + p.Reset() + } + }() + convey.Convey("got and locked", func() { + patch1 := gomonkey.ApplyFunc((*EtcdClient).Get, func(_ *EtcdClient, ctxInfo EtcdCtxInfo, key string, + opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + return &clientv3.GetResponse{Kvs: []*mvccpb.KeyValue{{Key: []byte("/test/key1")}}}, nil + }) + defer patch1.Reset() + ft.response = buildTxnResponse(true, 123, []*mvccpb.KeyValue{}, []*mvccpb.KeyValue{ + { + Key: []byte("/test/key1"), + CreateRevision: 100, + }, + }) + ft.err = nil + err := lock.TryLockWithPrefix("/test", func(k, v []byte) bool { return false }) + convey.So(err, convey.ShouldBeNil) + key := lock.GetLockedKey() + convey.So(key, convey.ShouldEqual, "/test/key1") + }) + convey.Convey("got error", func() { + patch1 := gomonkey.ApplyFunc((*EtcdClient).Get, func(_ *EtcdClient, ctxInfo EtcdCtxInfo, key string, + opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + return nil, errors.New("some error") + }) + defer patch1.Reset() + err := lock.TryLockWithPrefix("/test", func(k, v []byte) bool { return false }) + convey.So(err.Error(), convey.ShouldEqual, "some error") + }) + convey.Convey("lock key lost", func() { + patch1 := gomonkey.ApplyFunc((*EtcdClient).Get, func(_ *EtcdClient, ctxInfo EtcdCtxInfo, key string, + opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + return &clientv3.GetResponse{Kvs: []*mvccpb.KeyValue{{Key: []byte("/test/key1")}}}, nil + }) + defer patch1.Reset() + ft.response = buildTxnResponse(true, 123, []*mvccpb.KeyValue{}, nil) + ft.err = nil + err := lock.TryLockWithPrefix("/test", func(k, v []byte) bool { return false }) + convey.So(err, convey.ShouldNotBeNil) + ft.response = buildTxnResponse(true, 123, []*mvccpb.KeyValue{}, []*mvccpb.KeyValue{ + { + Key: []byte("/test/key1/123"), + CreateRevision: 100, + }, + }) + ft.err = nil + err = lock.TryLockWithPrefix("/test", func(k, v []byte) bool { return false }) + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("lock key locked by others", func() { + patch1 := gomonkey.ApplyFunc((*EtcdClient).Get, func(_ *EtcdClient, ctxInfo EtcdCtxInfo, key string, + opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + return &clientv3.GetResponse{Kvs: []*mvccpb.KeyValue{{Key: []byte("/test/key1")}}}, nil + }) + defer patch1.Reset() + ft.response = buildTxnResponse(true, 123, []*mvccpb.KeyValue{}, []*mvccpb.KeyValue{ + { + Key: []byte("/test/key1"), + CreateRevision: 100, + }, + { + Key: []byte("/test/key1/xxx"), + CreateRevision: 101, + }, + }) + ft.err = nil + err := lock.TryLockWithPrefix("/test", func(k, v []byte) bool { return false }) + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("lock callback", func() { + patch1 := gomonkey.ApplyFunc((*EtcdClient).Get, func(_ *EtcdClient, ctxInfo EtcdCtxInfo, key string, + opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + return &clientv3.GetResponse{Kvs: []*mvccpb.KeyValue{{Key: []byte("/test/key1")}}}, nil + }) + defer patch1.Reset() + ft.response = buildTxnResponse(true, 123, []*mvccpb.KeyValue{}, []*mvccpb.KeyValue{ + { + Key: []byte("/test/key1"), + CreateRevision: 100, + }, + }) + ft.err = nil + lock.LockCallback = func(l *EtcdLocker) error { return errors.New("some error") } + err := lock.TryLockWithPrefix("/test", func(k, v []byte) bool { return false }) + convey.So(err.Error(), convey.ShouldEqual, "some error") + }) + }) +} + +func TestUnlock(t *testing.T) { + stopCh := make(chan struct{}) + lock := &EtcdLocker{EtcdClient: &EtcdClient{}, StopCh: stopCh} + convey.Convey("test Unlock", t, func() { + convey.Convey("unlock ok", func() { + patch := gomonkey.ApplyFunc((*EtcdClient).Delete, func(_ *EtcdClient, ctxInfo EtcdCtxInfo, etcdKey string, + opts ...clientv3.OpOption) error { + return nil + }) + defer patch.Reset() + err := lock.Unlock() + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("unlock error", func() { + patch := gomonkey.ApplyFunc((*EtcdClient).Delete, func(_ *EtcdClient, ctxInfo EtcdCtxInfo, etcdKey string, + opts ...clientv3.OpOption) error { + return errors.New("some error") + }) + defer patch.Reset() + err := lock.Unlock() + convey.So(err.Error(), convey.ShouldEqual, "some error") + }) + convey.Convey("unlock callback", func() { + patch := gomonkey.ApplyFunc((*EtcdClient).Delete, func(_ *EtcdClient, ctxInfo EtcdCtxInfo, etcdKey string, + opts ...clientv3.OpOption) error { + return nil + }) + defer patch.Reset() + lock.UnlockCallback = func(l *EtcdLocker) error { return errors.New("some error") } + err := lock.Unlock() + convey.So(err.Error(), convey.ShouldEqual, "some error") + }) + }) +} + +func TestLockKeeperLoop(t *testing.T) { + convey.Convey("test lockKeeperLoop", t, func() { + stopCh := make(chan struct{}) + lock := &EtcdLocker{EtcdClient: &EtcdClient{}, LeaseTTL: 0, StopCh: stopCh} + getResp := &clientv3.GetResponse{} + getErr := error(nil) + patches := []*gomonkey.Patches{ + gomonkey.ApplyFunc((*EtcdClient).Get, func(_ *EtcdClient, ctxInfo EtcdCtxInfo, key string, + opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + return getResp, getErr + }), + gomonkey.ApplyFunc((*EtcdLocker).Unlock, func(_ *EtcdLocker) error { + return nil + }), + gomonkey.ApplyGlobalVar(&lockFailCountLimit, 0), + } + defer func() { + for _, p := range patches { + p.Reset() + } + }() + convey.Convey("ticker case 1", func() { + ticker := time.NewTicker(100 * time.Millisecond) + patch1 := gomonkey.ApplyFunc(time.NewTicker, func(d time.Duration) *time.Ticker { + return ticker + }) + defer patch1.Reset() + patch2 := gomonkey.ApplyFunc((*EtcdLocker).tryLock, func(_ *EtcdLocker, key string) error { + return ErrNoKeyCanBeFound + }) + defer patch2.Reset() + getErr = errors.New("get key error") + lock.leaseID = 123 + called := false + lock.FailCallback = func() { called = true } + lock.lockKeeperLoop() + convey.So(called, convey.ShouldBeTrue) + }) + convey.Convey("ticker case 2", func() { + ticker := time.NewTicker(100 * time.Millisecond) + patch1 := gomonkey.ApplyFunc(time.NewTicker, func(d time.Duration) *time.Ticker { + return ticker + }) + defer patch1.Reset() + patch2 := gomonkey.ApplyFunc((*EtcdClient).KeepAliveOnce, func(_ *EtcdClient, ctxInfo EtcdCtxInfo, + leaseID clientv3.LeaseID) error { + return errors.New("context deadline exceeded") + }) + defer patch2.Reset() + patch3 := gomonkey.ApplyFunc((*EtcdLocker).tryLock, func(_ *EtcdLocker, key string) error { + return ErrNoKeyCanBeFound + }) + defer patch3.Reset() + getErr = nil + getResp.Kvs = []*mvccpb.KeyValue{{}} + lock.leaseID = 123 + called := false + lock.FailCallback = func() { called = true } + lock.lockKeeperLoop() + convey.So(called, convey.ShouldBeTrue) + }) + convey.Convey("other case", func() { + called := false + patch1 := gomonkey.ApplyFunc((*EtcdLocker).Unlock, func(_ *EtcdLocker) error { + called = true + return nil + }) + defer patch1.Reset() + ticker := time.NewTicker(100 * time.Millisecond) + patch2 := gomonkey.ApplyFunc(time.NewTicker, func(d time.Duration) *time.Ticker { + return ticker + }) + defer patch2.Reset() + unlockCh := make(chan struct{}) + lock.unlockCh = unlockCh + close(unlockCh) + lock.lockKeeperLoop() + unlockCh = make(chan struct{}) + lock.unlockCh = unlockCh + close(stopCh) + lock.lockKeeperLoop() + convey.So(called, convey.ShouldBeTrue) + }) + }) +} diff --git a/frontend/pkg/common/faas_common/etcd3/type.go b/frontend/pkg/common/faas_common/etcd3/type.go new file mode 100644 index 0000000000000000000000000000000000000000..b54b67c0bfac50766f5714e9576783327deb51ac --- /dev/null +++ b/frontend/pkg/common/faas_common/etcd3/type.go @@ -0,0 +1,109 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd3 type +package etcd3 + +import ( + "sync" + "time" + + "go.etcd.io/etcd/client/v3" +) + +// EtcdWatcherFilter defines watch filter of etcd +type EtcdWatcherFilter func(*Event) bool + +// EtcdWatcherHandler defines watch handler of etcd +type EtcdWatcherHandler func(*Event) + +// EtcdClient wrapper etcd client +type EtcdClient struct { + Client *clientv3.Client + config *EtcdConfig + etcdTimer *time.Timer + rwMutex sync.RWMutex + cond *sync.Cond + // notify goroutine keepConnAlive exit + stopCh <-chan struct{} + clientExitCh chan struct{} + exitOnce sync.Once + etcdType string + // router etcd status lost contact after defaultEtcdLostContactTime, true is healthy, false is unhealthy + etcdStatusAfterLostContact bool + etcdStatusNow bool + isAlarmEnable bool + abnormalContinuouslyTimes int +} + +// EtcdWatcher - +type EtcdWatcher struct { + filter EtcdWatcherFilter + handler EtcdWatcherHandler + cacheConfig EtcdCacheConfig + watcher *EtcdClient + ResultChan chan *Event + CacheChan chan *Event + resultChanWG *sync.WaitGroup + configCh chan struct{} + stopCh <-chan struct{} + key string + etcdType string + initialRev int64 + historyRev int64 + cacheFlushing bool + sync.Mutex +} + +// EtcdInitParam - +type EtcdInitParam struct { + metaEtcdConfig *EtcdConfig + routeEtcdConfig *EtcdConfig + CAEMetaEtcdConfig *EtcdConfig + DataSystemEtcdConfig *EtcdConfig + stopCh <-chan struct{} + enableAlarm bool +} + +// EtcdConfig the info to get function instance +type EtcdConfig struct { + Servers []string `json:"servers" valid:"optional"` + AZPrefix string `json:"azPrefix" valid:"optional"` + User string `json:"user" valid:"optional"` + Password string `json:"password" valid:"optional"` + SslEnable bool `json:"sslEnable" valid:"optional"` + AuthType string `json:"authType" valid:"optional"` + UseSecret bool `json:"useSecret" valid:"optional"` + SecretName string `json:"secretName" valid:"optional"` + LimitRate int `json:"limitRate,omitempty" valid:"optional"` + LimitBurst int `json:"limitBurst,omitempty" valid:"optional"` + LimitTimeout int `json:"limitTimeout,omitempty" valid:"optional"` + CaFile string `json:"cafile,omitempty" valid:",optional"` + CertFile string `json:"certfile,omitempty" valid:",optional"` + KeyFile string `json:"keyfile,omitempty" valid:",optional"` + PassphraseFile string `json:"passphraseFile,omitempty" valid:",optional"` +} + +// EtcdCacheConfig - +type EtcdCacheConfig struct { + EnableCache bool `json:"enableCache"` + PersistPath string `json:"persistPath"` + FlushInterval int `json:"flushInterval"` + FlushThreshold int `json:"flushThreshold"` + MetaFilePath string + DataFilePath string + BackupFilePath string +} diff --git a/frontend/pkg/common/faas_common/etcd3/utils.go b/frontend/pkg/common/faas_common/etcd3/utils.go new file mode 100644 index 0000000000000000000000000000000000000000..1ef33de47bf7c7d874872f34f8bb9af52946e9c4 --- /dev/null +++ b/frontend/pkg/common/faas_common/etcd3/utils.go @@ -0,0 +1,170 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd3 - +package etcd3 + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "sync" + "time" + + "go.etcd.io/etcd/client/v3" + "k8s.io/api/core/v1" + + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/utils" +) + +const ( + // etcdDialTimeout is the timeout for establishing a connection. + etcdDialTimeout = 20 * time.Second + + // etcdKeepaliveTime is the time after which client pings the server to see if + etcdKeepaliveTime = 30 * time.Second + + // etcdKeepaliveTimeout is the time that the client waits for a response for the + etcdKeepaliveTimeout = 10 * time.Second + + etcdClientCerts = "etcd-client-certs" + + etcdCertsMountPath = "/home/snuser/resource/etcd" + + etcdCaFile = "/home/snuser/resource/etcd/ca.crt" + + etcdCertFile = "/home/snuser/resource/etcd/client.crt" + + etcdKeyFile = "/home/snuser/resource/etcd/client.key" + + etcdPassphraseFile = "/home/snuser/resource/etcd/passphrase" +) + +const ( + retrySleepTime = 100 * time.Millisecond + maxRetryTime = 3 +) + +var ( + etcdClientMap sync.Map +) + +// GetEtcdConfigKey generates key for etcd config +func GetEtcdConfigKey(etcdConfig *EtcdConfig) string { + sort.Strings(etcdConfig.Servers) + return strings.Join(etcdConfig.Servers, "#") +} + +func createETCDClient(config *EtcdConfig) (*clientv3.Client, error) { + cfg, err := GetEtcdAuthType(*config).GetEtcdConfig() + if err != nil { + log.GetLogger().Errorf("failed to create shared etcd client error %s", err.Error()) + return nil, err + } + cfg.DialTimeout = etcdDialTimeout + cfg.DialKeepAliveTime = etcdKeepaliveTime + cfg.DialKeepAliveTimeout = etcdKeepaliveTimeout + cfg.Endpoints = config.Servers + etcdClient, err := clientv3.New(*cfg) + if err != nil { + log.GetLogger().Errorf("failed to create shared etcd client error %s", err.Error()) + return nil, err + } + return etcdClient, nil +} + +// GetSharedEtcdClient returns a shared etcd client +func GetSharedEtcdClient(etcdConfig *EtcdConfig) (*clientv3.Client, error) { + etcdConfigKey := GetEtcdConfigKey(etcdConfig) + obj, exist := etcdClientMap.Load(etcdConfigKey) + var err error + if !exist { + if obj, err = createETCDClient(etcdConfig); err != nil { + return nil, err + } + } + etcdClient, ok := obj.(*clientv3.Client) + if !ok { + return nil, errors.New("etcd client type error") + } + etcdClientMap.Store(etcdConfigKey, etcdClient) + return etcdClient, nil +} + +// GetValueFromEtcdWithRetry query value from etcd and retry only in case of timeout +func GetValueFromEtcdWithRetry(key string, etcdClient *EtcdClient) ([]byte, error) { + if etcdClient.GetEtcdStatusLostContact() == false || etcdClient.Client == nil { + return nil, errors.New("etcd connection loss") + } + var ( + values []string + err error + ) + for i := 1; i <= maxRetryTime; i++ { + defaultEtcdCtx := CreateEtcdCtxInfoWithTimeout(context.Background(), DurationContextTimeout) + values, err = etcdClient.GetValues(defaultEtcdCtx, key) + if err == nil { + break + } + if err != context.DeadlineExceeded { + return nil, err + } + log.GetLogger().Errorf("get value from etcd with key %s timeout, try time %d", key, i) + time.Sleep(retrySleepTime) + } + + if len(values) == 0 { + log.GetLogger().Errorf("failed to get value from etcd, key: %s", key) + return nil, fmt.Errorf("the value got from etcd is empty") + } + + return []byte(values[0]), err +} + +// GenerateETCDClientCertsVolumesAndMounts - +func GenerateETCDClientCertsVolumesAndMounts(secretName string, builder *utils.VolumeBuilder) (string, string, error) { + if builder == nil { + return "", "", fmt.Errorf("etcd volume builder is nil") + } + builder.AddVolume(v1.Volume{Name: etcdClientCerts, + VolumeSource: v1.VolumeSource{Secret: &v1.SecretVolumeSource{SecretName: secretName}}}) + builder.AddVolumeMount(utils.ContainerRuntimeManager, + v1.VolumeMount{Name: etcdClientCerts, MountPath: etcdCertsMountPath}) + volumesData, err := json.Marshal(builder.Volumes) + if err != nil { + return "", "", err + } + volumesMountData, err := json.Marshal(builder.Mounts[utils.ContainerRuntimeManager]) + if err != nil { + return "", "", err + } + return string(volumesData), string(volumesMountData), nil +} + +// SetETCDTLSConfig - +func SetETCDTLSConfig(etcdConfig *EtcdConfig) { + if etcdConfig == nil { + return + } + etcdConfig.CaFile = etcdCaFile + etcdConfig.CertFile = etcdCertFile + etcdConfig.KeyFile = etcdKeyFile + etcdConfig.PassphraseFile = etcdPassphraseFile +} diff --git a/frontend/pkg/common/faas_common/etcd3/utils_test.go b/frontend/pkg/common/faas_common/etcd3/utils_test.go new file mode 100644 index 0000000000000000000000000000000000000000..e316c9cacc6c91eaacae38e58e4896f009d44877 --- /dev/null +++ b/frontend/pkg/common/faas_common/etcd3/utils_test.go @@ -0,0 +1,155 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd3 - +package etcd3 + +import ( + "encoding/json" + "errors" + "fmt" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + "go.etcd.io/etcd/client/v3" + "k8s.io/api/core/v1" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/utils" +) + +func TestGetSharedEtcdClient(t *testing.T) { + etcdConfig123 := &EtcdConfig{ + Servers: []string{"1", "2", "3"}, + } + convey.Convey("get client failed", t, func() { + defer gomonkey.ApplyFunc(clientv3.New, func(cfg clientv3.Config) (*clientv3.Client, error) { + return nil, errors.New("some error") + }).Reset() + _, err := GetSharedEtcdClient(etcdConfig123) + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("get client success", t, func() { + defer gomonkey.ApplyFunc(clientv3.New, func(cfg clientv3.Config) (*clientv3.Client, error) { + return &clientv3.Client{}, nil + }).Reset() + _, err := GetSharedEtcdClient(etcdConfig123) + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("load client", t, func() { + _, err := GetSharedEtcdClient(etcdConfig123) + convey.So(err, convey.ShouldBeNil) + }) +} + +func TestGetValueFromEtcdWithRetry(t *testing.T) { + funcKey := "123/testFunc/1" + tenantID, funcName, funcVersion := utils.ParseFuncKey(funcKey) + silentEtcdKey := fmt.Sprintf(constant.SilentFuncKey, tenantID, funcName, funcVersion) + convey.Convey("Test GetValueFromEtcdWithRetry", t, func() { + convey.Convey("etcd connection loss", func() { + etcdClient := &EtcdClient{} + _, err := GetValueFromEtcdWithRetry(silentEtcdKey, etcdClient) + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("get values error", func() { + etcdClient := &EtcdClient{ + etcdStatusAfterLostContact: true, + Client: &clientv3.Client{}, + } + defer gomonkey.ApplyMethod(reflect.TypeOf(&EtcdClient{}), "GetValues", + func(_ *EtcdClient, ctxInfo EtcdCtxInfo, key string, opts ...clientv3.OpOption) ([]string, error) { + return nil, errors.New("error") + }).Reset() + _, err := GetValueFromEtcdWithRetry(silentEtcdKey, etcdClient) + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("value got from etcd is empty", func() { + etcdClient := &EtcdClient{ + etcdStatusAfterLostContact: true, + Client: &clientv3.Client{}, + } + defer gomonkey.ApplyMethod(reflect.TypeOf(&EtcdClient{}), "GetValues", + func(_ *EtcdClient, ctxInfo EtcdCtxInfo, key string, opts ...clientv3.OpOption) ([]string, error) { + return []string{}, nil + }).Reset() + _, err := GetValueFromEtcdWithRetry(silentEtcdKey, etcdClient) + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("fetch success", func() { + etcdClient := &EtcdClient{ + etcdStatusAfterLostContact: true, + Client: &clientv3.Client{}, + } + defer gomonkey.ApplyMethod(reflect.TypeOf(&EtcdClient{}), "GetValues", + func(_ *EtcdClient, ctxInfo EtcdCtxInfo, key string, opts ...clientv3.OpOption) ([]string, error) { + return []string{"silent func"}, nil + }).Reset() + value, err := GetValueFromEtcdWithRetry(silentEtcdKey, etcdClient) + convey.So(err, convey.ShouldBeNil) + convey.So(string(value), convey.ShouldEqual, "silent func") + }) + }) +} + +func TestGenerateETCDClientCertsVolumesAndMounts(t *testing.T) { + t.Run("builder is nil", func(t *testing.T) { + volumesData, volumesMountData, err := GenerateETCDClientCertsVolumesAndMounts("test-secret", nil) + assert.Empty(t, volumesData) + assert.Empty(t, volumesMountData) + assert.EqualError(t, err, "etcd volume builder is nil") + }) + + t.Run("normal case", func(t *testing.T) { + builder := utils.NewVolumeBuilder() + + secretName := "test-secret" + volumesData, volumesMountData, err := GenerateETCDClientCertsVolumesAndMounts(secretName, builder) + assert.NoError(t, err) + + var volumes []v1.Volume + err = json.Unmarshal([]byte(volumesData), &volumes) + assert.NoError(t, err) + assert.Len(t, volumes, 1) + assert.Equal(t, etcdClientCerts, volumes[0].Name) + assert.Equal(t, secretName, volumes[0].VolumeSource.Secret.SecretName) + + var volumeMounts []v1.VolumeMount + err = json.Unmarshal([]byte(volumesMountData), &volumeMounts) + assert.NoError(t, err) + assert.Len(t, volumeMounts, 1) + assert.Equal(t, etcdClientCerts, volumeMounts[0].Name) + assert.Equal(t, etcdCertsMountPath, volumeMounts[0].MountPath) + }) +} + +func TestSetETCDTLSConfig(t *testing.T) { + t.Run("etcdConfig", func(t *testing.T) { + SetETCDTLSConfig(nil) + + etcdConfig := &EtcdConfig{} + + SetETCDTLSConfig(etcdConfig) + + assert.Equal(t, etcdCaFile, etcdConfig.CaFile, "CaFile should be set correctly") + assert.Equal(t, etcdCertFile, etcdConfig.CertFile, "CertFile should be set correctly") + assert.Equal(t, etcdKeyFile, etcdConfig.KeyFile, "KeyFile should be set correctly") + assert.Equal(t, etcdPassphraseFile, etcdConfig.PassphraseFile, "PassphraseFile should be set correctly") + }) +} diff --git a/frontend/pkg/common/faas_common/etcd3/watcher.go b/frontend/pkg/common/faas_common/etcd3/watcher.go new file mode 100644 index 0000000000000000000000000000000000000000..fc95155a91229ea61d54bfbe49fb37a31e12f8cd --- /dev/null +++ b/frontend/pkg/common/faas_common/etcd3/watcher.go @@ -0,0 +1,357 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd3 - +package etcd3 + +import ( + "context" + "fmt" + "sync" + "time" + + "go.etcd.io/etcd/client/v3" + + "frontend/pkg/common/faas_common/logger/log" +) + +const ( + defaultEventChanSize = 1000 + // DurationContextTimeout default context duration timeout + DurationContextTimeout = 5 * time.Second +) + +var ( + // keepConnAliveTTL - + keepConnAliveTTL = 10 * time.Second +) + +// EtcdCtxInfo etcd context info +type EtcdCtxInfo struct { + Ctx context.Context + Cancel context.CancelFunc +} + +// Watcher defines watcher of registry +type Watcher interface { + StartWatch() + StartList() + EtcdHistory(revision int64) +} + +// EtcdClientInterface is the interface of ETCD client +type EtcdClientInterface interface { + GetResponse(ctxInfo EtcdCtxInfo, etcdKey string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) + Put(ctxInfo EtcdCtxInfo, etcdKey string, value string, opts ...clientv3.OpOption) error + Delete(ctxInfo EtcdCtxInfo, etcdKey string, opts ...clientv3.OpOption) error +} + +// NewEtcdWatcher create a EtcdWatcher object +func NewEtcdWatcher(prefix string, filter EtcdWatcherFilter, handler EtcdWatcherHandler, stopCh <-chan struct{}, + etcdClient *EtcdClient) *EtcdWatcher { + ew := &EtcdWatcher{ + watcher: etcdClient, + ResultChan: make(chan *Event, defaultEventChanSize), + CacheChan: make(chan *Event, defaultEventChanSize), + filter: filter, + handler: handler, + key: etcdClient.AttachAZPrefix(prefix), + resultChanWG: &sync.WaitGroup{}, + configCh: make(chan struct{}, 1), + stopCh: stopCh, + } + if etcdClient != nil { + ew.etcdType = etcdClient.GetEtcdType() + } + ew.resultChanWG.Add(1) + go ew.processEventLoop() + return ew +} + +// etcdList get current events in etcd and handle these events +func (ew *EtcdWatcher) etcdList(handler func(*clientv3.GetResponse)) error { + opts := []clientv3.OpOption{clientv3.WithPrefix()} + response, err := ew.watcher.Client.KV.Get(context.TODO(), ew.key, opts...) + if err != nil { + log.GetLogger().Errorf("failed to get value from etcd, key: %s, err: %s", ew.key, err.Error()) + return err + } + ew.initialRev = response.Header.Revision + handler(response) + return nil +} + +// EtcdHistory find if delete event happened while recovering +func (ew *EtcdWatcher) EtcdHistory(revision int64) { + if revision == 0 || revision >= ew.initialRev { + return + } + log.GetLogger().Debugf("start to find key %s history event", ew.key) + watchOption := []clientv3.OpOption{clientv3.WithPrefix(), clientv3.WithPrevKV(), clientv3.WithRev(revision), + clientv3.WithProgressNotify()} + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + watchChan := clientv3.NewWatcher(ew.watcher.Client).Watch(ctx, ew.key, watchOption...) + if watchChan == nil { + log.GetLogger().Errorf("failed to watch %s, watch channel is empty", ew.key) + return + } + events, ok := <-watchChan + if !ok { + log.GetLogger().Warnf("the channel received the result may be closed") + return + } + for _, event := range events.Events { + ew.sendEvent(parseHistoryEvent(event, ew.etcdType)) + } +} + +// StartWatch start watch etcd event +func (ew *EtcdWatcher) StartWatch() { + go ew.recoverWatch() + if !ew.watcher.etcdStatusNow { + log.GetLogger().Warnf("no connection with etcd.") + return + } + go ew.run() +} + +// recoverWatch recover watch etcd event when etcd reconnected +func (ew *EtcdWatcher) recoverWatch() { +loop: + for { + if ew.watcher.cond == nil { + log.GetLogger().Warnf("etcd client condition lock is not initialized") + return + } + ew.watcher.cond.L.Lock() + ew.watcher.cond.Wait() + ew.watcher.cond.L.Unlock() + select { + case <-ew.stopCh: + break loop + default: + } + go ew.run() + } + ew.resultChanWG.Wait() + close(ew.ResultChan) +} + +func (ew *EtcdWatcher) run() { + log.GetLogger().Infof("start to watch etcd prefix %s", ew.key) + if ew.watcher.Client == nil { + log.GetLogger().Errorf("failed to watch %s, etcd client is nil", ew.key) + return + } + if ew.cacheConfig.EnableCache { + go ew.processETCDCache() + } + ew.StartList() + watchChan, cancel, err := createWatchChan(ew) + defer cancel() + if err != nil || watchChan == nil { + return + } + for { + select { + case events, ok := <-watchChan: + if !ok { + cancel() + log.GetLogger().Warnf("the channel received the result may be closed") + watchChan, cancel, err = createWatchChan(ew) + if err != nil { + return + } + continue + } + if events.Err() != nil { + log.GetLogger().Errorf("etcd receive err events, err:%s", events.Err().Error()) + } + if ew.historyRev > 0 && ew.historyRev < ew.initialRev { + ew.EtcdHistory(ew.historyRev) + } + for _, event := range events.Events { + e := parseEvent(event, ew.etcdType) + ew.initialRev = e.Rev + ew.historyRev = ew.initialRev + ew.sendEvent(e) + } + case <-ew.stopCh: + log.GetLogger().Infof("stop watching etcd prefix %s", ew.key) + return + case <-ew.watcher.clientExitCh: + log.GetLogger().Errorf("lost %s etcd client", ew.watcher.etcdType) + return + } + } +} + +func createWatchChan(ew *EtcdWatcher) (clientv3.WatchChan, context.CancelFunc, error) { + watchOption := []clientv3.OpOption{clientv3.WithPrefix(), clientv3.WithPrevKV(), + clientv3.WithRev(ew.initialRev), clientv3.WithProgressNotify()} + ctx, cancelFunc := context.WithCancel(context.Background()) + if err := ew.etcdList(func(_ *clientv3.GetResponse) {}); err != nil { + log.GetLogger().Errorf("failed to etcdList, err: %s", err.Error()) + return nil, cancelFunc, err + } + watchChan := clientv3.NewWatcher(ew.watcher.Client).Watch(ctx, ew.key, watchOption...) + if watchChan == nil { + log.GetLogger().Errorf("failed to watch %s, watch channel is empty", ew.key) + return nil, cancelFunc, fmt.Errorf("failed to watch %s, watch channel is empty", ew.key) + } + return watchChan, cancelFunc, nil +} + +// StartList performs a ETCD List and send corresponding events, revision will be set after list +func (ew *EtcdWatcher) StartList() { + if ew.initialRev == 0 { + var restoreErr error + if ew.cacheConfig.EnableCache { + restoreErr = ew.restoreCacheFromFile() + } + if !ew.cacheConfig.EnableCache || restoreErr != nil { + if err := ew.etcdList(func(response *clientv3.GetResponse) { + for _, event := range response.Kvs { + ew.sendEvent(parseKV(event, ew.etcdType)) + } + }); err != nil { + log.GetLogger().Errorf("failed to sync with latest state, error: %s", err.Error()) + } + } + // notice watcher, ready to watch etcd kv + ew.sendEvent(syncedEvent()) + ew.historyRev = ew.initialRev + } +} + +// processEventLoop receive etcd event and process +func (ew *EtcdWatcher) processEventLoop() { + defer ew.resultChanWG.Done() + for { + select { + case event, ok := <-ew.ResultChan: + if !ok { + log.GetLogger().Warnf("event channel is closed, stop processing event") + return + } + if event.Type == SYNCED || !ew.filter(event) { + ew.handler(event) + } + case <-ew.stopCh: + log.GetLogger().Warnf("stop processing etcd event loop") + return + } + } +} + +func (ew *EtcdWatcher) sendEvent(e *Event) { + if len(ew.ResultChan) == defaultEventChanSize { + log.GetLogger().Warnf("Fast watcher, slow processing. Number of buffered events: %d."+ + "Probably caused by slow decoding, user not receiving fast, or other processing logic", + defaultEventChanSize) + } + if ew.watcher != nil { + e.Key = ew.watcher.DetachAZPrefix(e.Key) + } + select { + case ew.ResultChan <- e: + case <-ew.stopCh: + log.GetLogger().Warnf("etcd watcher chan closed") + } + if ew.cacheConfig.EnableCache && (e.Type == PUT || e.Type == DELETE) { + select { + case ew.CacheChan <- e: + case <-ew.stopCh: + log.GetLogger().Warnf("etcd watcher chan closed") + } + } +} + +// GetResponse get etcd value and return pointer of GetResponse struct +func (e *EtcdClient) GetResponse(ctxInfo EtcdCtxInfo, etcdKey string, + opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + etcdKey = e.AttachAZPrefix(etcdKey) + ctx, cancel := ctxInfo.Ctx, ctxInfo.Cancel + defer cancel() + + kv := clientv3.NewKV(e.Client) + getResp, err := kv.Get(ctx, etcdKey, opts...) + + return getResp, err +} + +// Put put context key and value +func (e *EtcdClient) Put(ctxInfo EtcdCtxInfo, etcdKey string, value string, opts ...clientv3.OpOption) error { + etcdKey = e.AttachAZPrefix(etcdKey) + ctx, cancel := ctxInfo.Ctx, ctxInfo.Cancel + defer cancel() + + kv := clientv3.NewKV(e.Client) + _, err := kv.Put(ctx, etcdKey, value, opts...) + return err +} + +// Delete delete key +func (e *EtcdClient) Delete(ctxInfo EtcdCtxInfo, etcdKey string, opts ...clientv3.OpOption) error { + etcdKey = e.AttachAZPrefix(etcdKey) + ctx, cancel := ctxInfo.Ctx, ctxInfo.Cancel + defer cancel() + kv := clientv3.NewKV(e.Client) + _, err := kv.Delete(ctx, etcdKey, opts...) + return err +} + +// Get gets from etcd +func (e *EtcdClient) Get(ctxInfo EtcdCtxInfo, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + key = e.AttachAZPrefix(key) + ctx, cancel := ctxInfo.Ctx, ctxInfo.Cancel + defer cancel() + kv := clientv3.NewKV(e.Client) + response, err := kv.Get(ctx, key, opts...) + if err != nil { + return nil, err + } + return response, nil +} + +// GetValues return list of object for key +func (e *EtcdClient) GetValues(ctxInfo EtcdCtxInfo, key string, opts ...clientv3.OpOption) ([]string, error) { + key = e.AttachAZPrefix(key) + ctx, cancel := ctxInfo.Ctx, ctxInfo.Cancel + defer cancel() + + kv := clientv3.NewKV(e.Client) + response, err := kv.Get(ctx, key, opts...) + if err != nil { + return nil, err + } + values := make([]string, len(response.Kvs)) + + for index, v := range response.Kvs { + values[index] = string(v.Value) + } + return values, err +} + +// CreateEtcdCtxInfoWithTimeout create a context with timeout, default timeout is DurationContextTimeout +func CreateEtcdCtxInfoWithTimeout(ctx context.Context, duration time.Duration) EtcdCtxInfo { + ctx, cancel := context.WithTimeout(ctx, duration) + return EtcdCtxInfo{ + Ctx: ctx, + Cancel: cancel, + } +} diff --git a/frontend/pkg/common/faas_common/etcd3/watcher_test.go b/frontend/pkg/common/faas_common/etcd3/watcher_test.go new file mode 100644 index 0000000000000000000000000000000000000000..7350df3fdca76b42298e54a4bb5d2989e01a93d3 --- /dev/null +++ b/frontend/pkg/common/faas_common/etcd3/watcher_test.go @@ -0,0 +1,472 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package etcd3 + +import ( + "context" + "errors" + "fmt" + "os" + "reflect" + "sync" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.etcd.io/etcd/api/v3/mvccpb" + "go.etcd.io/etcd/client/v3" + + mockUtils "frontend/pkg/common/faas_common/utils" +) + +var watchChan clientv3.WatchChan +var resultCh chan *Event + +type EtcdWatcherMock struct { +} + +func (e EtcdWatcherMock) Watch(ctx context.Context, key string, opts ...clientv3.OpOption) clientv3.WatchChan { + watchChan = make(chan clientv3.WatchResponse, 1) + return watchChan +} + +func (e EtcdWatcherMock) RequestProgress(ctx context.Context) error { + //TODO implement me + panic("implement me") +} + +func (e EtcdWatcherMock) Close() error { + //TODO implement me + panic("implement me") +} + +func TestNewEtcdWatcher(t *testing.T) { + prefix := "" + filter := func(event *Event) bool { return true } + handler := func(event *Event) {} + stopCh := make(chan struct{}) + + convey.Convey("Test NewEtcdWatcher", t, func() { + + convey.Convey("Test NewEtcdWatcher for success", func() { + etcdClient := GetRouterEtcdClient() + watcher := NewEtcdWatcher(prefix, filter, handler, stopCh, etcdClient) + convey.So(watcher, convey.ShouldNotBeNil) + }) + }) +} + +func TestEtcdList(t *testing.T) { + convey.Convey("StartList", t, func() { + stopCh := make(chan struct{}) + kv := &KvMock{} + client := &clientv3.Client{KV: kv} + etcdClient := &EtcdClient{Client: client, clientExitCh: make(chan struct{}), etcdStatusNow: true, cond: sync.NewCond(&sync.Mutex{})} + resultCh = make(chan *Event, 2) + watcher := NewEtcdWatcher("/xxx", etcdFilter, etcdHandler, stopCh, etcdClient) + defer gomonkey.ApplyMethod(reflect.TypeOf(kv), "Get", + func(_ *KvMock, ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + c := &clientv3.GetResponse{ + Header: &etcdserverpb.ResponseHeader{}, + Kvs: []*mvccpb.KeyValue{{Key: []byte("/xxx1"), Value: []byte("value1")}}, + } + c.Header.Revision = 1 + return c, nil + }).Reset() + watcher.StartList() + event := <-resultCh + convey.So(event.Type, convey.ShouldEqual, PUT) + convey.So(event.Key, convey.ShouldEqual, "/xxx1") + convey.So(string(event.Value), convey.ShouldEqual, "value1") + event = <-resultCh + convey.So(event.Type, convey.ShouldEqual, SYNCED) + close(stopCh) + }) +} + +func etcdFilter(event *Event) bool { + return false +} + +func etcdHandler(event *Event) { + resultCh <- event +} + +func erFail(t *testing.T) { + convey.Convey("failed to watch", t, func() { + convey.Convey("no connection with etcd", func() { + stopCh := make(chan struct{}) + etcdClient := &EtcdClient{clientExitCh: make(chan struct{}), cond: sync.NewCond(&sync.Mutex{})} + watcher := NewEtcdWatcher("/xxx", etcdFilter, etcdHandler, stopCh, etcdClient) + watcher.StartWatch() + watcher.watcher.cond.Broadcast() + close(stopCh) + }) + convey.Convey("recover watcher", func() { + exitCh := make(chan struct{}, 1) + stopCh := make(chan struct{}, 1) + etcdClient := &EtcdClient{clientExitCh: exitCh, cond: sync.NewCond(&sync.Mutex{})} + etcdClient.etcdStatusNow = true + e := &EtcdWatcher{watcher: etcdClient, resultChanWG: &sync.WaitGroup{}, stopCh: stopCh, ResultChan: make(chan *Event)} + e.resultChanWG.Add(1) + go e.StartWatch() + exitCh <- struct{}{} + etcdClient.etcdStatusNow = true + time.Sleep(1 * time.Second) + close(stopCh) + e.watcher.cond.Broadcast() + e.resultChanWG.Done() + _, ok := <-e.ResultChan + convey.So(ok, convey.ShouldEqual, false) + }) + }) +} + +func TestEtcdWatcher(t *testing.T) { + stopCh := make(chan struct{}) + kv := &KvMock{} + client := &clientv3.Client{KV: kv} + etcdClient := &EtcdClient{Client: client, clientExitCh: make(chan struct{}), etcdStatusNow: true, cond: sync.NewCond(&sync.Mutex{})} + resultCh = make(chan *Event, 1) + watcher := NewEtcdWatcher("/xxx", etcdFilter, etcdHandler, stopCh, etcdClient) + watcher.initialRev = 1 + patches := []*gomonkey.Patches{ + gomonkey.ApplyMethod(reflect.TypeOf(kv), "Get", + func(_ *KvMock, ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + c := &clientv3.GetResponse{ + Header: &etcdserverpb.ResponseHeader{}, + Kvs: []*mvccpb.KeyValue{}, + } + c.Header.Revision = 1 + return c, nil + }), + gomonkey.ApplyFunc(clientv3.NewWatcher, func(c *clientv3.Client) clientv3.Watcher { + return &EtcdWatcherMock{} + }), + } + defer func() { + for _, patch := range patches { + patch.Reset() + } + }() + convey.Convey("watch etcd", t, func() { + go watcher.StartWatch() + e := &Event{ + Type: PUT, + Key: "/xxx", + Value: []byte("test"), + PrevValue: nil, + Rev: 0, + } + time.Sleep(500 * time.Millisecond) + watcher.sendEvent(e) + close(stopCh) + event := <-resultCh + convey.So(event, convey.ShouldEqual, e) + }) +} + +type fakeKV struct { + cache map[string]string +} + +func (f *fakeKV) Put(ctx context.Context, key, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error) { + f.cache[key] = val + return nil, nil +} + +func (f *fakeKV) Delete(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.DeleteResponse, error) { + delete(f.cache, key) + return nil, nil +} + +func (f *fakeKV) Do(ctx context.Context, op clientv3.Op) (clientv3.OpResponse, error) { + return clientv3.OpResponse{}, nil +} + +func (f *fakeKV) Txn(ctx context.Context) clientv3.Txn { + return nil +} + +func (f *fakeKV) Compact(ctx context.Context, rev int64, opts ...clientv3.CompactOption) (*clientv3.CompactResponse, error) { + return nil, nil +} + +func (f *fakeKV) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + if _, ok := f.cache[key]; !ok { + return nil, fmt.Errorf("Doesn't exist") + } + return &clientv3.GetResponse{Count: 1, Kvs: []*mvccpb.KeyValue{ + &mvccpb.KeyValue{Value: []byte(f.cache[key])}, + }}, nil +} + +func TestOptEtcd(t *testing.T) { + ew := &EtcdWatcher{ + watcher: &EtcdClient{ + Client: &clientv3.Client{}, + cond: sync.NewCond(&sync.Mutex{}), + }, + } + fakeKv := &fakeKV{cache: map[string]string{}} + defer gomonkey.ApplyFunc(clientv3.NewKV, func(c *clientv3.Client) clientv3.KV { + return fakeKv + }).Reset() + etcdCtx := EtcdCtxInfo{ + Cancel: func() {}, + } + key1 := "etcdKey" + val1 := "etcdValue" + key2 := "etcdKey2" + val2 := "etcdValue2" + + convey.Convey("Put", t, func() { + err := ew.watcher.Put(etcdCtx, key1, val1) + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("GetResponse", t, func() { + resp, err := ew.watcher.GetResponse(etcdCtx, key1) + convey.So(err, convey.ShouldBeNil) + convey.So(resp, convey.ShouldResemble, &clientv3.GetResponse{Count: 1, Kvs: []*mvccpb.KeyValue{ + &mvccpb.KeyValue{Value: []byte(val1)}, + }}) + }) + convey.Convey("Delete", t, func() { + err := ew.watcher.Delete(etcdCtx, key1) + convey.So(err, convey.ShouldBeNil) + + resp, err := ew.watcher.GetValues(etcdCtx, key1) + convey.So(err.Error(), convey.ShouldEqual, "Doesn't exist") + convey.So(resp, convey.ShouldBeNil) + }) + convey.Convey("GetValues", t, func() { + err := ew.watcher.Put(etcdCtx, key2, val2) + convey.So(err, convey.ShouldBeNil) + resp, err := ew.watcher.GetValues(etcdCtx, key2) + convey.So(err, convey.ShouldBeNil) + convey.So(resp, convey.ShouldResemble, []string{val2}) + }) +} + +func TestCreateEtcdCtxInfoWithTimeout(t *testing.T) { + convey.Convey("CreateEtcdCtxInfoWithTimeout", t, func() { + ctxInfoWithTimeout := CreateEtcdCtxInfoWithTimeout(context.TODO(), time.Second) + convey.So(ctxInfoWithTimeout, convey.ShouldNotBeNil) + }) +} + +func Test_run(t *testing.T) { + convey.Convey("run", t, func() { + defer gomonkey.ApplyFunc(clientv3.NewWatcher, func(c *clientv3.Client) clientv3.Watcher { + return &EtcdWatcherMock{} + }).Reset() + convey.Convey("the channel received the result may be closed", func() { + kv := &KvMock{} + client := &clientv3.Client{KV: kv} + etcdClient := &EtcdClient{Client: client, clientExitCh: make(chan struct{}), etcdStatusNow: true, cond: sync.NewCond(&sync.Mutex{})} + receiveCh := make(chan *Event, 1) + stopCh := make(chan struct{}) + e := &EtcdWatcher{watcher: etcdClient, ResultChan: receiveCh, stopCh: stopCh} + e.initialRev = 1 + watchCh := make(chan clientv3.WatchResponse, 1) + callCount := 0 + defer gomonkey.ApplyMethod(reflect.TypeOf(&EtcdWatcherMock{}), "Watch", + func(e *EtcdWatcherMock, ctx context.Context, key string, opts ...clientv3.OpOption) clientv3.WatchChan { + callCount++ + if callCount == 1 { + return watchCh + } + return make(chan clientv3.WatchResponse, 1) + }).Reset() + defer gomonkey.ApplyMethod(reflect.TypeOf(e.watcher), "GetEtcdStatusNow", func(e *EtcdClient) bool { + return false + }).Reset() + closeCount := 0 + defer gomonkey.ApplyMethod(reflect.TypeOf(kv), "Get", + func(_ *KvMock, ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + closeCount++ + return &clientv3.GetResponse{Header: &etcdserverpb.ResponseHeader{Revision: int64(closeCount)}}, nil + }).Reset() + go e.run() + time.Sleep(100 * time.Millisecond) + close(watchCh) + time.Sleep(100 * time.Millisecond) + convey.So(callCount, convey.ShouldEqual, 2) + close(stopCh) + }) + convey.Convey("sendEvent", func() { + eventCh := make(chan clientv3.WatchResponse, 1) + defer gomonkey.ApplyMethod(reflect.TypeOf(&EtcdWatcherMock{}), "Watch", + func(e *EtcdWatcherMock, ctx context.Context, key string, opts ...clientv3.OpOption) clientv3.WatchChan { + return eventCh + }).Reset() + stopCh := make(chan struct{}) + receiveCh := make(chan *Event, defaultEventChanSize) + kv := &KvMock{} + client := &clientv3.Client{KV: kv} + etcdClient := &EtcdClient{Client: client, clientExitCh: make(chan struct{}), etcdStatusNow: true, cond: sync.NewCond(&sync.Mutex{})} + e := &EtcdWatcher{watcher: etcdClient, stopCh: stopCh, ResultChan: receiveCh} + e.initialRev = 1 + closeCount := 0 + defer gomonkey.ApplyMethod(reflect.TypeOf(kv), "Get", + func(_ *KvMock, ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + closeCount++ + return &clientv3.GetResponse{Header: &etcdserverpb.ResponseHeader{Revision: int64(closeCount)}}, nil + }).Reset() + go e.run() + eventCh <- clientv3.WatchResponse{Events: []*clientv3.Event{{Kv: &mvccpb.KeyValue{Key: []byte("key1"), Value: []byte("value1")}}}} + event := <-receiveCh + convey.So(event.Key, convey.ShouldEqual, "key1") + convey.So(string(event.Value), convey.ShouldEqual, "value1") + close(stopCh) + }) + convey.Convey("enable cache", func() { + os.Remove("etcdCacheMeta_#sn#function") + os.Remove("etcdCacheData_#sn#function") + os.Remove("etcdCacheData_#sn#function_backup") + defer gomonkey.ApplyMethod(reflect.TypeOf(&KvMock{}), "Get", func(_ *KvMock, ctx context.Context, + key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + return nil, errors.New("some error") + }).Reset() + stopCh := make(chan struct{}) + receiveCh := make(chan *Event, defaultEventChanSize) + cacheCh := make(chan *Event, defaultEventChanSize) + kv := &KvMock{} + client := &clientv3.Client{KV: kv} + etcdClient := &EtcdClient{Client: client, clientExitCh: make(chan struct{}), etcdStatusNow: true} + e := &EtcdWatcher{watcher: etcdClient, stopCh: stopCh, ResultChan: receiveCh, CacheChan: cacheCh, + key: "/sn/function", cacheConfig: EtcdCacheConfig{ + EnableCache: true, + PersistPath: "./", + FlushInterval: 10, + }} + os.WriteFile("./etcdCacheMeta_#sn#function", []byte(`{"revision":101,"cacheMD5":"5642747b723c9497e2b7324b49fb0513"}`), 0600) + os.WriteFile("./etcdCacheData_#sn#function", []byte("/sn/function/123/goodbye/latest|101|{\"name\":\"goodbye\",\"version\":\"latest\"}\n/sn/function/123/hello/latest|100|{\"name\":\"hello\",\"version\":\"latest\"}\n"), 0600) + go e.run() + time.Sleep(500 * time.Millisecond) + convey.So(len(e.ResultChan), convey.ShouldEqual, 3) + event1 := <-e.ResultChan + event2 := <-e.ResultChan + convey.So(event1, convey.ShouldResemble, &Event{ + Rev: 101, + Type: PUT, + Key: "/sn/function/123/goodbye/latest", + Value: []byte(`{"name":"goodbye","version":"latest"}`), + }) + convey.So(event2, convey.ShouldResemble, &Event{ + Rev: 100, + Type: PUT, + Key: "/sn/function/123/hello/latest", + Value: []byte(`{"name":"hello","version":"latest"}`), + }) + close(stopCh) + }) + }) + os.Remove("etcdCacheMeta_#sn#function") + os.Remove("etcdCacheData_#sn#function") + os.Remove("etcdCacheData_#sn#function_backup") +} + +func TestEtcdWatcher_EtcdHistory(t *testing.T) { + type fields struct { + filter EtcdWatcherFilter + handler EtcdWatcherHandler + watcher *EtcdClient + ResultChan chan *Event + resultChanWG *sync.WaitGroup + stopCh <-chan struct{} + key string + initialRev int64 + } + type args struct { + revision int64 + } + tests := []struct { + name string + fields fields + args args + patchesFunc mockUtils.PatchesFunc + }{ + {"case1", fields{watcher: &EtcdClient{cond: sync.NewCond(&sync.Mutex{})}}, args{revision: -1}, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(clientv3.NewWatcher, func(c *clientv3.Client) clientv3.Watcher { + return &EtcdWatcherMock{} + }), + gomonkey.ApplyMethod(reflect.TypeOf(&EtcdWatcherMock{}), "Watch", + func(e *EtcdWatcherMock, ctx context.Context, key string, opts ...clientv3.OpOption) clientv3.WatchChan { + ch := make(chan clientv3.WatchResponse, 1) + go close(ch) + return ch + })}) + return patches + }}, + {"case2 watch chan nil", fields{watcher: &EtcdClient{cond: sync.NewCond(&sync.Mutex{})}}, args{revision: -1}, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(clientv3.NewWatcher, func(c *clientv3.Client) clientv3.Watcher { + return &EtcdWatcherMock{} + }), + gomonkey.ApplyMethod(reflect.TypeOf(&EtcdWatcherMock{}), "Watch", + func(e *EtcdWatcherMock, ctx context.Context, key string, opts ...clientv3.OpOption) clientv3.WatchChan { + return nil + })}) + return patches + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + patches := tt.patchesFunc() + ew := &EtcdWatcher{ + filter: tt.fields.filter, + handler: tt.fields.handler, + watcher: tt.fields.watcher, + ResultChan: tt.fields.ResultChan, + resultChanWG: tt.fields.resultChanWG, + stopCh: tt.fields.stopCh, + key: tt.fields.key, + initialRev: tt.fields.initialRev, + } + ew.EtcdHistory(tt.args.revision) + patches.ResetAll() + }) + } +} + +func TestEtcdClient_Get(t *testing.T) { + e := &EtcdClient{} + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctxInfo := EtcdCtxInfo{Ctx: ctx, Cancel: cancel} + + key := "test-key" + response := &clientv3.GetResponse{} + + patches := gomonkey.NewPatches() + defer patches.Reset() + + patches.ApplyFuncReturn(clientv3.NewKV, &clientv3.Client{}) + patches.ApplyMethodReturn(&clientv3.Client{}, "Get", response, nil) + + got, err := e.Get(ctxInfo, key) + + assert.NoError(t, err) + assert.Equal(t, response, got) +} diff --git a/frontend/pkg/common/faas_common/instance/util.go b/frontend/pkg/common/faas_common/instance/util.go new file mode 100644 index 0000000000000000000000000000000000000000..6c2fc929824b1f9694ba8d903d7e345b7574ddde --- /dev/null +++ b/frontend/pkg/common/faas_common/instance/util.go @@ -0,0 +1,58 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package instance +package instance + +import ( + "encoding/json" + "fmt" + "strings" + + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/types" +) + +const ( + keySeparator = "/" + + instanceIDValueIndex = 13 + validEtcdKeyLenForInstance = 14 +) + +// GetInstanceIDFromEtcdKey gets instance id from etcd key of instance +func GetInstanceIDFromEtcdKey(etcdKey string) string { + items := strings.Split(etcdKey, keySeparator) + if len(items) != validEtcdKeyLenForInstance { + return "" + } + return fmt.Sprintf("%s", items[instanceIDValueIndex]) +} + +// GetInsSpecFromEtcdValue gets InstanceSpecification from etcd value of instance +func GetInsSpecFromEtcdValue(etcdKey string, etcdValue []byte) *types.InstanceSpecification { + insSpec := &types.InstanceSpecification{} + if len(etcdValue) != 0 { + err := json.Unmarshal(etcdValue, insSpec) + if err != nil { + log.GetLogger().Errorf("failed to unmarshal etcd value to instance specification %s", err.Error()) + return nil + } + } else { + log.GetLogger().Warnf("etcd value is empty when get instance specification from key %s", etcdKey) + } + return insSpec +} diff --git a/frontend/pkg/common/faas_common/instance/util_test.go b/frontend/pkg/common/faas_common/instance/util_test.go new file mode 100644 index 0000000000000000000000000000000000000000..0690a7ebcb2205a2a0dd0553e370ac2410f9757e --- /dev/null +++ b/frontend/pkg/common/faas_common/instance/util_test.go @@ -0,0 +1,84 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package instance + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + commonTypes "frontend/pkg/common/faas_common/types" +) + +func TestGetInstanceIDFromEtcdKey(t *testing.T) { + etcdKey := "/sn/instance/business/yrk/tenant/123/function/faasscheduler/version/$latest/defaultaz/requestID/abc" + instanceID := GetInstanceIDFromEtcdKey(etcdKey) + assert.Equal(t, "abc", instanceID) + + instanceIDNil := GetInstanceIDFromEtcdKey("") + assert.Equal(t, "", instanceIDNil) +} + +func TestGetInsSpecFromEtcdValue(t *testing.T) { + etcdValue := []byte("{\"instanceID\":\"51f71580-3a07-4000-8000-004b56e7f471\",\"requestID\":\"7fb31" + + "b50-7c5a-11ed-a991-fa163e3523c8\",\"runtimeID\":\"runtime-e06fe343-0000-4000-8000-00bbad15e23" + + "8\",\"runtimeAddress\":\"10.244.162.129:33333\",\"functionAgentID\":\"function_agent_10.244.16" + + "2.129-33333\",\"functionProxyID\":\"dggphis35893-8490\",\"function\":\"12345678901234561234567" + + "890123456/0-system-hello/$latest\",\"resources\":{\"resources\":{\"Memory\":{\"name\":\"Memor" + + "y\",\"scalar\":{\"value\":500}},\"CPU\":{\"name\":\"CPU\",\"scalar\":{\"value\":500}}}},\"sched" + + "uleOption\":{\"affinity\":{\"instanceAffinity\":{}}},\"instanceStatus\":{\"code\":3,\"msg\":\"i" + + "nstance is running\"}}") + insSpecTrans := GetInsSpecFromEtcdValue("", etcdValue) + insSpecExpected := &commonTypes.InstanceSpecification{ + InstanceID: "51f71580-3a07-4000-8000-004b56e7f471", + RequestID: "7fb31b50-7c5a-11ed-a991-fa163e3523c8", + RuntimeID: "runtime-e06fe343-0000-4000-8000-00bbad15e238", + RuntimeAddress: "10.244.162.129:33333", + FunctionAgentID: "function_agent_10.244.162.129-33333", + FunctionProxyID: "dggphis35893-8490", + Function: "12345678901234561234567890123456/0-system-hello/$latest", + RestartPolicy: "", + Resources: commonTypes.Resources{ + Resources: map[string]commonTypes.Resource{ + "CPU": commonTypes.Resource{ + Name: "CPU", + Scalar: commonTypes.ValueScalar{Value: 500}, + }, + "Memory": commonTypes.Resource{ + Name: "Memory", + Scalar: commonTypes.ValueScalar{Value: 500}, + }, + }, + }, + ActualUse: commonTypes.Resources{}, + ScheduleOption: commonTypes.ScheduleOption{ + Affinity: commonTypes.Affinity{ + InstanceAffinity: commonTypes.InstanceAffinity{}, + }, + }, + CreateOptions: nil, + Labels: nil, + StartTime: "", + InstanceStatus: commonTypes.InstanceStatus{Code: 3, Msg: "instance is running"}, + JobID: "", + SchedulerChain: nil, + } + assert.Equal(t, insSpecExpected, insSpecTrans) + + insSpecNil := GetInsSpecFromEtcdValue("", []byte("")) + assert.Equal(t, true, insSpecNil != nil) +} diff --git a/frontend/pkg/common/faas_common/instanceconfig/util.go b/frontend/pkg/common/faas_common/instanceconfig/util.go new file mode 100644 index 0000000000000000000000000000000000000000..d607662b6838536539555e6041ee38882f034c5c --- /dev/null +++ b/frontend/pkg/common/faas_common/instanceconfig/util.go @@ -0,0 +1,122 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package instanceconfig - +package instanceconfig + +import ( + "encoding/json" + "fmt" + "strings" + + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/types" + wisecloudtypes "frontend/pkg/common/faas_common/wisecloudtool/types" +) + +const ( + keySeparator = "/" + insConfigTenantValueIndex = 7 + insConfigFuncNameValueIndex = 9 + insConfigVersionValueIndex = 11 + validEtcdKeyLenForInsConfig = 12 + insConfigLabelValueIndex = 13 + validEtcdKeyLenForInsWithLabelConf = 14 + + insConfigKeyIndex = 1 + insConfigClusterKeyIndex = 4 + insConfigClusterValueIndex = 5 + insConfigTenantKeyIndex = 6 + insConfigFunctionKeyIndex = 8 + insConfigLabelKeyIndex = 12 + + functionClusterKeyIdx = 5 + + // InsConfigEtcdPrefix - 函数实例配置项元数据key前缀 + InsConfigEtcdPrefix = "/instances" +) + +// GetLabelFromInstanceConfigEtcdKey - +func GetLabelFromInstanceConfigEtcdKey(etcdKey string) string { + items := strings.Split(etcdKey, keySeparator) + if len(items) != validEtcdKeyLenForInsWithLabelConf { + return "" + } + return items[insConfigLabelValueIndex] +} + +// ParseInstanceConfigFromEtcdEvent - +func ParseInstanceConfigFromEtcdEvent(etcdKey string, etcdValue []byte) (*Configuration, error) { + items := strings.Split(etcdKey, keySeparator) + if len(items) != validEtcdKeyLenForInsConfig && len(items) != validEtcdKeyLenForInsWithLabelConf { + return nil, fmt.Errorf("etcdKey format error") + } + + funcKey := fmt.Sprintf("%s/%s/%s", items[insConfigTenantValueIndex], items[insConfigFuncNameValueIndex], + items[insConfigVersionValueIndex]) + + label := "" + if len(items) == validEtcdKeyLenForInsWithLabelConf { + label = items[insConfigLabelValueIndex] + } + + if len(etcdValue) == 0 { + return nil, fmt.Errorf("etcdValue is empty") + } + insConfig := &Configuration{} + err := json.Unmarshal(etcdValue, insConfig) + if err != nil { + return nil, fmt.Errorf("unmarshal etcdValue failed, err: %s", err.Error()) + } + + insConfig.FuncKey = funcKey + insConfig.InstanceLabel = label + return insConfig, nil +} + +// Configuration - +type Configuration struct { + FuncKey string + InstanceLabel string + InstanceMetaData types.InstanceMetaData `json:"instanceMetaData" valid:",optional"` + NuwaRuntimeInfo wisecloudtypes.NuwaRuntimeInfo `json:"nuwaRuntimeInfo" valid:",optional"` +} + +// DeepCopy return a Configuration Copy +func (i *Configuration) DeepCopy() *Configuration { + return &(*i) +} + +// GetWatcherFilter - +func GetWatcherFilter(clusterId string) func(event *etcd3.Event) bool { + return func(event *etcd3.Event) bool { + items := strings.Split(event.Key, keySeparator) + if len(items) != validEtcdKeyLenForInsConfig && len(items) != validEtcdKeyLenForInsWithLabelConf { + return true + } + if items[insConfigKeyIndex] != "instances" || items[insConfigClusterKeyIndex] != "cluster" || + items[insConfigTenantKeyIndex] != "tenant" || items[insConfigFunctionKeyIndex] != "function" { + return true + } + if len(items) == validEtcdKeyLenForInsWithLabelConf && items[insConfigLabelKeyIndex] != "label" { + return true + } + if clusterId != items[insConfigClusterValueIndex] { + return true + } + return false + } +} diff --git a/frontend/pkg/common/faas_common/instanceconfig/util_test.go b/frontend/pkg/common/faas_common/instanceconfig/util_test.go new file mode 100644 index 0000000000000000000000000000000000000000..a3de3d67f7ddb585051b93873d665d3dedb2bed6 --- /dev/null +++ b/frontend/pkg/common/faas_common/instanceconfig/util_test.go @@ -0,0 +1,84 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package instanceconfig + +import ( + "encoding/json" + "testing" + + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/instance" + "frontend/pkg/common/faas_common/types" + wisecloudtypes "frontend/pkg/common/faas_common/wisecloudtool/types" +) + +func TestParseInstanceConfigFromEtcdEvent(t *testing.T) { + convey.Convey("Test ParseInstanceConfigFromEtcdEvent", t, func() { + testConfig := &Configuration{ + InstanceMetaData: types.InstanceMetaData{PoolID: "test"}, + NuwaRuntimeInfo: wisecloudtypes.NuwaRuntimeInfo{WisecloudRuntimeId: "runtime1"}, + } + testConfigData, _ := json.Marshal(testConfig) + + convey.Convey("should parse config with label successfully", func() { + etcdKey := "/instances/business/yrk/cluster/cluster001/tenant/12345678901234561234567890123456/function/0@test111@yrfunc111/version/latest/label/aaa" + config, err := ParseInstanceConfigFromEtcdEvent(etcdKey, testConfigData) + convey.So(err, convey.ShouldBeNil) + convey.So(config.FuncKey, convey.ShouldEqual, "12345678901234561234567890123456/0@test111@yrfunc111/latest") + convey.So(config.InstanceLabel, convey.ShouldEqual, "aaa") + }) + + convey.Convey("should return error for invalid key format", func() { + key := "/invalid/key" + _, err := ParseInstanceConfigFromEtcdEvent(key, testConfigData) + convey.So(err, convey.ShouldNotBeNil) + }) + }) +} + +func TestGetWatcherFilter(t *testing.T) { + convey.Convey("Test GetWatcherFilter", t, func() { + filter := GetWatcherFilter("cluster1") + + convey.Convey("should filter matching cluster key", func() { + event := &etcd3.Event{ + Key: "/instances/business/yrk/cluster/cluster1/tenant/t1/function/f1/version/v1", + } + convey.So(filter(event), convey.ShouldBeFalse) + }) + + convey.Convey("should not filter invalid key structure", func() { + event := &etcd3.Event{ + Key: "/invalid/key", + } + convey.So(filter(event), convey.ShouldBeTrue) + }) + }) +} + +func TestGetLabelFromInstanceConfigEtcdKey(t *testing.T) { + etcdKey := "/instances/business/yrk/cluster/cluster001/tenant/12345678901234561234567890123456/function/0@test111@yrfunc111/version/latest" + label := GetLabelFromInstanceConfigEtcdKey(etcdKey) + assert.Equal(t, "", label) + + etcdKey = "/instances/business/yrk/cluster/cluster001/tenant/12345678901234561234567890123456/function/0@test111@yrfunc111/version/latest/label/aaa" + label = instance.GetInstanceIDFromEtcdKey(etcdKey) + assert.Equal(t, "aaa", label) +} diff --git a/frontend/pkg/common/faas_common/k8sclient/tools.go b/frontend/pkg/common/faas_common/k8sclient/tools.go new file mode 100644 index 0000000000000000000000000000000000000000..619a617b4f760de12d059ff279587e892faa1628 --- /dev/null +++ b/frontend/pkg/common/faas_common/k8sclient/tools.go @@ -0,0 +1,287 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package k8sclient include some k8s Client operation +package k8sclient + +import ( + "context" + "fmt" + "reflect" + "sync" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + "frontend/pkg/common/faas_common/logger/log" +) + +// KubeClient - +type KubeClient struct { + Client kubernetes.Interface +} + +var ( + // KubeClientSet - + KubeClientSet *KubeClient + kubeClientOnce sync.Once +) + +// GetkubeClient is used to obtain a K8S Client +func GetkubeClient() *KubeClient { + kubeClientOnce.Do(func() { + // create Kubernetes config + config, err := rest.InClusterConfig() + if err != nil { + log.GetLogger().Errorf("Failed to create Kubernetes config: %v", err) + return + } + + // create Kubernetes Client + client, err := kubernetes.NewForConfig(config) + if err != nil { + log.GetLogger().Errorf("Failed to create Kubernetes Client: %v", err) + return + } + + KubeClientSet = &KubeClient{ + Client: client, + } + }) + return KubeClientSet +} + +// DeleteK8sService - +func (kc *KubeClient) DeleteK8sService(namespace string, serviceName string) error { + if kc == nil { + return fmt.Errorf("kubeclient is nil") + } + err := kc.Client.CoreV1().Services(namespace).Delete(context.TODO(), serviceName, metav1.DeleteOptions{}) + if err != nil { + if errors.IsNotFound(err) { + log.GetLogger().Infof("Service %s in namespace %s not found", serviceName, namespace) + return nil + } + return err + } + log.GetLogger().Infof("Service %s in namespace %s deleted", serviceName, namespace) + return nil +} + +// CreateK8sService - +func (kc *KubeClient) CreateK8sService(service *v1.Service) error { + if kc == nil { + return fmt.Errorf("kubeclient is nil") + } + + // delete service + if err := kc.DeleteK8sService(service.Namespace, service.Name); err != nil { + return err + } + // create Service + result, err := kc.Client.CoreV1().Services(service.Namespace).Create(context.TODO(), service, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create Service: %s", err.Error()) + } + + log.GetLogger().Infof("created Service %q with IP %q", result.GetObjectMeta().GetName(), result.Spec.ClusterIP) + return nil +} + +// CreateK8sConfigMap - +func (kc *KubeClient) CreateK8sConfigMap(configMap *v1.ConfigMap) error { + if kc == nil { + return fmt.Errorf("kubeclient is nil") + } + + // delete configMap + if err := kc.DeleteK8sConfigMap(configMap.Namespace, configMap.Name); err != nil { + return err + } + // create configMap + result, err := kc.Client.CoreV1().ConfigMaps(configMap.Namespace).Create(context.TODO(), + configMap, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create ConfigMap: %s", err.Error()) + } + + log.GetLogger().Infof("created ConfigMap: %s", result.GetObjectMeta().GetName()) + return nil +} + +// DeleteK8sConfigMap - +func (kc *KubeClient) DeleteK8sConfigMap(namespace string, configMapName string) error { + if kc == nil { + return fmt.Errorf("kubeclient is nil") + } + + // delete configMap + err := kc.Client.CoreV1().ConfigMaps(namespace).Delete(context.TODO(), configMapName, metav1.DeleteOptions{}) + if err != nil { + if errors.IsNotFound(err) { + log.GetLogger().Infof("configMap %s in namespace %s not found", configMapName, namespace) + return nil + } + return err + } + log.GetLogger().Infof("configMap %s in namespace %s deleted", configMapName, namespace) + return nil +} + +// UpdateK8sConfigMap - +func (kc *KubeClient) UpdateK8sConfigMap(configMap *v1.ConfigMap) error { + if kc == nil { + return fmt.Errorf("kubeclient is nil") + } + _, err := kc.Client.CoreV1().ConfigMaps(configMap.Namespace).Get(context.TODO(), configMap.Name, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + log.GetLogger().Infof("configMap %s in namespace %s not found", configMap.Name, configMap.Namespace) + } + return err + } + _, err = kc.Client.CoreV1().ConfigMaps(configMap.Namespace).Update(context.TODO(), configMap, metav1.UpdateOptions{}) + if err != nil { + log.GetLogger().Errorf("update configmap failed, error is %s", err.Error()) + return err + } + log.GetLogger().Infof("configMap %s in namespace %s updated", configMap.Name, configMap.Namespace) + return nil +} + +// GetK8sConfigMap - +func (kc *KubeClient) GetK8sConfigMap(namespace string, configMapName string) (*v1.ConfigMap, error) { + if kc == nil { + return nil, fmt.Errorf("kubeclient is nil") + } + configmap, err := kc.Client.CoreV1().ConfigMaps(namespace).Get(context.TODO(), configMapName, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + log.GetLogger().Infof("configMap %s in namespace %s not found", configMapName, namespace) + } + return nil, err + } + log.GetLogger().Infof("Get configMap %s in namespace %s updated", configMapName, namespace) + return configmap, nil +} + +// GetK8sSecret - +func (kc *KubeClient) GetK8sSecret(namespace string, secretName string) (*v1.Secret, error) { + if kc == nil { + return nil, fmt.Errorf("kubeclient is nil") + } + ctx := context.TODO() + secret, err := kc.Client.CoreV1().Secrets(namespace).Get(ctx, secretName, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + log.GetLogger().Infof("secret %s not found", secretName) + return nil, err + } + log.GetLogger().Errorf("secret %s get failed, err is: %s", secretName, err) + return nil, err + } + log.GetLogger().Errorf("secret %s already exists, no need create.", secretName) + return secret, nil +} + +// CreateK8sSecret - +func (kc *KubeClient) CreateK8sSecret(namespace string, s *v1.Secret) (*v1.Secret, error) { + if kc == nil { + return nil, fmt.Errorf("kubeclient is nil") + } + ctx := context.TODO() + secret, err := kc.Client.CoreV1().Secrets(namespace).Create(ctx, s, metav1.CreateOptions{}) + if err != nil { + log.GetLogger().Errorf("k8s failed to create secret: %s, secretName: %s", err.Error(), s.Name) + return nil, err + } + log.GetLogger().Infof("secret %s in namespace %s created", secret.Name, namespace) + + return secret, nil +} + +// UpdateK8sSecret - +func (kc *KubeClient) UpdateK8sSecret(namespace string, s *v1.Secret) (*v1.Secret, error) { + if kc == nil { + return nil, fmt.Errorf("kubeclient is nil") + } + ctx := context.TODO() + secret, err := kc.Client.CoreV1().Secrets(namespace).Update(ctx, s, metav1.UpdateOptions{}) + if err != nil { + log.GetLogger().Errorf("k8s failed to update secret: %s, secretName: %s", err.Error(), s.Name) + return nil, err + } + log.GetLogger().Infof("secret %s in namespace %s updated", secret.Name, namespace) + + return secret, nil +} + +// DeleteK8sSecret - +func (kc *KubeClient) DeleteK8sSecret(namespace string, secretName string) error { + if kc == nil { + return fmt.Errorf("kubeclient is nil") + } + ctx := context.TODO() + err := kc.Client.CoreV1().Secrets(namespace).Delete(ctx, secretName, metav1.DeleteOptions{}) + + if err != nil { + if errors.IsNotFound(err) { + log.GetLogger().Infof("secret %s in namespace %s not found", secretName, namespace) + return nil + } + log.GetLogger().Errorf("k8s failed to delete secret: %s, secretName: %s", err.Error(), secretName) + return err + } + log.GetLogger().Infof("secret %s in namespace %s deleted successfully", secretName, namespace) + + return nil +} + +// CreateOrUpdateConfigMap - +func (kc *KubeClient) CreateOrUpdateConfigMap(c *v1.ConfigMap) error { + if kc == nil { + return fmt.Errorf("kubeclient is nil") + } + ctx := context.TODO() + oldConfig, getErr := kc.Client.CoreV1().ConfigMaps(c.Namespace).Get(ctx, c.Name, metav1.GetOptions{}) + if getErr != nil && errors.IsNotFound(getErr) { + log.GetLogger().Infof("Creating a new Configmap, Configmap.Name: %s", c.Name) + _, createErr := kc.Client.CoreV1().ConfigMaps(c.Namespace).Create(ctx, c, metav1.CreateOptions{}) + if createErr != nil { + log.GetLogger().Errorf("k8s failed to create configmap: %s, traceID: %s", + createErr.Error(), "TraceID") + return createErr + } + return nil + } + if getErr != nil { + log.GetLogger().Errorf("failed to get configmap: %s, err:%v", c.Name, getErr.Error()) + return getErr + } + + if !reflect.DeepEqual(oldConfig, c) { + _, updateErr := kc.Client.CoreV1().ConfigMaps(c.Namespace).Update(ctx, c, metav1.UpdateOptions{}) + if updateErr != nil { + log.GetLogger().Errorf("k8s failed to update configmap: %s, traceID: %s", updateErr.Error(), + "TraceID") + return updateErr + } + } + return nil +} diff --git a/frontend/pkg/common/faas_common/k8sclient/tools_test.go b/frontend/pkg/common/faas_common/k8sclient/tools_test.go new file mode 100644 index 0000000000000000000000000000000000000000..0950d341fe5f165db588985bc37e26d00f2450a7 --- /dev/null +++ b/frontend/pkg/common/faas_common/k8sclient/tools_test.go @@ -0,0 +1,479 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package k8sclient include some k8s client operation +package k8sclient + +import ( + "context" + "fmt" + "sync" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" + k8testing "k8s.io/client-go/testing" +) + +func TestGetkubeClient(t *testing.T) { + defer gomonkey.ApplyFunc(rest.InClusterConfig, func() (*rest.Config, error) { + return &rest.Config{}, nil + }).Reset() + convey.Convey("get client success", t, func() { + defer gomonkey.ApplyFunc(kubernetes.NewForConfig, func(c *rest.Config) (*kubernetes.Clientset, error) { + return &kubernetes.Clientset{}, nil + }).Reset() + client := GetkubeClient() + convey.So(client, convey.ShouldNotBeNil) + }) + KubeClientSet = nil + kubeClientOnce = sync.Once{} + convey.Convey("get client error", t, func() { + defer gomonkey.ApplyFunc(kubernetes.NewForConfig, func(c *rest.Config) (*kubernetes.Clientset, error) { + return nil, fmt.Errorf("get client error") + }).Reset() + client := GetkubeClient() + convey.So(client, convey.ShouldBeNil) + }) + kubeClientOnce = sync.Once{} + convey.Convey("get cfg error", t, func() { + defer gomonkey.ApplyFunc(rest.InClusterConfig, func() (*rest.Config, error) { + return nil, fmt.Errorf("get cfg error") + }).Reset() + client := GetkubeClient() + convey.So(client, convey.ShouldBeNil) + }) +} + +func TestDeleteK8sService(t *testing.T) { + client := fake.NewSimpleClientset() + KubeClientSet = &KubeClient{client} + namespace := "default" + serviceName := "frontend" + service := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName, + Namespace: namespace, + }, + Spec: v1.ServiceSpec{ + Selector: map[string]string{ + "app": "frontend", + }, + Ports: []v1.ServicePort{ + { + Name: "http", + Protocol: "TCP", + Port: 8888, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 32104, + }, + NodePort: 31222, + }, + }, + Type: v1.ServiceTypeNodePort, + }, + } + client.PrependReactor("delete", "services", func(action k8testing.Action) (handled bool, ret runtime.Object, err error) { + deleteAction := action.(k8testing.DeleteAction) + if deleteAction.GetName() == service.Name && deleteAction.GetNamespace() == service.Namespace { + return true, service, nil + } + return true, nil, fmt.Errorf("Not found") + }) + + convey.Convey("delete service success", t, func() { + err := KubeClientSet.DeleteK8sService(namespace, serviceName) + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("delete service not found", t, func() { + err := KubeClientSet.DeleteK8sService(namespace, "error service name") + convey.So(err.Error(), convey.ShouldContainSubstring, "Not found") + }) + convey.Convey("delete service not found", t, func() { + client.PrependReactor("delete", "services", func(action k8testing.Action) (handled bool, ret runtime.Object, err error) { + deleteAction := action.(k8testing.DeleteAction) + if deleteAction.GetName() == service.Name && deleteAction.GetNamespace() == service.Namespace { + return true, service, fmt.Errorf("delete error") + } + return false, nil, nil + }) + + err := KubeClientSet.DeleteK8sService(namespace, serviceName) + convey.So(err.Error(), convey.ShouldContainSubstring, "delete error") + }) + convey.Convey("KubeClientSet is nil", t, func() { + KubeClientSet = nil + err := KubeClientSet.DeleteK8sService(namespace, serviceName) + convey.So(err, convey.ShouldNotBeNil) + }) +} + +func TestCreateK8sService(t *testing.T) { + client := fake.NewSimpleClientset() + KubeClientSet = &KubeClient{client} + service := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "frontend", + Namespace: "default", + }, + Spec: v1.ServiceSpec{ + Selector: map[string]string{ + "app": "frontend", + }, + Ports: []v1.ServicePort{ + { + Name: "http", + Protocol: "TCP", + Port: 8888, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 32104, + }, + NodePort: 31222, + }, + }, + Type: v1.ServiceTypeNodePort, + }, + } + + convey.Convey("create service success", t, func() { + err := KubeClientSet.CreateK8sService(service) + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("create service error", t, func() { + client.PrependReactor("create", "services", func(action k8testing.Action) (handled bool, ret runtime.Object, err error) { + createAction := action.(k8testing.CreateAction) + if createAction.GetObject().(*v1.Service).Name == service.Name && createAction.GetNamespace() == service.Namespace { + return true, service, fmt.Errorf("failed to create service") + } + return false, nil, nil + }) + err := KubeClientSet.CreateK8sService(service) + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("KubeClientSet is nil", t, func() { + KubeClientSet = nil + err := KubeClientSet.CreateK8sService(service) + convey.So(err, convey.ShouldNotBeNil) + }) +} + +func TestCreateK8sConfigMap(t *testing.T) { + client := fake.NewSimpleClientset() + KubeClientSet = &KubeClient{client} + configmap := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test_configmap", + Namespace: "default", + }, + Data: map[string]string{"key": "value"}, + } + + convey.Convey("create configmap success", t, func() { + err := KubeClientSet.CreateK8sConfigMap(configmap) + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("create configmap error", t, func() { + client.PrependReactor("create", "configmaps", func(action k8testing.Action) (handled bool, ret runtime.Object, err error) { + createAction := action.(k8testing.CreateAction) + if createAction.GetObject().(*v1.ConfigMap).Name == configmap.Name && createAction.GetNamespace() == configmap.Namespace { + return true, configmap, fmt.Errorf("failed to create configmap") + } + return false, nil, nil + }) + err := KubeClientSet.CreateK8sConfigMap(configmap) + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("KubeClientSet is nil", t, func() { + KubeClientSet = nil + err := KubeClientSet.CreateK8sConfigMap(configmap) + convey.So(err, convey.ShouldNotBeNil) + }) +} + +func TestDeleteK8sConfigMap(t *testing.T) { + client := fake.NewSimpleClientset() + KubeClientSet = &KubeClient{client} + namespace := "default" + configmapName := "test_configmap" + configmap := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configmapName, + Namespace: namespace, + }, + Data: map[string]string{"key": "value"}, + } + + convey.Convey("delete configmap success", t, func() { + err := KubeClientSet.DeleteK8sConfigMap(namespace, configmapName) + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("delete configmap error", t, func() { + client.PrependReactor("delete", "configmaps", func(action k8testing.Action) (handled bool, ret runtime.Object, err error) { + deleteAction := action.(k8testing.DeleteAction) + if deleteAction.GetName() == configmap.Name && deleteAction.GetNamespace() == configmap.Namespace { + return true, configmap, fmt.Errorf("failed to delete service") + } + return false, nil, nil + }) + err := KubeClientSet.CreateK8sConfigMap(configmap) + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("KubeClientSet is nil", t, func() { + KubeClientSet = nil + err := KubeClientSet.DeleteK8sConfigMap(namespace, configmapName) + convey.So(err, convey.ShouldNotBeNil) + }) +} + +func TestUpdateK8sConfigMap(t *testing.T) { + client := fake.NewSimpleClientset() + KubeClientSet = &KubeClient{client} + namespace := "default" + configmapName := "test_configmap" + configmap := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configmapName, + Namespace: namespace, + }, + Data: map[string]string{"key": "value"}, + } + _, err := client.CoreV1().ConfigMaps(namespace).Create(context.TODO(), configmap, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create configmap: %v", err) + } + convey.Convey("update configmap success", t, func() { + err := KubeClientSet.UpdateK8sConfigMap(configmap) + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("update configmap error", t, func() { + client.PrependReactor("update", "configmaps", func(action k8testing.Action) (handled bool, ret runtime.Object, err error) { + updateAction := action.(k8testing.UpdateAction) + if updateAction.GetObject().(*v1.ConfigMap).Name == configmap.Name && updateAction.GetNamespace() == configmap.Namespace { + return true, configmap, fmt.Errorf("failed to update configmap") + } + return false, nil, nil + }) + err := KubeClientSet.UpdateK8sConfigMap(configmap) + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("KubeClientSet is nil", t, func() { + KubeClientSet = nil + err := KubeClientSet.UpdateK8sConfigMap(configmap) + convey.So(err, convey.ShouldNotBeNil) + }) +} + +func TestGetK8sConfigMap(t *testing.T) { + client := fake.NewSimpleClientset() + KubeClientSet = &KubeClient{client} + namespace := "default" + configmapName := "test_configmap" + configmap := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configmapName, + Namespace: namespace, + }, + Data: map[string]string{"key": "value"}, + } + _, err := client.CoreV1().ConfigMaps(namespace).Create(context.TODO(), configmap, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create configmap: %v", err) + } + convey.Convey("get configmap success", t, func() { + _, err := KubeClientSet.GetK8sConfigMap(namespace, namespace) + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("KubeClientSet is nil", t, func() { + KubeClientSet = nil + err := KubeClientSet.UpdateK8sConfigMap(configmap) + convey.So(err, convey.ShouldNotBeNil) + }) +} + +func TestKubeClient_GetK8sSecret(t *testing.T) { + client := fake.NewSimpleClientset() + KubeClientSet = &KubeClient{client} + namespace := "default" + secretName := "test_secret" + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: namespace, + }, + Data: map[string][]byte{"key": []byte("value")}, + } + _, err := client.CoreV1().Secrets(namespace).Create(context.TODO(), secret, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create secret: %v", err) + } + convey.Convey("get secret success", t, func() { + _, err := KubeClientSet.GetK8sSecret(namespace, namespace) + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("KubeClientSet is nil", t, func() { + KubeClientSet = nil + _, err := KubeClientSet.GetK8sSecret(namespace, secretName) + convey.So(err, convey.ShouldNotBeNil) + }) +} + +func TestKubeClient_CreateK8sSecret(t *testing.T) { + client := fake.NewSimpleClientset() + KubeClientSet = &KubeClient{client} + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test_secret", + Namespace: "default", + }, + Data: map[string][]byte{"key": []byte("value")}, + } + + convey.Convey("create secret success", t, func() { + _, err := KubeClientSet.CreateK8sSecret("default", secret) + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("create secret error", t, func() { + client.PrependReactor("create", "secrets", func(action k8testing.Action) (handled bool, ret runtime.Object, err error) { + createAction := action.(k8testing.CreateAction) + if createAction.GetObject().(*v1.Secret).Name == secret.Name && createAction.GetNamespace() == secret.Namespace { + return true, secret, fmt.Errorf("failed to create secret") + } + return false, nil, nil + }) + _, err := KubeClientSet.CreateK8sSecret("default", secret) + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("KubeClientSet is nil", t, func() { + KubeClientSet = nil + _, err := KubeClientSet.CreateK8sSecret("default", secret) + convey.So(err, convey.ShouldNotBeNil) + }) +} + +func TestKubeClient_UpdateK8sSecret(t *testing.T) { + client := fake.NewSimpleClientset() + KubeClientSet = &KubeClient{client} + namespace := "default" + secretName := "test_secret" + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: namespace, + }, + Data: map[string][]byte{"key": []byte("value")}, + } + _, err := client.CoreV1().Secrets(namespace).Create(context.TODO(), secret, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create secret: %v", err) + } + convey.Convey("update secret success", t, func() { + _, err := KubeClientSet.UpdateK8sSecret(namespace, secret) + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("update secret error", t, func() { + client.PrependReactor("update", "secrets", func(action k8testing.Action) (handled bool, ret runtime.Object, err error) { + updateAction := action.(k8testing.UpdateAction) + if updateAction.GetObject().(*v1.Secret).Name == secret.Name && updateAction.GetNamespace() == secret.Namespace { + return true, secret, fmt.Errorf("failed to update secret") + } + return false, nil, nil + }) + _, err := KubeClientSet.UpdateK8sSecret(namespace, secret) + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("KubeClientSet is nil", t, func() { + KubeClientSet = nil + _, err := KubeClientSet.UpdateK8sSecret(namespace, secret) + convey.So(err, convey.ShouldNotBeNil) + }) +} + +func TestKubeClient_DeleteK8sSecret(t *testing.T) { + client := fake.NewSimpleClientset() + KubeClientSet = &KubeClient{client} + namespace := "default" + secretName := "test_secret" + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: namespace, + }, + Data: map[string][]byte{"key": []byte("value")}, + } + + convey.Convey("delete secret success", t, func() { + err := KubeClientSet.DeleteK8sSecret(namespace, secretName) + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("delete secret error", t, func() { + client.PrependReactor("delete", "secrets", func(action k8testing.Action) (handled bool, ret runtime.Object, err error) { + deleteAction := action.(k8testing.DeleteAction) + if deleteAction.GetName() == secret.Name && deleteAction.GetNamespace() == secret.Namespace { + return true, secret, fmt.Errorf("failed to delete service") + } + return false, nil, nil + }) + err := KubeClientSet.DeleteK8sSecret(namespace, secretName) + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("KubeClientSet is nil", t, func() { + KubeClientSet = nil + err := KubeClientSet.DeleteK8sSecret(namespace, secretName) + convey.So(err, convey.ShouldNotBeNil) + }) +} + +func TestKubeClient_CreateOrUpdateConfigMap(t *testing.T) { + client := fake.NewSimpleClientset() + KubeClientSet = &KubeClient{client} + configmap := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test_configmap", + Namespace: "default", + }, + Data: map[string]string{"key": "value"}, + } + configmap2 := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test_configmap", + Namespace: "default", + }, + Data: map[string]string{"key": "value1"}, + } + _, err := client.CoreV1().ConfigMaps("default").Create(context.TODO(), configmap2, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create configmap: %v", err) + } + + convey.Convey("create configmap success", t, func() { + err := KubeClientSet.CreateOrUpdateConfigMap(configmap) + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("KubeClientSet is nil", t, func() { + KubeClientSet = nil + err := KubeClientSet.CreateOrUpdateConfigMap(configmap) + convey.So(err, convey.ShouldNotBeNil) + }) +} diff --git a/frontend/pkg/common/faas_common/kernelrpc/connection/connection.go b/frontend/pkg/common/faas_common/kernelrpc/connection/connection.go new file mode 100644 index 0000000000000000000000000000000000000000..2da51eae566d3726bc7b99f6d78b3276a26d2efe --- /dev/null +++ b/frontend/pkg/common/faas_common/kernelrpc/connection/connection.go @@ -0,0 +1,41 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package connection - +package connection + +import ( + "time" + + "frontend/pkg/common/faas_common/grpc/pb" // production: package api + "frontend/pkg/common/faas_common/grpc/pb/runtime" +) + +// SendOption - +type SendOption struct { + Timeout time.Duration +} + +// SendCallback - +type SendCallback func(message *runtime.NotifyRequest) + +// Connection defines basic grpc connection +type Connection interface { + Send(message *api.StreamingMessage, option SendOption, callback SendCallback) (*api.StreamingMessage, error) + Recv() (*api.StreamingMessage, error) + Close() + CheckClose() chan struct{} +} diff --git a/frontend/pkg/common/faas_common/kernelrpc/connection/stream_connection.go b/frontend/pkg/common/faas_common/kernelrpc/connection/stream_connection.go new file mode 100644 index 0000000000000000000000000000000000000000..d029f441bbfb036e0d6cf475e540343b3abd51c8 --- /dev/null +++ b/frontend/pkg/common/faas_common/kernelrpc/connection/stream_connection.go @@ -0,0 +1,450 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package connection - +package connection + +import ( + "errors" + "reflect" + "sync" + "time" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/grpc/pb" // production: package api + "frontend/pkg/common/faas_common/grpc/pb/runtime" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/uuid" +) + +var ( + defaultChannelSize = 300000 +) + +var ( + // ErrStreamConnectionBroken is the error of stream connection broken + ErrStreamConnectionBroken = errors.New("stream connection is broken") + // ErrStreamConnectionClosed is the error of stream connection closed + ErrStreamConnectionClosed = errors.New("stream connection is closed") + // ErrRequestIDAlreadyExist is the error of requestID already exist + ErrRequestIDAlreadyExist = errors.New("requestID already exist") +) + +// StreamParams - +type StreamParams struct { + PeerAddr string + SendReqConcurrentNum int + SendRspConcurrentNum int + RecvConcurrentNum int +} + +// Stream is the common interface of stream for both server and client +type Stream interface { + Send(*api.StreamingMessage) error + Recv() (*api.StreamingMessage, error) +} + +// RepairStreamFunc repairs stream +type RepairStreamFunc func() Stream + +// HealthCheckFunc checks stream health +type HealthCheckFunc func() bool + +type sendAckPack struct { + rsp *api.StreamingMessage + err error +} + +type sendCbPack struct { + t time.Time + cb SendCallback +} + +// StreamConnection is an implementation of Connection with stream +type StreamConnection struct { + stream Stream + sendAckRecord map[string]chan sendAckPack + sendCbRecord map[string]sendCbPack + peerAddr string + closed bool + repairing bool + repairFunc RepairStreamFunc + healthFunc HealthCheckFunc + sendReqCh chan *api.StreamingMessage + sendRspCh chan *api.StreamingMessage + recvCh chan *api.StreamingMessage + repairCh chan struct{} + closeCh chan struct{} + *sync.RWMutex + *sync.Cond +} + +// CreateStreamConnection creates a StreamConnection +func CreateStreamConnection(stream Stream, params StreamParams, healthFunc HealthCheckFunc, + repairFunc RepairStreamFunc) Connection { + calibrateParams(¶ms) + mutex := new(sync.RWMutex) + sc := &StreamConnection{ + stream: stream, + sendAckRecord: make(map[string]chan sendAckPack, constant.DefaultMapSize), + sendCbRecord: make(map[string]sendCbPack, constant.DefaultMapSize), + peerAddr: params.PeerAddr, + healthFunc: healthFunc, + repairFunc: repairFunc, + sendReqCh: make(chan *api.StreamingMessage, defaultChannelSize), + sendRspCh: make(chan *api.StreamingMessage, defaultChannelSize), + recvCh: make(chan *api.StreamingMessage, defaultChannelSize), + repairCh: make(chan struct{}, 1), + closeCh: make(chan struct{}), + RWMutex: mutex, + Cond: sync.NewCond(mutex), + } + startLoopProcess(func() { sc.sendLoop(sc.sendReqCh) }, params.SendReqConcurrentNum) + startLoopProcess(func() { sc.sendLoop(sc.sendRspCh) }, params.SendRspConcurrentNum) + startLoopProcess(sc.recvLoop, params.RecvConcurrentNum) + if repairFunc != nil { + startLoopProcess(sc.repairLoop, 1) + } + return sc +} + +// Send sends stream message +func (sc *StreamConnection) Send(message *api.StreamingMessage, option SendOption, callback SendCallback) ( + *api.StreamingMessage, error) { + select { + case <-sc.closeCh: + return nil, ErrStreamConnectionClosed + default: + } + if sc.healthFunc != nil && !sc.healthFunc() { + return nil, ErrStreamConnectionBroken + } + if len(message.MessageID) == 0 { + message.MessageID = uuid.New().String() + } + sc.Lock() + ackCh := make(chan sendAckPack, 1) + sc.sendAckRecord[message.MessageID] = ackCh + // message with requestID is an async message which needs a callback + requestID := getRequestID(message.GetBody()) + if len(requestID) != 0 && callback != nil { + if _, exist := sc.sendCbRecord[requestID]; exist { + sc.Unlock() + return nil, ErrRequestIDAlreadyExist + } + sc.sendCbRecord[requestID] = sendCbPack{t: time.Now(), cb: callback} + } + sc.Unlock() + defer func() { + sc.Lock() + delete(sc.sendAckRecord, message.MessageID) + sc.Unlock() + }() + select { + case sc.sendReqCh <- message: + default: + log.GetLogger().Warnf("send channel reach limit %d for connection of %s", defaultChannelSize, sc.peerAddr) + sc.Lock() + delete(sc.sendCbRecord, requestID) + sc.Unlock() + return nil, errors.New("stream send is blocked") + } + timer := time.NewTimer(option.Timeout) + select { + case <-timer.C: + // send failed, no need to record callback + sc.Lock() + delete(sc.sendCbRecord, requestID) + sc.Unlock() + return nil, errors.New("send timeout") + case ackPack, ok := <-ackCh: + // consider to add retry here + if !ok { + return nil, errors.New("send response channel closed") + } + return ackPack.rsp, ackPack.err + } +} + +// Recv receives stream message +func (sc *StreamConnection) Recv() (*api.StreamingMessage, error) { + select { + case <-sc.closeCh: + return nil, ErrStreamConnectionClosed + case msg, ok := <-sc.recvCh: + if !ok { + return nil, errors.New("recv channel is closed") + } + return msg, nil + } +} + +// Close closes stream +func (sc *StreamConnection) Close() { + sc.Lock() + if sc.closed { + sc.Unlock() + return + } + sc.closed = true + sc.Unlock() + close(sc.closeCh) +} + +// CheckClose checks if stream is closed +func (sc *StreamConnection) CheckClose() chan struct{} { + return sc.closeCh +} + +func (sc *StreamConnection) sendLoop(sendCh chan *api.StreamingMessage) { + for { + select { + case <-sc.closeCh: + log.GetLogger().Debugf("stop send loop for connection of %s", sc.peerAddr) + return + case msg, ok := <-sendCh: + if !ok { + log.GetLogger().Warnf("close stream, send channel closed for connection of %s", sc.peerAddr) + return + } + if !sc.waitForStreamFix() { + log.GetLogger().Warnf("cannot fix stream, stop send loop for connection of %s", sc.peerAddr) + return + } + err := sc.stream.Send(msg) + sc.RLock() + ackCh, exist := sc.sendAckRecord[msg.GetMessageID()] + sc.RUnlock() + if err != nil { + if exist && ackCh != nil { + ackCh <- sendAckPack{ + rsp: nil, + err: err, + } + } else { + log.GetLogger().Warnf("response channel for sending message %s doesn't exist for connection %s", + msg.MessageID, sc.peerAddr) + } + sc.repairStream() + continue + } + if !expectResponse(msg) { + if exist && ackCh != nil { + ackCh <- sendAckPack{ + rsp: nil, + err: nil, + } + } else { + log.GetLogger().Warnf("response channel for sending message %s doesn't exist for connection %s", + msg.MessageID, sc.peerAddr) + } + } + } + } +} + +func (sc *StreamConnection) recvLoop() { + for { + select { + case <-sc.closeCh: + log.GetLogger().Debugf("close stream, stop recv loop for connection of %s", sc.peerAddr) + return + default: + if !sc.waitForStreamFix() { + log.GetLogger().Warnf("cannot fix stream, stop recv loop for connection of %s", sc.peerAddr) + return + } + msg, err := sc.stream.Recv() + if err != nil { + log.GetLogger().Errorf("receive error %s for connection of %s", err.Error(), sc.peerAddr) + sc.repairStream() + continue + } + switch msg.GetBody().(type) { + case *api.StreamingMessage_CreateRsp, *api.StreamingMessage_InvokeRsp, *api.StreamingMessage_ExitRsp, + *api.StreamingMessage_SaveRsp, *api.StreamingMessage_LoadRsp, *api.StreamingMessage_KillRsp, + *api.StreamingMessage_NotifyRsp: + sc.Lock() + askCh, exist := sc.sendAckRecord[msg.GetMessageID()] + if exist { + delete(sc.sendAckRecord, msg.GetMessageID()) + } else { + log.GetLogger().Warnf("receive unexpected response messageID %s for connection %s", + msg.GetMessageID(), sc.peerAddr) + } + sc.Unlock() + if exist { + askCh <- sendAckPack{ + rsp: msg, + err: nil, + } + continue + } + + case *api.StreamingMessage_CallReq, *api.StreamingMessage_CheckpointReq, *api.StreamingMessage_RecoverReq, + *api.StreamingMessage_ShutdownReq, *api.StreamingMessage_SignalReq, *api.StreamingMessage_InvokeReq: + // StreamingMessage_InvokeReq is used in simplified server mode + select { + case sc.recvCh <- msg: + default: + log.GetLogger().Warnf("receive channel reaches limit %d for connection %s", defaultChannelSize, + sc.peerAddr) + } + case *api.StreamingMessage_NotifyReq: + notifyReq := msg.GetNotifyReq() + requestID := notifyReq.GetRequestID() + sc.Lock() + cbPack, exist := sc.sendCbRecord[requestID] + if exist { + delete(sc.sendCbRecord, requestID) + } else { + log.GetLogger().Warnf("receive unexpected notify requestID %s for connection %s", requestID, + sc.peerAddr) + } + sc.Unlock() + if exist { + go cbPack.cb(notifyReq) + } + select { + case sc.sendRspCh <- &api.StreamingMessage{ + MessageID: msg.GetMessageID(), + Body: &api.StreamingMessage_NotifyRsp{ + NotifyRsp: &runtime.NotifyResponse{}, + }, + }: + default: + log.GetLogger().Warnf("sendRsp channel reaches limit %d for connection %s", defaultChannelSize, + sc.peerAddr) + } + case *api.StreamingMessage_HeartbeatReq: + select { + case sc.sendRspCh <- &api.StreamingMessage{ + MessageID: msg.GetMessageID(), + Body: &api.StreamingMessage_HeartbeatRsp{ + HeartbeatRsp: &runtime.HeartbeatResponse{}, + }, + }: + default: + log.GetLogger().Warnf("sendRsp channel reaches limit %d for connection %s", defaultChannelSize, + sc.peerAddr) + } + default: + log.GetLogger().Warnf("receive unknown type message %s", reflect.TypeOf(msg.GetBody()).String()) + } + + } + } +} + +func (sc *StreamConnection) repairLoop() { + for { + select { + case <-sc.closeCh: + log.GetLogger().Debugf("stop recv loop for connection of %s", sc.peerAddr) + return + case _, ok := <-sc.repairCh: + if !ok { + log.GetLogger().Warnf("repair channel closed for connection of %s", sc.peerAddr) + return + } + stream := sc.repairFunc() + if stream == nil { + log.GetLogger().Warnf("failed to fix stream during fix loop") + continue + } + sc.Lock() + sc.repairing = false + sc.stream = stream + sc.Unlock() + sc.Broadcast() + } + } +} + +func (sc *StreamConnection) waitForStreamFix() bool { + sc.L.Lock() + if sc.repairing || (sc.healthFunc != nil && !sc.healthFunc()) { + sc.Wait() + } + if sc.closed { + sc.L.Unlock() + return false + } + sc.L.Unlock() + return true +} + +func (sc *StreamConnection) repairStream() { + sc.Lock() + // close stream if there is no way to fix it + if sc.repairFunc == nil { + sc.Unlock() + sc.Close() + return + } + if sc.repairing { + sc.Unlock() + return + } + sc.repairing = true + sc.Unlock() + select { + case sc.repairCh <- struct{}{}: + default: + } +} + +func calibrateParams(params *StreamParams) { + if params.SendReqConcurrentNum < 1 { + params.SendReqConcurrentNum = 1 + } + if params.SendRspConcurrentNum < 1 { + params.SendRspConcurrentNum = 1 + } + if params.RecvConcurrentNum < 1 { + params.RecvConcurrentNum = 1 + } +} + +func startLoopProcess(loop func(), num int) { + for i := 0; i < num; i++ { + go loop() + } +} + +// only async request contains requestID (create and invoke) +func getRequestID(req interface{}) string { + switch req.(type) { + case *api.StreamingMessage_CreateReq: + return req.(*api.StreamingMessage_CreateReq).CreateReq.GetRequestID() + case *api.StreamingMessage_InvokeReq: + return req.(*api.StreamingMessage_InvokeReq).InvokeReq.GetRequestID() + default: + return "" + } +} + +// server may send some message which doesn't expect response +func expectResponse(msg *api.StreamingMessage) bool { + switch msg.GetBody().(type) { + case *api.StreamingMessage_CreateReq, *api.StreamingMessage_InvokeReq, *api.StreamingMessage_ExitReq, + *api.StreamingMessage_SaveReq, *api.StreamingMessage_LoadReq, *api.StreamingMessage_KillReq, + *api.StreamingMessage_NotifyReq: + return true + default: + return false + } +} diff --git a/frontend/pkg/common/faas_common/kernelrpc/connection/stream_connection_test.go b/frontend/pkg/common/faas_common/kernelrpc/connection/stream_connection_test.go new file mode 100644 index 0000000000000000000000000000000000000000..3bdfd511d551f1a43831769ebe487db85d7b56ed --- /dev/null +++ b/frontend/pkg/common/faas_common/kernelrpc/connection/stream_connection_test.go @@ -0,0 +1,410 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package connection - +package connection + +import ( + "errors" + "io" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + + api "frontend/pkg/common/faas_common/grpc/pb" + "frontend/pkg/common/faas_common/grpc/pb/core" + "frontend/pkg/common/faas_common/grpc/pb/runtime" + "github.com/smartystreets/goconvey/convey" +) + +type fakeStream struct { + sendDelay time.Duration + sendErrCh chan error + recvErrCh chan error + sendCh chan *api.StreamingMessage + recvCh chan *api.StreamingMessage +} + +func createFakeStream(sendDelay time.Duration) *fakeStream { + return &fakeStream{ + sendDelay: sendDelay, + sendErrCh: make(chan error, 1), + recvErrCh: make(chan error, 1), + sendCh: make(chan *api.StreamingMessage, 1), + recvCh: make(chan *api.StreamingMessage, 1), + } +} + +func (f *fakeStream) Send(msg *api.StreamingMessage) error { + if f.sendDelay != 0 { + <-time.After(f.sendDelay) + } + select { + case err := <-f.sendErrCh: + return err + default: + f.sendCh <- msg + return nil + } +} + +func (f *fakeStream) Recv() (*api.StreamingMessage, error) { + select { + case err := <-f.recvErrCh: + return nil, err + case msg := <-f.recvCh: + return msg, nil + } +} + +func TestStreamSend(t *testing.T) { + convey.Convey("test steam send", t, func() { + healthReturn := true + healthFunc := func() bool { + return healthReturn + } + repairCount := 0 + repairFunc := func() Stream { + repairCount++ + return &fakeStream{} + } + var callbackRes *runtime.NotifyRequest + callbackFunc := func(message *runtime.NotifyRequest) { + callbackRes = message + } + convey.Convey("unhealthy stream", func() { + healthReturn = false + repairCount = 0 + stream := createFakeStream(0) + sc := CreateStreamConnection(stream, StreamParams{}, healthFunc, repairFunc) + _, err := sc.Send(&api.StreamingMessage{ + Body: &api.StreamingMessage_InvokeReq{ + InvokeReq: &core.InvokeRequest{ + RequestID: "reqID-123", + }, + }, + }, SendOption{Timeout: 200 * time.Millisecond}, nil) + convey.So(err, convey.ShouldEqual, ErrStreamConnectionBroken) + }) + convey.Convey("requestID already exist", func() { + healthReturn = true + repairCount = 0 + stream := createFakeStream(1 * time.Second) + sc := CreateStreamConnection(stream, StreamParams{}, healthFunc, repairFunc) + go sc.Send(&api.StreamingMessage{ + Body: &api.StreamingMessage_InvokeReq{ + InvokeReq: &core.InvokeRequest{ + RequestID: "reqID-123", + }, + }, + }, SendOption{Timeout: 200 * time.Millisecond}, callbackFunc) + time.Sleep(100 * time.Millisecond) + _, err := sc.Send(&api.StreamingMessage{ + Body: &api.StreamingMessage_InvokeReq{ + InvokeReq: &core.InvokeRequest{ + RequestID: "reqID-123", + }, + }, + }, SendOption{Timeout: 200 * time.Millisecond}, callbackFunc) + convey.So(err, convey.ShouldEqual, ErrRequestIDAlreadyExist) + }) + convey.Convey("stream send blocked", func() { + healthReturn = true + repairCount = 0 + patch := gomonkey.ApplyGlobalVar(&defaultChannelSize, 1) + stream := createFakeStream(1 * time.Second) + sc := CreateStreamConnection(stream, StreamParams{}, healthFunc, repairFunc) + sc.(*StreamConnection).sendReqCh <- &api.StreamingMessage{} + sc.(*StreamConnection).sendReqCh <- &api.StreamingMessage{} + _, err := sc.Send(&api.StreamingMessage{ + Body: &api.StreamingMessage_InvokeReq{ + InvokeReq: &core.InvokeRequest{ + RequestID: "reqID-789", + }, + }, + }, SendOption{Timeout: 200 * time.Millisecond}, nil) + convey.So(err.Error(), convey.ShouldEqual, "stream send is blocked") + patch.Reset() + }) + convey.Convey("stream send error and repair", func() { + healthReturn = true + repairCount = 0 + stream := createFakeStream(0) + var newStream *fakeStream + repairFunc = func() Stream { + repairCount++ + newStream = createFakeStream(0) + return newStream + } + sc := CreateStreamConnection(stream, StreamParams{}, healthFunc, repairFunc) + stream.sendErrCh <- io.EOF + _, err := sc.Send(&api.StreamingMessage{ + Body: &api.StreamingMessage_InvokeReq{ + InvokeReq: &core.InvokeRequest{ + RequestID: "reqID-123", + }, + }, + }, SendOption{Timeout: 200 * time.Millisecond}, nil) + time.Sleep(100 * time.Millisecond) + convey.So(err, convey.ShouldEqual, io.EOF) + convey.So(repairCount, convey.ShouldEqual, 1) + stream.recvErrCh <- io.EOF + time.Sleep(100 * time.Millisecond) + go func() { + time.Sleep(100 * time.Millisecond) + msg := <-newStream.sendCh + newStream.recvCh <- &api.StreamingMessage{ + MessageID: msg.GetMessageID(), + Body: &api.StreamingMessage_InvokeRsp{ + InvokeRsp: &core.InvokeResponse{}, + }, + } + }() + _, err = sc.Send(&api.StreamingMessage{ + Body: &api.StreamingMessage_InvokeReq{ + InvokeReq: &core.InvokeRequest{ + RequestID: "reqID-123", + }, + }, + }, SendOption{Timeout: 200 * time.Millisecond}, nil) + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("stream send error and no repair", func() { + healthReturn = true + repairCount = 0 + stream := createFakeStream(0) + sc := CreateStreamConnection(stream, StreamParams{}, healthFunc, nil) + stream.sendErrCh <- io.EOF + _, err := sc.Send(&api.StreamingMessage{ + Body: &api.StreamingMessage_InvokeReq{ + InvokeReq: &core.InvokeRequest{ + RequestID: "reqID-123", + }, + }, + }, SendOption{Timeout: 200 * time.Millisecond}, nil) + convey.So(err, convey.ShouldEqual, io.EOF) + convey.So(repairCount, convey.ShouldEqual, 0) + _, err = sc.Send(&api.StreamingMessage{ + Body: &api.StreamingMessage_InvokeReq{ + InvokeReq: &core.InvokeRequest{ + RequestID: "reqID-123", + }, + }, + }, SendOption{Timeout: 200 * time.Millisecond}, nil) + convey.So(err, convey.ShouldEqual, ErrStreamConnectionClosed) + convey.So(repairCount, convey.ShouldEqual, 0) + }) + convey.Convey("stream send timeout", func() { + healthReturn = true + repairCount = 0 + stream := createFakeStream(0) + sc := CreateStreamConnection(stream, StreamParams{}, healthFunc, repairFunc) + _, err := sc.Send(&api.StreamingMessage{ + MessageID: "msgID-123", + Body: &api.StreamingMessage_InvokeReq{ + InvokeReq: &core.InvokeRequest{ + RequestID: "reqID-123", + }, + }, + }, SendOption{Timeout: 100 * time.Millisecond}, nil) + convey.So(err.Error(), convey.ShouldEqual, "send timeout") + convey.So(repairCount, convey.ShouldEqual, 0) + }) + convey.Convey("stream send expect no response", func() { + healthReturn = true + repairCount = 0 + stream := createFakeStream(0) + sc := CreateStreamConnection(stream, StreamParams{}, healthFunc, repairFunc) + _, err := sc.Send(&api.StreamingMessage{ + MessageID: "msgID-123", + Body: &api.StreamingMessage_InvokeRsp{ + InvokeRsp: &core.InvokeResponse{}, + }, + }, SendOption{Timeout: 100 * time.Millisecond}, nil) + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("stream send expect response", func() { + healthReturn = true + repairCount = 0 + stream := createFakeStream(0) + sc := CreateStreamConnection(stream, StreamParams{}, healthFunc, repairFunc) + go func() { + time.Sleep(100 * time.Millisecond) + msg := <-stream.sendCh + stream.recvCh <- &api.StreamingMessage{ + MessageID: msg.GetMessageID(), + Body: &api.StreamingMessage_InvokeRsp{ + InvokeRsp: &core.InvokeResponse{}, + }, + } + stream.recvCh <- &api.StreamingMessage{ + Body: &api.StreamingMessage_NotifyReq{ + NotifyReq: &runtime.NotifyRequest{ + RequestID: "reqID-123", + }, + }, + } + }() + _, err := sc.Send(&api.StreamingMessage{ + Body: &api.StreamingMessage_InvokeReq{ + InvokeReq: &core.InvokeRequest{ + RequestID: "reqID-123", + }, + }, + }, SendOption{Timeout: 200 * time.Minute}, callbackFunc) + time.Sleep(100 * time.Millisecond) + convey.So(err, convey.ShouldBeNil) + convey.So(repairCount, convey.ShouldEqual, 0) + convey.So(callbackRes, convey.ShouldNotBeNil) + }) + }) +} + +func TestStreamRecv(t *testing.T) { + convey.Convey("test steam receive", t, func() { + healthReturn := true + healthFunc := func() bool { + return healthReturn + } + repairCount := 0 + repairFunc := func() Stream { + repairCount++ + return createFakeStream(0) + } + convey.Convey("stream receive error and no repair", func() { + healthReturn = true + repairCount = 0 + stream := createFakeStream(0) + sc := CreateStreamConnection(stream, StreamParams{}, healthFunc, nil) + stream.recvErrCh <- errors.New("some error") + time.Sleep(100 * time.Millisecond) + _, err := sc.Recv() + convey.So(err, convey.ShouldEqual, ErrStreamConnectionClosed) + }) + convey.Convey("stream receive error and repair", func() { + healthReturn = true + repairCount = 0 + stream := createFakeStream(0) + var newStream *fakeStream + repairFunc = func() Stream { + repairCount++ + newStream = createFakeStream(0) + return newStream + } + sc := CreateStreamConnection(stream, StreamParams{}, healthFunc, repairFunc) + stream.recvErrCh <- io.EOF + time.Sleep(100 * time.Millisecond) + newStream.recvCh <- &api.StreamingMessage{ + Body: &api.StreamingMessage_CallReq{ + CallReq: &runtime.CallRequest{ + RequestID: "reqID-123", + }, + }, + } + time.Sleep(100 * time.Millisecond) + _, err := sc.Recv() + convey.So(err, convey.ShouldBeNil) + convey.So(repairCount, convey.ShouldEqual, 1) + }) + convey.Convey("receive call message", func() { + healthReturn = true + repairCount = 0 + stream := createFakeStream(0) + sc := CreateStreamConnection(stream, StreamParams{}, healthFunc, repairFunc) + stream.recvCh <- &api.StreamingMessage{ + Body: &api.StreamingMessage_CallReq{ + CallReq: &runtime.CallRequest{ + RequestID: "reqID-123", + }, + }, + } + msg, err := sc.Recv() + convey.So(err, convey.ShouldBeNil) + callReq := msg.GetCallReq() + convey.So(callReq, convey.ShouldNotBeNil) + convey.So(callReq.GetRequestID(), convey.ShouldEqual, "reqID-123") + }) + convey.Convey("receive heartbeat message", func() { + healthReturn = true + repairCount = 0 + stream := createFakeStream(0) + CreateStreamConnection(stream, StreamParams{}, healthFunc, repairFunc) + stream.recvCh <- &api.StreamingMessage{ + Body: &api.StreamingMessage_HeartbeatReq{}, + } + msg := <-stream.sendCh + heartbeatRsp := msg.GetHeartbeatRsp() + convey.So(heartbeatRsp, convey.ShouldNotBeNil) + }) + convey.Convey("receive unexpected id", func() { + healthReturn = true + repairCount = 0 + stream := createFakeStream(0) + CreateStreamConnection(stream, StreamParams{}, healthFunc, repairFunc) + stream.recvCh <- &api.StreamingMessage{ + MessageID: "msgID-123", + Body: &api.StreamingMessage_NotifyRsp{ + NotifyRsp: &runtime.NotifyResponse{}, + }, + } + stream.recvCh <- &api.StreamingMessage{ + MessageID: "msgID-123", + Body: &api.StreamingMessage_NotifyReq{ + NotifyReq: &runtime.NotifyRequest{ + RequestID: "reqID-123", + }, + }, + } + time.Sleep(100 * time.Millisecond) + convey.So(len(stream.sendCh), convey.ShouldEqual, 1) + }) + convey.Convey("receive unsupported message", func() { + healthReturn = true + repairCount = 0 + stream := createFakeStream(0) + CreateStreamConnection(stream, StreamParams{}, healthFunc, repairFunc) + stream.recvCh <- &api.StreamingMessage{ + Body: &api.StreamingMessage_SignalRsp{}, + } + convey.So(len(stream.sendCh), convey.ShouldEqual, 0) + }) + }) +} + +func TestStreamClose(t *testing.T) { + convey.Convey("test steam close", t, func() { + repairFunc := func() Stream { + return createFakeStream(0) + } + stream := createFakeStream(0) + sc := CreateStreamConnection(stream, StreamParams{}, nil, repairFunc) + closeCh := sc.CheckClose() + sc.Close() + sc.Close() + convey.So(sc.(*StreamConnection).closed, convey.ShouldEqual, true) + select { + case <-closeCh: + default: + t.Errorf("closeCh is not closed") + } + _, err := sc.Send(nil, SendOption{}, nil) + convey.So(err, convey.ShouldEqual, ErrStreamConnectionClosed) + stream.recvErrCh <- io.EOF + _, err = sc.Recv() + convey.So(err, convey.ShouldEqual, ErrStreamConnectionClosed) + }) +} diff --git a/frontend/pkg/common/faas_common/kernelrpc/rpcclient/basic_stream_client.go b/frontend/pkg/common/faas_common/kernelrpc/rpcclient/basic_stream_client.go new file mode 100644 index 0000000000000000000000000000000000000000..5aee1b7066b0b73de429b281e2e31fa1d873c6bc --- /dev/null +++ b/frontend/pkg/common/faas_common/kernelrpc/rpcclient/basic_stream_client.go @@ -0,0 +1,298 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package rpcclient - +package rpcclient + +import ( + "context" + "errors" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/backoff" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/metadata" + + rtapi "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/grpc/pb" // production: package api + "frontend/pkg/common/faas_common/grpc/pb/common" + "frontend/pkg/common/faas_common/grpc/pb/core" + "frontend/pkg/common/faas_common/grpc/pb/runtime" + "frontend/pkg/common/faas_common/kernelrpc/connection" + "frontend/pkg/common/faas_common/kernelrpc/utils" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/snerror" + "frontend/pkg/common/faas_common/statuscode" +) + +const ( + maxMsgSize = 1024 * 1024 * 10 + maxWindowSize = 1024 * 1024 * 10 + maxBufferSize = 1024 * 1024 * 10 + dialBaseDelay = 300 * time.Millisecond + dialMultiplier = 1.2 + dialJitter = 0.1 + runtimeDialMaxDelay = 100 * time.Second +) + +var ( + // ErrUnsupportedMethod - + ErrUnsupportedMethod = snerror.New(statuscode.InternalErrorCode, "unsupported method") + dialTimeout = 5 * time.Second + dialRetryTime = 10 + dialRetryInterval = 3 * time.Second + setupStreamRetryInterval = 3 * time.Second + streamMessagePool = sync.Pool{} + invokeRequestPool = sync.Pool{} +) + +// StreamClientParams - +type StreamClientParams struct { + SendReqConcurrentNum int + SendRspConcurrentNum int + RecvConcurrentNum int +} + +// BasicSteamClient is basic implementation of KernelClient which only sends POSIX calls as a runtime +type BasicSteamClient struct { + clientConn *grpc.ClientConn + streamConn connection.Connection + peerAddr string +} + +// CreateBasicStreamClient creates BasicSteamClient +func CreateBasicStreamClient(peerAddr string, params StreamClientParams) (KernelClient, error) { + conn, err := dialConnection(peerAddr) + if err != nil { + log.GetLogger().Errorf("failed to dial connection to %s error %s", peerAddr, err.Error()) + return nil, err + } + stream, err := createStream(conn, nil) + if err != nil { + log.GetLogger().Errorf("failed to create stream to %s error %s", peerAddr, err.Error()) + return nil, err + } + client := &BasicSteamClient{ + peerAddr: peerAddr, + clientConn: conn, + } + streamConn := connection.CreateStreamConnection(stream, + connection.StreamParams{ + PeerAddr: peerAddr, + SendReqConcurrentNum: params.SendReqConcurrentNum, + SendRspConcurrentNum: params.SendRspConcurrentNum, + RecvConcurrentNum: params.RecvConcurrentNum, + }, + client.checkClientConnHealth, client.repairStream) + client.streamConn = streamConn + return client, nil +} + +// Create - +func (k *BasicSteamClient) Create(funcKey string, args []*rtapi.Arg, createParams CreateParams, + callback KernelClientCallback) (string, snerror.SNError) { + return "", ErrUnsupportedMethod +} + +// Invoke - +func (k *BasicSteamClient) Invoke(funcKey string, instanceID string, args []*rtapi.Arg, invokeParams InvokeParams, + callback KernelClientCallback) (string, snerror.SNError) { + CalibrateTransportParams(&invokeParams.TransportParams) + message := acquireStreamMessageInvokeRequest() + defer releaseStreamMessageInvokeRequest(message) + invokeReq := message.GetInvokeReq() + invokeReq.Function = funcKey + invokeReq.Args = pb2Arg(args) + invokeReq.InstanceID = instanceID + if len(invokeParams.RequestID) != 0 { + invokeReq.RequestID = invokeParams.RequestID + } else { + invokeReq.RequestID = utils.GenTaskID() + } + if len(invokeParams.TraceID) != 0 { + invokeReq.TraceID = invokeParams.TraceID + } else { + invokeReq.TraceID = utils.GenTaskID() + } + sendOption := connection.SendOption{ + Timeout: invokeParams.Timeout, + } + sendCallback := func(notifyReq *runtime.NotifyRequest) { + var ( + notifyMsg []byte + notifyErr snerror.SNError + ) + if notifyReq.Code != common.ErrorCode_ERR_NONE { + notifyErr = snerror.New(int(notifyReq.Code), notifyReq.Message) + } else { + notifyMsg = []byte(notifyReq.Message) + } + callback(notifyMsg, notifyErr) + } + msg, err := k.streamConn.Send(message, sendOption, sendCallback) + if err != nil { + return "", snerror.New(statuscode.InternalErrorCode, err.Error()) + } + sendRsp, ok := msg.GetBody().(*api.StreamingMessage_InvokeRsp) + if !ok { + return "", snerror.New(statuscode.InternalErrorCode, "invoke response type error") + } + if sendRsp.InvokeRsp.Code != common.ErrorCode_ERR_NONE { + return "", snerror.New(int(sendRsp.InvokeRsp.Code), sendRsp.InvokeRsp.Message) + } + return sendRsp.InvokeRsp.Message, nil +} + +// SaveState - +func (k *BasicSteamClient) SaveState(state []byte) (string, snerror.SNError) { + return "", ErrUnsupportedMethod +} + +// LoadState - +func (k *BasicSteamClient) LoadState(checkpointID string) ([]byte, snerror.SNError) { + return nil, ErrUnsupportedMethod +} + +// Kill - +func (k *BasicSteamClient) Kill(instanceID string, signal int32, payload []byte) snerror.SNError { + return ErrUnsupportedMethod +} + +// Exit - +func (k *BasicSteamClient) Exit() { +} + +func (k *BasicSteamClient) checkClientConnHealth() bool { + return checkClientConnHealth(k.clientConn) +} + +func (k *BasicSteamClient) repairStream() connection.Stream { + if !k.checkClientConnHealth() { + conn, err := dialConnection(k.peerAddr) + if err != nil { + log.GetLogger().Errorf("failed to repair stream, dial connection to %s error %s", k.peerAddr, err.Error()) + return nil + } + k.clientConn = conn + } + stream, err := createStream(k.clientConn, nil) + if err != nil { + log.GetLogger().Errorf("failed to repair stream, create stream to %s error %s", k.peerAddr, err.Error()) + return nil + } + return stream +} + +func dialConnection(addr string) (*grpc.ClientConn, error) { + ctx, cancel := context.WithTimeout(context.TODO(), dialTimeout) + defer cancel() + dialFunc := func() (*grpc.ClientConn, error) { + return grpc.DialContext(ctx, addr, grpc.WithInsecure(), grpc.WithBlock(), + grpc.WithInitialWindowSize(maxWindowSize), + grpc.WithInitialConnWindowSize(maxWindowSize), + grpc.WithWriteBufferSize(maxBufferSize), + grpc.WithReadBufferSize(maxBufferSize), + grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(maxMsgSize), grpc.MaxCallRecvMsgSize(maxMsgSize)), + grpc.WithConnectParams(grpc.ConnectParams{ + Backoff: backoff.Config{BaseDelay: dialBaseDelay, Multiplier: dialMultiplier, Jitter: dialJitter, + MaxDelay: runtimeDialMaxDelay}, MinConnectTimeout: dialBaseDelay, + })) + } + var ( + conn *grpc.ClientConn + err error + ) + for i := 0; i < dialRetryTime; i++ { + conn, err = dialFunc() + if err == nil { + return conn, err + } + log.GetLogger().Warnf("failed to dial connection to %s error %s", addr, err.Error()) + time.Sleep(time.Duration(i+1) * dialRetryInterval) + } + log.GetLogger().Errorf("failed to dial connection to %s after %d retries error %s", addr, dialRetryTime, + err.Error()) + return nil, err +} + +func createStream(conn *grpc.ClientConn, mdMap map[string]string) (api.RuntimeRPC_MessageStreamClient, error) { + if !checkClientConnHealth(conn) { + log.GetLogger().Errorf("grpc connection is nil, failed to create stream rpcclient") + return nil, errors.New("conn is unhealthy") + } + client := api.NewRuntimeRPCClient(conn) + md := metadata.New(mdMap) + var ( + stream api.RuntimeRPC_MessageStreamClient + err error + ) + var retryTimes int + for i := 0; i < dialRetryTime; i++ { + stream, err = client.MessageStream(metadata.NewOutgoingContext(context.Background(), md)) + if err == nil { + log.GetLogger().Infof("succeed to get stream from function proxy") + break + } + log.GetLogger().Errorf("failed to get stream from function proxy for %d times, err: %s", + retryTimes, err.Error()) + time.Sleep(setupStreamRetryInterval) + } + if err != nil { + log.GetLogger().Errorf("failed to create stream rpcclient to %s when setup message stream error %s", conn.Target(), + err.Error()) + return nil, err + } + return stream, nil +} + +func checkClientConnHealth(conn *grpc.ClientConn) bool { + if conn == nil { + return false + } + return conn.GetState() == connectivity.Idle || conn.GetState() == connectivity.Ready +} + +func acquireStreamMessageInvokeRequest() *api.StreamingMessage { + var ( + streamMsg *api.StreamingMessage + invokeReq *api.StreamingMessage_InvokeReq + ok bool + ) + streamMsg, ok = streamMessagePool.Get().(*api.StreamingMessage) + if !ok { + streamMsg = &api.StreamingMessage{} + } + invokeReq, ok = invokeRequestPool.Get().(*api.StreamingMessage_InvokeReq) + if !ok { + invokeReq = &api.StreamingMessage_InvokeReq{InvokeReq: &core.InvokeRequest{}} + } + streamMsg.Body = invokeReq + return streamMsg +} + +func releaseStreamMessageInvokeRequest(streamMsg *api.StreamingMessage) { + invokeReq, ok := streamMsg.GetBody().(*api.StreamingMessage_InvokeReq) + if !ok { + return + } + invokeReq.InvokeReq.Reset() + invokeRequestPool.Put(invokeReq) + streamMsg.Reset() + streamMessagePool.Put(streamMsg) +} diff --git a/frontend/pkg/common/faas_common/kernelrpc/rpcclient/basic_stream_client_test.go b/frontend/pkg/common/faas_common/kernelrpc/rpcclient/basic_stream_client_test.go new file mode 100644 index 0000000000000000000000000000000000000000..655d991aa7e7899032ddc11be94fe273e3ac2738 --- /dev/null +++ b/frontend/pkg/common/faas_common/kernelrpc/rpcclient/basic_stream_client_test.go @@ -0,0 +1,75 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rpcclient + +import ( + "testing" + + "github.com/stretchr/testify/assert" + rtapi "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/snerror" +) + +func TestBasicSteamClient_Create(t *testing.T) { + client := &BasicSteamClient{} + + funcKey := "testFuncKey" + args := []*rtapi.Arg{} + createParams := CreateParams{} + callback := func(result []byte, err snerror.SNError) { + } + + result, err := client.Create(funcKey, args, createParams, callback) + + assert.Equal(t, "", result) + assert.Equal(t, ErrUnsupportedMethod, err) +} + +func TestBasicSteamClient_SaveState(t *testing.T) { + client := &BasicSteamClient{} + + state := []byte("test state") + + result, err := client.SaveState(state) + + assert.Equal(t, "", result, "Expected empty string as result") + assert.Equal(t, ErrUnsupportedMethod, err, "Expected ErrUnsupportedMethod error") +} + +func TestBasicSteamClient_LoadState(t *testing.T) { + client := &BasicSteamClient{} + + checkpointID := "testCheckpointID" + + result, err := client.LoadState(checkpointID) + + assert.Nil(t, result, "Expected nil as result") + assert.Equal(t, ErrUnsupportedMethod, err, "Expected ErrUnsupportedMethod error") +} + +func TestBasicSteamClient_Kill(t *testing.T) { + client := &BasicSteamClient{} + + instanceID := "testInstanceID" + signal := int32(9) + payload := []byte("test payload") + + err := client.Kill(instanceID, signal, payload) + + assert.Equal(t, ErrUnsupportedMethod, err, "Expected ErrUnsupportedMethod error") +} diff --git a/frontend/pkg/common/faas_common/kernelrpc/rpcclient/client.go b/frontend/pkg/common/faas_common/kernelrpc/rpcclient/client.go new file mode 100644 index 0000000000000000000000000000000000000000..1aa5c3c785529c5eec2067ac6753e024e13c6d46 --- /dev/null +++ b/frontend/pkg/common/faas_common/kernelrpc/rpcclient/client.go @@ -0,0 +1,167 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package rpcclient - +package rpcclient + +import ( + "time" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/grpc/pb/common" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/snerror" + "frontend/pkg/common/faas_common/statuscode" +) + +const ( + defaultTimeout = 900 * time.Second +) + +var ( + // ErrKernelClientTimeout - + ErrKernelClientTimeout = snerror.New(statuscode.InternalErrorCode, "kernel rpcclient timeout") +) + +// TransportParams - +type TransportParams struct { + Timeout time.Duration + RetryInterval time.Duration + RetryNumber int +} + +// AffinityType - +type AffinityType int32 + +// SchedulingOptions - +type SchedulingOptions struct { + Priority int32 + Resources map[string]float64 + Extension map[string]string + Affinity map[string]AffinityType + ScheduleAffinity []byte +} + +// CreateParams - +type CreateParams struct { + TransportParams + DesignatedInstanceID string + Label []string + CreateOption map[string]string + ScheduleOption SchedulingOptions +} + +// InvokeParams - +type InvokeParams struct { + TransportParams + InvokeOptions map[string]string + RequestID string + TraceID string +} + +// KernelClientCallback - +type KernelClientCallback = func(result []byte, err snerror.SNError) + +// KernelClientAsyncCreate - +type KernelClientAsyncCreate = func(function string, args []string, createParams CreateParams, + callback KernelClientCallback) (string, snerror.SNError) + +// KernelClientAsyncInvoke _ +type KernelClientAsyncInvoke = func(function string, instanceID string, args []string, invokeParams InvokeParams, + callback KernelClientCallback) snerror.SNError + +// KernelClient defines basic POSIX client methods, it's worth noting that Create and +// Invoke are original async calls while others are sync calls +type KernelClient interface { + Create(funcKey string, args []*api.Arg, createParams CreateParams, callback KernelClientCallback) (string, + snerror.SNError) + + Invoke(funcKey string, instanceID string, args []*api.Arg, invokeParams InvokeParams, + callback KernelClientCallback) (string, snerror.SNError) + + SaveState(state []byte) (string, snerror.SNError) + + LoadState(checkpointID string) ([]byte, snerror.SNError) + + Kill(instanceID string, signal int32, payload []byte) snerror.SNError + + Exit() +} + +// SyncInvoke will call invoke synchronously +func SyncInvoke(asyncInvoke KernelClientAsyncInvoke, funcKey string, instanceID string, args []string, + invokeParams InvokeParams) ([]byte, snerror.SNError) { + CalibrateTransportParams(&invokeParams.TransportParams) + var ( + resultData []byte + resultError snerror.SNError + ) + waitCh := make(chan struct{}, 1) + callback := func(result []byte, err snerror.SNError) { + resultData, resultError = result, err + waitCh <- struct{}{} + } + invokeErr := asyncInvoke(funcKey, instanceID, args, invokeParams, callback) + if invokeErr != nil { + return nil, invokeErr + } + timer := time.NewTimer(invokeParams.Timeout) + defer timer.Stop() + retryCount := 0 + for { + select { + case <-timer.C: + log.GetLogger().Errorf("sync invoke times out after %ds for function %s traceID %s", + invokeParams.Timeout.Seconds(), funcKey, invokeParams.TraceID) + return nil, ErrKernelClientTimeout + case <-waitCh: + if resultError == nil { + return resultData, nil + } + retryCount++ + if retryCount <= invokeParams.RetryNumber { + time.Sleep(invokeParams.RetryInterval) + log.GetLogger().Errorf("sync invoke reties count %d after %ds for function %s traceID %s", + retryCount, invokeParams.RetryInterval.Seconds(), funcKey, invokeParams.TraceID) + invokeErr = asyncInvoke(funcKey, instanceID, args, invokeParams, callback) + if invokeErr != nil { + return nil, invokeErr + } + continue + } + log.GetLogger().Errorf("sync invoke reties reach limit %d for function %s traceID %s", + invokeParams.RetryNumber, funcKey, invokeParams.TraceID) + return resultData, resultError + } + } +} + +func pb2Arg(args []*api.Arg) []*common.Arg { + length := len(args) + newArgs := make([]*common.Arg, 0, length) + for _, arg := range args { + newArgs = append(newArgs, &common.Arg{Type: common.Arg_ArgType(arg.Type), Value: arg.Data}) + } + return newArgs +} + +// CalibrateTransportParams calibrates transport params +func CalibrateTransportParams(params *TransportParams) { + if params.Timeout == 0 { + params.Timeout = defaultTimeout + } +} diff --git a/frontend/pkg/common/faas_common/kernelrpc/rpcclient/client_test.go b/frontend/pkg/common/faas_common/kernelrpc/rpcclient/client_test.go new file mode 100644 index 0000000000000000000000000000000000000000..43c494dc7b972d458a5f4ab3a0e25b46f5163168 --- /dev/null +++ b/frontend/pkg/common/faas_common/kernelrpc/rpcclient/client_test.go @@ -0,0 +1,110 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rpcclient + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "frontend/pkg/common/faas_common/snerror" +) + +func TestSyncInvoke(t *testing.T) { + funcKey := "testFuncKey" + instanceID := "testInstanceID" + args := []string{"arg1", "arg2"} + invokeParams := InvokeParams{ + TransportParams: TransportParams{ + Timeout: 500 * time.Millisecond, + RetryInterval: 100 * time.Millisecond, + RetryNumber: 2, + }, + InvokeOptions: map[string]string{"option1": "value1"}, + RequestID: "testRequestID", + TraceID: "testTraceID", + } + + t.Run("Success without retries", func(t *testing.T) { + asyncInvoke := func(funcKey, instanceID string, args []string, invokeParams InvokeParams, + callback func(result []byte, err snerror.SNError)) snerror.SNError { + go func() { + time.Sleep(50 * time.Millisecond) // 模拟异步调用的延迟 + callback([]byte("success"), nil) + }() + return nil + } + + resultData, resultError := SyncInvoke(asyncInvoke, funcKey, instanceID, args, invokeParams) + + assert.Nil(t, resultError) + assert.Equal(t, []byte("success"), resultData) + }) + + t.Run("Error then success after retries", func(t *testing.T) { + var callCount int + asyncInvoke := func(funcKey, instanceID string, args []string, invokeParams InvokeParams, + callback func(result []byte, err snerror.SNError)) snerror.SNError { + go func() { + time.Sleep(50 * time.Millisecond) + if callCount < 1 { + callCount++ + callback(nil, snerror.New(1, "temporary error")) + } else { + callback([]byte("recovered success"), nil) + } + }() + return nil + } + + resultData, resultError := SyncInvoke(asyncInvoke, funcKey, instanceID, args, invokeParams) + + assert.Nil(t, resultError) + assert.Equal(t, []byte("recovered success"), resultData) + }) + + t.Run("Error with retries exhausted", func(t *testing.T) { + asyncInvoke := func(funcKey, instanceID string, args []string, invokeParams InvokeParams, + callback func(result []byte, err snerror.SNError)) snerror.SNError { + go func() { + time.Sleep(50 * time.Millisecond) + callback(nil, snerror.New(1, "persistent error")) + }() + return nil + } + + resultData, resultError := SyncInvoke(asyncInvoke, funcKey, instanceID, args, invokeParams) + + assert.NotNil(t, resultError) + assert.Equal(t, "persistent error", resultError.Error()) + assert.Nil(t, resultData) + }) + + t.Run("Timeout without response", func(t *testing.T) { + asyncInvoke := func(funcKey, instanceID string, args []string, invokeParams InvokeParams, + callback func(result []byte, err snerror.SNError)) snerror.SNError { + return nil + } + + resultData, resultError := SyncInvoke(asyncInvoke, funcKey, instanceID, args, invokeParams) + + assert.NotNil(t, resultError) + assert.Equal(t, ErrKernelClientTimeout, resultError) + assert.Nil(t, resultData) + }) +} diff --git a/frontend/pkg/common/faas_common/kernelrpc/rpcserver/server.go b/frontend/pkg/common/faas_common/kernelrpc/rpcserver/server.go new file mode 100644 index 0000000000000000000000000000000000000000..3d28fcc2e93cf38132609fab33e9c7c524058e22 --- /dev/null +++ b/frontend/pkg/common/faas_common/kernelrpc/rpcserver/server.go @@ -0,0 +1,49 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package rpcserver - +package rpcserver + +import ( + "time" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/grpc/pb/common" +) + +const ( + defaultSendTimeout = 5 * time.Second +) + +// KernelInvokeHandler - +type KernelInvokeHandler func(args []*api.Arg, traceID string) (string, error) + +// KernelServer defines basic POSIX server methods, currently only RegisterInvokeHandler is needed +type KernelServer interface { + RegisterInvokeHandler(handler KernelInvokeHandler) + Serve() error + Stop() +} + +func pb2Arg(args []*common.Arg) []*api.Arg { + length := len(args) + newArgs := make([]*api.Arg, 0, length) + for _, value := range args { + newArgs = append(newArgs, &api.Arg{Type: api.ArgType(value.Type), Data: value.Value}) + } + return newArgs +} diff --git a/frontend/pkg/common/faas_common/kernelrpc/rpcserver/simplified_stream_server.go b/frontend/pkg/common/faas_common/kernelrpc/rpcserver/simplified_stream_server.go new file mode 100644 index 0000000000000000000000000000000000000000..34e69b985187a5cea880ccfbb5cd0aaf7a7ad09d --- /dev/null +++ b/frontend/pkg/common/faas_common/kernelrpc/rpcserver/simplified_stream_server.go @@ -0,0 +1,217 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package rpcserver - +package rpcserver + +import ( + "net" + "reflect" + "sync" + + "github.com/panjf2000/ants/v2" + "google.golang.org/grpc" + "google.golang.org/grpc/peer" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/grpc/pb" + "frontend/pkg/common/faas_common/grpc/pb/common" + "frontend/pkg/common/faas_common/grpc/pb/core" + "frontend/pkg/common/faas_common/kernelrpc/connection" + "frontend/pkg/common/faas_common/logger/log" +) + +var ( + streamMessagePool = sync.Pool{} + invokeResponsePool = sync.Pool{} +) + +type requestPack struct { + msg *api.StreamingMessage + conn connection.Connection +} + +// SimplifiedStreamServer is a simplified stream server which can respond to invokeRequest +type SimplifiedStreamServer struct { + api.UnimplementedRuntimeRPCServer + grpcServer *grpc.Server + taskPool *ants.PoolWithFunc + streamConnMap map[string]connection.Connection + invokeHandler KernelInvokeHandler + listenAddr string + stopped bool + stopCh chan struct{} + sync.Mutex +} + +// CreateSimplifiedStreamServer creates SimplifiedStreamServer +func CreateSimplifiedStreamServer(listenAddr string, concurrentNum int) (KernelServer, error) { + server := &SimplifiedStreamServer{ + grpcServer: grpc.NewServer(), + listenAddr: listenAddr, + streamConnMap: make(map[string]connection.Connection, constant.DefaultMapSize), + stopCh: make(chan struct{}), + } + taskPool, err := ants.NewPoolWithFunc(concurrentNum, func(arg interface{}) { + reqPack, ok := arg.(requestPack) + if !ok { + return + } + server.handleRequest(reqPack.msg, reqPack.conn) + }) + if err != nil { + log.GetLogger().Errorf("failed to create task pool error %s", err.Error()) + return nil, err + } + server.taskPool = taskPool + api.RegisterRuntimeRPCServer(server.grpcServer, server) + return server, nil +} + +// MessageStream handles stream from grpc server +func (s *SimplifiedStreamServer) MessageStream(stream api.RuntimeRPC_MessageStreamServer) error { + peerObj, _ := peer.FromContext(stream.Context()) + peerAddr := peerObj.Addr.String() + streamConn := connection.CreateStreamConnection(stream, connection.StreamParams{PeerAddr: peerAddr}, nil, nil) + closeCh := streamConn.CheckClose() + s.Lock() + s.streamConnMap[peerAddr] = streamConn + s.Unlock() + log.GetLogger().Infof("create streamConn success,peer:%s", peerAddr) + defer func() { + s.Lock() + delete(s.streamConnMap, peerAddr) + s.Unlock() + }() + for { + select { + case <-s.stopCh: + log.GetLogger().Warnf("server stops, closing stream connection to %s", peerAddr) + streamConn.Close() + return nil + case <-closeCh: + log.GetLogger().Warnf("stream connection to %s is closed", peerAddr) + return nil + default: + msg, err := streamConn.Recv() + if err != nil { + log.GetLogger().Errorf("failed to receive stream message from %s error %s", peerAddr, err.Error()) + continue + } + if err = s.taskPool.Invoke(requestPack{msg: msg, conn: streamConn}); err != nil { + log.GetLogger().Errorf("failed to invoke task pool error %s", err.Error()) + } + } + } +} + +// Serve starts serving on listenAddr +func (s *SimplifiedStreamServer) Serve() error { + lis, err := net.Listen("tcp", s.listenAddr) + if err != nil { + log.GetLogger().Errorf("failed to listen to address %s error %s\n", s.listenAddr, err.Error()) + return err + } + if err = s.grpcServer.Serve(lis); err != nil { + log.GetLogger().Errorf("failed to serve on address %s error %s", s.listenAddr, err.Error()) + return err + } + log.GetLogger().Infof("stop serve on address %s", s.listenAddr) + return nil +} + +// Stop stops server +func (s *SimplifiedStreamServer) Stop() { + s.Lock() + if s.stopped { + s.Unlock() + return + } + s.stopped = true + s.Unlock() + s.grpcServer.GracefulStop() + s.Lock() + for _, stream := range s.streamConnMap { + stream.Close() + } + s.Unlock() +} + +func (s *SimplifiedStreamServer) handleRequest(msg *api.StreamingMessage, conn connection.Connection) { + switch msg.GetBody().(type) { + case *api.StreamingMessage_InvokeReq: + invokeReq := msg.GetInvokeReq() + message := acquireStreamMessageInvokeResponse() + message.MessageID = msg.GetMessageID() + InvokeRsp := message.GetInvokeRsp() + defer func() { + if _, err := conn.Send(message, connection.SendOption{Timeout: defaultSendTimeout}, nil); err != nil { + log.GetLogger().Errorf("failed to send invoke response error %s", err.Error()) + } + releaseStreamMessageInvokeResponse(message) + }() + if s.invokeHandler == nil { + log.GetLogger().Errorf("invoke handler is nil") + InvokeRsp.Code = common.ErrorCode_ERR_USER_FUNCTION_EXCEPTION + InvokeRsp.Message = "invoke handler is nil" + return + } + rsp, err := s.invokeHandler(pb2Arg(invokeReq.GetArgs()), invokeReq.TraceID) + if err != nil { + InvokeRsp.Code = common.ErrorCode_ERR_USER_FUNCTION_EXCEPTION + InvokeRsp.Message = err.Error() + } else { + InvokeRsp.Code = common.ErrorCode_ERR_NONE + InvokeRsp.Message = rsp + } + default: + log.GetLogger().Warnf("receive unknown type message %s", reflect.TypeOf(msg.GetBody()).String()) + } +} + +// RegisterInvokeHandler registers invokeHandler +func (s *SimplifiedStreamServer) RegisterInvokeHandler(handler KernelInvokeHandler) { + s.invokeHandler = handler +} + +func acquireStreamMessageInvokeResponse() *api.StreamingMessage { + var ( + streamMsg *api.StreamingMessage + invokeRsp *api.StreamingMessage_InvokeRsp + ok bool + ) + streamMsg, ok = streamMessagePool.Get().(*api.StreamingMessage) + if !ok { + streamMsg = &api.StreamingMessage{} + } + invokeRsp, ok = invokeResponsePool.Get().(*api.StreamingMessage_InvokeRsp) + if !ok { + invokeRsp = &api.StreamingMessage_InvokeRsp{InvokeRsp: &core.InvokeResponse{}} + } + streamMsg.Body = invokeRsp + return streamMsg +} + +func releaseStreamMessageInvokeResponse(streamMsg *api.StreamingMessage) { + invokeRsp, ok := streamMsg.GetBody().(*api.StreamingMessage_InvokeRsp) + if !ok { + return + } + invokeRsp.InvokeRsp.Reset() + invokeResponsePool.Put(invokeRsp) + streamMsg.Reset() + streamMessagePool.Put(streamMsg) +} diff --git a/frontend/pkg/common/faas_common/kernelrpc/rpcserver/simplified_stream_server_test.go b/frontend/pkg/common/faas_common/kernelrpc/rpcserver/simplified_stream_server_test.go new file mode 100644 index 0000000000000000000000000000000000000000..4957d3c57fae092abd12d47eed7d2ce1dbeb2572 --- /dev/null +++ b/frontend/pkg/common/faas_common/kernelrpc/rpcserver/simplified_stream_server_test.go @@ -0,0 +1,116 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package rpcserver - +package rpcserver + +import ( + "errors" + "net" + "testing" + "time" + + gomonkey "github.com/agiledragon/gomonkey/v2" + ants "github.com/panjf2000/ants/v2" + "github.com/smartystreets/goconvey/convey" + "google.golang.org/grpc" + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/grpc/pb/common" + "frontend/pkg/common/faas_common/kernelrpc/rpcclient" +) + +func TestCreateSimplifiedStreamServer(t *testing.T) { + convey.Convey("test CreateSimplifiedStreamServer", t, func() { + patch := gomonkey.ApplyFunc(ants.NewPoolWithFunc, func(size int, pf func(interface{}), options ...ants.Option) ( + *ants.PoolWithFunc, error) { + return nil, errors.New("some error") + }) + server, err := CreateSimplifiedStreamServer("0.0.0.0:0", 100) + convey.So(err, convey.ShouldNotBeNil) + convey.So(server, convey.ShouldBeNil) + patch.Reset() + server, err = CreateSimplifiedStreamServer("0.0.0.0:0", 100) + convey.So(err, convey.ShouldBeNil) + convey.So(server, convey.ShouldNotBeNil) + }) +} + +func TestSimplifiedStreamServerServeAndClose(t *testing.T) { + convey.Convey("test SimplifiedStreamServer serve", t, func() { + patch := gomonkey.ApplyFunc(net.Listen, func(network, address string) (net.Listener, error) { + return nil, errors.New("some error") + }) + server, _ := CreateSimplifiedStreamServer("0.0.0.0:0", 100) + err := server.Serve() + convey.So(err, convey.ShouldNotBeNil) + patch.Reset() + patch = gomonkey.ApplyFunc((*grpc.Server).Serve, func(_ *grpc.Server, lis net.Listener) error { + return errors.New("some error") + }) + server, _ = CreateSimplifiedStreamServer("0.0.0.0:0", 100) + err = server.Serve() + convey.So(err, convey.ShouldNotBeNil) + patch.Reset() + server, _ = CreateSimplifiedStreamServer("0.0.0.0:0", 100) + go func() { + time.Sleep(100 * time.Millisecond) + server.Stop() + server.Stop() + }() + err = server.Serve() + convey.So(err, convey.ShouldBeNil) + server.Stop() + }) +} + +func TestSimplifiedStreamServerHandleInvoke(t *testing.T) { + convey.Convey("test SimplifiedStreamServer handleInvoke", t, func() { + server, _ := CreateSimplifiedStreamServer("0.0.0.0:5678", 100) + go server.Serve() + client, _ := rpcclient.CreateBasicStreamClient("0.0.0.0:5678", rpcclient.StreamClientParams{}) + args := []*api.Arg{{ + Type: api.Value, + Data: []byte("123"), + }} + convey.Convey("invokeHandler is nil", func() { + _, err := client.Invoke("testFunc", "testIns", args, rpcclient.InvokeParams{}, nil) + convey.So(err, convey.ShouldNotBeNil) + convey.So(err.Code(), convey.ShouldEqual, common.ErrorCode_ERR_USER_FUNCTION_EXCEPTION) + }) + convey.Convey("invokeHandler return error", func() { + invokeHandler := func(args []*api.Arg, traceID string) (string, error) { + return "", errors.New("some error") + } + server.RegisterInvokeHandler(invokeHandler) + msg, err := client.Invoke("testFunc", "testIns", args, rpcclient.InvokeParams{}, nil) + convey.So(err, convey.ShouldNotBeNil) + convey.So(err.Code(), convey.ShouldEqual, common.ErrorCode_ERR_USER_FUNCTION_EXCEPTION) + convey.So(msg, convey.ShouldBeEmpty) + }) + convey.Convey("invokeHandler return ok", func() { + invokeHandler := func(args []*api.Arg, traceID string) (string, error) { + return "abc", nil + } + server.RegisterInvokeHandler(invokeHandler) + msg, err := client.Invoke("testFunc", "testIns", args, rpcclient.InvokeParams{}, nil) + convey.So(err, convey.ShouldBeNil) + convey.So(msg, convey.ShouldEqual, "abc") + }) + grpcServer, _ := server.(*SimplifiedStreamServer) + grpcServer.grpcServer.Stop() + }) +} diff --git a/frontend/pkg/common/faas_common/kernelrpc/utils/utils.go b/frontend/pkg/common/faas_common/kernelrpc/utils/utils.go new file mode 100644 index 0000000000000000000000000000000000000000..00390ca0029b219776ba6388757fa95eda68b864 --- /dev/null +++ b/frontend/pkg/common/faas_common/kernelrpc/utils/utils.go @@ -0,0 +1,27 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package utils - +package utils + +import ( + "frontend/pkg/common/uuid" +) + +// GenTaskID for create a task id +func GenTaskID() string { + return "task-" + uuid.New().String() +} diff --git a/frontend/pkg/common/faas_common/kernelrpc/utils/utils_test.go b/frontend/pkg/common/faas_common/kernelrpc/utils/utils_test.go new file mode 100644 index 0000000000000000000000000000000000000000..d41509f901ad8bdbbf655753d70423f82cc33bef --- /dev/null +++ b/frontend/pkg/common/faas_common/kernelrpc/utils/utils_test.go @@ -0,0 +1,30 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package utils - +package utils + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGenTaskID(t *testing.T) { + taskID := GenTaskID() + assert.Equal(t, true, strings.Contains(taskID, "task")) +} diff --git a/frontend/pkg/common/faas_common/loadbalance/hash.go b/frontend/pkg/common/faas_common/loadbalance/hash.go new file mode 100644 index 0000000000000000000000000000000000000000..e97d3e71b0b3579821337b7c406fc752331c7869 --- /dev/null +++ b/frontend/pkg/common/faas_common/loadbalance/hash.go @@ -0,0 +1,454 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package loadbalance provides consistent hash alogrithm +package loadbalance + +import ( + "hash/crc32" + "sort" + "sync" + "time" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/log" +) + +const ( + // MaxInstanceSize is the max instance size be stored in hash ring + MaxInstanceSize = 100 + defaultMapSize = 100 +) + +type uint32Slice []uint32 + +// Len returns the size +func (u uint32Slice) Len() int { + return len(u) +} + +// Swap will swap two elements +func (u uint32Slice) Swap(i, j int) { + if i < 0 || i >= len(u) || j < 0 || j >= len(u) { + return + } + u[i], u[j] = u[j], u[i] +} + +// Less returns true if i less than j +func (u uint32Slice) Less(i, j int) bool { + if i < 0 || i >= len(u) || j < 0 || j >= len(u) { + return false + } + return u[i] < u[j] +} + +type anchorInfo struct { + instanceHash uint32 + instanceKey string +} + +// CHGeneric is the generic consistent hash +type CHGeneric struct { + anchorPoint map[string]*anchorInfo + instanceMap map[uint32]string + hashPool uint32Slice + insMutex sync.RWMutex + anchorMutex sync.Mutex +} + +// NewCHGeneric creates generic consistent hash +func NewCHGeneric() *CHGeneric { + return &CHGeneric{ + hashPool: make([]uint32, 0, MaxInstanceSize), + instanceMap: make(map[uint32]string, defaultMapSize), + anchorPoint: make(map[string]*anchorInfo, defaultMapSize), + } +} + +// Next returns the next scheduled node of a function +func (c *CHGeneric) Next(name string, move bool) interface{} { + c.anchorMutex.Lock() + anchor, exist := c.anchorPoint[name] + if !exist { + anchor = c.addAnchorPoint(name) + c.anchorMutex.Unlock() + return anchor.instanceKey + } + if move { + c.moveAnchorPoint(name, anchor.instanceHash) + } + c.insMutex.RLock() + _, exist = c.instanceMap[anchor.instanceHash] + c.insMutex.RUnlock() + // check if node still exists, no maxReqCount limitation + if !exist { + c.moveAnchorPoint(name, anchor.instanceHash) + } + c.anchorMutex.Unlock() + return anchor.instanceKey +} + +// Previous - returns the previous scheduled node of a function +func (c *CHGeneric) Previous(name string, move bool) interface{} { + previous := c.getPreviousHashKey(getHashKeyCRC32([]byte(name))) + if move { + previous = c.getPreviousHashKey(previous) + } + c.insMutex.RLock() + _, exist := c.instanceMap[previous] + c.insMutex.RUnlock() + if !exist { + previous = c.getPreviousHashKey(previous) + } + return c.instanceMap[previous] +} + +// Add will add a node into hash ring +func (c *CHGeneric) Add(node interface{}, weight int) { + c.insMutex.Lock() + defer c.insMutex.Unlock() + name, ok := node.(string) + if !ok { + log.GetLogger().Errorf("unable to convert %T to string", node) + return + } + hashKey := getHashKeyCRC32([]byte(name)) + _, exist := c.instanceMap[hashKey] + if exist { + return + } + c.instanceMap[hashKey] = name + c.hashPool = append(c.hashPool, hashKey) + sort.Sort(c.hashPool) + log.GetLogger().Infof("add node %s, hashKey %d to hash ring, hashPool is %v", name, hashKey, c.hashPool) +} + +// Remove will remove a node from hash ring +func (c *CHGeneric) Remove(node interface{}) { + name, assertOK := node.(string) + if !assertOK { + log.GetLogger().Errorf("unable to convert %T to string", node) + return + } + hashKey := getHashKeyCRC32([]byte(name)) + c.insMutex.Lock() + delete(c.instanceMap, hashKey) + for i, hash := range c.hashPool { + if hash == hashKey { + copy(c.hashPool[i:], c.hashPool[i+1:]) + c.hashPool[len(c.hashPool)-1] = 0 + c.hashPool = c.hashPool[:len(c.hashPool)-1] + break + } + } + log.GetLogger().Infof("delete node %s from hash ring", name) + c.insMutex.Unlock() + +} + +// RemoveAll will remove all nodes from hash ring +func (c *CHGeneric) RemoveAll() { + c.insMutex.Lock() + c.hashPool = make([]uint32, 0, MaxInstanceSize) + c.instanceMap = make(map[uint32]string, defaultMapSize) + c.insMutex.Unlock() + return +} + +// Reset will clean all anchor infos +func (c *CHGeneric) Reset() { + c.anchorMutex.Lock() + c.anchorPoint = make(map[string]*anchorInfo, defaultMapSize) + c.anchorMutex.Unlock() + log.GetLogger().Infof("reset hash ring anchorPoint") + return +} + +// DeleteBalancer - +func (c *CHGeneric) DeleteBalancer(name string) { + c.anchorMutex.Lock() + defer c.anchorMutex.Unlock() + delete(c.anchorPoint, name) +} + +func (c *CHGeneric) addAnchorPoint(name string) *anchorInfo { + // need to be called in a thread safe context + hashKey := getHashKeyCRC32([]byte(name)) + c.insMutex.RLock() + instanceHash := c.getNextHashKey(hashKey) + c.insMutex.RUnlock() + newAnchor := &anchorInfo{ + instanceHash: instanceHash, + instanceKey: c.instanceMap[instanceHash], + } + c.anchorPoint[name] = newAnchor + log.GetLogger().Debugf("name %s hashKey %d", name, hashKey) + return newAnchor +} + +func (c *CHGeneric) moveAnchorPoint(name string, curHash uint32) { + c.insMutex.Lock() + instanceHash := c.getNextHashKey(curHash) + c.anchorPoint[name].instanceHash = instanceHash + c.anchorPoint[name].instanceKey = c.instanceMap[instanceHash] + c.insMutex.Unlock() +} + +func (c *CHGeneric) getNextHashKey(hashKey uint32) uint32 { + // need to be called with insMutex locked + if len(c.hashPool) == 0 { + return 0 + } + nextHashKey := c.hashPool[0] + for _, v := range c.hashPool { + if v > hashKey { + nextHashKey = v + break + } + } + return nextHashKey +} + +func (c *CHGeneric) getPreviousHashKey(hashKey uint32) uint32 { + // need to be called with insMutex locked + if len(c.hashPool) == 0 { + return 0 + } + hashLen := len(c.hashPool) + previousHashKey := c.hashPool[hashLen-1] + for i := hashLen - 1; i >= 0; i-- { + if c.hashPool[i] < hashKey { + previousHashKey = c.hashPool[i] + break + } + } + return previousHashKey +} + +func getHashKeyCRC32(key []byte) uint32 { + return crc32.ChecksumIEEE(key) +} + +// NewConcurrentCHGeneric return ConcurrentCHGeneric with given concurrency +func NewConcurrentCHGeneric(concurrency int) *ConcurrentCHGeneric { + return &ConcurrentCHGeneric{ + CHGeneric: NewCHGeneric(), + concurrency: concurrency, + counter: make(map[string]*concurrentCounter, constant.DefaultMapSize), + } +} + +type concurrentCounter struct { + count int + last time.Time +} + +// ConcurrentCHGeneric is concurrency balanced +type ConcurrentCHGeneric struct { + *CHGeneric + counter map[string]*concurrentCounter + countMutex sync.Mutex + concurrency int +} + +// Next returns the next scheduled node +func (c *ConcurrentCHGeneric) Next(name string, move bool) interface{} { + c.countMutex.Lock() + defer c.countMutex.Unlock() + l, ok := c.counter[name] + if !ok { + c.counter[name] = &concurrentCounter{ + last: time.Now(), + } + return c.CHGeneric.Next(name, move) + } + l.count++ + if l.count >= c.concurrency { + now := time.Now() + l.count = 0 + if now.Sub(l.last) < 1*time.Second { + move = true + } + l.last = now + } + return c.CHGeneric.Next(name, move) +} + +// Previous - returns the previous scheduled node of a function +func (c *ConcurrentCHGeneric) Previous(name string, move bool) interface{} { + return c.CHGeneric.Previous(name, move) +} + +// Add a node to hash ring +func (c *ConcurrentCHGeneric) Add(node interface{}, weight int) { + c.CHGeneric.Add(node, weight) +} + +// Remove a node from hash ring +func (c *ConcurrentCHGeneric) Remove(node interface{}) { + c.countMutex.Lock() + defer c.countMutex.Unlock() + c.CHGeneric.Remove(node) +} + +// RemoveAll remove all nodes from hash ring +func (c *ConcurrentCHGeneric) RemoveAll() { + c.countMutex.Lock() + defer c.countMutex.Unlock() + c.counter = make(map[string]*concurrentCounter, constant.DefaultMapSize) + c.CHGeneric.RemoveAll() +} + +// Reset clean all anchor infos and counters +func (c *ConcurrentCHGeneric) Reset() { + c.countMutex.Lock() + defer c.countMutex.Unlock() + c.counter = make(map[string]*concurrentCounter, constant.DefaultMapSize) + c.CHGeneric.Reset() +} + +// DeleteBalancer - +func (c *ConcurrentCHGeneric) DeleteBalancer(name string) { + c.countMutex.Lock() + delete(c.counter, name) + c.countMutex.Unlock() +} + +// NewLimiterCHGeneric return limiterCHGeneric with given concurrency +func NewLimiterCHGeneric(limiterTime time.Duration) *LimiterCHGeneric { + return &LimiterCHGeneric{ + CHGeneric: NewCHGeneric(), + limiterTime: limiterTime, + limiter: make(map[string]*concurrentLimiter, constant.DefaultMapSize), + } +} + +type concurrentLimiter struct { + head *limiterNode +} + +type limiterNode struct { + instanceKey interface{} + lastTime time.Time + next *limiterNode +} + +// LimiterCHGeneric is limiter balanced +type LimiterCHGeneric struct { + *CHGeneric + limiter map[string]*concurrentLimiter + nodeCount int + limiterMutex sync.Mutex + limiterTime time.Duration +} + +// Next returns the next scheduled node +func (c *LimiterCHGeneric) Next(name string, move bool) interface{} { + c.limiterMutex.Lock() + defer c.limiterMutex.Unlock() + if _, ok := c.limiter[name]; !ok { + c.limiter[name] = &concurrentLimiter{ + head: &limiterNode{}, + } + } + + moveFlag := move +label: + for exitFlag := 0; exitFlag <= c.nodeCount; exitFlag++ { + instanceKey := c.CHGeneric.Next(name, moveFlag) + h := c.limiter[name].head + n := h.next + for ; n != nil; n = n.next { + if n.instanceKey == instanceKey && !n.lastTime.IsZero() && time.Now().Sub(n.lastTime) < c.limiterTime { + moveFlag = true + continue label + } + if n.instanceKey == instanceKey && (n.lastTime.IsZero() || time.Now().Sub(n.lastTime) >= c.limiterTime) { + break + } + } + if n == nil { + h.next = &limiterNode{ + instanceKey: instanceKey, + next: h.next, + } + } + return instanceKey + } + return nil +} + +// Previous - returns the previous scheduled node of a function +func (c *LimiterCHGeneric) Previous(name string, move bool) interface{} { + return nil +} + +// Add a node to hash ring +func (c *LimiterCHGeneric) Add(node interface{}, weight int) { + c.limiterMutex.Lock() + defer c.limiterMutex.Unlock() + c.nodeCount++ + c.CHGeneric.Add(node, weight) +} + +// Remove a node from hash ring +func (c *LimiterCHGeneric) Remove(node interface{}) { + c.limiterMutex.Lock() + defer c.limiterMutex.Unlock() + c.nodeCount-- + c.CHGeneric.Remove(node) +} + +// RemoveAll remove all nodes from hash ring +func (c *LimiterCHGeneric) RemoveAll() { + c.limiterMutex.Lock() + defer c.limiterMutex.Unlock() + c.nodeCount = 0 + c.CHGeneric.RemoveAll() +} + +// Reset clean all anchor infos and counters +func (c *LimiterCHGeneric) Reset() { + c.limiterMutex.Lock() + defer c.limiterMutex.Unlock() + c.limiter = make(map[string]*concurrentLimiter, constant.DefaultMapSize) + c.CHGeneric.Reset() +} + +// DeleteBalancer - +func (c *LimiterCHGeneric) DeleteBalancer(name string) { + c.limiterMutex.Lock() + defer c.limiterMutex.Unlock() + c.CHGeneric.DeleteBalancer(name) + delete(c.limiter, name) +} + +// SetStain give the specified function, specify the node to set the stain +func (c *LimiterCHGeneric) SetStain(function string, node interface{}) { + c.limiterMutex.Lock() + defer c.limiterMutex.Unlock() + if _, ok := c.limiter[function]; !ok { + return + } + n := c.limiter[function].head + for ; n != nil; n = n.next { + if n.instanceKey == node { + n.lastTime = time.Now() + return + } + } +} diff --git a/frontend/pkg/common/faas_common/loadbalance/hash_test.go b/frontend/pkg/common/faas_common/loadbalance/hash_test.go new file mode 100644 index 0000000000000000000000000000000000000000..c9ed8f204f37f55e22302db39da25e7cec62b1e8 --- /dev/null +++ b/frontend/pkg/common/faas_common/loadbalance/hash_test.go @@ -0,0 +1,83 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package loadbalance provides consistent hash algorithm +package loadbalance + +import ( + "github.com/smartystreets/goconvey/convey" + "testing" + "time" +) + +func TestConcurrentCHGeneric_Next(t *testing.T) { + convey.Convey("concurrentCHGeneric next", t, func() { + generic := NewConcurrentCHGeneric(10) + + generic.Add("node1", 0) + generic.Add("node2", 0) + next1 := generic.Next("function1", false) + next2 := generic.Next("function1", false) + convey.So(next1, convey.ShouldResemble, next2) + + generic = NewConcurrentCHGeneric(1) + generic.Add("node1", 0) + generic.Add("node2", 0) + next3 := generic.Next("function1", false) + next4 := generic.Next("function1", false) + convey.So(next3, convey.ShouldNotResemble, next4) + }) +} + +func TestCHGeneric_Previous(t *testing.T) { + convey.Convey("CHGeneric previous", t, func() { + generic := NewCHGeneric() + generic.Add("node1", 0) + generic.Add("node2", 0) + generic.Add("node3", 0) + + previous := generic.Previous("node2", false) + convey.So(previous, convey.ShouldEqual, "node1") + + previous = generic.Previous("node2", true) + convey.So(previous, convey.ShouldEqual, "node3") + }) +} + +func TestLimiterCHGeneric_DeleteBalancer(t *testing.T) { + convey.Convey("LimiterCHGeneric_DeleteBalancer", t, func() { + generic := NewLimiterCHGeneric(1 * time.Second) + generic.Add("node1", 0) + generic.Add("node2", 0) + generic.Add("node3", 0) + + next1 := generic.Next("function1", false) + convey.So(next1, convey.ShouldEqual, "node2") + next2 := generic.Next("function2", false) + convey.So(next2, convey.ShouldEqual, "node3") + + _, ok := generic.limiter["function1"] + _, exist := generic.anchorPoint["function1"] + convey.So(ok, convey.ShouldBeTrue) + convey.So(exist, convey.ShouldBeTrue) + + generic.DeleteBalancer("function1") + _, ok = generic.limiter["function1"] + _, exist = generic.anchorPoint["function1"] + convey.So(ok, convey.ShouldBeFalse) + convey.So(exist, convey.ShouldBeFalse) + }) +} diff --git a/frontend/pkg/common/faas_common/loadbalance/hashcache.go b/frontend/pkg/common/faas_common/loadbalance/hashcache.go new file mode 100644 index 0000000000000000000000000000000000000000..66a0f9dc5c1e56e2a29e71a25759d03f637cc868 --- /dev/null +++ b/frontend/pkg/common/faas_common/loadbalance/hashcache.go @@ -0,0 +1,43 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package loadbalance + +import "sync" + +type hashCache struct { + hashes sync.Map +} + +func createHashCache() *hashCache { + return &hashCache{ + hashes: sync.Map{}, + } +} + +func (cache *hashCache) getHash(key string) uint32 { + hashIf, ok := cache.hashes.Load(key) + if ok { + hash, ok := hashIf.(uint32) + if ok { + return hash + } + return 0 + } + hash := getHashKeyCRC32([]byte(key)) + cache.hashes.Store(key, hash) + return hash +} diff --git a/frontend/pkg/common/faas_common/loadbalance/loadbalance.go b/frontend/pkg/common/faas_common/loadbalance/loadbalance.go new file mode 100644 index 0000000000000000000000000000000000000000..7b3365eceedd1a57d29300b79972baeaa11cd045 --- /dev/null +++ b/frontend/pkg/common/faas_common/loadbalance/loadbalance.go @@ -0,0 +1,68 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package loadbalance provides load balancing algorithm +package loadbalance + +import "time" + +const ( + // RoundRobinNginx represents type of Round Robin Nginx + RoundRobinNginx LBType = iota + // RoundRobinLVS represents type of Round Robin LVS + RoundRobinLVS + // ConsistentHashGeneric represents type of Generic Consistent Hash + ConsistentHashGeneric + // ConcurrentConsistentHashGeneric represents type of concurrent Consistent + ConcurrentConsistentHashGeneric +) + +// Request - +type Request struct { + Name string + TraceID string + Timestamp time.Time +} + +// LBType is the type of load loadbalance algorithm +type LBType int + +const defaultCHGenericConcurrency = 100 + +// LoadBalance is the interface of loadbalance algorithm +type LoadBalance interface { + Next(name string, move bool) interface{} // move parameter controls whether the hash loop moves + Previous(name string, move bool) interface{} + Add(node interface{}, weight int) + Remove(node interface{}) + RemoveAll() + Reset() + DeleteBalancer(name string) +} + +// LBFactory is the factory of loadbalance algorithm +func LBFactory(t LBType) LoadBalance { + switch t { + case RoundRobinNginx: + return &WNGINX{} + case ConsistentHashGeneric: + return NewCHGeneric() + case ConcurrentConsistentHashGeneric: + return NewConcurrentCHGeneric(defaultCHGenericConcurrency) + default: + return NewCHGeneric() + } +} diff --git a/frontend/pkg/common/faas_common/loadbalance/loadbalance_test.go b/frontend/pkg/common/faas_common/loadbalance/loadbalance_test.go new file mode 100644 index 0000000000000000000000000000000000000000..17e7d8761f57d8b368514cc5dfa18d5b97015415 --- /dev/null +++ b/frontend/pkg/common/faas_common/loadbalance/loadbalance_test.go @@ -0,0 +1,238 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package loadbalance provides consistent hash algorithm +package loadbalance + +import ( + "strconv" + "sync" + "testing" + "time" + + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type LBTestSuite struct { + suite.Suite + LoadBalance + lbType LBType + m sync.RWMutex + emptyNode interface{} +} + +func (lbs *LBTestSuite) SetupSuite() { + switch lbs.lbType { + case RoundRobinNginx, RoundRobinLVS: + lbs.emptyNode = nil + case ConsistentHashGeneric: + lbs.emptyNode = "" + default: + lbs.emptyNode = "" + } +} + +func (lbs *LBTestSuite) SetupTest() { + lbs.m = sync.RWMutex{} + lbs.LoadBalance = LBFactory(lbs.lbType) +} + +func (lbs *LBTestSuite) TearDownTest() { + lbs.LoadBalance = nil +} + +func (lbs *LBTestSuite) AddToLB(workerInstance interface{}, weight int) { + switch lbs.lbType { + case RoundRobinNginx, RoundRobinLVS: + lbs.m.Lock() + lbs.Add(workerInstance, weight) + lbs.Reset() + lbs.m.Unlock() + case ConsistentHashGeneric: + lbs.Add(workerInstance, 0) + default: + } +} + +func (lbs *LBTestSuite) DelFromLB(workerInstance interface{}) { + switch lbs.lbType { + case RoundRobinNginx, RoundRobinLVS: + lbs.m.Lock() + lbs.Remove(workerInstance) + lbs.Reset() + defer lbs.m.Unlock() + case ConsistentHashGeneric: + lbs.Remove(workerInstance) + default: + } +} + +func (lbs *LBTestSuite) TestAdd() { + lbs.AddToLB("new-node-01", 0) + lbs.AddToLB("new-node-01", 1) // test duplicate + lbs.AddToLB("new-node-02", 2) + lbs.AddToLB("new-node-03", 5) + lbs.AddToLB("", 6) + lbs.AddToLB(nil, 4) + next := lbs.Next("fn-urn-01", true) + assert.NotEqual(lbs.T(), lbs.emptyNode, next) + lbs.Reset() + next = lbs.Next("fn-urn-01", true) + assert.NotEqual(lbs.T(), lbs.emptyNode, next) +} + +func (lbs *LBTestSuite) TestNext() { + var wg sync.WaitGroup + next := lbs.Next("fn-urn-01", false) + assert.Equal(lbs.T(), lbs.emptyNode, next) + + lbs.AddToLB("new-node-01", 5) + next = lbs.Next("fn-urn-01", true) + assert.Equal(lbs.T(), "new-node-01", next) + + for i := 2; i < 5; i++ { + wg.Add(1) + go func(i int, wg *sync.WaitGroup) { + lbs.AddToLB("new-node-0"+strconv.Itoa(i), 5) + wg.Done() + }(i, &wg) + } + wg.Wait() + next = lbs.Next("fn-urn-01", true) + assert.NotEqual(lbs.T(), lbs.emptyNode, next) +} + +func (lbs *LBTestSuite) TestRemove() { + var wg sync.WaitGroup + for i := 1; i < 5; i++ { + wg.Add(1) + go func(i int, wg *sync.WaitGroup) { + lbs.AddToLB("new-node-0"+strconv.Itoa(i), 5) + wg.Done() + }(i, &wg) + } + wg.Wait() + for i := 1; i < 4; i++ { + wg.Add(1) + go func(i int, wg *sync.WaitGroup) { + lbs.DelFromLB("new-node-0" + strconv.Itoa(i)) + wg.Done() + }(i, &wg) + } + wg.Wait() + next := lbs.Next("fn-urn-01", true) + assert.Equal(lbs.T(), "new-node-04", next) +} + +func (lbs *LBTestSuite) TestRemoveAll() { + var wg sync.WaitGroup + for i := 1; i < 5; i++ { + wg.Add(1) + go func(i int, wg *sync.WaitGroup) { + lbs.Add("new-node-0"+strconv.Itoa(i), 5) + wg.Done() + }(i, &wg) + } + wg.Wait() + lbs.RemoveAll() + next := lbs.Next("fn-urn-01", true) + assert.Equal(lbs.T(), lbs.emptyNode, next) +} + +func TestLBTestSuite(t *testing.T) { + suite.Run(t, &LBTestSuite{lbType: ConsistentHashGeneric}) +} + +func TestConcurrentCHGeneric_Add(t *testing.T) { + con := NewConcurrentCHGeneric(2) + con.Add("n1", 0) + con.Add("n2", 0) + + next := con.Next("n1", false) + assert.Equal(t, "n2", next) + + con.Remove("n2") + con.RemoveAll() + con.Reset() + + next = con.Next("n1", false) + assert.Equal(t, "", next) +} + +func TestLimiterCHGeneric(t *testing.T) { + limiter := NewLimiterCHGeneric(5 * time.Second) + limiter.Add("n1", 0) + limiter.Add("n2", 0) + limiter.Add("n3", 0) + + next := limiter.Next("func1", false) + assert.Equal(t, "n1", next) + + limiter.SetStain("func1", "n1") + + next = limiter.Next("func1", false) + assert.Equal(t, "n3", next) + + limiter.SetStain("func1", "n3") + + next = limiter.Next("func1", false) + assert.Equal(t, "n2", next) + + limiter.SetStain("func1", "n2") + + next = limiter.Next("func1", false) + assert.Equal(t, nil, next) + + time.Sleep(5 * time.Second) + + next = limiter.Next("func1", false) + assert.Equal(t, "n2", next) + + limiter.Remove("n2") + + next = limiter.Next("func1", false) + assert.Equal(t, "n1", next) + + limiter.RemoveAll() + + next = limiter.Next("func1", false) + assert.Equal(t, "", next) + + limiter.Reset() +} + +func TestLBFactory(t *testing.T) { + convey.Convey("LBFactory", t, func() { + convey.Convey("RoundRobinNginx", func() { + factory := LBFactory(LBType(0)) + convey.So(factory, convey.ShouldNotBeNil) + }) + convey.Convey("ConsistentHashGeneric", func() { + factory := LBFactory(LBType(2)) + convey.So(factory, convey.ShouldNotBeNil) + }) + convey.Convey("ConcurrentConsistentHashGeneric", func() { + factory := LBFactory(LBType(3)) + convey.So(factory, convey.ShouldNotBeNil) + }) + convey.Convey("default", func() { + factory := LBFactory(LBType(1)) + convey.So(factory, convey.ShouldNotBeNil) + }) + }) +} diff --git a/frontend/pkg/common/faas_common/loadbalance/nolockconsistenthash.go b/frontend/pkg/common/faas_common/loadbalance/nolockconsistenthash.go new file mode 100644 index 0000000000000000000000000000000000000000..852df80764ea751299fdbbf7dc2376afe548eff6 --- /dev/null +++ b/frontend/pkg/common/faas_common/loadbalance/nolockconsistenthash.go @@ -0,0 +1,126 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package loadbalance + +import ( + "errors" + "sort" +) + +// Node - +type Node struct { + Obj interface{} + Key string + hash uint32 +} + +// NoLockLoadBalance - +type NoLockLoadBalance interface { + Add(node *Node) error + Next(key string) *Node + Delete(nodeKey string) *Node +} + +// CreateNoLockLB - +func CreateNoLockLB() NoLockLoadBalance { + return &ConsistentHash{ + nodes: make([]*Node, 0), + cache: createHashCache(), + } +} + +type nodeSlice []*Node + +// Len returns the size +func (s nodeSlice) Len() int { + return len(s) +} + +// Swap will swap two elements +func (s nodeSlice) Swap(i, j int) { + if i < 0 || i >= len(s) || j < 0 || j >= len(s) { + return + } + s[i], s[j] = s[j], s[i] +} + +// Less returns true if i less than j +func (s nodeSlice) Less(i, j int) bool { + if i < 0 || i >= len(s) || j < 0 || j >= len(s) { + return false + } + return s[i].hash < s[j].hash +} + +// ConsistentHash - +type ConsistentHash struct { + cache *hashCache + nodes nodeSlice +} + +// Add - +func (c *ConsistentHash) Add(newNode *Node) error { + newNode.hash = getHashKeyCRC32([]byte(newNode.Key)) + for _, node := range c.nodes { + if node.Key == newNode.Key { + return errors.New("node already exist") + } + if node.hash == newNode.hash { + return errors.New("node hash already exist") + } + } + + c.nodes = append(c.nodes, newNode) + sort.Sort(c.nodes) + return nil +} + +// Next - +func (c *ConsistentHash) Next(key string) *Node { + if len(c.nodes) == 0 { + return nil + } + + keyHash := c.cache.getHash(key) + index := c.search(keyHash) + return c.nodes[index] +} + +func (c *ConsistentHash) search(keyHash uint32) int { + f := func(x int) bool { + if x >= len(c.nodes) { + return false + } + return c.nodes[x].hash > keyHash + } + index := sort.Search(len(c.nodes), f) + if index >= len(c.nodes) { + return 0 + } + return index +} + +// Delete - +func (c *ConsistentHash) Delete(nodeKey string) *Node { + for i, node := range c.nodes { + if node.Key == nodeKey { + c.nodes = append(c.nodes[:i], c.nodes[i+1:]...) + return node + } + } + return nil +} diff --git a/frontend/pkg/common/faas_common/loadbalance/nolockconsistenthash_test.go b/frontend/pkg/common/faas_common/loadbalance/nolockconsistenthash_test.go new file mode 100644 index 0000000000000000000000000000000000000000..c57de8812dc5f453a3c305d7a1a5bded2bd1374e --- /dev/null +++ b/frontend/pkg/common/faas_common/loadbalance/nolockconsistenthash_test.go @@ -0,0 +1,96 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package loadbalance + +import ( + "testing" + + "github.com/smartystreets/goconvey/convey" +) + +const ( + nodeKey = "faas-scheduler-6b758c8b74-5zdwv" + funcKeyWithRes = "7e186a/0@base@testresourcepython36768/latest/300-128" +) + +var ( + node1 = &Node{ + Key: nodeKey, + } + + node2 = &Node{ + Key: nodeKey + "1", + } + + node3 = &Node{ + Key: nodeKey + "2", + } +) + +type mockRealNode struct { + state bool +} + +func (node *mockRealNode) IsEnable() bool { + return node.state +} + +func TestStatefulConsistent(t *testing.T) { + convey.Convey("TestStatefulConsistentHashWithOneNode", t, func() { + lb := CreateNoLockLB() + outNode := lb.Next(funcKeyWithRes) + convey.So(outNode, convey.ShouldBeNil) + + lb.Add(node1) + lb.Add(node1) + + outNode = lb.Next(funcKeyWithRes) + convey.So(outNode, convey.ShouldNotBeNil) + convey.So(outNode.Key, convey.ShouldEqual, node1.Key) + + outNode = lb.Delete(nodeKey) + convey.So(outNode, convey.ShouldNotBeNil) + convey.So(outNode.Key, convey.ShouldEqual, node1.Key) + + outNode = lb.Delete(nodeKey) + convey.So(outNode, convey.ShouldBeNil) + + outNode = lb.Next(funcKeyWithRes) + convey.So(outNode, convey.ShouldBeNil) + + lb.Add(node2) + lb.Add(node3) + lb.Add(node1) + + outNode = lb.Next(funcKeyWithRes) + convey.So(outNode, convey.ShouldNotBeNil) + convey.So(outNode.Key, convey.ShouldEqual, node2.Key) + + }) +} + +func BenchmarkStatefulConsistentHashWithThreeNode(b *testing.B) { + lb := CreateNoLockLB() + + lb.Add(node1) + lb.Add(node2) + lb.Add(node3) + + for i := 0; i < b.N; i++ { + lb.Next(funcKeyWithRes + "3") + } +} diff --git a/frontend/pkg/common/faas_common/loadbalance/roundrobin.go b/frontend/pkg/common/faas_common/loadbalance/roundrobin.go new file mode 100644 index 0000000000000000000000000000000000000000..448d5666ecfe207ddf83836191cf6abde82e4655 --- /dev/null +++ b/frontend/pkg/common/faas_common/loadbalance/roundrobin.go @@ -0,0 +1,129 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package loadbalance provides roundrobin algorithm +package loadbalance + +// WeightNginx weight nginx +type WeightNginx struct { + Node interface{} + Weight int + CurrentWeight int + EffectiveWeight int +} + +// WNGINX w nginx +type WNGINX struct { + nodes []*WeightNginx +} + +// Add add node +func (w *WNGINX) Add(node interface{}, weight int) { + weightNginx := &WeightNginx{ + Node: node, + Weight: weight, + EffectiveWeight: weight} + w.nodes = append(w.nodes, weightNginx) +} + +// Remove removes a node +func (w *WNGINX) Remove(node interface{}) { + for i, weighted := range w.nodes { + if weighted.Node == node { + w.nodes = append(w.nodes[:i], w.nodes[i+1:]...) + break + } + } +} + +// RemoveAll remove all nodes +func (w *WNGINX) RemoveAll() { + w.nodes = w.nodes[:0] +} + +// Next get next node +func (w *WNGINX) Next(_ string, _ bool) interface{} { + if len(w.nodes) == 0 { + return nil + } + if len(w.nodes) == 1 { + return w.nodes[0].Node + } + return nextWeightedNode(w.nodes).Node +} + +// Previous - returns the previous scheduled node of a function +func (w *WNGINX) Previous(name string, move bool) interface{} { + return nil +} + +// DeleteBalancer - +func (w *WNGINX) DeleteBalancer(name string) { +} + +// nextWeightedNode get best next node info +func nextWeightedNode(nodes []*WeightNginx) *WeightNginx { + total := 0 + if len(nodes) == 0 { + return nil + } + best := nodes[0] + for _, w := range nodes { + w.CurrentWeight += w.EffectiveWeight + total += w.EffectiveWeight + if w.CurrentWeight > best.CurrentWeight { + best = w + } + } + best.CurrentWeight -= total + return best +} + +// Reset reset all nodes +func (w *WNGINX) Reset() { + for _, s := range w.nodes { + s.EffectiveWeight = s.Weight + s.CurrentWeight = 0 + } +} + +// Done - +func (w *WNGINX) Done(node interface{}) {} + +// NextWithRequest - +func (w *WNGINX) NextWithRequest(req *Request, move bool) interface{} { + return w.Next(req.Name, move) +} + +// SetConcurrency - +func (w *WNGINX) SetConcurrency(concurrency int) {} + +// Start - +func (w *WNGINX) Start() {} + +// Stop - +func (w *WNGINX) Stop() {} + +// NoLock - +func (w *WNGINX) NoLock() bool { + return false +} + +// WeightLvs weight lv5 +type WeightLvs struct { + Node interface{} + Weight int +} diff --git a/frontend/pkg/common/faas_common/loadbalance/roundrobin_test.go b/frontend/pkg/common/faas_common/loadbalance/roundrobin_test.go new file mode 100644 index 0000000000000000000000000000000000000000..6ec3abf13e9e1943e854834208f0b9ad05e732e0 --- /dev/null +++ b/frontend/pkg/common/faas_common/loadbalance/roundrobin_test.go @@ -0,0 +1,95 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package loadbalance + +import ( + "testing" + + "github.com/smartystreets/goconvey/convey" +) + +func TestNext(t *testing.T) { + convey.Convey("node length is 0", t, func() { + node := []*WeightNginx{} + wnginx := WNGINX{node} + + res := wnginx.Next("", true) + convey.So(res, convey.ShouldBeNil) + }) + convey.Convey("node length is 1", t, func() { + node := []*WeightNginx{ + {"Node1", 30, 10, 20}, + } + wnginx := WNGINX{node} + + res := wnginx.Next("", true) + convey.So(res, convey.ShouldNotBeNil) + }) + convey.Convey("node length > 1", t, func() { + node := []*WeightNginx{ + {"Node1", 30, 10, 20}, + {"Node2", 30, 60, 20}, + } + wnginx := WNGINX{node} + + res := wnginx.Next("", true) + resStr, ok := res.(string) + convey.So(ok, convey.ShouldBeTrue) + convey.So(resStr, convey.ShouldEqual, "Node2") + }) + + convey.Convey("remove", t, func() { + node := []*WeightNginx{ + {"Node1", 30, 10, 20}, + } + wnginx := WNGINX{node} + wnginx.Add("Node2", 60) + res := wnginx.Next("", true) + resStr, ok := res.(string) + convey.So(ok, convey.ShouldBeTrue) + convey.So(resStr, convey.ShouldEqual, "Node2") + + wnginx.Remove("Node2") + res = wnginx.Next("", true) + resStr, ok = res.(string) + convey.So(ok, convey.ShouldBeTrue) + convey.So(resStr, convey.ShouldEqual, "Node1") + }) + + convey.Convey("remove", t, func() { + node := []*WeightNginx{ + {"Node1", 30, 10, 20}, + {"Node2", 30, 60, 20}, + } + wnginx := WNGINX{node} + wnginx.RemoveAll() + convey.So(len(wnginx.nodes), convey.ShouldEqual, 0) + }) +} + +func TestReset(t *testing.T) { + convey.Convey("Reset success", t, func() { + weightNginx := &WeightNginx{"Node1", 30, 10, 20} + var node []*WeightNginx + node = append(node, weightNginx) + wnginx := WNGINX{node} + + wnginx.Reset() + convey.So(weightNginx.EffectiveWeight, convey.ShouldEqual, weightNginx.Weight) + }) + +} diff --git a/frontend/pkg/common/faas_common/localauth/authcache.go b/frontend/pkg/common/faas_common/localauth/authcache.go new file mode 100644 index 0000000000000000000000000000000000000000..4a1cc50ef5e044e5cd1da9de6f2ad09b673d2c5f --- /dev/null +++ b/frontend/pkg/common/faas_common/localauth/authcache.go @@ -0,0 +1,195 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package localauth authenticates requests by local configmaps +package localauth + +import ( + "errors" + "sync" + "sync/atomic" + "time" + + "k8s.io/client-go/tools/cache" + + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/signals" +) + +const ( + senderCacheDuration = 1 * time.Minute +) + +// AuthCache cache interface +type AuthCache interface { + GetSignForSender() (string, string, error) + GetSignForReceiver(auth string) (string, bool, error) + updateReceiver(string) error + updateSender(string, string) +} + +type authCache struct { + // use an atomic value to promise concurrent safety, which stores the authorization token and time. + senderCache *atomic.Value + // sign-time + receiverCache cache.Store + appID string + AuthConfig +} + +type senderValue struct { + auth string + time string +} + +var localCache *authCache +var doOnce sync.Once + +// GetLocalAuthCache you have to create it before you get it. +func GetLocalAuthCache(aKey, sKey, appID string, duration int) AuthCache { + doOnce.Do(func() { + var c cache.Store + stopCh := signals.WaitForSignal() + // cache.Store ttl the minimum valid value is 1 second. If this parameter is set to 0, + // the cache does not need to be increased. Therefore, set the cache to nil. + if duration == 0 { + c = nil + } else { + c = cache.NewTTLStore(receiverCacheKey, time.Duration(duration)*time.Minute) + } + atom := &atomic.Value{} + localCache = &authCache{ + senderCache: atom, + receiverCache: c, + } + localCache.appID = appID + localCache.AKey = aKey + localCache.SKey = sKey + localCache.initSenderCache(stopCh) + localCache.Duration = duration + // clean expired keys by ticker could avoid worker-manager oom problem + // because receiver cache clean expired keys is lazy by calling GetByKeys method or List method + go localCache.startCleanExpiredKeysByTicker(stopCh, time.Duration(duration)*time.Minute) + }) + if localCache == nil { + return nil + } + return localCache +} + +func (c *authCache) startCleanExpiredKeysByTicker(stopCh <-chan struct{}, duration time.Duration) { + if stopCh == nil || c.receiverCache == nil { + return + } + log.GetLogger().Infof("start to clean expired keys by ticker duration %s", duration.String()) + ticker := time.NewTicker(duration) + defer ticker.Stop() + for { + select { + case <-ticker.C: + // call receiver cache list method will clean all expired keys + length := len(c.receiverCache.List()) + log.GetLogger().Debugf("receiver cache length is %d after clean expired keys once by ticker", length) + case <-stopCh: + log.GetLogger().Infof("stop channel is closed") + return + } + } +} + +// GetSignForSender return time auth error +func (c *authCache) GetSignForSender() (string, string, error) { + loaded := c.senderCache.Load() + value, ok := loaded.(senderValue) + if !ok { + return "", "", errors.New("no sender cache") + } + if value.time == "" || value.auth == "" { + return "", "", errors.New("no sender time") + } + return value.time, value.auth, nil +} + +// GetSignForReceiver value exit error +func (c *authCache) GetSignForReceiver(auth string) (string, bool, error) { + if c.receiverCache == nil { + return "", false, nil + } + key, b, err := c.receiverCache.GetByKey(auth) + if !b { + key = "" + } + return key.(string), b, err +} + +func (c *authCache) updateReceiver(sign string) error { + if c.receiverCache == nil { + return nil + } + err := c.receiverCache.Add(sign) + if err != nil { + return err + } + return nil +} + +func (c *authCache) updateSender(auth, time string) { + c.senderCache.Store(senderValue{ + auth: auth, + time: time, + }) +} + +func (c *authCache) waitForDoneSignal(stopCh <-chan struct{}) { + if stopCh == nil { + return + } + ticker := time.NewTicker(senderCacheDuration) + for { + select { + case <-ticker.C: + // update senderCache + c.createAndUpdateSender() + case <-stopCh: + ticker.Stop() + return + } + } +} + +func (c *authCache) initSenderCache(stopCh <-chan struct{}) { + c.createAndUpdateSender() + go c.waitForDoneSignal(stopCh) +} + +func (c *authCache) createAndUpdateSender() { + var data []byte + authorization, t := CreateAuthorization( + c.AKey, + c.SKey, + "", + c.appID, + data, + ) + c.updateSender(authorization, t) + if c.Duration != 0 { + log.GetLogger().Debugf("the length of receiver cache is: %d", len(c.receiverCache.ListKeys())) + } +} + +func receiverCacheKey(obj interface{}) (string, error) { + return obj.(string), nil +} diff --git a/frontend/pkg/common/faas_common/localauth/authcache_test.go b/frontend/pkg/common/faas_common/localauth/authcache_test.go new file mode 100644 index 0000000000000000000000000000000000000000..82afb746676db7dc0bc889e8661489816fe26ffe --- /dev/null +++ b/frontend/pkg/common/faas_common/localauth/authcache_test.go @@ -0,0 +1,221 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package localauth + +import ( + "reflect" + "sync/atomic" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/stretchr/testify/assert" + "k8s.io/client-go/tools/cache" +) + +func Test_receiverCacheKey(t *testing.T) { + type args struct { + obj interface{} + } + var a args + a.obj = "aaa" + tests := []struct { + name string + args args + want string + wantErr bool + }{ + {"case1", a, "aaa", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := receiverCacheKey(tt.args.obj) + if (err != nil) != tt.wantErr { + t.Errorf("receiverCacheKey() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("receiverCacheKey() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_authCache_GetSignForSender(t *testing.T) { + type fields struct { + senderCache *atomic.Value + receiverCache cache.Store + appID string + AuthConfig AuthConfig + } + var f fields + senderCache := &atomic.Value{} + f.senderCache = senderCache + + var f2 fields + senderCache2 := &atomic.Value{} + senderCache2.Store(senderValue{ + time: "aaa", + auth: "aaa", + }) + f2.senderCache = senderCache2 + tests := []struct { + name string + fields fields + wantS string + wantD string + wantErr bool + }{ + {"case1", f, "", "", true}, + {"case2", f2, "aaa", "aaa", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &authCache{ + senderCache: tt.fields.senderCache, + receiverCache: tt.fields.receiverCache, + appID: tt.fields.appID, + AuthConfig: tt.fields.AuthConfig, + } + gotS, gotD, err := c.GetSignForSender() + if (err != nil) != tt.wantErr { + t.Errorf("GetSignForSender() error = %v, wantErr %v", err, tt.wantErr) + return + } + if gotS != tt.wantS { + t.Errorf("GetSignForSender() gotS = %v, want %v", gotS, tt.wantS) + } + if gotD != tt.wantD { + t.Errorf("GetSignForSender() gotD = %v, want %v", gotD, tt.wantD) + } + }) + } +} + +func Test_authCache_updateReceiver(t *testing.T) { + type fields struct { + senderCache *atomic.Value + receiverCache cache.Store + appID string + AuthConfig AuthConfig + } + type args struct { + sign string + } + var f fields + var a args + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + {"case1", f, a, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &authCache{ + senderCache: tt.fields.senderCache, + receiverCache: tt.fields.receiverCache, + appID: tt.fields.appID, + AuthConfig: tt.fields.AuthConfig, + } + if err := c.updateReceiver(tt.args.sign); (err != nil) != tt.wantErr { + t.Errorf("updateReceiver() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_authCache_createAndUpdateSender(t *testing.T) { + receiverCache := cache.NewTTLStore(receiverCacheKey, time.Duration(5)*time.Second) + patches := [...]*gomonkey.Patches{ + gomonkey.ApplyMethod(reflect.TypeOf(receiverCache), "ListKeys", + func(_ *cache.ExpirationCache) []string { + return []string{} + }), + gomonkey.ApplyFunc(DecryptKeys, func(inputAKey string, inputSKey string) ([]byte, []byte, error) { + return []byte("aaa"), []byte("aaa"), nil + }), + } + + defer func() { + for idx := range patches { + patches[idx].Reset() + } + }() + type fields struct { + senderCache *atomic.Value + receiverCache cache.Store + appID string + AuthConfig AuthConfig + } + var f fields + f.AuthConfig.Duration = 1 + f.receiverCache = receiverCache + f.senderCache = &atomic.Value{} + f.senderCache.Store(senderValue{ + time: "aaa", + auth: "aaa", + }) + tests := []struct { + name string + fields fields + }{ + {"case1", f}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &authCache{ + senderCache: tt.fields.senderCache, + receiverCache: tt.fields.receiverCache, + appID: tt.fields.appID, + AuthConfig: tt.fields.AuthConfig, + } + c.createAndUpdateSender() + }) + } +} + +func TestWaitForDoneSignal(t *testing.T) { + c := &authCache{ + senderCache: &atomic.Value{}, + } + c.senderCache.Store(senderValue{ + time: "aaa", + auth: "bbb", + }) + c.waitForDoneSignal(nil) + stopChan := make(chan struct{}) + go c.waitForDoneSignal(stopChan) + close(stopChan) + assert.NotEqual(t, c, nil) +} + +func TestGetSignForReceiver(t *testing.T) { + c := &authCache{ + senderCache: &atomic.Value{}, + receiverCache: cache.NewTTLStore(receiverCacheKey, time.Duration(1)*time.Minute), + } + c.senderCache.Store(senderValue{ + time: "aaa", + auth: "aaa", + }) + c.updateReceiver("sign") + _, _, err := c.GetSignForReceiver("auth") + assert.Equal(t, err, nil) +} diff --git a/frontend/pkg/common/faas_common/localauth/authcheck.go b/frontend/pkg/common/faas_common/localauth/authcheck.go new file mode 100644 index 0000000000000000000000000000000000000000..6e56be25e1592de30024b3dde64d6fc980553300 --- /dev/null +++ b/frontend/pkg/common/faas_common/localauth/authcheck.go @@ -0,0 +1,407 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package localauth authenticates requests by local configmaps +package localauth + +import ( + "bytes" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "fmt" + "math" + "net/http" + "net/url" + "os" + "sort" + "strconv" + "strings" + "time" + + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/utils" +) + +const ( + modeSDK = "SDKMode" + modeHWS = "HWSMode" + // the difference limit of a timestamp + defaultTimestampDiffLimit = 5 + // 7 days + maxTimestampDiffLimit = 10080 + maxHeaderLength = 20 + minLengthOfAuthValue = 2 + base = 10 + bitSize = 64 +) + +var timestampDiffLimit = getTimestampDiffLimit() + +type modeOptions struct { + authHeaderPrefix string + timeFormat string + shortTimeFormat string + terminalString string + name string + date string +} + +var modeOption = &modeOptions{ + authHeaderPrefix: "", + timeFormat: "", + shortTimeFormat: "", + terminalString: "", + name: "", + date: "", +} + +// Signer is a struct of +type Signer struct { + signTime time.Time + serviceName string + region string +} + +// AuthConfig represents configurations of local auth +type AuthConfig struct { + AKey string `json:"aKey" yaml:"aKey" valid:"optional"` + SKey string `json:"sKey" yaml:"sKey" valid:"optional"` + Duration int `json:"duration" yaml:"duration" valid:"optional"` +} + +// Authentication represents aKey and sKey Decrypted from ak and sk +type Authentication struct { + AKey []byte + SKey []byte +} + +// signLocalAuthRequest returns the authentication header +func signLocalAuthRequest(rawURL, timeStamp, appID string, key *Authentication, data []byte) (string, []byte) { + signer := getSigner("SDKMode", "", "") + timeStampInt, err := strconv.ParseInt(timeStamp, base, bitSize) + if err != nil { + log.GetLogger().Errorf("failed to parse the timestamp string") + return "", data + } + signer.signTime = time.Unix(timeStampInt, 0) + // default text of data + if len(data) == 0 { + data = []byte(`signature verification`) + } + header := make(map[string][]string, maxHeaderLength) + header["Content-Type"] = []string{"application/json"} + + parsedURL, err := url.Parse(rawURL) + if err != nil { + log.GetLogger().Errorf("failed to parse a URL") + return "", data + } + request := &http.Request{Method: "POST", URL: parsedURL, Header: header} + signerHeader := signer.sign(request, key.AKey, key.SKey, data, appID) + return signerHeader["X-Identity-Sign"], data +} + +func getSigner(mode, serviceName, region string) *Signer { + if mode == modeSDK { + setSDKMode() + } else { + setHWSMode() + } + return &Signer{ + signTime: time.Now(), + serviceName: serviceName, + region: region, + } +} + +func setSDKMode() { + modeOption = &modeOptions{ + authHeaderPrefix: "SDK-HMAC-SHA256", + timeFormat: "20060102T150405Z", + shortTimeFormat: "20060102", + terminalString: "sdk_request", + name: "SDK", + date: "X-Sdk-Date", + } +} + +func setHWSMode() { + modeOption = &modeOptions{ + authHeaderPrefix: "HWS-HMAC-SHA256", + timeFormat: "20060102T150405Z", + shortTimeFormat: "20060102", + terminalString: "hws_request", + name: "HWS", + date: "X-Hws-Date", + } +} + +func (sig *Signer) sign(request *http.Request, aKey, sKey []byte, body []byte, + appID string) map[string]string { + header := map[string]string{} + request.Header.Add(modeOption.date, sig.signTime.UTC().Format(modeOption.timeFormat)) + contentSha256 := makeSha256Hex(body) + canonicalString := sig.buildCanonicalRequest(request, contentSha256) + stringToSign := sig.buildStringToSign(canonicalString) + signatureStr := sig.buildSignature(sKey, stringToSign) + credentialString := sig.buildCredentialString() + signedHeaders := sig.buildSignedHeadersString(request) + aKeyString := string(aKey) + utils.ClearByteMemory(aKey) + parts := []string{ + modeOption.authHeaderPrefix + " Credential=" + aKeyString + "/" + credentialString, + "SignedHeaders=" + signedHeaders, + "Signature=" + signatureStr, + } + if appID != "" { + parts = append(parts, "appid="+appID) + } + utils.ClearStringMemory(aKeyString) + + signResult := strings.Join(parts, ", ") + header["host"] = request.Host + header[modeOption.date] = sig.signTime.UTC().Format(modeOption.timeFormat) + header["Content-Type"] = "application/json;charset=UTF-8" + header["Accept"] = "application/json" + header["X-Identity-Sign"] = signResult + return header +} + +// buildSignature generate a signature with request and secret key +func (sig *Signer) buildSignature(sKey []byte, stringtoSign string) string { + var secretBuf bytes.Buffer + secretBuf.Write([]byte(modeOption.name)) + secretBuf.Write(sKey) + utils.ClearByteMemory(sKey) + sigTime := []byte(sig.signTime.UTC().Format(modeOption.shortTimeFormat)) + date := makeHmac(secretBuf.Bytes(), sigTime) + secretBuf.Reset() + region := makeHmac(date, []byte(sig.region)) + service := makeHmac(region, []byte(sig.serviceName)) + credentials := makeHmac(service, []byte(modeOption.terminalString)) + toSignature := makeHmac(credentials, []byte(stringtoSign)) + signature := hex.EncodeToString(toSignature) + return signature +} + +// buildStringToSign prepare data for building signature +func (sig *Signer) buildStringToSign(canonicalString string) string { + stringToSign := strings.Join([]string{ + modeOption.authHeaderPrefix, + sig.signTime.UTC().Format(modeOption.timeFormat), + sig.buildCredentialString(), + hex.EncodeToString(makeSha256([]byte(canonicalString))), + }, "\n") + return stringToSign +} + +// buildCanonicalRequest converts the request info into canonical format +func (sig *Signer) buildCanonicalRequest(request *http.Request, hexbody string) string { + canonicalHeadersOut := sig.buildCanonicalHeaders(request) + signedHeaders := sig.buildSignedHeadersString(request) + canonicalRequestStr := strings.Join([]string{ + request.Method, + request.URL.Path + "/", + request.URL.RawQuery, + canonicalHeadersOut, + signedHeaders, + hexbody, + }, "\n") + return canonicalRequestStr +} + +// buildCanonicalHeaders generate canonical headers +func (sig *Signer) buildCanonicalHeaders(request *http.Request) string { + var headers []string + + for header := range request.Header { + standardized := strings.ToLower(strings.TrimSpace(header)) + headers = append(headers, standardized) + } + sort.Strings(headers) + + for i, header := range headers { + headers[i] = header + ":" + strings.Replace(request.Header.Get(header), "\n", " ", -1) + } + + if len(headers) > 0 { + return strings.Join(headers, "\n") + "\n" + } + + return "" +} + +// buildSignedHeadersString convert the header in request to a certain format +func (sig *Signer) buildSignedHeadersString(request *http.Request) string { + var headers []string + for header := range request.Header { + headers = append(headers, strings.ToLower(header)) + } + sort.Strings(headers) + return strings.Join(headers, ";") +} + +// buildCredentialString add date and several other information to signature header +func (sig *Signer) buildCredentialString() string { + credentialString := strings.Join([]string{ + sig.signTime.UTC().Format(modeOption.shortTimeFormat), + sig.region, + sig.serviceName, + modeOption.terminalString, + }, "/") + return credentialString +} + +// makeHmac convert data into sha256 format with certain key +func makeHmac(key []byte, data []byte) []byte { + hash := hmac.New(sha256.New, key) + _, err := hash.Write(data) + if err != nil { + log.GetLogger().Errorf("failed to write in makeHmac, error: %s", err.Error()) + } + return hash.Sum(nil) + +} + +// makeHmac convert data into sha256 format +func makeSha256(data []byte) []byte { + hash := sha256.New() + _, err := hash.Write(data) + if err != nil { + log.GetLogger().Errorf("failed to write in makeSha256, error: %s", err.Error()) + } + return hash.Sum(nil) +} + +// makeHmac convert data into Hex format +func makeSha256Hex(data []byte) string { + hash := sha256.New() + _, err := hash.Write(data) + if err != nil { + log.GetLogger().Errorf("failed to write in makeSha256Hex, error: %s", err.Error()) + } + md := hash.Sum(nil) + hexBody := hex.EncodeToString(md) + return hexBody +} + +func getTimestampDiffLimit() float64 { + var tsDiffLimit float64 + envTimestampDiffLimit, err := strconv.Atoi(os.Getenv("AUTH_VALID_TIME_MINUTE")) + if err == nil && envTimestampDiffLimit > 0 && envTimestampDiffLimit <= maxTimestampDiffLimit { + tsDiffLimit = float64(envTimestampDiffLimit) + } else { + tsDiffLimit = float64(defaultTimestampDiffLimit) + } + log.GetLogger().Infof("current timestampDiffLimit is %f", tsDiffLimit) + return tsDiffLimit +} + +// AuthCheckLocally authenticates requests by local auth +func AuthCheckLocally(ak string, sk string, requestSign string, timestamp string, duration int) error { + if len(requestSign) == 0 { + return fmt.Errorf("authentication string is nil") + } + curTime := time.Now().Unix() + timeUnix, err := strconv.ParseInt(timestamp, base, bitSize) + if err != nil { + return fmt.Errorf("invalid timestamp") + } + // the default timestamp limit is 5 minutes + if math.Abs(float64(curTime-timeUnix)) >= timestampDiffLimit*time.Minute.Seconds() { + return fmt.Errorf("the request is timeout") + } + appID, err := getAppIDFromRequestSign(requestSign) + if err != nil { + return err + } + _, exist, err := GetLocalAuthCache(ak, sk, appID, duration).GetSignForReceiver(requestSign) + if err != nil { + log.GetLogger().Errorf("failed to get sign from receiver cache") + return err + } + if exist { + return nil + } + aKey, sKey, err := DecryptKeys(ak, sk) + if err != nil { + utils.ClearByteMemory(aKey) + utils.ClearByteMemory(sKey) + return err + } + key := &Authentication{ + AKey: aKey, + SKey: sKey, + } + var data []byte + signature, _ := signLocalAuthRequest("", timestamp, appID, key, data) + utils.ClearByteMemory(aKey) + utils.ClearByteMemory(sKey) + if signature == "" || signature != requestSign { + return fmt.Errorf("auth check failed") + } + if err := GetLocalAuthCache(ak, sk, appID, duration).updateReceiver(signature); err != nil { + log.GetLogger().Errorf("failed to update receiver cache") + return err + } + return nil +} + +func getAppIDFromRequestSign(sign string) (string, error) { + arrays := strings.Split(sign, "appid=") + if len(arrays) < minLengthOfAuthValue { + return "", fmt.Errorf("failed to parse authorization appid= %s", "*****") + } + arrays = strings.Split(arrays[1], ", ") + return arrays[0], nil +} + +// SignLocally makes signatures by local auth +func SignLocally(ak, sk, appID string, duration int) (string, string) { + t, auth, err := GetLocalAuthCache(ak, sk, appID, duration).GetSignForSender() + if err != nil { + var data []byte + log.GetLogger().Warnf("failed to get sender cache: %s", err.Error()) + return CreateAuthorization(ak, sk, "", appID, data) + } + return auth, t +} + +// SignOMSVC make signatures for request send to OMSVC +func SignOMSVC(ak, sk, url string, data []byte) (string, string) { + return CreateAuthorization(ak, sk, url, "", data) +} + +// CreateAuthorization create Authentication Information +func CreateAuthorization(ak, sk, url, appID string, data []byte) (string, string) { + timestamp := strconv.FormatInt(time.Now().Unix(), base) + aKey, sKey, err := DecryptKeys(ak, sk) + if err != nil { + utils.ClearByteMemory(aKey) + utils.ClearByteMemory(sKey) + log.GetLogger().Errorf("failed to decrypt SKey when create auth, error: %s", err.Error()) + return "", "" + } + key := &Authentication{ + AKey: aKey, + SKey: sKey, + } + authorization, _ := signLocalAuthRequest(url, timestamp, appID, key, data) + utils.ClearByteMemory(aKey) + utils.ClearByteMemory(sKey) + return authorization, timestamp +} diff --git a/frontend/pkg/common/faas_common/localauth/authcheck_test.go b/frontend/pkg/common/faas_common/localauth/authcheck_test.go new file mode 100644 index 0000000000000000000000000000000000000000..365748f770cb2d6891f2a3aa1fb0bfc25ffb0cf3 --- /dev/null +++ b/frontend/pkg/common/faas_common/localauth/authcheck_test.go @@ -0,0 +1,293 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package localauth + +import ( + "errors" + + "net/http" + "net/url" + "os" + "reflect" + "strconv" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" +) + +func TestAuthCheckLocally(t *testing.T) { + type args struct { + ak string + sk string + requestSign string + timestamp string + duration int + } + var a args + var b args + b.requestSign = "aaa" + var c args + c.requestSign = "aaa" + c.timestamp = strconv.FormatInt(time.Now().AddDate(1, 0, 0).Unix(), 10) + var d args + d.requestSign = "aaa" + d.timestamp = strconv.FormatInt(time.Now().Unix(), 10) + var e args + e.requestSign = "aaa,appid=aaa" + e.timestamp = strconv.FormatInt(time.Now().Unix(), 10) + tests := []struct { + name string + args args + wantErr bool + }{ + {"case1", a, true}, + {"case2", b, true}, + {"case3", c, true}, + {"case4", d, true}, + {"case5", e, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := AuthCheckLocally(tt.args.ak, tt.args.sk, tt.args.requestSign, tt.args.timestamp, tt.args.duration); (err != nil) != tt.wantErr { + t.Errorf("AuthCheckLocally() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_GetTimestampDiffLimit(t *testing.T) { + tsDiffLimit := getTimestampDiffLimit() + assert.Equal(t, 5, int(tsDiffLimit)) + patches := gomonkey.ApplyFunc(os.Getenv, func(key string) string { + return "100" + }) + tsDiffLimit = getTimestampDiffLimit() + assert.Equal(t, 100, int(tsDiffLimit)) + defer patches.Reset() +} + +func TestCreateAuthorization(t *testing.T) { + patches := [...]*gomonkey.Patches{ + gomonkey.ApplyFunc(DecryptKeys, + func(_ string, _ string) ([]byte, []byte, error) { + return []byte{}, []byte{}, errors.New("aaa") + }), + } + defer func() { + for idx := range patches { + patches[idx].Reset() + } + }() + type args struct { + ak string + sk string + url string + appID string + data []byte + } + var a args + tests := []struct { + name string + args args + want string + want1 string + }{ + {"case1", a, "", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := CreateAuthorization(tt.args.ak, tt.args.sk, tt.args.url, tt.args.appID, tt.args.data) + if got != tt.want { + t.Errorf("CreateAuthorization() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("CreateAuthorization() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func TestSignOMSVC(t *testing.T) { + patches := [...]*gomonkey.Patches{ + gomonkey.ApplyFunc(DecryptKeys, + func(_ string, _ string) ([]byte, []byte, error) { + return []byte{}, []byte{}, errors.New("aaa") + }), + } + defer func() { + for idx := range patches { + patches[idx].Reset() + } + }() + type args struct { + ak string + sk string + url string + data []byte + } + var a args + tests := []struct { + name string + args args + want string + want1 string + }{ + {"case1", a, "", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := SignOMSVC(tt.args.ak, tt.args.sk, tt.args.url, tt.args.data) + if got != tt.want { + t.Errorf("SignOMSVC() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("SignOMSVC() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func TestSigner_buildCanonicalHeaders(t *testing.T) { + type fields struct { + signTime time.Time + serviceName string + region string + } + type args struct { + request *http.Request + } + var f fields + var a args + request := &http.Request{ + Method: "", + URL: nil, + Proto: "", + ProtoMajor: 0, + ProtoMinor: 0, + Header: nil, + Body: nil, + GetBody: nil, + ContentLength: 0, + TransferEncoding: nil, + Close: false, + Host: "", + Form: nil, + PostForm: nil, + MultipartForm: nil, + Trailer: nil, + RemoteAddr: "", + RequestURI: "", + TLS: nil, + Response: nil, + } + a.request = request + tests := []struct { + name string + fields fields + args args + want string + }{ + {"case1", f, a, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sig := &Signer{ + signTime: tt.fields.signTime, + serviceName: tt.fields.serviceName, + region: tt.fields.region, + } + if got := sig.buildCanonicalHeaders(tt.args.request); got != tt.want { + t.Errorf("buildCanonicalHeaders() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getSigner(t *testing.T) { + type args struct { + mode string + serviceName string + region string + } + var a args + a.mode = "aaa" + signer := &Signer{} + tests := []struct { + name string + args args + want *Signer + }{ + {"case1", a, signer}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getSigner(tt.args.mode, tt.args.serviceName, tt.args.region); !reflect.DeepEqual(got.serviceName, tt.want.serviceName) { + t.Errorf("getSigner() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_signLocalAuthRequest(t *testing.T) { + patches := [...]*gomonkey.Patches{ + gomonkey.ApplyFunc(url.Parse, + func(_ string) (*url.URL, error) { + return nil, errors.New("aaa") + }), + } + defer func() { + for idx := range patches { + patches[idx].Reset() + } + }() + type args struct { + rawURL string + timeStamp string + appID string + key *Authentication + data []byte + } + var a args + var b args + b.timeStamp = strconv.FormatInt(time.Now().Unix(), 10) + tests := []struct { + name string + args args + want string + }{ + {"case1", a, ""}, + {"case2", b, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, _ := signLocalAuthRequest(tt.args.rawURL, tt.args.timeStamp, tt.args.appID, tt.args.key, tt.args.data); got != tt.want { + t.Errorf("signLocalAuthRequest() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestSignLocally(t *testing.T) { + convey.Convey("TestSignLocally", t, func() { + auth, time := SignLocally("ak", "sk", "appID", 0) + convey.So(auth, convey.ShouldNotBeEmpty) + convey.So(time, convey.ShouldNotBeEmpty) + }) +} diff --git a/frontend/pkg/common/faas_common/localauth/crypto.go b/frontend/pkg/common/faas_common/localauth/crypto.go new file mode 100644 index 0000000000000000000000000000000000000000..5016b8a5aa589577051c381c51cc9daba0f5a30c --- /dev/null +++ b/frontend/pkg/common/faas_common/localauth/crypto.go @@ -0,0 +1,50 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package localauth authenticates requests by local configmaps +package localauth + +import ( + "sync" +) + +var ( + algorithm = "aeswithkey" + once sync.Once +) + +func initCrypto() error { + return nil +} + +// Decrypt decrypts a cypher text using a certain algorithm +func Decrypt(src string) ([]byte, error) { + var text []byte + return text, nil +} + +// Encrypt encrypts a cypher text using a certain algorithm +func Encrypt(src string) (string, error) { + var ciperText string + return ciperText, nil +} + +// DecryptKeys decrypts a set of aKey and sKey +func DecryptKeys(inputAKey string, inputSKey string) ([]byte, []byte, error) { + var aKey []byte + var sKey []byte + return aKey, sKey, nil +} diff --git a/frontend/pkg/common/faas_common/localauth/crypto_test.go b/frontend/pkg/common/faas_common/localauth/crypto_test.go new file mode 100644 index 0000000000000000000000000000000000000000..f7da6d37011cf495f813af8452dfb95dabda63ac --- /dev/null +++ b/frontend/pkg/common/faas_common/localauth/crypto_test.go @@ -0,0 +1,35 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package localauth + +import ( + "sync" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/common/faas_common/utils" +) + +func (m mockEngine) Encrypt(domainId int, encData string) (string, error) { + return encData, nil +} + +func (m mockEngine) Decrypt(domainId int, encData string) (string, error) { + return encData, nil +} diff --git a/frontend/pkg/common/faas_common/localauth/env.go b/frontend/pkg/common/faas_common/localauth/env.go new file mode 100644 index 0000000000000000000000000000000000000000..c7ff2e0fadbf26fdac54411c3a59e2097b0452e0 --- /dev/null +++ b/frontend/pkg/common/faas_common/localauth/env.go @@ -0,0 +1,36 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package localauth authenticates requests by local configmaps +package localauth + +import ( + "encoding/json" + "os" + + "frontend/pkg/common/faas_common/logger/log" +) + +// GetDecryptFromEnv - +func GetDecryptFromEnv() (map[string]string, error) { + res := make(map[string]string) + value := os.Getenv("ENV_DELEGATE_DECRYPT") + err := json.Unmarshal([]byte(value), &res) + if err != nil { + log.GetLogger().Warnf("ENV_DELEGATE_DECRYPT unmarshal error, it is null") + } + return res, nil +} diff --git a/frontend/pkg/common/faas_common/localauth/env_test.go b/frontend/pkg/common/faas_common/localauth/env_test.go new file mode 100644 index 0000000000000000000000000000000000000000..7e4c9db3b166689cf9ed912e68b25d55bbda991c --- /dev/null +++ b/frontend/pkg/common/faas_common/localauth/env_test.go @@ -0,0 +1,73 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package localauth authenticates requests by local configmaps +package localauth + +import ( + "encoding/json" + "errors" + "os" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + + mockUtils "frontend/pkg/common/faas_common/utils" +) + +func TestGetDecryptFromEnv(t *testing.T) { + tests := []struct { + name string + want map[string]string + wantErr bool + patchesFunc mockUtils.PatchesFunc + }{ + {"case1 failed to unmarshal", make(map[string]string), false, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(json.Unmarshal, func(data []byte, v interface{}) error { + return errors.New("failed to unmarshal json") + }), + }) + return patches + }}, + {"case2 succeed to unmarshal", map[string]string{"test": "test"}, + false, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(os.Getenv, func(key string) string { + return `{"test":"test"}` + }), + }) + return patches + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + patches := tt.patchesFunc() + got, err := GetDecryptFromEnv() + if (err != nil) != tt.wantErr { + t.Errorf("GetDecryptFromEnv() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetDecryptFromEnv() got = %v, want %v", got, tt.want) + } + patches.ResetAll() + }) + } +} diff --git a/frontend/pkg/common/faas_common/logger/async/writer.go b/frontend/pkg/common/faas_common/logger/async/writer.go new file mode 100644 index 0000000000000000000000000000000000000000..bc2e9841758b69cc66a7b5a2054347c6cc85c0c6 --- /dev/null +++ b/frontend/pkg/common/faas_common/logger/async/writer.go @@ -0,0 +1,176 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package async makes io.Writer write async +package async + +import ( + "bytes" + "fmt" + "io" + "sync/atomic" + "time" + + "go.uber.org/zap/buffer" +) + +const ( + diskBufferSize = 1024 * 1024 + diskFlushSize = diskBufferSize >> 1 + diskFlushTime = 500 * time.Millisecond + defaultChannelSize = 200000 + softLimitFactor = 0.8 // must be smaller than 1 +) + +var ( + linePool = buffer.NewPool() +) + +// Opt - +type Opt func(*Writer) + +// WithCachedLimit - +func WithCachedLimit(limit int) Opt { + return func(w *Writer) { + w.cachedLimit = limit + w.cachedSoftLimit = int(float64(limit) * softLimitFactor) + w.cachedLow = w.cachedSoftLimit >> 1 + } +} + +// NewAsyncWriteSyncer wrappers io.Writer to async zapcore.WriteSyncer +func NewAsyncWriteSyncer(w io.Writer, opts ...Opt) *Writer { + writer := &Writer{ + w: w, + diskBuf: bytes.NewBuffer(make([]byte, 0, diskBufferSize)), + lines: make(chan *buffer.Buffer, defaultChannelSize), + sync: make(chan struct{}), + syncDone: make(chan struct{}), + } + for _, opt := range opts { + opt(writer) + } + go writer.logConsumer() + return writer +} + +// Writer - +type Writer struct { + diskBuf *bytes.Buffer + lines chan *buffer.Buffer + w io.Writer + sync chan struct{} + syncDone chan struct{} + + cachedLimit int + cachedSoftLimit int + cachedLow int + cached int64 // atomic +} + +// Write sends data to channel non-blocking +func (w *Writer) Write(data []byte) (int, error) { + // note: data will be put back to zap's inner pool after Write, so we couldn't send it to channel directly + lp := linePool.Get() + lp.Write(data) + select { + case w.lines <- lp: + if w.cachedLimit != 0 && atomic.AddInt64(&w.cached, int64(len(data))) > int64(w.cachedLimit) { + w.doSync() + } + default: + fmt.Println("failed to push log to channel, skip") + lp.Free() + } + return len(data), nil +} + +// Sync implements zapcore.WriteSyncer. Current do nothing. +func (w *Writer) Sync() error { + w.doSync() + return nil +} + +func (w *Writer) doSync() { + w.sync <- struct{}{} + <-w.syncDone +} + +func (w *Writer) logConsumer() { + ticker := time.NewTicker(diskFlushTime) +loop: + for { + select { + case line := <-w.lines: + w.write(line) + if w.cachedLimit != 0 && atomic.LoadInt64(&w.cached) > int64(w.cachedSoftLimit) { + w.flushLines(len(w.lines), w.cachedLow) + } + case <-ticker.C: + if w.diskBuf.Len() == 0 { + continue + } + if _, err := w.w.Write(w.diskBuf.Bytes()); err != nil { + fmt.Println("failed to write", err.Error()) + } + w.diskBuf.Reset() + case _, ok := <-w.sync: + if !ok { + close(w.syncDone) + break loop + } + nLines := len(w.lines) + if nLines == 0 && w.diskBuf.Len() == 0 { + w.syncDone <- struct{}{} + continue + } + w.flushLines(nLines, -1) + if _, err := w.w.Write(w.diskBuf.Bytes()); err != nil { + fmt.Println("failed to write", err.Error()) + } + w.diskBuf.Reset() + w.syncDone <- struct{}{} + } + } + ticker.Stop() +} + +func (w *Writer) flushLines(nLines int, upTo int) { + nBytes := 0 + for i := 0; i < nLines; i++ { + line := <-w.lines + nBytes += line.Len() + w.write(line) + if upTo >= 0 && nBytes > upTo { + break + } + } +} + +func (w *Writer) write(line *buffer.Buffer) { + w.diskBuf.Write(line.Bytes()) + if w.cachedLimit != 0 { + atomic.AddInt64(&w.cached, -int64(line.Len())) + } + line.Free() + if w.diskBuf.Len() < diskFlushSize { + return + } + if _, err := w.w.Write(w.diskBuf.Bytes()); err != nil { + fmt.Println("failed to write", err.Error()) + } + w.diskBuf.Reset() +} diff --git a/frontend/pkg/common/faas_common/logger/async/writer_test.go b/frontend/pkg/common/faas_common/logger/async/writer_test.go new file mode 100644 index 0000000000000000000000000000000000000000..3f34a76ece916c759a87b30c822eb78bb16ff8cb --- /dev/null +++ b/frontend/pkg/common/faas_common/logger/async/writer_test.go @@ -0,0 +1,125 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package async + +import ( + "io" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +type mockWriter struct { + buf []byte + delay time.Duration + sync.Mutex +} + +func (m *mockWriter) Write(data []byte) (int, error) { + m.Lock() + m.buf = data + if m.delay != 0 { + time.Sleep(m.delay) + } + m.Unlock() + return len(data), nil +} + +func (m *mockWriter) Clear() []byte { + m.Lock() + ret := m.buf + m.buf = nil + m.Unlock() + return ret +} + +func (m *mockWriter) SetWriteDelay(delay time.Duration) { + m.delay = delay +} + +func TestWriter_Write(t *testing.T) { + w := &mockWriter{} + + asyncWriter := NewAsyncWriteSyncer(w) + + data := []byte("hello world") + + // write small data, will be cached in inner buffer + asyncWriter.Write(data) + time.Sleep(100 * time.Millisecond) + assert.Equal(t, 0, len(w.Clear())) + + // small data will be written after flush time + time.Sleep(diskFlushTime) + assert.Equal(t, data, w.Clear()) + + // big data will be flushed immediately + asyncWriter.Write(make([]byte, diskFlushSize+1)) + time.Sleep(100 * time.Millisecond) + assert.Equal(t, diskFlushSize+1, len(w.Clear())) + + // Sync() will flush buffer immediately + asyncWriter.Write(data) + assert.Equal(t, 0, len(w.Clear())) + asyncWriter.Sync() + assert.Equal(t, len(data), len(w.Clear())) + + for i := 0; i < 100; i++ { + go asyncWriter.Sync() + } + time.Sleep(10 * time.Millisecond) + asyncWriter.Sync() +} + +func TestCachedLimit(t *testing.T) { + w := &mockWriter{} + w.SetWriteDelay(150 * time.Millisecond) + + asyncWriter := NewAsyncWriteSyncer(w, WithCachedLimit(diskFlushSize*4)) // softLimit = 512kb * 4 * 0.8 = 1.6mb + + size := float64(diskFlushSize)*2*softLimitFactor + 1 + data := make([]byte, int(size)) + + // big data will be flushed immediately and triggers the mockWriter's write + asyncWriter.Write(make([]byte, diskFlushSize+1)) + + // logConsumer is blocked in mockWriter's write + asyncWriter.Write(data) + time.Sleep(100 * time.Millisecond) + assert.Equal(t, 1, len(asyncWriter.lines)) + + // this write should hit the soft limit + asyncWriter.Write(data) + time.Sleep(100 * time.Millisecond) // mockWriter's write finishes + assert.Equal(t, 1, len(asyncWriter.lines)) + + time.Sleep(200 * time.Millisecond) + assert.Equal(t, 0, len(asyncWriter.lines)) +} + +func BenchmarkWrite(b *testing.B) { + asyncWriter := NewAsyncWriteSyncer(io.Discard, WithCachedLimit(diskFlushSize*4)) + data := []byte("hello world") + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + asyncWriter.Write(data) + } + }) +} diff --git a/frontend/pkg/common/faas_common/logger/config/config.go b/frontend/pkg/common/faas_common/logger/config/config.go new file mode 100644 index 0000000000000000000000000000000000000000..4f9b2af24b6a5b7de5e706608ae75ebaaca45878 --- /dev/null +++ b/frontend/pkg/common/faas_common/logger/config/config.go @@ -0,0 +1,121 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package config is common logger client +package config + +import ( + "encoding/json" + "errors" + "os" + + "github.com/asaskevich/govalidator/v11" + "go.uber.org/zap/zapcore" + + "frontend/pkg/common/faas_common/utils" +) + +const ( + configPath = "/home/sn/config/log.json" + fileMode = 0750 + logConfigKey = "LOG_CONFIG" +) + +var ( + defaultCoreInfo CoreInfo + // LogLevel - + LogLevel zapcore.Level = zapcore.InfoLevel +) + +func init() { + defaultFilePath := os.Getenv("GLOG_log_dir") + if defaultFilePath == "" { + defaultFilePath = "/home/snuser/log" + } + defaultLevel := "INFO" + // defaultCoreInfo default logger config + defaultCoreInfo = CoreInfo{ + FilePath: defaultFilePath, + Level: defaultLevel, + Tick: 0, // Unit: Second + First: 0, // Unit: Number of logs + Thereafter: 0, // Unit: Number of logs + SingleSize: 100, + Threshold: 10, + Tracing: false, // tracing log switch + Disable: false, // Disable file logger + } +} + +// CoreInfo contains the core info +type CoreInfo struct { + FilePath string `json:"filepath" valid:",optional"` + Level string `json:"level" valid:",optional"` + Tick int `json:"tick" valid:"range(0|86400),optional"` + First int `json:"first" valid:"range(0|20000),optional"` + Thereafter int `json:"thereafter" valid:"range(0|1000),optional"` + Tracing bool `json:"tracing" valid:",optional"` + Disable bool `json:"disable" valid:",optional"` + SingleSize int64 `json:"singlesize" valid:",optional"` + Threshold int `json:"threshold" valid:",optional"` + IsUserLog bool `json:"-"` + IsWiseCloudAlarmLog bool `json:"isWiseCloudAlarmLog" valid:",optional"` +} + +// GetDefaultCoreInfo get defaultCoreInfo +func GetDefaultCoreInfo() CoreInfo { + return defaultCoreInfo +} + +// GetCoreInfoFromEnv extracts the logger config and ensures that the log file is available +func GetCoreInfoFromEnv() (CoreInfo, error) { + coreInfo, err := ExtractCoreInfoFromEnv(logConfigKey) + if err != nil { + return defaultCoreInfo, err + } + if err = utils.ValidateFilePath(coreInfo.FilePath); err != nil { + return defaultCoreInfo, err + } + if err = os.MkdirAll(coreInfo.FilePath, fileMode); err != nil && !os.IsExist(err) { + return defaultCoreInfo, err + } + + return coreInfo, nil +} + +// ExtractCoreInfoFromEnv extracts the logger config from ENV +func ExtractCoreInfoFromEnv(env string) (CoreInfo, error) { + var coreInfo CoreInfo + conf := os.Getenv(env) + if conf == "" { + return defaultCoreInfo, errors.New(env + " is empty") + } + err := json.Unmarshal([]byte(conf), &coreInfo) + if err != nil { + return defaultCoreInfo, err + } + + // if the file path is empty, return error + // if the log file is not writable, zap will create a new file with the configured file path and file name + if coreInfo.FilePath == "" { + return defaultCoreInfo, errors.New("the log file path is empty") + } + if _, err = govalidator.ValidateStruct(coreInfo); err != nil { + return defaultCoreInfo, err + } + + return coreInfo, nil +} diff --git a/frontend/pkg/common/faas_common/logger/config/config_test.go b/frontend/pkg/common/faas_common/logger/config/config_test.go new file mode 100644 index 0000000000000000000000000000000000000000..a7f6aec02f1399327d8e3a3b243d303091dd7b36 --- /dev/null +++ b/frontend/pkg/common/faas_common/logger/config/config_test.go @@ -0,0 +1,315 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package config is common logger client +package config + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/common/faas_common/utils" + mockUtils "frontend/pkg/common/faas_common/utils" +) + +func TestInitConfig(t *testing.T) { + convey.Convey("TestInitConfig", t, func() { + convey.Convey("test 1", func() { + patches := gomonkey.ApplyFunc(GetCoreInfoFromEnv, func() (CoreInfo, error) { + return defaultCoreInfo, nil + }) + defer patches.Reset() + coreInfo, err := GetCoreInfoFromEnv() + fmt.Printf("log config:%+v\n", coreInfo) + convey.So(err, convey.ShouldEqual, nil) + }) + }) +} + +func TestInitConfigWithReadFileError(t *testing.T) { + convey.Convey("TestInitConfigWithEmptyPath", t, func() { + convey.Convey("test 1", func() { + patches := [...]*gomonkey.Patches{ + gomonkey.ApplyFunc(ioutil.ReadFile, + func(filename string) ([]byte, error) { + return nil, errors.New("mock read file error") + }), + } + defer func() { + for _, p := range patches { + p.Reset() + } + }() + coreInfo, err := GetCoreInfoFromEnv() + fmt.Printf("error:%s\n", err) + fmt.Printf("log config:%+v\n", coreInfo) + convey.So(err, convey.ShouldNotEqual, nil) + }) + }) +} + +func TestInitConfigWithErrorJson(t *testing.T) { + convey.Convey("TestInitConfigWithEmptyPath", t, func() { + convey.Convey("test 1", func() { + mockErrorJson := "{\n\"filepath\": \"/home/sn/mock\",\n\"level\": \"INFO\",\n\"maxsize\": " + + "500,\n\"maxbackups\": 1,\n\"maxage\": 1,\n\"compress\": true\n" + patches := [...]*gomonkey.Patches{ + gomonkey.ApplyFunc(ioutil.ReadFile, + func(filename string) ([]byte, error) { + return []byte(mockErrorJson), nil + }), + } + defer func() { + for _, p := range patches { + p.Reset() + } + }() + coreInfo, err := GetCoreInfoFromEnv() + fmt.Printf("error:%s\n", err) + fmt.Printf("log config:%+v\n", coreInfo) + convey.So(err, convey.ShouldNotEqual, nil) + }) + }) +} + +func TestInitConfigWithEmptyPath(t *testing.T) { + convey.Convey("TestInitConfigWithEmptyPath", t, func() { + convey.Convey("test 1", func() { + mockCfgInfo := "{\n\"filepath\": \"\",\n\"level\": \"INFO\",\n\"maxsize\": " + + "500,\n\"maxbackups\": 1,\n\"maxage\": 1,\n\"compress\": true\n}" + patches := [...]*gomonkey.Patches{ + gomonkey.ApplyFunc(ioutil.ReadFile, + func(filename string) ([]byte, error) { + return []byte(mockCfgInfo), nil + }), + } + defer func() { + for _, p := range patches { + p.Reset() + } + }() + coreInfo, err := GetCoreInfoFromEnv() + fmt.Printf("error:%s\n", err) + fmt.Printf("log config:%+v\n", coreInfo) + convey.So(err, convey.ShouldNotEqual, nil) + }) + }) +} + +func TestInitConfigWithValidateError(t *testing.T) { + convey.Convey("TestInitConfigWithEmptyPath", t, func() { + convey.Convey("test 1", func() { + mockErrorJson := "{\n\"filepath\": \"some_relative_path\",\n\"level\": \"INFO\",\n\"maxsize\": " + + "500,\n\"maxbackups\": 1,\n\"maxage\": 1}" + patches := [...]*gomonkey.Patches{ + gomonkey.ApplyFunc(ioutil.ReadFile, + func(filename string) ([]byte, error) { + return []byte(mockErrorJson), nil + }), + } + defer func() { + for _, p := range patches { + p.Reset() + } + }() + coreInfo, err := GetCoreInfoFromEnv() + fmt.Printf("error:%s\n", err) + fmt.Printf("log config:%+v\n", coreInfo) + convey.So(err, convey.ShouldNotEqual, nil) + }) + }) +} + +func TestGetDefaultCoreInfo(t *testing.T) { + tests := []struct { + name string + want CoreInfo + }{ + { + name: "test001", + want: CoreInfo{ + FilePath: "/home/snuser/log", + Level: "INFO", + Tick: 0, // Unit: Second + First: 0, // Unit: Number of logs + Thereafter: 0, // Unit: Number of logs + SingleSize: 100, + Threshold: 10, + Tracing: false, // tracing log switch + Disable: false, // Disable file logger + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GetDefaultCoreInfo(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetDefaultCoreInfo() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestExtractCoreInfoFromEnv(t *testing.T) { + normalInfo, _ := json.Marshal(defaultCoreInfo) + abnormal1 := mockUtils.PatchSlice{} + abnormalInfo1, _ := json.Marshal(abnormal1) + abnormal2 := CoreInfo{ + FilePath: "", + Level: "INFO", + Tick: 10, // Unit: Second + First: 10, // Unit: Number of logs + Thereafter: 5, // Unit: Number of logs + Tracing: false, // tracing log switch + Disable: false, // Disable file logger + } + abnormalInfo2, _ := json.Marshal(abnormal2) + type args struct { + env string + } + tests := []struct { + name string + args args + want CoreInfo + wantErr bool + patchesFunc mockUtils.PatchesFunc + }{ + { + name: "case1", + args: args{logConfigKey}, + want: defaultCoreInfo, + wantErr: false, + patchesFunc: func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(os.Getenv, + func(key string) string { + return string(normalInfo) + }), + }) + return patches + }, + }, + { + name: "case2", + args: args{logConfigKey}, + want: defaultCoreInfo, + wantErr: true, + patchesFunc: func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(os.Getenv, + func(key string) string { + return string(abnormalInfo1) + }), + }) + return patches + }, + }, + { + name: "case3", + args: args{logConfigKey}, + want: defaultCoreInfo, + wantErr: true, + patchesFunc: func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(os.Getenv, + func(key string) string { + return string(abnormalInfo2) + }), + }) + return patches + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + patches := tt.patchesFunc() + got, err := ExtractCoreInfoFromEnv(tt.args.env) + if (err != nil) != tt.wantErr { + t.Errorf("ExtractCoreInfoFromEnv() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ExtractCoreInfoFromEnv() got = %v, want %v", got, tt.want) + } + patches.ResetAll() + }) + } +} + +func TestGetCoreInfoFromEnv(t *testing.T) { + convey.Convey("GetCoreInfoFromEnv", t, func() { + convey.Convey("ValidateFilePath error", func() { + defer gomonkey.ApplyFunc(ExtractCoreInfoFromEnv, func(env string) (CoreInfo, error) { + return CoreInfo{FilePath: "../test"}, nil + }).Reset() + _, err := GetCoreInfoFromEnv() + convey.So(err, convey.ShouldBeError) + }) + + convey.Convey("MkdirAll error", func() { + patches := []*gomonkey.Patches{ + gomonkey.ApplyFunc(ExtractCoreInfoFromEnv, func(env string) (CoreInfo, error) { + return CoreInfo{FilePath: "/home/test"}, nil + }), + gomonkey.ApplyFunc(utils.ValidateFilePath, func(path string) error { + return nil + }), + gomonkey.ApplyFunc(os.MkdirAll, func(path string, perm os.FileMode) error { + return errors.New("create dir error") + }), + } + defer func() { + for _, patch := range patches { + patch.Reset() + } + }() + _, err := GetCoreInfoFromEnv() + convey.So(err, convey.ShouldBeError) + }) + + convey.Convey("success", func() { + patches := []*gomonkey.Patches{ + gomonkey.ApplyFunc(ExtractCoreInfoFromEnv, func(env string) (CoreInfo, error) { + return CoreInfo{FilePath: "/home/test"}, nil + }), + gomonkey.ApplyFunc(utils.ValidateFilePath, func(path string) error { + return nil + }), + gomonkey.ApplyFunc(os.MkdirAll, func(path string, perm os.FileMode) error { + return nil + }), + } + defer func() { + for _, patch := range patches { + patch.Reset() + } + }() + env, err := GetCoreInfoFromEnv() + convey.So(err, convey.ShouldBeNil) + convey.So(env.FilePath, convey.ShouldEqual, "/home/test") + }) + }) +} diff --git a/frontend/pkg/common/faas_common/logger/custom_encoder.go b/frontend/pkg/common/faas_common/logger/custom_encoder.go new file mode 100644 index 0000000000000000000000000000000000000000..39e37c14c10898a9fd018161c5be0c0e5f8bd652 --- /dev/null +++ b/frontend/pkg/common/faas_common/logger/custom_encoder.go @@ -0,0 +1,391 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package logger log +package logger + +import ( + "math" + "os" + "regexp" + "strings" + "sync" + "time" + + "go.uber.org/zap/buffer" + "go.uber.org/zap/zapcore" + + "frontend/pkg/common/faas_common/constant" +) + +const ( + float64bitSize = 64 + float32bitSize = 32 + headerSeparator = ' ' + elementSeparator = " " + customDefaultLineEnding = "\n" + logMsgMaxLen = 1024 + fieldSeparator = " | " +) + +var ( + _customBufferPool = buffer.NewPool() + + _customPool = sync.Pool{New: func() interface{} { + return &customEncoder{} + }} + + replComp = regexp.MustCompile(`\s+`) + + clusterName = os.Getenv("CLUSTER_ID") +) + +// customEncoder represents the encoder for zap logger +// project's interface log +type customEncoder struct { + *zapcore.EncoderConfig + buf *buffer.Buffer + podName string +} + +// NewConsoleEncoder new custom console encoder to zap log module +func NewConsoleEncoder(cfg zapcore.EncoderConfig) (zapcore.Encoder, error) { + return &customEncoder{ + EncoderConfig: &cfg, + buf: _customBufferPool.Get(), + podName: os.Getenv(constant.HostNameEnvKey), + }, nil +} + +// NewCustomEncoder new custom encoder to zap log module +func NewCustomEncoder(cfg *zapcore.EncoderConfig) zapcore.Encoder { + return &customEncoder{ + EncoderConfig: cfg, + buf: _customBufferPool.Get(), + podName: os.Getenv(constant.HostNameEnvKey), + } +} + +// Clone return zap core Encoder +func (enc *customEncoder) Clone() zapcore.Encoder { + clone := enc.clone() + if enc.buf.Len() > 0 { + _, _ = clone.buf.Write(enc.buf.Bytes()) + } + return clone +} + +// EncodeEntry - +func (enc *customEncoder) EncodeEntry(ent zapcore.Entry, fields []zapcore.Field) (*buffer.Buffer, error) { + final := enc.clone() + // add time + final.AppendString(ent.Time.UTC().Format("2006-01-02 15:04:05.000")) + final.buf.AppendString(fieldSeparator) + + final.EncodeLevel(ent.Level, final) + final.buf.AppendString(fieldSeparator) + + // add caller + if ent.Caller.Defined { + final.EncodeCaller(ent.Caller, final) + final.buf.AppendString(fieldSeparator) + } + // add podName + if enc.podName != "" { + final.buf.AppendString(enc.podName) + final.buf.AppendString(fieldSeparator) + } + // add clusterName + if clusterName != "" { + final.buf.AppendString(clusterName) + final.buf.AppendString(fieldSeparator) + } + if enc.buf.Len() > 0 { + final.buf.Write(enc.buf.Bytes()) + } + // add msg + if len(ent.Message) > logMsgMaxLen { + final.AppendString(ent.Message[0:logMsgMaxLen]) + } else { + final.AppendString(ent.Message) + } + if ent.Stack != "" && final.StacktraceKey != "" { + final.buf.AppendString(elementSeparator) + final.AddString(final.StacktraceKey, ent.Stack) + } + for _, field := range fields { + field.AddTo(final) + } + final.buf.AppendString(customDefaultLineEnding) + ret := final.buf + putCustomEncoder(final) + return ret, nil +} + +func putCustomEncoder(enc *customEncoder) { + enc.EncoderConfig = nil + enc.buf = nil + _customPool.Put(enc) +} + +func getCustomEncoder() *customEncoder { + return _customPool.Get().(*customEncoder) +} + +func (enc *customEncoder) clone() *customEncoder { + clone := getCustomEncoder() + clone.buf = _customBufferPool.Get() + clone.EncoderConfig = enc.EncoderConfig + clone.podName = enc.podName + return clone +} + +func (enc *customEncoder) writeField(k string, writeVal func()) *customEncoder { + enc.buf.AppendString("(" + k + ":") + writeVal() + enc.buf.AppendString(")") + return enc +} + +// AddArray Add Array +func (enc *customEncoder) AddArray(k string, marshaler zapcore.ArrayMarshaler) error { + return nil +} + +// AddObject Add Object +func (enc *customEncoder) AddObject(k string, marshaler zapcore.ObjectMarshaler) error { + return nil +} + +// AddBinary Add Binary +func (enc *customEncoder) AddBinary(k string, v []byte) { + enc.AddString(k, string(v)) +} + +// AddByteString Add Byte String +func (enc *customEncoder) AddByteString(k string, v []byte) { + enc.AddString(k, string(v)) +} + +// AddBool Add Bool +func (enc *customEncoder) AddBool(k string, v bool) { + enc.writeField(k, func() { + enc.AppendBool(v) + }) +} + +// AddComplex128 Add Complex128 +func (enc *customEncoder) AddComplex128(k string, val complex128) {} + +// AddComplex64 Add Complex64 +func (enc *customEncoder) AddComplex64(k string, v complex64) {} + +// AddDuration Add Duration +func (enc *customEncoder) AddDuration(k string, val time.Duration) { + enc.writeField(k, func() { + enc.AppendString(val.String()) + }) +} + +// AddFloat64 Add Float64 +func (enc *customEncoder) AddFloat64(k string, val float64) { + enc.writeField(k, func() { + enc.AppendFloat64(val) + }) +} + +// AddFloat32 Add Float32 +func (enc *customEncoder) AddFloat32(k string, v float32) { + enc.writeField(k, func() { + enc.AppendFloat64(float64(v)) + }) +} + +// AddInt Add Int +func (enc *customEncoder) AddInt(k string, v int) { + enc.writeField(k, func() { + enc.AppendInt64(int64(v)) + }) +} + +// AddInt64 Add Int64 +func (enc *customEncoder) AddInt64(k string, val int64) { + enc.writeField(k, func() { + enc.AppendInt64(val) + }) +} + +// AddInt32 Add Int32 +func (enc *customEncoder) AddInt32(k string, v int32) { + enc.writeField(k, func() { + enc.AppendInt64(int64(v)) + }) +} + +// AddInt16 Add Int16 +func (enc *customEncoder) AddInt16(k string, v int16) { + enc.writeField(k, func() { + enc.AppendInt64(int64(v)) + }) +} + +// AddInt8 Add Int8 +func (enc *customEncoder) AddInt8(k string, v int8) { + enc.writeField(k, func() { + enc.AppendInt64(int64(v)) + }) +} + +// AddString Append String +func (enc *customEncoder) AddString(k, v string) { + enc.writeField(k, func() { + v = replComp.ReplaceAllString(v, " ") + if strings.Contains(v, " ") { + enc.buf.AppendString("(" + v + ")") + return + } + enc.AppendString(v) + }) +} + +// AddTime Add Time +func (enc *customEncoder) AddTime(k string, v time.Time) { + enc.writeField(k, func() { + enc.AppendString(v.UTC().Format("2006-01-02 15:04:05.000")) + }) +} + +// AddUint Add Uint +func (enc *customEncoder) AddUint(k string, v uint) { + enc.writeField(k, func() { + enc.AppendUint64(uint64(v)) + }) +} + +// AddUint64 Add Uint64 +func (enc *customEncoder) AddUint64(k string, v uint64) { + enc.writeField(k, func() { + enc.AppendUint64(v) + }) +} + +// AddUint32 Add Uint32 +func (enc *customEncoder) AddUint32(k string, v uint32) { + enc.writeField(k, func() { + enc.AppendUint64(uint64(v)) + }) +} + +// AddUint16 Add Uint16 +func (enc *customEncoder) AddUint16(k string, v uint16) { + enc.writeField(k, func() { + enc.AppendUint64(uint64(v)) + }) +} + +// AddUint8 Add Uint8 +func (enc *customEncoder) AddUint8(k string, v uint8) { + enc.writeField(k, func() { + enc.AppendUint64(uint64(v)) + }) +} + +// AddUintptr Add Uint ptr +func (enc *customEncoder) AddUintptr(k string, v uintptr) { + enc.writeField(k, func() { + enc.AppendUint64(uint64(v)) + }) +} + +// AddReflected uses reflection to serialize arbitrary objects, so it's slow +// and allocation-heavy. +func (enc *customEncoder) AddReflected(k string, v interface{}) error { + return nil +} + +// OpenNamespace opens an isolated namespace where all subsequent fields will +// be added. Applications can use namespaces to prevent key collisions when +// injecting loggers into sub-components or third-party libraries. +func (enc *customEncoder) OpenNamespace(k string) {} + +// AppendBool Append Bool +func (enc *customEncoder) AppendBool(v bool) { enc.buf.AppendBool(v) } + +// AppendByteString Append Byte String +func (enc *customEncoder) AppendByteString(v []byte) { enc.AppendString(string(v)) } + +// AppendComplex128 Append Complex128 +func (enc *customEncoder) AppendComplex128(v complex128) {} + +// AppendComplex64 Append Complex64 +func (enc *customEncoder) AppendComplex64(v complex64) {} + +// AppendFloat64 Append Float64 +func (enc *customEncoder) AppendFloat64(v float64) { enc.appendFloat(v, float64bitSize) } + +// AppendFloat32 Append Float32 +func (enc *customEncoder) AppendFloat32(v float32) { enc.appendFloat(float64(v), float32bitSize) } + +func (enc *customEncoder) appendFloat(v float64, bitSize int) { + switch { + // If the condition is not met, a string is returned to prevent blankness. + // IsNaN reports whether f is an IEEE 754 ``not-a-number'' value. + case math.IsNaN(v): + enc.buf.AppendString(`"NaN"`) + case math.IsInf(v, 1): + enc.buf.AppendString(`"+Inf"`) + case math.IsInf(v, -1): + enc.buf.AppendString(`"-Inf"`) + default: + enc.buf.AppendFloat(v, bitSize) + } +} + +// AppendInt Append Int +func (enc *customEncoder) AppendInt(v int) { enc.buf.AppendInt(int64(v)) } + +// AppendInt64 Append Int64 +func (enc *customEncoder) AppendInt64(v int64) { enc.buf.AppendInt(v) } + +// AppendInt32 Append Int32 +func (enc *customEncoder) AppendInt32(v int32) { enc.buf.AppendInt(int64(v)) } + +// AppendInt16 Append Int16 +func (enc *customEncoder) AppendInt16(v int16) { enc.buf.AppendInt(int64(v)) } + +// AppendInt8 Append Int8 +func (enc *customEncoder) AppendInt8(v int8) { enc.buf.AppendInt(int64(v)) } + +// AppendString Append String +func (enc *customEncoder) AppendString(val string) { enc.buf.AppendString(val) } + +// AppendUint Append Uint +func (enc *customEncoder) AppendUint(v uint) { enc.buf.AppendUint(uint64(v)) } + +// AppendUint64 Append Uint64 +func (enc *customEncoder) AppendUint64(v uint64) { enc.buf.AppendUint(v) } + +// AppendUint32 Append Uint32 +func (enc *customEncoder) AppendUint32(v uint32) { enc.buf.AppendUint(uint64(v)) } + +// AppendUint16 Append Uint16 +func (enc *customEncoder) AppendUint16(v uint16) { enc.buf.AppendUint(uint64(v)) } + +// AppendUint8 Append Uint8 +func (enc *customEncoder) AppendUint8(v uint8) { enc.buf.AppendUint(uint64(v)) } + +// AppendUintptr Append Uint ptr +func (enc *customEncoder) AppendUintptr(v uintptr) { enc.buf.AppendUint(uint64(v)) } diff --git a/frontend/pkg/common/faas_common/logger/custom_encoder_test.go b/frontend/pkg/common/faas_common/logger/custom_encoder_test.go new file mode 100644 index 0000000000000000000000000000000000000000..10cb623f821c8ffb5d1e2b2e0c0a4f4e2237987c --- /dev/null +++ b/frontend/pkg/common/faas_common/logger/custom_encoder_test.go @@ -0,0 +1,113 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package logger + +import ( + "math" + "os" + "testing" + "time" + "unsafe" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap/zapcore" + + "frontend/pkg/common/faas_common/constant" +) + +func TestNewCustomEncoder(t *testing.T) { + encoderConfig := zapcore.EncoderConfig{ + TimeKey: "T", + LevelKey: "L", + NameKey: "Logger", + MessageKey: "M", + CallerKey: "C", + LineEnding: zapcore.DefaultLineEnding, + EncodeLevel: zapcore.CapitalLevelEncoder, + EncodeTime: zapcore.ISO8601TimeEncoder, + EncodeCaller: zapcore.ShortCallerEncoder, + } + encoder := NewCustomEncoder(&encoderConfig) + clone := encoder.Clone() + assert.NotEmpty(t, clone) + encoder.AddBool("3", true) + err := encoder.AddArray("4", nil) + assert.Empty(t, err) + err = encoder.AddObject("4", nil) + assert.Empty(t, err) + encoder.AddBinary("4", []byte{}) + encoder.AddComplex128("4", complex(1, 2)) + encoder.AddComplex64("4", complex(1, 2)) + encoder.AddDuration("4", time.Second) + encoder.AddByteString("4", []byte{}) + encoder.AddFloat64("2", 3.14) + encoder.AddFloat32("2", 3.14) + encoder.AddInt("1", 1) + encoder.AddInt8("1", 1) + encoder.AddInt16("1", 1) + encoder.AddInt32("1", 1) + encoder.AddInt64("1", 1) + encoder.AddString("5", "12") + encoder.AddString("5", "1 2") + encoder.AddTime("6", time.Time{}) + encoder.AddUint("1", uint(1)) + encoder.AddUint8("1", uint8(10)) + encoder.AddUint16("1", uint16(100)) + encoder.AddUint32("1", uint32(1000)) + encoder.AddUint64("1", uint64(1000)) + b := make([]int, 1) + encoder.AddUintptr("12", uintptr(unsafe.Pointer(&b[0]))) + encoder.OpenNamespace("3") + err = encoder.AddReflected("3", 1) + assert.Empty(t, err) + +} + +func Test_customEncoder_Append(t *testing.T) { + encoderConfig := zapcore.EncoderConfig{ + TimeKey: "T", + LevelKey: "L", + NameKey: "Logger", + MessageKey: "M", + CallerKey: "C", + LineEnding: zapcore.DefaultLineEnding, + EncodeLevel: zapcore.CapitalLevelEncoder, + EncodeTime: zapcore.ISO8601TimeEncoder, + EncodeCaller: zapcore.ShortCallerEncoder, + } + encoder := &customEncoder{ + EncoderConfig: &encoderConfig, + buf: _customBufferPool.Get(), + podName: os.Getenv(constant.HostNameEnvKey), + } + encoder.AppendInt16(1) + encoder.AppendUint32(2) + encoder.AppendByteString([]byte("abc")) + encoder.AppendFloat32(3) + encoder.appendFloat(math.Inf(1), 10) + encoder.appendFloat(math.Inf(-1), 10) + encoder.AppendComplex64(1 + 2i) + encoder.AppendUintptr(uintptr(1)) + encoder.AppendUint8(1) + encoder.AppendInt32(2) + encoder.AppendUint16(3) + encoder.AppendUint(4) + encoder.AppendInt8(0) + encoder.AppendInt(5) + encoder.AppendInt32(7) + assert.NotEmpty(t, encoder.buf.Len()) +} diff --git a/frontend/pkg/common/faas_common/logger/healthlog/healthlog.go b/frontend/pkg/common/faas_common/logger/healthlog/healthlog.go new file mode 100644 index 0000000000000000000000000000000000000000..bcf88fc66a9ea6a504dfb37688be901fb1c3de35 --- /dev/null +++ b/frontend/pkg/common/faas_common/logger/healthlog/healthlog.go @@ -0,0 +1,46 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package healthlog is for printing health logs +package healthlog + +import ( + "time" + + "frontend/pkg/common/faas_common/logger/log" +) + +const logInterval = 5 * time.Minute + +// PrintHealthLog prints timing health logs of components +func PrintHealthLog(stopCh <-chan struct{}, inputLog func(), name string) { + if stopCh == nil { + log.GetLogger().Errorf("stop channel is nil") + return + } + ticker := time.NewTicker(logInterval) + defer ticker.Stop() + time.After(logInterval) + for { + select { + case <-ticker.C: + inputLog() + case <-stopCh: + log.GetLogger().Warnf("%s receives a terminating signal", name) + return + } + } +} diff --git a/frontend/pkg/common/faas_common/logger/healthlog/healthlog_test.go b/frontend/pkg/common/faas_common/logger/healthlog/healthlog_test.go new file mode 100644 index 0000000000000000000000000000000000000000..bca8a005cbc008f13f09ccf113115093d4cb0258 --- /dev/null +++ b/frontend/pkg/common/faas_common/logger/healthlog/healthlog_test.go @@ -0,0 +1,65 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package healthlog + +import "testing" + +func TestPrintHealthLog(t *testing.T) { + type args struct { + stopCh chan struct{} + inputLog func() + name string + } + var a args + a.stopCh = nil + a.inputLog = func() { + return + } + + tests := []struct { + name string + args args + }{ + { + name: "case1", + args: args{ + stopCh: nil, + inputLog: func() { + return + }, + }, + }, + { + name: "case2", + args: args{ + stopCh: make(chan struct{}), + inputLog: func() { + return + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.args.stopCh != nil { + close(tt.args.stopCh) + } + PrintHealthLog(tt.args.stopCh, tt.args.inputLog, tt.args.name) + }) + } +} diff --git a/frontend/pkg/common/faas_common/logger/interface_encoder.go b/frontend/pkg/common/faas_common/logger/interface_encoder.go new file mode 100644 index 0000000000000000000000000000000000000000..a32c5eea1764a1bc5d9f4455c85f8c5ab313b271 --- /dev/null +++ b/frontend/pkg/common/faas_common/logger/interface_encoder.go @@ -0,0 +1,346 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package logger log +package logger + +import ( + "errors" + "math" + "os" + "sync" + "time" + + "go.uber.org/zap/buffer" + "go.uber.org/zap/zapcore" + + "frontend/pkg/common/faas_common/constant" +) + +var ( + _bufferPool = buffer.NewPool() + + _interfacePool = sync.Pool{New: func() interface{} { + return &interfaceEncoder{} + }} +) + +// InterfaceEncoderConfig holds interface log encoder config +type InterfaceEncoderConfig struct { + ModuleName string + HTTPMethod string + ModuleFrom string + TenantID string + FuncName string + FuncVer string + EncodeCaller zapcore.CallerEncoder +} + +// interfaceEncoder represents the encoder for interface log +// project's interface log +type interfaceEncoder struct { + *InterfaceEncoderConfig + buf *buffer.Buffer + podName string + spaced bool +} + +func getInterfaceEncoder() *interfaceEncoder { + return _interfacePool.Get().(*interfaceEncoder) +} + +func putInterfaceEncoder(enc *interfaceEncoder) { + enc.InterfaceEncoderConfig = nil + enc.spaced = false + enc.buf = nil + _interfacePool.Put(enc) +} + +// NewInterfaceEncoder create a new interface log encoder +func NewInterfaceEncoder(cfg InterfaceEncoderConfig, spaced bool) zapcore.Encoder { + return newInterfaceEncoder(cfg, spaced) +} + +func newInterfaceEncoder(cfg InterfaceEncoderConfig, spaced bool) *interfaceEncoder { + return &interfaceEncoder{ + InterfaceEncoderConfig: &cfg, + buf: _bufferPool.Get(), + spaced: spaced, + podName: os.Getenv(constant.PodNameEnvKey), + } +} + +// Clone return zap core Encoder +func (enc *interfaceEncoder) Clone() zapcore.Encoder { + return enc.clone() +} + +func (enc *interfaceEncoder) clone() *interfaceEncoder { + clone := getInterfaceEncoder() + clone.InterfaceEncoderConfig = enc.InterfaceEncoderConfig + clone.spaced = enc.spaced + clone.buf = _bufferPool.Get() + return clone +} + +// EncodeEntry Encode Entry +func (enc *interfaceEncoder) EncodeEntry(ent zapcore.Entry, fields []zapcore.Field) (*buffer.Buffer, error) { + final := enc.clone() + // add time + final.AppendString(ent.Time.UTC().Format("2006-01-02 15:04:05.000")) + // add level + // Level of interfaceLog is eternally INFO + final.buf.AppendString(fieldSeparator) + final.AppendString("INFO") + // add caller + if ent.Caller.Defined { + final.buf.AppendString(fieldSeparator) + final.EncodeCaller(ent.Caller, final) + } + final.buf.AppendString(fieldSeparator) + // add podName + if enc.podName != "" { + final.buf.AppendString(enc.podName) + } + final.buf.AppendString(fieldSeparator) + if enc.buf.Len() > 0 { + _, err := final.buf.Write(enc.buf.Bytes()) + if err != nil { + return nil, err + } + } + // add msg + final.AppendString(ent.Message) + for _, field := range fields { + field.AddTo(final) + } + final.buf.AppendString(customDefaultLineEnding) + ret := final.buf + putInterfaceEncoder(final) + return ret, nil +} + +// AddString Append String +func (enc *interfaceEncoder) AddString(key, val string) { + enc.buf.AppendString(val) +} + +// AppendString Append String +func (enc *interfaceEncoder) AppendString(val string) { + enc.buf.AppendString(val) +} + +// AddDuration Add Duration +func (enc *interfaceEncoder) AddDuration(key string, val time.Duration) { + enc.AppendDuration(val) +} + +func (enc *interfaceEncoder) addElementSeparator() { + last := enc.buf.Len() - 1 + if last < 0 { + return + } + switch enc.buf.Bytes()[last] { + case headerSeparator: + return + default: + enc.buf.AppendByte(headerSeparator) + if enc.spaced { + enc.buf.AppendByte(' ') + } + } +} + +// AppendTime Append Time +func (enc *interfaceEncoder) AppendTime(val time.Time) { + cur := enc.buf.Len() + interfaceTimeEncode(val, enc) + if cur == enc.buf.Len() { + // User-supplied EncodeTime is a no-op. Fall back to nanos since epoch to keep + // output JSON valid. + enc.AppendInt64(val.UnixNano()) + } +} + +// AddArray Add Array +func (enc *interfaceEncoder) AddArray(key string, marshaler zapcore.ArrayMarshaler) error { + return errors.New("unsupported method") +} + +// AddObject Add Object +func (enc *interfaceEncoder) AddObject(key string, marshaler zapcore.ObjectMarshaler) error { + return errors.New("unsupported method") +} + +// AddBinary Add Binary +func (enc *interfaceEncoder) AddBinary(key string, value []byte) {} + +// AddByteString Add Byte String +func (enc *interfaceEncoder) AddByteString(key string, val []byte) { + enc.AppendByteString(val) +} + +// AddBool Add Bool +func (enc *interfaceEncoder) AddBool(key string, value bool) {} + +// AddComplex64 Add Complex64 +func (enc *interfaceEncoder) AddComplex64(k string, v complex64) { enc.AddComplex128(k, complex128(v)) } + +// AddFloat32 Add Float32 +func (enc *interfaceEncoder) AddFloat32(k string, v float32) { enc.AddFloat64(k, float64(v)) } + +// AddInt Add Int +func (enc *interfaceEncoder) AddInt(k string, v int) { enc.AddInt64(k, int64(v)) } + +// AddInt32 Add Int32 +func (enc *interfaceEncoder) AddInt32(k string, v int32) { enc.AddInt64(k, int64(v)) } + +// AddInt16 Add Int16 +func (enc *interfaceEncoder) AddInt16(k string, v int16) { enc.AddInt64(k, int64(v)) } + +// AddInt8 Add Int8 +func (enc *interfaceEncoder) AddInt8(k string, v int8) { enc.AddInt64(k, int64(v)) } + +// AddUint Add Uint +func (enc *interfaceEncoder) AddUint(k string, v uint) { enc.AddUint64(k, uint64(v)) } + +// AddUint32 Add Uint32 +func (enc *interfaceEncoder) AddUint32(k string, v uint32) { enc.AddUint64(k, uint64(v)) } + +// AddUint16 Add Uint16 +func (enc *interfaceEncoder) AddUint16(k string, v uint16) { enc.AddUint64(k, uint64(v)) } + +// AddUint8 Add Uint8 +func (enc *interfaceEncoder) AddUint8(k string, v uint8) { enc.AddUint64(k, uint64(v)) } + +// AddUintptr Add Uint ptr +func (enc *interfaceEncoder) AddUintptr(k string, v uintptr) { enc.AddUint64(k, uint64(v)) } + +// AddComplex128 Add Complex128 +func (enc *interfaceEncoder) AddComplex128(key string, val complex128) { + enc.AppendComplex128(val) +} + +// AddFloat64 Add Float64 +func (enc *interfaceEncoder) AddFloat64(key string, val float64) { + enc.AppendFloat64(val) +} + +// AddInt64 Add Int64 +func (enc *interfaceEncoder) AddInt64(key string, val int64) { + enc.AppendInt64(val) +} + +// AddTime Add Time +func (enc *interfaceEncoder) AddTime(key string, value time.Time) { + enc.AppendTime(value) +} + +// AddUint64 Add Uint64 +func (enc *interfaceEncoder) AddUint64(key string, value uint64) {} + +// AddReflected uses reflection to serialize arbitrary objects, so it's slow +// and allocation-heavy. +func (enc *interfaceEncoder) AddReflected(key string, value interface{}) error { + return nil +} + +// OpenNamespace opens an isolated namespace where all subsequent fields will +// be added. Applications can use namespaces to prevent key collisions when +// injecting loggers into sub-components or third-party libraries. +func (enc *interfaceEncoder) OpenNamespace(key string) {} + +// AppendComplex128 Append Complex128 +func (enc *interfaceEncoder) AppendComplex128(val complex128) {} + +// AppendInt64 Append Int64 +func (enc *interfaceEncoder) AppendInt64(val int64) { + enc.addElementSeparator() + enc.buf.AppendInt(val) +} + +// AppendBool Append Bool +func (enc *interfaceEncoder) AppendBool(val bool) { + enc.addElementSeparator() + enc.buf.AppendBool(val) +} + +func (enc *interfaceEncoder) appendFloat(val float64, bitSize int) { + enc.addElementSeparator() + switch { + case math.IsNaN(val): + enc.buf.AppendString(`"NaN"`) + case math.IsInf(val, 1): + enc.buf.AppendString(`"+Inf"`) + case math.IsInf(val, -1): + enc.buf.AppendString(`"-Inf"`) + default: + enc.buf.AppendFloat(val, bitSize) + } +} + +// AppendUint64 Append Uint64 +func (enc *interfaceEncoder) AppendUint64(val uint64) { + enc.addElementSeparator() + enc.buf.AppendUint(val) +} + +// AppendByteString Append Byte String +func (enc *interfaceEncoder) AppendByteString(val []byte) {} + +// AppendDuration Append Duration +func (enc *interfaceEncoder) AppendDuration(val time.Duration) {} + +// AppendComplex64 Append Complex64 +func (enc *interfaceEncoder) AppendComplex64(v complex64) { enc.AppendComplex128(complex128(v)) } + +// AppendFloat64 Append Float64 +func (enc *interfaceEncoder) AppendFloat64(v float64) { enc.appendFloat(v, float64bitSize) } + +// AppendFloat32 Append Float32 +func (enc *interfaceEncoder) AppendFloat32(v float32) { enc.appendFloat(float64(v), float32bitSize) } + +// AppendInt Append Int +func (enc *interfaceEncoder) AppendInt(v int) { enc.AppendInt64(int64(v)) } + +// AppendInt32 Append Int32 +func (enc *interfaceEncoder) AppendInt32(v int32) { enc.AppendInt64(int64(v)) } + +// AppendInt16 Append Int16 +func (enc *interfaceEncoder) AppendInt16(v int16) { enc.AppendInt64(int64(v)) } + +// AppendInt8 Append Int8 +func (enc *interfaceEncoder) AppendInt8(v int8) { enc.AppendInt64(int64(v)) } + +// AppendUint Append Uint +func (enc *interfaceEncoder) AppendUint(v uint) { enc.AppendUint64(uint64(v)) } + +// AppendUint32 Append Uint32 +func (enc *interfaceEncoder) AppendUint32(v uint32) { enc.AppendUint64(uint64(v)) } + +// AppendUint16 Append Uint16 +func (enc *interfaceEncoder) AppendUint16(v uint16) { enc.AppendUint64(uint64(v)) } + +// AppendUint8 Append Uint8 +func (enc *interfaceEncoder) AppendUint8(v uint8) { enc.AppendUint64(uint64(v)) } + +// AppendUintptr Append Uint ptr +func (enc *interfaceEncoder) AppendUintptr(v uintptr) { enc.AppendUint64(uint64(v)) } + +func interfaceTimeEncode(t time.Time, enc zapcore.PrimitiveArrayEncoder) { + t = t.UTC() + enc.AppendString(t.Format("2006-01-02 15:04:05.000")) +} diff --git a/frontend/pkg/common/faas_common/logger/interface_encoder_test.go b/frontend/pkg/common/faas_common/logger/interface_encoder_test.go new file mode 100644 index 0000000000000000000000000000000000000000..486acc1256b9ac60728f05dc3a7ee0f25b293682 --- /dev/null +++ b/frontend/pkg/common/faas_common/logger/interface_encoder_test.go @@ -0,0 +1,122 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package logger + +import ( + "math" + "os" + "testing" + "time" + "unsafe" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "frontend/pkg/common/faas_common/constant" +) + +// TestNewInterfaceEncoder Test New Interface Encoder +func TestNewInterfaceEncoder(t *testing.T) { + cfg := InterfaceEncoderConfig{ + ModuleName: "FunctionWorker", + HTTPMethod: "POST", + ModuleFrom: "FrontendInvoke", + TenantID: "tenant2", + FuncName: "myFunction", + FuncVer: "latest", + } + + encoder := NewInterfaceEncoder(cfg, false) + + priority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool { + return lvl >= zapcore.InfoLevel + }) + + sink := zapcore.Lock(os.Stdout) + core := zapcore.NewCore(encoder, sink, priority) + logger := zap.New(core) + + logger.Info("e1b71add-cb24-4ef8-93eb-af8d3ceb74e8|0|success|1") + + clone := encoder.Clone() + assert.NotEmpty(t, clone) + encoder.AddBool("3", true) + err := encoder.AddArray("4", nil) + assert.NotEmpty(t, err) + err = encoder.AddObject("4", nil) + assert.NotEmpty(t, err) + encoder.AddBinary("4", []byte{}) + encoder.AddComplex128("4", complex(1, 2)) + encoder.AddComplex64("4", complex(1, 2)) + encoder.AddDuration("4", time.Second) + encoder.AddByteString("4", []byte{}) + encoder.AddFloat64("2", 3.14) + encoder.AddFloat32("2", 3.14) + encoder.AddInt("1", 1) + encoder.AddInt8("1", 1) + encoder.AddInt16("1", 1) + encoder.AddInt32("1", 1) + encoder.AddInt64("1", 1) + encoder.AddString("5", "12") + encoder.AddString("5", "1 2") + encoder.AddTime("6", time.Time{}) + encoder.AddUint("1", uint(1)) + encoder.AddUint8("1", uint8(10)) + encoder.AddUint16("1", uint16(100)) + encoder.AddUint32("1", uint32(1000)) + encoder.AddUint64("1", uint64(1000)) + b := make([]int, 1) + encoder.AddUintptr("12", uintptr(unsafe.Pointer(&b[0]))) + encoder.OpenNamespace("3") + err = encoder.AddReflected("3", 1) + assert.Empty(t, err) +} + +func Test_interfaceEncoder_Append(t *testing.T) { + encoderConfig := InterfaceEncoderConfig{ + ModuleName: "FunctionWorker", + HTTPMethod: "POST", + ModuleFrom: "FrontendInvoke", + TenantID: "tenant2", + FuncName: "myFunction", + FuncVer: "latest", + } + encoder := &interfaceEncoder{ + InterfaceEncoderConfig: &encoderConfig, + buf: _bufferPool.Get(), + spaced: false, + podName: os.Getenv(constant.HostNameEnvKey), + } + encoder.AppendInt16(1) + encoder.AppendUint32(2) + encoder.AppendByteString([]byte("abc")) + encoder.AppendFloat32(3) + encoder.appendFloat(math.Inf(1), 10) + encoder.appendFloat(math.Inf(-1), 10) + encoder.AppendComplex64(1 + 2i) + encoder.AppendUintptr(uintptr(1)) + encoder.AppendUint8(1) + encoder.AppendInt32(2) + encoder.AppendUint16(3) + encoder.AppendUint(4) + encoder.AppendInt8(0) + encoder.AppendInt(5) + encoder.AppendInt32(7) + encoder.AppendBool(false) + assert.NotEmpty(t, encoder.buf.Len()) +} diff --git a/frontend/pkg/common/faas_common/logger/interfacelogger.go b/frontend/pkg/common/faas_common/logger/interfacelogger.go new file mode 100644 index 0000000000000000000000000000000000000000..2b7b506b7fde0860c36080ede80d72009ee56e24 --- /dev/null +++ b/frontend/pkg/common/faas_common/logger/interfacelogger.go @@ -0,0 +1,100 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package logger log +package logger + +import ( + "fmt" + "io" + "os" + "path/filepath" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "frontend/pkg/common/faas_common/logger/config" +) + +const defaultPerm = 0666 + +// NewInterfaceLogger returns a new interface logger +func NewInterfaceLogger(logPath, fileName string, cfg InterfaceEncoderConfig) (*InterfaceLogger, error) { + coreInfo, err := config.GetCoreInfoFromEnv() + if err != nil { + coreInfo = config.GetDefaultCoreInfo() + } + filePath := filepath.Join(coreInfo.FilePath, fileName+".log") + + coreInfo.FilePath = filePath + cfg.EncodeCaller = zapcore.ShortCallerEncoder + // skip level to print caller line of origin log + const skipLevel = 3 + core, err := newCore(coreInfo, cfg) + if err != nil { + return nil, err + } + logger := zap.New(core, zap.AddCaller(), zap.AddCallerSkip(skipLevel)) + + return &InterfaceLogger{log: logger}, nil +} + +// InterfaceLogger interface logger which implements by zap logger +type InterfaceLogger struct { + log *zap.Logger +} + +// Write writes message information +func (logger *InterfaceLogger) Write(msg string) { + logger.log.Info(msg) +} + +func newCore(coreInfo config.CoreInfo, cfg InterfaceEncoderConfig) (zapcore.Core, error) { + w, err := CreateSink(coreInfo) + if err != nil { + return nil, err + } + syncer := zapcore.AddSync(w) + + encoder := NewInterfaceEncoder(cfg, false) + + priority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool { + var customLevel zapcore.Level + if err := customLevel.UnmarshalText([]byte(coreInfo.Level)); err != nil { + customLevel = zapcore.InfoLevel + } + return lvl >= customLevel + }) + + return zapcore.NewCore(encoder, syncer, priority), nil +} + +// CreateSink creates a new zap log sink +func CreateSink(coreInfo config.CoreInfo) (io.Writer, error) { + // create directory if not already exist + dir := filepath.Dir(coreInfo.FilePath) + err := os.MkdirAll(dir, os.ModePerm) + if err != nil { + fmt.Printf("failed to mkdir: %s", dir) + return nil, err + } + w, err := initRollingLog(coreInfo, os.O_WRONLY|os.O_APPEND|os.O_CREATE, defaultPerm) + if err != nil { + fmt.Printf("failed to open log file: %s, err: %s\n", coreInfo.FilePath, err.Error()) + return nil, err + } + return w, nil +} diff --git a/frontend/pkg/common/faas_common/logger/interfacelogger_test.go b/frontend/pkg/common/faas_common/logger/interfacelogger_test.go new file mode 100644 index 0000000000000000000000000000000000000000..8949cc294a5b68d4c646752c310f63dff9eb17c4 --- /dev/null +++ b/frontend/pkg/common/faas_common/logger/interfacelogger_test.go @@ -0,0 +1,57 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package logger + +import ( + "errors" + "os" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + + "frontend/pkg/common/faas_common/logger/config" +) + +func TestInterfaceLogger(t *testing.T) { + cfg := InterfaceEncoderConfig{ModuleName: "WorkerManager"} + interfaceLog, err := NewInterfaceLogger("", "worker-manager-interface", cfg) + interfaceLog.Write("123") + assert.Empty(t, err) + assert.NotEmpty(t, interfaceLog) +} + +func TestCreateSink(t *testing.T) { + convey.Convey("Test Create Sink Error", t, func() { + coreInfo := config.CoreInfo{} + w, err := CreateSink(coreInfo) + convey.So(w, convey.ShouldBeNil) + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("Test Create Sink Error 2", t, func() { + patch := gomonkey.ApplyFunc(os.MkdirAll, func(path string, perm os.FileMode) error { + return errors.New("err") + }) + defer patch.Reset() + coreInfo := config.CoreInfo{} + w, err := CreateSink(coreInfo) + convey.So(w, convey.ShouldBeNil) + convey.So(err, convey.ShouldNotBeNil) + }) + +} diff --git a/frontend/pkg/common/faas_common/logger/log/logger.go b/frontend/pkg/common/faas_common/logger/log/logger.go new file mode 100644 index 0000000000000000000000000000000000000000..aed75dd065302d630d9ae658c19475facd5f7056 --- /dev/null +++ b/frontend/pkg/common/faas_common/logger/log/logger.go @@ -0,0 +1,263 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package log - +package log + +import ( + "fmt" + "path/filepath" + "strings" + "sync" + + uberZap "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/config" + "frontend/pkg/common/faas_common/logger/zap" +) + +const ( + skipLevel = 1 + snuserLogPath = "/home/snuser/log" +) + +type loggerWrapper struct { + real api.FormatLogger +} + +func (l *loggerWrapper) With(fields ...zapcore.Field) api.FormatLogger { + return &loggerWrapper{ + real: l.real.With(fields...), + } +} + +func (l *loggerWrapper) Infof(format string, paras ...interface{}) { + l.real.Infof(format, paras...) +} +func (l *loggerWrapper) Errorf(format string, paras ...interface{}) { + l.real.Errorf(format, paras...) +} +func (l *loggerWrapper) Warnf(format string, paras ...interface{}) { + l.real.Warnf(format, paras...) +} +func (l *loggerWrapper) Debugf(format string, paras ...interface{}) { + l.real.Debugf(format, paras...) +} +func (l *loggerWrapper) Fatalf(format string, paras ...interface{}) { + l.real.Fatalf(format, paras...) +} +func (l *loggerWrapper) Info(msg string, fields ...uberZap.Field) { + l.real.Info(msg, fields...) +} +func (l *loggerWrapper) Error(msg string, fields ...uberZap.Field) { + l.real.Error(msg, fields...) +} +func (l *loggerWrapper) Warn(msg string, fields ...uberZap.Field) { + l.real.Warn(msg, fields...) +} +func (l *loggerWrapper) Debug(msg string, fields ...uberZap.Field) { + l.real.Debug(msg, fields...) +} +func (l *loggerWrapper) Fatal(msg string, fields ...uberZap.Field) { + l.real.Fatal(msg, fields...) +} +func (l *loggerWrapper) Sync() { + l.real.Sync() +} + +var ( + once sync.Once + formatLogger api.FormatLogger + defaultLogger, _ = uberZap.NewProduction() +) + +// InitRunLog init run log with log.json file +func InitRunLog(fileName string, isAsync bool) error { + coreInfo, err := config.GetCoreInfoFromEnv() + if err != nil { + return err + } + if coreInfo.Disable { + return nil + } + formatLogger, err = NewFormatLogger(fileName, isAsync, coreInfo) + return err +} + +// SetupLoggerLibruntime setup logger +func SetupLoggerLibruntime(runtimeLogger api.FormatLogger) { + if runtimeLogger == nil { + return + } + wrapLogger := &loggerWrapper{real: runtimeLogger} + formatLogger = wrapLogger +} + +// SetupLogger setup logger +func SetupLogger(runtimeLogger api.FormatLogger) { + if runtimeLogger == nil { + return + } + formatLogger = runtimeLogger +} + +// GetLogger get logger directly +func GetLogger() api.FormatLogger { + if formatLogger == nil { + once.Do(func() { + formatLogger = NewConsoleLogger() + }) + } + return formatLogger +} + +// NewConsoleLogger returns a console logger +func NewConsoleLogger() api.FormatLogger { + logger, err := newConsoleLog() + if err != nil { + fmt.Println("new console log error", err) + logger = defaultLogger + } + return &zapLoggerWithFormat{ + Logger: logger, + SLogger: logger.Sugar(), + } +} + +// NewFormatLogger new formatLogger with log config info +func NewFormatLogger(fileName string, isAsync bool, coreInfo config.CoreInfo) (api.FormatLogger, error) { + if strings.Compare(constant.MonitorFileName, fileName) == 0 { + coreInfo.FilePath = snuserLogPath + } + coreInfo.FilePath = filepath.Join(coreInfo.FilePath, fileName+"-run.log") + logger, err := zap.NewWithLevel(coreInfo, isAsync) + if err != nil { + return nil, err + } + + return &zapLoggerWithFormat{ + Logger: logger, + SLogger: logger.Sugar(), + }, nil +} + +// newConsoleLog returns a console logger based on uber zap +func newConsoleLog() (*uberZap.Logger, error) { + outputPaths := []string{"stdout"} + cfg := uberZap.Config{ + Level: uberZap.NewAtomicLevelAt(uberZap.InfoLevel), + Development: false, + DisableCaller: false, + DisableStacktrace: true, + Encoding: "custom_console", + OutputPaths: outputPaths, + ErrorOutputPaths: outputPaths, + EncoderConfig: zapcore.EncoderConfig{ + TimeKey: "T", + LevelKey: "L", + NameKey: "Logger", + MessageKey: "M", + CallerKey: "C", + LineEnding: zapcore.DefaultLineEnding, + EncodeLevel: zapcore.CapitalLevelEncoder, + EncodeTime: zapcore.ISO8601TimeEncoder, + EncodeCaller: zapcore.ShortCallerEncoder, + }, + } + consoleLogger, err := cfg.Build() + if err != nil { + return nil, err + } + return consoleLogger.WithOptions(uberZap.AddCaller(), uberZap.AddCallerSkip(skipLevel)), nil +} + +// zapLoggerWithFormat define logger +type zapLoggerWithFormat struct { + Logger *uberZap.Logger + SLogger *uberZap.SugaredLogger +} + +// With add fields to log header +func (z *zapLoggerWithFormat) With(fields ...zapcore.Field) api.FormatLogger { + logger := z.Logger.With(fields...) + return &zapLoggerWithFormat{ + Logger: logger, + SLogger: logger.Sugar(), + } +} + +// Infof stdout format and paras +func (z *zapLoggerWithFormat) Infof(format string, paras ...interface{}) { + z.SLogger.Infof(format, paras...) +} + +// Errorf stdout format and paras +func (z *zapLoggerWithFormat) Errorf(format string, paras ...interface{}) { + z.SLogger.Errorf(format, paras...) +} + +// Warnf stdout format and paras +func (z *zapLoggerWithFormat) Warnf(format string, paras ...interface{}) { + z.SLogger.Warnf(format, paras...) +} + +// Debugf stdout format and paras +func (z *zapLoggerWithFormat) Debugf(format string, paras ...interface{}) { + z.SLogger.Debugf(format, paras...) +} + +// Fatalf stdout format and paras +func (z *zapLoggerWithFormat) Fatalf(format string, paras ...interface{}) { + z.SLogger.Fatalf(format, paras...) +} + +// Info stdout format and paras +func (z *zapLoggerWithFormat) Info(msg string, fields ...uberZap.Field) { + z.Logger.Info(msg, fields...) +} + +// Error stdout format and paras +func (z *zapLoggerWithFormat) Error(msg string, fields ...uberZap.Field) { + z.Logger.Error(msg, fields...) +} + +// Warn stdout format and paras +func (z *zapLoggerWithFormat) Warn(msg string, fields ...uberZap.Field) { + z.Logger.Warn(msg, fields...) +} + +// Debug stdout format and paras +func (z *zapLoggerWithFormat) Debug(msg string, fields ...uberZap.Field) { + z.Logger.Debug(msg, fields...) +} + +// Fatal stdout format and paras +func (z *zapLoggerWithFormat) Fatal(msg string, fields ...uberZap.Field) { + z.Logger.Fatal(msg, fields...) +} + +// Sync calls the underlying Core's Sync method, flushing any buffered log +// entries. Applications should take care to call Sync before exiting. +func (z *zapLoggerWithFormat) Sync() { + err := z.Logger.Sync() + if err != nil { + return + } +} diff --git a/frontend/pkg/common/faas_common/logger/log/logger_test.go b/frontend/pkg/common/faas_common/logger/log/logger_test.go new file mode 100644 index 0000000000000000000000000000000000000000..6661b77d51aae37675421267272e14a078fea44b --- /dev/null +++ b/frontend/pkg/common/faas_common/logger/log/logger_test.go @@ -0,0 +1,98 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package log + +import ( + "errors" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + uberZap "go.uber.org/zap" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/config" + "frontend/pkg/common/faas_common/logger/zap" +) + +func TestSetupLoggerRuntime(t *testing.T) { + SetupLoggerLibruntime(nil) + assert.Equal(t, formatLogger, nil) +} + +func TestInitLogger(t *testing.T) { + errCtrl := "" + patch := gomonkey.ApplyFunc(config.GetCoreInfoFromEnv, func() (config.CoreInfo, error) { + if errCtrl == "returnError" { + return config.CoreInfo{}, errors.New("some error") + } + return config.CoreInfo{}, nil + }) + defer patch.Reset() + SetupLogger(nil) + SetupLogger(NewConsoleLogger()) + assert.NotNil(t, formatLogger) + errCtrl = "returnError" + err := InitRunLog("test", false) + assert.NotNil(t, err) + errCtrl = "" + err = InitRunLog("test", false) + assert.Nil(t, err) +} + +func TestGetLogger(t *testing.T) { + convey.Convey("log", t, func() { + logger := GetLogger() + logger.With(uberZap.Any("name", "test-log")) + logger.Info("info log") + logger.Infof("info log") + logger.Debug("debug log") + logger.Debugf("debug log") + logger.Warn("warn log") + logger.Warnf("warn log") + logger.Error("error log") + logger.Errorf("error log") + }) +} + +func TestFormatLogger(t *testing.T) { + convey.Convey("new log error", t, func() { + patch := gomonkey.ApplyFunc(zap.NewWithLevel, func(coreInfo config.CoreInfo, isAsync bool) (*uberZap.Logger, error) { + return nil, errors.New("1") + }) + defer patch.Reset() + _, err := NewFormatLogger(constant.MonitorFileName, true, config.CoreInfo{}) + assert.NotNil(t, err) + }) + convey.Convey("new log success", t, func() { + logger, err := NewFormatLogger(constant.MonitorFileName, true, config.CoreInfo{}) + assert.Nil(t, err) + logger.With(uberZap.Any("name", "test-log")) + logger.Info("info log") + logger.Infof("info log") + logger.Debug("debug log") + logger.Debugf("debug log") + //logger.Fatal("fatal log") + //logger.Fatalf("fatal log") + logger.Warn("warn log") + logger.Warnf("warn log") + logger.Error("error log") + logger.Errorf("error log") + logger.Sync() + }) +} diff --git a/frontend/pkg/common/faas_common/logger/rollinglog.go b/frontend/pkg/common/faas_common/logger/rollinglog.go new file mode 100644 index 0000000000000000000000000000000000000000..2349d7629e38fe903295fcd61e67a2275cef57e9 --- /dev/null +++ b/frontend/pkg/common/faas_common/logger/rollinglog.go @@ -0,0 +1,276 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package logger rollingLog +package logger + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "sync" + "time" + + "frontend/pkg/common/faas_common/logger/config" +) + +const ( + megabyte = 1024 * 1024 + defaultFileSize = 100 + defaultBackups = 20 +) + +var logNameCache = struct { + m map[string]string + sync.Mutex +}{ + m: make(map[string]string, 1), + Mutex: sync.Mutex{}, +} + +type rollingLog struct { + file *os.File + reg *regexp.Regexp + mu sync.RWMutex + sinks []string + dir string + nameTemplate string + maxSize int64 + size int64 + maxBackups int + flag int + perm os.FileMode + isUserLog bool + isWiseCloudAlarmLog bool +} + +func initRollingLog(coreInfo config.CoreInfo, flag int, perm os.FileMode) (*rollingLog, error) { + if coreInfo.FilePath == "" { + return nil, errors.New("empty log file path") + } + log := &rollingLog{ + dir: filepath.Dir(coreInfo.FilePath), + nameTemplate: filepath.Base(coreInfo.FilePath), + flag: flag, + perm: perm, + maxSize: coreInfo.SingleSize * megabyte, + maxBackups: coreInfo.Threshold, + isUserLog: coreInfo.IsUserLog, + isWiseCloudAlarmLog: coreInfo.IsWiseCloudAlarmLog, + } + if log.maxBackups < 1 { + log.maxBackups = defaultBackups + } + if log.maxSize < megabyte { + log.maxSize = defaultFileSize * megabyte + } + if log.isUserLog { + return log, log.tidySinks() + } + extension := filepath.Ext(log.nameTemplate) + regExp := fmt.Sprintf(`^%s(?:(?:-|\.)\d*)?\%s$`, + log.nameTemplate[:len(log.nameTemplate)-len(extension)], extension) + reg, err := regexp.Compile(regExp) + if err != nil { + return nil, err + } + log.reg = reg + return log, log.tidySinks() +} + +func (r *rollingLog) tidySinks() error { + if r.isUserLog || r.file != nil { + return r.newSink() + } + // scan and reuse past log file when service restarted + r.scanLogFiles() + if len(r.sinks) > 0 { + fullName := r.sinks[len(r.sinks)-1] + info, err := os.Stat(fullName) + if err != nil || info.Size() >= r.maxSize { + return r.newSink() + } + file, err := os.OpenFile(fullName, r.flag, r.perm) + if err == nil { + r.file = file + r.size = info.Size() + return nil + } + } + return r.newSink() +} + +func (r *rollingLog) scanLogFiles() { + dirEntrys, err := os.ReadDir(r.dir) + if err != nil { + fmt.Printf("failed to read dir: %s\n", r.dir) + return + } + infos := make([]os.FileInfo, 0, r.maxBackups) + for _, entry := range dirEntrys { + if r.reg.MatchString(entry.Name()) { + info, err := entry.Info() + if err == nil { + infos = append(infos, info) + } + } + } + if len(infos) > 0 { + sort.Slice(infos, func(i, j int) bool { + return infos[i].ModTime().Before(infos[j].ModTime()) + }) + for i := range infos { + r.sinks = append(r.sinks, filepath.Join(r.dir, infos[i].Name())) + } + r.cleanRedundantSinks() + } +} + +func (r *rollingLog) cleanRedundantSinks() { + if len(r.sinks) < r.maxBackups { + return + } + curSinks := make([]string, 0, len(r.sinks)) + for _, name := range r.sinks { + if isAvailable(name) { + curSinks = append(curSinks, name) + } + + } + r.sinks = curSinks + sinkNum := len(r.sinks) + if sinkNum > r.maxBackups { + removes := r.sinks[:sinkNum-r.maxBackups] + go removeFiles(removes) + r.sinks = r.sinks[sinkNum-r.maxBackups:] + } + return +} + +func removeFiles(paths []string) { + for _, path := range paths { + err := os.Remove(path) + if err != nil && !os.IsNotExist(err) { + fmt.Printf("failed remove file %s\n", path) + } + } +} + +func isAvailable(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +func (r *rollingLog) newSink() error { + fullName := filepath.Join(r.dir, r.newName()) + if isAvailable(fullName) && r.file != nil && r.file.Name() == filepath.Base(fullName) { + return errors.New("log file already opened: " + fullName) + } + file, err := os.OpenFile(fullName, r.flag, r.perm) + if err != nil { + return err + } + if r.file != nil { + err = r.file.Close() + } + if err != nil { + fmt.Printf("failed to close file: %s\n", err.Error()) + } + r.file = file + info, err := file.Stat() + if err != nil { + r.size = 0 + } else { + r.size = info.Size() + } + r.sinks = append(r.sinks, fullName) + r.cleanRedundantSinks() + if r.isUserLog { + logNameCache.Lock() + logNameCache.m[r.nameTemplate] = fullName + logNameCache.Unlock() + } + return nil +} + +func (r *rollingLog) newName() string { + if r.isWiseCloudAlarmLog { + timeNow := time.Now().Format("2006010215040506") + ext := filepath.Ext(r.nameTemplate) + return fmt.Sprintf("%s.%s%s", timeNow, r.nameTemplate[:len(r.nameTemplate)-len(ext)], ext) + } + if !r.isUserLog { + timeNow := time.Now().Format("2006010215040506") + ext := filepath.Ext(r.nameTemplate) + return fmt.Sprintf("%s.%s%s", r.nameTemplate[:len(r.nameTemplate)-len(ext)], timeNow, ext) + } + if r.file == nil { + return r.nameTemplate + } + timeNow := time.Now().Format("2006010215040506") + var prefix, suffix string + if index := strings.LastIndex(r.nameTemplate, "@") + 1; index <= len(r.nameTemplate) { + prefix = r.nameTemplate[:index] + } + if index := strings.Index(r.nameTemplate, "#"); index >= 0 { + suffix = r.nameTemplate[index:] + } + if prefix == "" || suffix == "" { + return "" + } + return fmt.Sprintf("%s%s%s", prefix, timeNow, suffix) +} + +// Write data to file and check whether to rotate log +func (r *rollingLog) Write(data []byte) (int, error) { + r.mu.Lock() + defer r.mu.Unlock() + if r == nil || r.file == nil { + return 0, errors.New("log file is nil") + } + n, err := r.file.Write(data) + r.size += int64(n) + if r.size > r.maxSize { + r.tryRotate() + } + if syncErr := r.file.Sync(); syncErr != nil { + fmt.Printf("failed to sync log err: %s\n", syncErr.Error()) + } + return n, err +} + +func (r *rollingLog) tryRotate() { + if info, err := r.file.Stat(); err == nil && info.Size() < r.maxSize { + return + } + err := r.tidySinks() + if err != nil { + fmt.Printf("failed to rotate log err: %s\n", err.Error()) + } + return +} + +// GetLogName get current log name when refreshing user log mod time +func GetLogName(nameTemplate string) string { + logNameCache.Lock() + name := logNameCache.m[nameTemplate] + logNameCache.Unlock() + return name +} diff --git a/frontend/pkg/common/faas_common/logger/rollinglog_test.go b/frontend/pkg/common/faas_common/logger/rollinglog_test.go new file mode 100644 index 0000000000000000000000000000000000000000..57ff9cfaa7d83152e7730cbd3be02f0d40ca9d9c --- /dev/null +++ b/frontend/pkg/common/faas_common/logger/rollinglog_test.go @@ -0,0 +1,163 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package logger + +import ( + "errors" + "io/fs" + "os" + "reflect" + "strings" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/common/faas_common/logger/config" +) + +type mockInfo struct { + name string + isDir bool + size int64 +} + +func (m mockInfo) Name() string { + return m.name +} + +func (m mockInfo) IsDir() bool { + return m.isDir +} + +func (m mockInfo) Type() fs.FileMode { + return 0 +} + +func (m mockInfo) Info() (fs.FileInfo, error) { + return m, nil +} + +func (m mockInfo) Size() int64 { + return m.size +} + +func (m mockInfo) Mode() fs.FileMode { + return 0 +} + +func (m mockInfo) ModTime() time.Time { + return time.Now() +} + +func (m mockInfo) Sys() interface{} { + return nil +} + +func Test_initRollingLog(t *testing.T) { + coreInfo := config.CoreInfo{ + FilePath: "./test-run.log", + } + defer gomonkey.ApplyFunc(os.ReadDir, func(string) ([]os.DirEntry, error) { + return []os.DirEntry{ + mockInfo{name: "test-run.2006010215040507.log"}, + mockInfo{name: "test-run.2006010215040508.log"}, + mockInfo{name: "{funcName}@ABCabc@latest@pool22-300-128-fusion-85c55c66d7-zzj9x@{timeNow}#{logGroupID}#{logStreamID}#cff-log.log"}, + }, nil + }).ApplyFunc(os.OpenFile, func(string, int, os.FileMode) (*os.File, error) { + return nil, nil + }).Reset() + convey.Convey("init service log", t, func() { + defer gomonkey.ApplyFunc(os.Stat, func(name string) (os.FileInfo, error) { + return mockInfo{name: strings.TrimPrefix(name, "./")}, nil + }).Reset() + log, err := initRollingLog(coreInfo, os.O_WRONLY|os.O_APPEND|os.O_CREATE, defaultPerm) + convey.So(log, convey.ShouldNotBeNil) + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("init user log", t, func() { + defer gomonkey.ApplyFunc(os.Stat, func(name string) (os.FileInfo, error) { + return nil, &os.PathError{} + }).Reset() + coreInfo.FilePath = "{funcName}@ABCabc@latest@pool22-300-128-fusion-85c55c66d7-zzj9x@{timeNow}#{logGroupID}#{logStreamID}#cff-log.log" + coreInfo.IsUserLog = true + log, _ := initRollingLog(coreInfo, os.O_WRONLY|os.O_APPEND|os.O_CREATE, defaultPerm) + convey.So(log, convey.ShouldNotBeNil) + convey.So(GetLogName(coreInfo.FilePath), convey.ShouldNotBeEmpty) + }) + convey.Convey("init wisecloud alarm log", t, func() { + defer gomonkey.ApplyFunc(os.Stat, func(name string) (os.FileInfo, error) { + return nil, &os.PathError{} + }).Reset() + coreInfo.FilePath = "{funcName}@ABCabc@latest@pool22-300-128-fusion-85c55c66d7-zzj9x@{timeNow}#{logGroupID}#{logStreamID}#cff-log.log" + coreInfo.IsWiseCloudAlarmLog = true + log, _ := initRollingLog(coreInfo, os.O_WRONLY|os.O_APPEND|os.O_CREATE, defaultPerm) + convey.So(log, convey.ShouldNotBeNil) + convey.So(GetLogName(coreInfo.FilePath), convey.ShouldNotBeEmpty) + }) +} + +func Test_rollingLog_Write(t *testing.T) { + log := &rollingLog{} + log.maxSize = 0 + log.isUserLog = true + log.file = &os.File{} + log.nameTemplate = "{funcName}@ABCabc@latest@pool22-300-128-fusion-85c55c66d7-zzj9x@{timeNow}#{logGroupID}#{logStreamID}#cff-log.log" + convey.Convey("write rolling log", t, func() { + convey.Convey("case1: failed to write rolling log", func() { + defer gomonkey.ApplyMethod(reflect.TypeOf(log.file), "Write", func(f *os.File, b []byte) (n int, err error) { + return len(b), nil + }).ApplyMethod(reflect.TypeOf(log.file), "Stat", func(f *os.File) (info os.FileInfo, err error) { + return mockInfo{size: 3}, nil + }).ApplyMethod(reflect.TypeOf(log.file), "Sync", func(f *os.File) error { + return nil + }).Reset() + n, err := log.Write([]byte("abc")) + convey.So(n, convey.ShouldEqual, 3) + convey.So(err, convey.ShouldBeNil) + }) + + convey.Convey("case2: failed to write rolling log", func() { + defer gomonkey.ApplyMethod(reflect.TypeOf(log.file), "Write", func(f *os.File, b []byte) (n int, err error) { + return len(b), nil + }).ApplyMethod(reflect.TypeOf(log.file), "Stat", func(f *os.File) (info os.FileInfo, err error) { + return mockInfo{size: 3}, nil + }).ApplyMethod(reflect.TypeOf(log.file), "Sync", func(f *os.File) error { + return errors.New("test") + }).Reset() + n, err := log.Write([]byte("abc")) + convey.So(n, convey.ShouldEqual, 3) + convey.So(err, convey.ShouldBeNil) + }) + }) +} + +func Test_rollingLog_cleanRedundantSinks(t *testing.T) { + log := &rollingLog{} + log.maxBackups = 0 + tn := time.Now().String() + os.Create("test_log_1#" + tn) + os.Create("test_log_2#" + tn) + log.sinks = []string{"test_log_1#" + tn, "test_log_2#" + tn} + convey.Convey("rollingLog_cleanRedundantSinks", t, func() { + log.cleanRedundantSinks() + time.Sleep(50 * time.Millisecond) + convey.So(isAvailable("test_log_1#"+tn), convey.ShouldEqual, false) + convey.So(isAvailable("test_log_2#"+tn), convey.ShouldEqual, false) + }) +} diff --git a/frontend/pkg/common/faas_common/logger/zap/zaplog.go b/frontend/pkg/common/faas_common/logger/zap/zaplog.go new file mode 100644 index 0000000000000000000000000000000000000000..fa67963fb60f7314c57577677f7ce271243a6804 --- /dev/null +++ b/frontend/pkg/common/faas_common/logger/zap/zaplog.go @@ -0,0 +1,160 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package zap zapper log +package zap + +import ( + "fmt" + "time" + + uberZap "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "frontend/pkg/common/faas_common/logger" + "frontend/pkg/common/faas_common/logger/async" + "frontend/pkg/common/faas_common/logger/config" +) + +const ( + skipLevel = 1 +) + +func init() { + uberZap.RegisterEncoder("custom_console", logger.NewConsoleEncoder) +} + +// NewDevelopmentLog returns a development logger based on uber zap and it output entry to stdout and stderr +func NewDevelopmentLog() (*uberZap.Logger, error) { + cfg := uberZap.NewDevelopmentConfig() + cfg.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder + return cfg.Build() +} + +// NewConsoleLog returns a console logger based on uber zap +func NewConsoleLog() (*uberZap.Logger, error) { + outputPaths := []string{"stdout"} + cfg := uberZap.Config{ + Level: uberZap.NewAtomicLevelAt(uberZap.InfoLevel), + Development: false, + DisableCaller: false, + DisableStacktrace: true, + Encoding: "custom_console", + OutputPaths: outputPaths, + ErrorOutputPaths: outputPaths, + EncoderConfig: zapcore.EncoderConfig{ + TimeKey: "T", + LevelKey: "L", + NameKey: "Logger", + MessageKey: "M", + CallerKey: "C", + LineEnding: zapcore.DefaultLineEnding, + EncodeLevel: zapcore.CapitalLevelEncoder, + EncodeTime: zapcore.ISO8601TimeEncoder, + EncodeCaller: zapcore.ShortCallerEncoder, + }, + } + consoleLogger, err := cfg.Build() + if err != nil { + return nil, err + } + return consoleLogger.WithOptions(uberZap.AddCaller(), uberZap.AddCallerSkip(skipLevel)), nil +} + +// NewWithLevel returns a log based on zap with Level +func NewWithLevel(coreInfo config.CoreInfo, isAsync bool) (*uberZap.Logger, error) { + core, err := newCore(coreInfo, isAsync) + if err != nil { + return nil, err + } + + return uberZap.New(core, uberZap.AddCaller(), uberZap.AddCallerSkip(skipLevel)), nil +} + +func newCore(coreInfo config.CoreInfo, isAsync bool) (zapcore.Core, error) { + w, err := logger.CreateSink(coreInfo) + if err != nil { + return nil, err + } + + var syncer zapcore.WriteSyncer + if isAsync { + syncer = async.NewAsyncWriteSyncer(w) + } else { + syncer = zapcore.AddSync(w) + } + + encoderConfig := zapcore.EncoderConfig{ + TimeKey: "T", + LevelKey: "L", + NameKey: "Logger", + MessageKey: "M", + CallerKey: "C", + LineEnding: zapcore.DefaultLineEnding, + EncodeLevel: zapcore.CapitalLevelEncoder, + EncodeTime: zapcore.ISO8601TimeEncoder, + EncodeCaller: zapcore.ShortCallerEncoder, + } + + fileEncoder := logger.NewCustomEncoder(&encoderConfig) + + if err := config.LogLevel.UnmarshalText([]byte(coreInfo.Level)); err != nil { + config.LogLevel = zapcore.InfoLevel + } + + priority := uberZap.LevelEnablerFunc(func(lvl zapcore.Level) bool { + return lvl >= config.LogLevel + }) + + if coreInfo.Tick == 0 || coreInfo.First == 0 || coreInfo.Thereafter == 0 { + return zapcore.NewCore(fileEncoder, syncer, priority), nil + } + return zapcore.NewSamplerWithOptions(zapcore.NewCore(fileEncoder, syncer, priority), + time.Duration(coreInfo.Tick)*time.Second, coreInfo.First, coreInfo.Thereafter), nil +} + +// LoggerWithFormat zap logger +type LoggerWithFormat struct { + *uberZap.Logger +} + +// Infof stdout format and paras +func (z *LoggerWithFormat) Infof(format string, paras ...interface{}) { + z.Logger.Info(fmt.Sprintf(format, paras...)) +} + +// Errorf stdout format and paras +func (z *LoggerWithFormat) Errorf(format string, paras ...interface{}) { + z.Logger.Error(fmt.Sprintf(format, paras...)) +} + +// Warnf stdout format and paras +func (z *LoggerWithFormat) Warnf(format string, paras ...interface{}) { + z.Logger.Warn(fmt.Sprintf(format, paras...)) +} + +// Debugf stdout format and paras +func (z *LoggerWithFormat) Debugf(format string, paras ...interface{}) { + if config.LogLevel > zapcore.DebugLevel { + return + } + z.Logger.Debug(fmt.Sprintf(format, paras...)) +} + +// Fatalf stdout format and paras +func (z *LoggerWithFormat) Fatalf(format string, paras ...interface{}) { + z.Logger.Fatal(fmt.Sprintf(format, paras...)) +} diff --git a/frontend/pkg/common/faas_common/logger/zap/zaplog_test.go b/frontend/pkg/common/faas_common/logger/zap/zaplog_test.go new file mode 100644 index 0000000000000000000000000000000000000000..bf78aab857b1244ff5d850ae7b3a0577e8b3c98b --- /dev/null +++ b/frontend/pkg/common/faas_common/logger/zap/zaplog_test.go @@ -0,0 +1,184 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package zap + +import ( + "fmt" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + uberZap "go.uber.org/zap" + + "frontend/pkg/common/faas_common/logger/config" +) + +// TestNewDvelopmentLog Test New Dvelopment Log +func TestNewDvelopmentLog(t *testing.T) { + if _, err := NewDevelopmentLog(); err != nil { + t.Errorf("NewDevelopmentLog() = %q, wants *logger", err) + } +} + +func TestNewConsoleLog(t *testing.T) { + tests := []struct { + name string + want *uberZap.Logger + wantErr bool + }{ + {"case1", nil, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := NewConsoleLog() + if (err != nil) != tt.wantErr { + t.Errorf("NewConsoleLog() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} + +func TestNewWithLevel(t *testing.T) { + type args struct { + coreInfo config.CoreInfo + isAsync bool + } + var a args + tests := []struct { + name string + args args + want *uberZap.Logger + wantErr bool + }{ + {"case1", a, nil, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := NewWithLevel(tt.args.coreInfo, tt.args.isAsync) + if (err != nil) != tt.wantErr { + t.Errorf("NewWithLevel() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewWithLevel() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestLoggerWithFormat_Infof(t *testing.T) { + type fields struct { + Logger *uberZap.Logger + } + type args struct { + format string + paras []interface{} + } + coreInfo := config.CoreInfo{ + FilePath: "tmp", + Level: "DEBUG", + Tick: 0, + First: 0, + Thereafter: 0, + Tracing: false, + Disable: false, + } + logger, err := NewWithLevel(coreInfo, true) + if err != nil { + fmt.Println(err) + } + var f fields + f.Logger = logger + var a args + tests := []struct { + name string + fields fields + args args + }{ + {"case1", f, a}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + z := &LoggerWithFormat{ + Logger: tt.fields.Logger, + } + z.Infof(tt.args.format, tt.args.paras...) + }) + } +} + +func TestNewCoreWithDebugLevel(t *testing.T) { + convey.Convey("TestNewCoreWithInfoLevel", t, func() { + coreInfo := config.CoreInfo{ + FilePath: "tmp", + Level: "DEBUG", + Tick: 0, + First: 0, + Thereafter: 0, + Tracing: false, + Disable: false, + } + logger, err := NewWithLevel(coreInfo, true) + convey.So(logger, convey.ShouldNotBeNil) + convey.So(err, convey.ShouldBeNil) + type fields struct { + Logger *uberZap.Logger + } + z := &LoggerWithFormat{ + Logger: logger, + } + cnt := 0 + gomonkey.ApplyMethod(reflect.TypeOf(logger), "Debug", + func(log *uberZap.Logger, msg string, fields ...uberZap.Field) { + cnt += 1 + }) + z.Debugf("should print") + convey.So(cnt, convey.ShouldEqual, 1) + }) +} + +func TestNewCoreWithInfoLevel(t *testing.T) { + convey.Convey("TestNewCoreWithInfoLevel", t, func() { + coreInfo := config.CoreInfo{ + FilePath: "tmp", + Level: "INFO", + Tick: 0, + First: 0, + Thereafter: 0, + Tracing: false, + Disable: false, + } + logger, err := NewWithLevel(coreInfo, true) + convey.So(logger, convey.ShouldNotBeNil) + convey.So(err, convey.ShouldBeNil) + type fields struct { + Logger *uberZap.Logger + } + z := &LoggerWithFormat{ + Logger: logger, + } + cnt := 0 + gomonkey.ApplyMethod(reflect.TypeOf(logger), "Debug", + func(log *uberZap.Logger, msg string, fields ...uberZap.Field) { + cnt += 1 + }) + z.Debugf("should not print") + convey.So(cnt, convey.ShouldEqual, 0) + }) +} diff --git a/frontend/pkg/common/faas_common/monitor/defaultfilewatcher.go b/frontend/pkg/common/faas_common/monitor/defaultfilewatcher.go new file mode 100644 index 0000000000000000000000000000000000000000..16ab5bed76f550cfca79cdc00c47ee2f1d137776 --- /dev/null +++ b/frontend/pkg/common/faas_common/monitor/defaultfilewatcher.go @@ -0,0 +1,176 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package monitor provide memory and file monitor +package monitor + +import ( + "crypto/sha256" + "encoding/hex" + "io/ioutil" + "os" + "path/filepath" + "time" + + "github.com/fsnotify/fsnotify" + + "frontend/pkg/common/faas_common/logger/log" +) + +var ( + hashRetry = 60 +) + +type defaultFileWatcher struct { + watcher *fsnotify.Watcher + filename string + callback FileChangedCallback + hash string + stopCh <-chan struct{} +} + +func createDefaultFileWatcher(stopCh <-chan struct{}) (FileWatcher, error) { + fsWatcher, err := fsnotify.NewWatcher() + if err != nil { + return nil, err + } + + w := &defaultFileWatcher{ + watcher: fsWatcher, + stopCh: stopCh, + } + return w, nil +} + +// RegisterCallback impl +func (w *defaultFileWatcher) RegisterCallback(filename string, callback FileChangedCallback) { + realPath, err := w.getRealPath(filename) + if err != nil { + log.GetLogger().Errorf("filename %s getRealPath failed err %s", filename, err.Error()) + return + } + + if callback == nil { + log.GetLogger().Errorf("filename %s callback is nil", filename) + return + } + + hash := w.getFileHashRetry(filename) + w.filename = filename + w.callback = callback + w.hash = hash + if err := w.watcher.Add(realPath); err != nil { + log.GetLogger().Warnf("watch file %s failed", filename) + } else { + log.GetLogger().Infof("file %s RegisterCallback, success", filename) + } +} + +func (w *defaultFileWatcher) getRealPath(filename string) (string, error) { + realPath, err := filepath.EvalSymlinks(filename) + if err != nil { + return "", err + } + + if _, err := os.Stat(realPath); err != nil { + return "", err + } + return realPath, nil +} + +func (w *defaultFileWatcher) handleFileRemove(event fsnotify.Event) { + // remove old watcher + w.watcher.Remove(event.Name) + w.watcher.Remove(w.filename) + + // re-add new watcher + realPath, err := w.getRealPath(w.filename) + if err != nil { + log.GetLogger().Warnf("filename %s getRealPath failed err %s", w.filename, err.Error()) + } else { + if err := w.watcher.Add(realPath); err != nil { + log.GetLogger().Warnf("re-add watcher %s failed", realPath) + } else { + log.GetLogger().Infof("re-add watcher %s success", realPath) + } + } + + if err := w.watcher.Add(w.filename); err != nil { + log.GetLogger().Warnf("re-add watcher %s failed", w.filename) + } else { + log.GetLogger().Infof("re-add watcher %s success", w.filename) + } +} + +// Start impl +func (w *defaultFileWatcher) Start() { + for { + select { + case event, ok := <-w.watcher.Events: + if !ok { + log.GetLogger().Errorf("watcher event chan not ok") + continue + } + w.invokeCallback(event) + if event.Op == fsnotify.Remove { + w.handleFileRemove(event) + } + case err, ok := <-w.watcher.Errors: + if !ok { + log.GetLogger().Errorf("errors chan not ok, err %s", err.Error()) + } + case <-w.stopCh: + w.watcher.Close() + return + } + } +} + +func (w *defaultFileWatcher) invokeCallback(event fsnotify.Event) { + newHash := w.getFileHashRetry(w.filename) + if newHash != w.hash { + begin := time.Now() + log.GetLogger().Infof("file event %s happen, start invoke callback", event.String()) + w.hash = newHash + w.callback(w.filename, OpType(event.Op)) + log.GetLogger().Infof("file event %s invoke callback success, cost %v", + event.String(), time.Since(begin)) + } +} + +func (w *defaultFileWatcher) getFileHashRetry(filename string) string { + for i := 0; i < hashRetry; i++ { + hash := w.getFileHash(filename) + if len(hash) > 0 { + return hash + } + time.Sleep(1 * time.Second) + } + return "" +} + +func (w *defaultFileWatcher) getFileHash(filename string) string { + content, err := ioutil.ReadFile(filename) + if err != nil { + return "" + } + hash := sha256.New() + _, err = hash.Write(content) + if err != nil { + return "" + } + return hex.EncodeToString(hash.Sum(nil)) +} diff --git a/frontend/pkg/common/faas_common/monitor/defaultfilewatcher_test.go b/frontend/pkg/common/faas_common/monitor/defaultfilewatcher_test.go new file mode 100644 index 0000000000000000000000000000000000000000..7fea059d2a6d23d99e164bd720f6fae36005ac7e --- /dev/null +++ b/frontend/pkg/common/faas_common/monitor/defaultfilewatcher_test.go @@ -0,0 +1,90 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package monitor + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/fsnotify/fsnotify" + "github.com/stretchr/testify/assert" +) + +func TestDefaultFileWatcherStart(t *testing.T) { + eventChan := make(chan fsnotify.Event) + stopCh := make(chan struct{}) + errorChan := make(chan error) + mockWatcher := &fsnotify.Watcher{ + Events: eventChan, + Errors: errorChan, + } + + invokeCallbackCh := make(chan bool) + closeCh := make(chan bool) + + watcher := &defaultFileWatcher{ + watcher: mockWatcher, + filename: "/path/testfile.txt", + stopCh: stopCh, + callback: func(filename string, opType OpType) { + invokeCallbackCh <- true + }, + } + patches := []*gomonkey.Patches{ + gomonkey.ApplyFunc(ioutil.ReadFile, func(filename string) ([]byte, error) { + return []byte("mock content"), nil + }), + gomonkey.ApplyMethod(reflect.TypeOf(watcher.watcher), "Remove", func(_ *fsnotify.Watcher, _ string) error { + return nil + }), + gomonkey.ApplyFunc(filepath.EvalSymlinks, func(path string) (string, error) { + return "/mock/symlink/path", nil + }), + gomonkey.ApplyFunc(os.Stat, func(name string) (os.FileInfo, error) { + return nil, nil + }), + gomonkey.ApplyMethod(reflect.TypeOf(watcher.watcher), "Add", func(_ *fsnotify.Watcher, _ string) error { + return nil + }), + gomonkey.ApplyMethod(reflect.TypeOf(watcher.watcher), "Close", func(_ *fsnotify.Watcher) error { + closeCh <- true + return nil + }), + } + defer func() { + for _, patch := range patches { + patch.Reset() + } + }() + + go watcher.Start() + + errorChan <- fmt.Errorf("err") + eventChan <- fsnotify.Event{Name: "/path/test1.txt", Op: fsnotify.Remove} + + invokeCallback := <-invokeCallbackCh + close(stopCh) + + assert.Equal(t, invokeCallback, true) + isClosed := <-closeCh + assert.Equal(t, isClosed, true) +} diff --git a/frontend/pkg/common/faas_common/monitor/filewatcher.go b/frontend/pkg/common/faas_common/monitor/filewatcher.go new file mode 100644 index 0000000000000000000000000000000000000000..9e28e2e5392dbaed8bcab3fdafb91952eaee9f2e --- /dev/null +++ b/frontend/pkg/common/faas_common/monitor/filewatcher.go @@ -0,0 +1,77 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package monitor provide memory and file monitor +package monitor + +import ( + "errors" + + "frontend/pkg/common/faas_common/logger/log" +) + +// OpType describes file operation type +type OpType uint32 + +const ( + // Create op type + Create OpType = 1 << iota + // Write op type + Write + // Remove op type + Remove + // Rename op type + Rename + // Chmod op type + Chmod +) + +var ( + creator Creator = createDefaultFileWatcher +) + +// FileChangedCallback describes callback function, when file changed, callback function will be invoked +type FileChangedCallback func(filename string, opType OpType) + +// Creator describes watcher create function +type Creator func(stopCh <-chan struct{}) (FileWatcher, error) + +// FileWatcher describes interface of general FileWatcher +type FileWatcher interface { + Start() + RegisterCallback(filename string, callback FileChangedCallback) +} + +// SetCreator set file watcher creator func, if not set, use createDefaultFileWatcher +func SetCreator(newCreator Creator) { + creator = newCreator +} + +// CreateFileWatcher create a file watcher +// notice: one FileWatcher can only watcher one file +func CreateFileWatcher(stopCh <-chan struct{}) (FileWatcher, error) { + watcher, err := creator(stopCh) + if err != nil { + log.GetLogger().Errorf("create watcher failed %s", err.Error()) + return nil, err + } + if watcher == nil { + log.GetLogger().Errorf("watcher is nil") + return nil, errors.New("watcher is nil") + } + go watcher.Start() + return watcher, nil +} diff --git a/frontend/pkg/common/faas_common/monitor/filewatcher_test.go b/frontend/pkg/common/faas_common/monitor/filewatcher_test.go new file mode 100644 index 0000000000000000000000000000000000000000..73ccf97ededf9c1f1cd71ee7a9b130804ffae2ca --- /dev/null +++ b/frontend/pkg/common/faas_common/monitor/filewatcher_test.go @@ -0,0 +1,116 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package monitor + +import ( + "errors" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" +) + +func buildTestFile() string { + path, _ := os.Getwd() + if strings.Contains(path, "\\") { + path = path + "\\test.json" + } else { + path = path + "/test.json" + } + + return path +} + +func TestCreateFileWatcher(t *testing.T) { + convey.Convey("TestCreateFileWatcher error", t, func() { + defer gomonkey.ApplyFunc(createDefaultFileWatcher, func(stopCh <-chan struct{}) (FileWatcher, error) { + return nil, fmt.Errorf("fsnotify.NewWatcher error") + }).Reset() + stopCh := make(chan struct{}, 1) + _, err := CreateFileWatcher(stopCh) + convey.So(err, convey.ShouldNotBeNil) + }) +} + +func TestInitFileWatcher(t *testing.T) { + convey.Convey("TestInitFileWatcher", t, func() { + stopCh := make(chan struct{}, 1) + watcher, err := CreateFileWatcher(stopCh) + convey.So(err, convey.ShouldBeNil) + + filename := buildTestFile() + handler, _ := os.Create(filename) + defer func() { + handler.Close() + os.Remove(filename) + }() + callbackChan := make(chan int, 5) + watcher.RegisterCallback("", nil) + watcher.RegisterCallback(filename, func(filename string, t OpType) { + callbackChan <- 1 + }) + + os.WriteFile(filename, []byte{'a'}, os.ModePerm) + res := <-callbackChan + convey.So(res, convey.ShouldBeGreaterThan, 0) + time.Sleep(5 * time.Millisecond) + close(stopCh) + }) +} + +func TestInitFileWatcherWithInvalidCreator(t *testing.T) { + convey.Convey("TestInitFileWatcherWithInvalidCreator", t, func() { + defer SetCreator(createDefaultFileWatcher) + SetCreator(func(stopCh <-chan struct{}) (FileWatcher, error) { + return nil, errors.New("error") + }) + + stopCh := make(chan struct{}, 1) + _, err := CreateFileWatcher(stopCh) + convey.So(err, convey.ShouldNotBeNil) + }) +} + +func TestFileWatcher_Start(t *testing.T) { + convey.Convey("TestFileWatcher_Start", t, func() { + stopCh := make(chan struct{}, 1) + watcher, _ := CreateFileWatcher(stopCh) + filename := "./TestFileWatcher_Start.tmp" + f, _ := os.Create(filename) + f.Close() + tmp := hashRetry + hashRetry = 1 + defer func() { + hashRetry = tmp + }() + callbackChan := make(chan int, 1) + watcher.RegisterCallback(filename, func(filename string, t OpType) { + callbackChan <- 1 + }) + os.Remove(filename) + time.AfterFunc(3*time.Second, func() { + close(callbackChan) + }) + convey.So(<-callbackChan, convey.ShouldEqual, 1) + time.Sleep(50 * time.Millisecond) + close(stopCh) + }) +} diff --git a/frontend/pkg/common/faas_common/monitor/memory.go b/frontend/pkg/common/faas_common/monitor/memory.go new file mode 100644 index 0000000000000000000000000000000000000000..a32cdb6dfb0477a2e390fc442b42cdd720a5f9ec --- /dev/null +++ b/frontend/pkg/common/faas_common/monitor/memory.go @@ -0,0 +1,353 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package monitor monitors and controls resource usage +package monitor + +import ( + "io/ioutil" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/types" +) + +// MemMonitor monitor memory usage +type MemMonitor interface { + // Allow returns whether you can take some memory to use (in bytes) + Allow(uint64) bool + // AllowByLowerThreshold - + AllowByLowerThreshold(string, string, uint64) bool + // ReleaseFunctionMem function mem when request finished + ReleaseFunctionMem(urn string, size uint64) +} + +const ( + defaultMemoryRefreshInterval = 50 + highMemoryPercent = 0.9 + statefulHighMemPercent = 0.9 + base = 10 + bitSize = 64 + lowerMemoryPercent = 0.7 + bodyThreshold = 10000 + zero = 0 + defaultFuncNum = 1 +) + +var ( + memory = struct { + sync.Once + monitor *memMonitor + err error + }{ + monitor: &memMonitor{}, + } + mu sync.Mutex +) + +var ( + config = &types.MemoryControlConfig{ + LowerMemoryPercent: lowerMemoryPercent, + HighMemoryPercent: highMemoryPercent, + StatefulHighMemPercent: statefulHighMemPercent, + BodyThreshold: bodyThreshold, + MemDetectIntervalMs: defaultMemoryRefreshInterval, + } +) + +type memMonitor struct { + enable bool + used uint64 + threshold uint64 + statefulThreshold uint64 + stopCh <-chan struct{} + lowerThreshold uint64 + memMapMutex sync.Mutex + functionMemMap map[string]uint64 + totalMemCnt uint64 + isStateful bool +} + +// SetMemoryControlConfig set memory control config from different service +func SetMemoryControlConfig(memoryControlConfig *types.MemoryControlConfig) { + if memoryControlConfig == nil { + return + } + if memoryControlConfig.LowerMemoryPercent > 0 { + config.LowerMemoryPercent = memoryControlConfig.LowerMemoryPercent + } + if memoryControlConfig.BodyThreshold > 0 { + config.BodyThreshold = memoryControlConfig.BodyThreshold + } + if memoryControlConfig.MemDetectIntervalMs > 0 { + config.MemDetectIntervalMs = memoryControlConfig.MemDetectIntervalMs + } + if memoryControlConfig.HighMemoryPercent > 0 { + config.HighMemoryPercent = memoryControlConfig.HighMemoryPercent + } + if memoryControlConfig.StatefulHighMemPercent > 0 { + config.StatefulHighMemPercent = memoryControlConfig.StatefulHighMemPercent + } + log.GetLogger().Infof("LowerMemoryPercent %f, HighMemoryPercent %f, "+ + "StatefulHighMemPercent %f, BodyThreshold %d, MemDetectIntervalMs %d", + config.LowerMemoryPercent, config.HighMemoryPercent, + config.StatefulHighMemPercent, config.BodyThreshold, config.MemDetectIntervalMs) + + if memory.monitor != nil { + memory.monitor.updateConfig() + } +} + +// InitMemMonitor initialize global memory monitor +func InitMemMonitor(stopCh <-chan struct{}) error { + memory.Do(func() { + memory.err = memory.monitor.init(stopCh) + }) + return memory.err +} + +// GetMemInstance returns global memory monitor +func GetMemInstance() MemMonitor { + return memory.monitor +} + +func readValue(path string) (uint64, error) { + v, err := ioutil.ReadFile(path) + if err != nil { + return 0, err + } + return parseValue(strings.TrimSpace(string(v)), base, bitSize) +} + +func parseValue(s string, base, bitSize int) (uint64, error) { + v, err := strconv.ParseUint(s, base, bitSize) + if err != nil { + intValue, intErr := strconv.ParseInt(s, base, bitSize) + if intErr == nil && intValue < 0 { + return 0, nil + } + if intErr != nil && + intErr.(*strconv.NumError).Err == strconv.ErrRange && + intValue < 0 { + return 0, nil + } + return 0, err + } + return v, nil +} + +// refresh actual memory usage +func (m *memMonitor) refreshActualMemoryUsage() { + interval := config.MemDetectIntervalMs + parser, err := NewCGroupMemoryParser() + if err != nil { + log.GetLogger().Warnf("failed to create cgroup memory parser: %s", err.Error()) + return + } + defer parser.Close() + ticker := time.NewTicker(time.Duration(interval) * time.Millisecond) + for { + select { + case <-ticker.C: + val, err := parser.Read() + if err != nil { + log.GetLogger().Errorf("GetSystemMemoryUsed failed, err: %s", err.Error()) + continue + } + used, ok := val.(uint64) + if !ok { + log.GetLogger().Errorf("GetSystemMemoryUsed failed, err: failed to assert parser data") + continue + } + atomic.StoreUint64(&m.used, used) + if interval != config.MemDetectIntervalMs { + log.GetLogger().Infof("MemDetectIntervalMs updated, old: %d, new: %d, reset timer", + interval, config.MemDetectIntervalMs) + interval = config.MemDetectIntervalMs + ticker.Reset(time.Duration(interval) * time.Millisecond) + } + case <-m.stopCh: + log.GetLogger().Info("memory monitor stopped") + ticker.Stop() + return + } + } +} + +func (m *memMonitor) init(stopCh <-chan struct{}) error { + memLimit, err := readValue("/sys/fs/cgroup/memory/memory.limit_in_bytes") + if err != nil { + log.GetLogger().Warn("failed to read limit_in_bytes") + return nil + } + m.threshold = uint64(float64(memLimit) * config.HighMemoryPercent) + m.statefulThreshold = uint64(float64(memLimit) * config.StatefulHighMemPercent) + m.enable = true + m.memMapMutex = sync.Mutex{} + m.functionMemMap = map[string]uint64{} + m.lowerThreshold = uint64(float64(memLimit) * config.LowerMemoryPercent) + log.GetLogger().Infof("memory threshold is %d, stateful memory threshold is %d, lowerThreshold is %d", + m.threshold, m.statefulThreshold, m.lowerThreshold) + m.stopCh = stopCh + go m.refreshActualMemoryUsage() + return nil +} + +// Allow returns whether you can take some memory to use (in bytes) +func (m *memMonitor) Allow(want uint64) bool { + if !m.enable { + return true + } + for { + threshold := m.threshold + if m.isStateful { + threshold = m.statefulThreshold + } + current := atomic.LoadUint64(&m.used) + if current > threshold || want > threshold-current { + log.GetLogger().Errorf("memory threshold triggered, current=%d want=%d threshold=%d", + current, want, threshold) + return false + } + if atomic.CompareAndSwapUint64(&m.used, current, current+want) { + return true + } + } +} + +func (m *memMonitor) increaseMemCnt(size uint64) { + m.totalMemCnt += size +} + +func (m *memMonitor) decreaseMemCnt(size uint64) { + if m.totalMemCnt < size { + log.GetLogger().Warnf("invalid mem cnt %d, size %d", m.totalMemCnt, size) + m.totalMemCnt = 0 + } else { + m.totalMemCnt -= size + } +} + +// ReleaseFunctionMem release function mem when function req finished +func (m *memMonitor) ReleaseFunctionMem(urn string, size uint64) { + if !m.enable || size <= config.BodyThreshold { + return + } + + m.memMapMutex.Lock() + defer m.memMapMutex.Unlock() + + memUsed, ok := m.functionMemMap[urn] + if !ok { + return + } + + m.decreaseMemCnt(size) + if memUsed <= size { + delete(m.functionMemMap, urn) + } else { + m.functionMemMap[urn] = memUsed - size + } +} + +// mallocFunctionMem malloc function mem when function req enter +func (m *memMonitor) mallocFunctionMem(urn string, realSize uint64) { + m.increaseMemCnt(realSize) + memUsed, ok := m.functionMemMap[urn] + if !ok { + m.functionMemMap[urn] = realSize + } else { + m.functionMemMap[urn] = memUsed + realSize + } +} + +// AllowByLowerThreshold control memory use by LowerThreshold +// if used memory > LowerThreshold and function mem use > average, this function just return heavy load +func (m *memMonitor) AllowByLowerThreshold(urn string, traceID string, size uint64) bool { + if !m.enable || size <= config.BodyThreshold { + return true + } + + m.memMapMutex.Lock() + defer m.memMapMutex.Unlock() + // if current mem lower than lowerThreshold, allow + current := atomic.LoadUint64(&m.used) + if current <= m.lowerThreshold && m.totalMemCnt <= m.lowerThreshold { + m.mallocFunctionMem(urn, size) + return true + } + + memUsed, ok := m.functionMemMap[urn] + // if it's new function, allow + if !ok { + m.increaseMemCnt(size) + m.functionMemMap[urn] = size + return true + } + + functionNum := uint64(len(m.functionMemMap)) + if functionNum <= zero { + functionNum = defaultFuncNum + } + // if function use mem lower than averageMem allow + averageMem := m.totalMemCnt / functionNum + if memUsed <= averageMem { + m.increaseMemCnt(size) + m.functionMemMap[urn] = memUsed + size + return true + } + + log.GetLogger().Errorf("lower memory threshold triggered, currentFromSys=%d,currentFromEvaluator=%d,"+ + "lowerThreshold=%d,functionUsed=%d,functionNum=%d,traceID=%s,bodyLength=%d", + current, m.totalMemCnt, m.lowerThreshold, memUsed, functionNum, traceID, size) + return false +} + +func (m *memMonitor) updateConfig() { + memLimit, err := readValue("/sys/fs/cgroup/memory/memory.limit_in_bytes") + if err != nil { + log.GetLogger().Warn("failed to read limit_in_bytes") + return + } + + m.threshold = uint64(float64(memLimit) * config.HighMemoryPercent) + m.statefulThreshold = uint64(float64(memLimit) * config.StatefulHighMemPercent) + m.lowerThreshold = uint64(float64(memLimit) * config.LowerMemoryPercent) + + log.GetLogger().Infof("config updated, memory threshold is %d, stateful memory threshold is %d,lowerThreshold is %d", + m.threshold, m.statefulThreshold, m.lowerThreshold) +} + +// IsAllowByMemory returns whether you can take some memory to use +func IsAllowByMemory(urn string, memoryWant uint64, traceID string) bool { + if !GetMemInstance().Allow(memoryWant) { + log.GetLogger().Errorf("request is limited by higher threshold, urn %s traceID %s want %d", + urn, traceID, memoryWant) + return false + } + + if !GetMemInstance().AllowByLowerThreshold(urn, traceID, memoryWant) { + log.GetLogger().Errorf("request is limited by lower threshold, urn %s traceID %s want %d", + urn, traceID, memoryWant) + return false + } + + return true +} diff --git a/frontend/pkg/common/faas_common/monitor/memory_test.go b/frontend/pkg/common/faas_common/monitor/memory_test.go new file mode 100644 index 0000000000000000000000000000000000000000..6f72f693940e972cbda6a21d7770a1dbf655173d --- /dev/null +++ b/frontend/pkg/common/faas_common/monitor/memory_test.go @@ -0,0 +1,164 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package monitor + +import ( + "reflect" + "sync" + "testing" + "time" + + . "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + + "frontend/pkg/common/faas_common/types" +) + +func TestInitMemMonitor(t *testing.T) { + convey.Convey("TestInitMemMonitor", t, func() { + convey.Convey("success", func() { + patches := [...]*Patches{ + ApplyMethod(reflect.TypeOf(new(Parser)), "Read", func(_ *Parser) (interface{}, error) { + return uint64(100), nil + }), + } + defer func() { + for idx := range patches { + patches[idx].Reset() + } + }() + stopCh := make(chan struct{}) + err := InitMemMonitor(stopCh) + assert.Nil(t, err) + assert.Equal(t, uint64(0x0), memory.monitor.used) + + time.Sleep(2 * time.Second) + assert.NotEqual(t, uint64(0x0), memory.monitor.used) + }) + }) + +} + +func TestMemMonitor_Allow(t *testing.T) { + memMonitor := &memMonitor{enable: true, threshold: 1024, used: 10} + result := memMonitor.Allow(1000) + assert.Equal(t, true, result) + result = memMonitor.Allow(15) + assert.Equal(t, false, result) +} + +func TestAllowByLowerThreshold(t *testing.T) { + memMonitor := &memMonitor{ + enable: true, + threshold: 200000, + lowerThreshold: 140000, + memMapMutex: sync.Mutex{}, + functionMemMap: map[string]uint64{}, + } + + allow := memMonitor.Allow(100) + assert.Equal(t, true, allow) + allow = memMonitor.AllowByLowerThreshold("1", "1", 100) + assert.Equal(t, true, allow) + + allow = memMonitor.Allow(100000) + assert.Equal(t, true, allow) + allow = memMonitor.AllowByLowerThreshold("1", "1", 100000) + assert.Equal(t, true, allow) + + allow = memMonitor.Allow(20000) + assert.Equal(t, true, allow) + allow = memMonitor.AllowByLowerThreshold("2", "2", 20000) + assert.Equal(t, true, allow) + + allow = memMonitor.Allow(30000) + assert.Equal(t, true, allow) + allow = memMonitor.AllowByLowerThreshold("1", "1", 30000) + assert.Equal(t, false, allow) + + memMonitor.ReleaseFunctionMem("1", 100000) + memMonitor.used = memMonitor.used - 100000 + + allow = memMonitor.Allow(100000) + assert.Equal(t, true, allow) + allow = memMonitor.AllowByLowerThreshold("1", "1", 100000) + assert.Equal(t, true, allow) + + allow = memMonitor.Allow(20000) + assert.Equal(t, true, allow) + allow = memMonitor.AllowByLowerThreshold("2", "2", 20000) + assert.Equal(t, true, allow) +} + +func TestSetMemoryControlConfig(t *testing.T) { + convey.Convey("TestSetMemoryControlConfig", t, func() { + convey.Convey("nil config", func() { + SetMemoryControlConfig(nil) + }) + convey.Convey("SetMemoryControlConfig", func() { + cfg := &types.MemoryControlConfig{ + LowerMemoryPercent: 0.5, + BodyThreshold: 1024, + MemDetectIntervalMs: 3, + HighMemoryPercent: 0.5, + StatefulHighMemPercent: 0.9, + } + SetMemoryControlConfig(cfg) + convey.So(*config == *cfg, convey.ShouldEqual, true) + }) + }) +} + +func Test_parseValue(t *testing.T) { + convey.Convey("parseValue", t, func() { + v, err := parseValue("100", 10, 64) + convey.So(v, convey.ShouldEqual, 100) + convey.So(err, convey.ShouldBeNil) + v, err = parseValue("-100", 10, 64) + convey.So(v, convey.ShouldEqual, 0) + convey.So(err, convey.ShouldBeNil) + v, err = parseValue("1.01", 10, 64) + convey.So(v, convey.ShouldEqual, 0) + convey.So(err, convey.ShouldNotBeNil) + }) +} + +func TestIsAllowByMemory(t *testing.T) { + convey.Convey("TestIsAllowByMemory", t, func() { + memory.monitor = &memMonitor{ + enable: true, + threshold: 200000, + lowerThreshold: 140000, + memMapMutex: sync.Mutex{}, + functionMemMap: map[string]uint64{}, + } + memory.monitor.decreaseMemCnt(100) + + allow := IsAllowByMemory("1", 200001, "") + convey.So(allow, convey.ShouldBeFalse) + + allow = IsAllowByMemory("1", 100000, "") + convey.So(allow, convey.ShouldBeTrue) + + allow = IsAllowByMemory("2", 50000, "") + convey.So(allow, convey.ShouldBeTrue) + + allow = IsAllowByMemory("1", 20000, "") + convey.So(allow, convey.ShouldBeFalse) + }) +} diff --git a/frontend/pkg/common/faas_common/monitor/mockfilewatcher.go b/frontend/pkg/common/faas_common/monitor/mockfilewatcher.go new file mode 100644 index 0000000000000000000000000000000000000000..97340d901f97ae02449c73576993ca9fb2629f8a --- /dev/null +++ b/frontend/pkg/common/faas_common/monitor/mockfilewatcher.go @@ -0,0 +1,42 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package monitor + +// MockFileWatcher - +type MockFileWatcher struct { + Callbacks map[string]FileChangedCallback + StopCh <-chan struct{} + EventChan chan string +} + +// Start - +func (watcher *MockFileWatcher) Start() { + for { + select { + case event, _ := <-watcher.EventChan: + callback, _ := watcher.Callbacks[event] + callback(event, Write) + case <-watcher.StopCh: + return + } + } +} + +// RegisterCallback - +func (watcher *MockFileWatcher) RegisterCallback(filename string, callback FileChangedCallback) { + watcher.Callbacks[filename] = callback +} diff --git a/frontend/pkg/common/faas_common/monitor/mockfilewatcher_test.go b/frontend/pkg/common/faas_common/monitor/mockfilewatcher_test.go new file mode 100644 index 0000000000000000000000000000000000000000..f2f5f5ccc72f5d8f68ad6369fb8089313dbaa256 --- /dev/null +++ b/frontend/pkg/common/faas_common/monitor/mockfilewatcher_test.go @@ -0,0 +1,51 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package monitor + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestMockFileWatcherStart(t *testing.T) { + stopCh := make(chan struct{}) + eventChan := make(chan string) + + watcher := &MockFileWatcher{ + EventChan: eventChan, + Callbacks: make(map[string]FileChangedCallback), + StopCh: stopCh, + } + + callbackCalled := false + watcher.RegisterCallback("test_event", func(filename string, opType OpType) { + assert.Equal(t, "test_event", filename) + assert.Equal(t, Write, opType) + callbackCalled = true + }) + + go watcher.Start() + + watcher.EventChan <- "test_event" + + assert.Eventually(t, func() bool { return callbackCalled }, 1*time.Second, 10*time.Millisecond, + "Callback function should be called") + + close(stopCh) +} diff --git a/frontend/pkg/common/faas_common/monitor/parser.go b/frontend/pkg/common/faas_common/monitor/parser.go new file mode 100644 index 0000000000000000000000000000000000000000..cf293e422661314b542e0fce5bfddd5b2b1bada3 --- /dev/null +++ b/frontend/pkg/common/faas_common/monitor/parser.go @@ -0,0 +1,110 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package monitor + +import ( + "bufio" + "bytes" + "io" + "os" + "strconv" +) + +const ( + cgroupMemoryPath = "/sys/fs/cgroup/memory/memory.stat" +) + +var ( + rssPrefix = []byte("rss ") +) + +// NewCGroupMemoryParser creates parser of /sys/fs/cgroup/memory/memory.stat +func NewCGroupMemoryParser() (*Parser, error) { + return NewParser(cgroupMemoryPath, cgroupMemoryParserFunc) +} + +var cgroupMemoryParserFunc = func(reader *bufio.Reader) (interface{}, error) { + for { + lineBytes, _, err := reader.ReadLine() + if err != nil { + return uint64(0), err + } + + if bytes.HasPrefix(lineBytes, rssPrefix) { + lineBytes = bytes.TrimSpace(lineBytes[len(rssPrefix):]) + return strconv.ParseUint(string(lineBytes), base, bitSize) + } + } +} + +// ParserFunc func that parser content of reader to uint64 +type ParserFunc func(reader *bufio.Reader) (interface{}, error) + +// NewParser creates new Parser with file path and ParserFunc +func NewParser(path string, parserFunc ParserFunc) (*Parser, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + return &Parser{ + f: f, + reader: bufio.NewReader(nil), + parser: parserFunc, + }, nil +} + +type nopCloser struct { + io.ReadSeeker +} + +// Close does nothing. It wraps io.ReadSeeker to io.ReadSeekCloser +func (nopCloser) Close() error { return nil } + +// NewReadSeekerParser creates new Parser with io.ReadSeeker and ParserFunc +func NewReadSeekerParser(reader io.ReadSeeker, parserFunc ParserFunc) *Parser { + return &Parser{ + f: nopCloser{reader}, + reader: bufio.NewReader(nil), + parser: parserFunc, + } +} + +// Parser aims to parse file content that updated frequently (such as cgroup file) with high performance. +// It opens file only once and seek to start every time before read. +// NOTICE: Parser is not thread safe +type Parser struct { + reader *bufio.Reader + f io.ReadSeekCloser + parser ParserFunc +} + +// Close closes file to parse +func (p *Parser) Close() error { + p.reader.Reset(nil) + return p.f.Close() +} + +// Read resets reader to the start of the file and parses it. +// This method is not thread safe +func (p *Parser) Read() (interface{}, error) { + _, err := p.f.Seek(0, io.SeekStart) + if err != nil { + return uint64(0), err + } + p.reader.Reset(p.f) + return p.parser(p.reader) +} diff --git a/frontend/pkg/common/faas_common/monitor/parser_test.go b/frontend/pkg/common/faas_common/monitor/parser_test.go new file mode 100644 index 0000000000000000000000000000000000000000..2ef7c650c5fd992d6303df80c5386ab43bc644a6 --- /dev/null +++ b/frontend/pkg/common/faas_common/monitor/parser_test.go @@ -0,0 +1,95 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package monitor + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewReadSeekerParser(t *testing.T) { + tests := []struct { + name string + content []byte + parser ParserFunc + hasError bool + expected uint64 + }{ + { + name: "parse cgroup memory", + content: []byte(`cache 10150707200 +rss 880640 +rss_huge 0 +shmem 0 +mapped_file 946176 +dirty 135168 +writeback 270336 +swap 0 +pgpgin 3158595 +pgpgout 680215 +pgfault 992277 +pgmajfault 0 +inactive_anon 0 +active_anon 0 +inactive_file 8343023616 +active_file 1808744448 +unevictable 0 +hierarchical_memory_limit 9223372036854771712 +hierarchical_memsw_limit 9223372036854771712 +total_cache 21492334592 +total_rss 9384980480 +total_rss_huge 5515509760 +total_shmem 654385152 +total_mapped_file 2744586240 +total_dirty 8110080 +total_writeback 2027520 +total_swap 0 +total_pgpgin 1336448421 +total_pgpgout 1354048239 +total_pgfault 1405894809 +total_pgmajfault 50622 +total_inactive_anon 199806976 +total_active_anon 8360579072 +total_inactive_file 19150966784 +total_active_file 3246854144 +total_unevictable 0`), + parser: cgroupMemoryParserFunc, + expected: 880640, + }, + { + name: "parse cgroup memory no such line", + content: []byte(`880640`), + parser: cgroupMemoryParserFunc, + hasError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + parser := NewReadSeekerParser(bytes.NewReader(tt.content), tt.parser) + data, err := parser.Read() + if tt.hasError { + assert.Error(t, err) + } else { + assert.Equal(t, tt.expected, data) + } + assert.Nil(t, parser.Close()) + }) + } +} diff --git a/frontend/pkg/common/faas_common/protobuf/common_args.proto b/frontend/pkg/common/faas_common/protobuf/common_args.proto new file mode 100644 index 0000000000000000000000000000000000000000..5c880843716e18045406a204dc781d469c2e18d0 --- /dev/null +++ b/frontend/pkg/common/faas_common/protobuf/common_args.proto @@ -0,0 +1,123 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package commonargs; + +option go_package = "frontend/pkg/common/faas_common/grpc/pb/commonargs;commonargs"; + +message Arg { + enum ArgType { + VALUE = 0; + OBJECT_REF = 1; + } + ArgType type = 1; + bytes value = 2; + repeated string nested_refs = 3; +} + + +// IN: Labels has the Requirement's key and Labels' value is in Requirement's value set +message LabelIn { + repeated string values = 1; +} + +// NOT_IN: contrary to the operator In +message LabelNotIn { + repeated string values = 1; +} + +// EXISTS: Labels has the Requirement's key +message LabelExists {} + +// DOES_NOT_EXIST: contrary to the operator Exists +message LabelDoesNotExist {} + +message LabelOperator { + oneof LabelOperator { + LabelIn in = 1; + LabelNotIn notIn = 2; + LabelExists exists = 3; + LabelDoesNotExist notExist = 4; + } +} + +message LabelMatchExpression { + string key = 1; + LabelOperator op = 2; +} + +message LabelMatchExpressions { + repeated LabelMatchExpression expressions = 1; + int64 weight = 2; +} + +message RequiredSelector { + repeated LabelMatchExpression expressions = 1; +} + +message PreferredSelector { + repeated LabelMatchExpressions matchLabels = 1; // max length = 10 + bool priority = 2; // ONLY valid while preffered, default is true +} + +message ResourceAffinity { + // key: PreferredAffinity PreferredAntiAffinity + map preferred = 1; + // key: RequiredAffinity RequiredAntiAffinity + map required = 2; +} + +message InstanceAffinity { + // key: PreferredAffinity PreferredAntiAffinity + map preferred = 1; + // key: RequiredAffinity RequiredAntiAffinity + map required = 2; + string topologyKey = 3; // By default, only the node level or function_proxy deamonSet level is supported. +} + +message Affinity { + ResourceAffinity resource = 1; + InstanceAffinity instance = 2; +} + +enum ErrorCode { + ERR_NONE = 0; + ERR_PARAM_INVALID = 1001; + ERR_RESOURCE_NOT_ENOUGH = 1002; + ERR_INSTANCE_NOT_FOUND = 1003; + ERR_INSTANCE_DUPLICATED = 1004; + ERR_INVOKE_RATE_LIMITED = 1005; + ERR_RESOURCE_CONFIG_ERROR = 1006; + ERR_INSTANCE_EXITED = 1007; + ERR_EXTENSION_META_ERROR = 1008; + ERR_USER_CODE_LOAD = 2001; + ERR_USER_FUNCTION_EXCEPTION = 2002; + ERR_REQUEST_BETWEEN_RUNTIME_BUS = 3001; + ERR_INNER_COMMUNICATION = 3002; + ERR_INNER_SYSTEM_ERROR = 3003; + ERR_DISCONNECT_FRONTEND_BUS = 3004; + ERR_ETCD_OPERATION_ERROR = 3005; + ERR_BUS_DISCONNECTION = 3006; + ERR_REDIS_OPERATION_ERROR = 3007; + ERR_NPU_FAULT_ERROR = 3016; +} + +message SmallObject { + string id = 1; + bytes value = 2; // sbuffer +} \ No newline at end of file diff --git a/frontend/pkg/common/faas_common/protobuf/data_service.proto b/frontend/pkg/common/faas_common/protobuf/data_service.proto new file mode 100644 index 0000000000000000000000000000000000000000..a1f94e26bad45c1a7ac44b53d709bdf6e59c9338 --- /dev/null +++ b/frontend/pkg/common/faas_common/protobuf/data_service.proto @@ -0,0 +1,117 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package data_service; + +option go_package = "frontend/pkg/common/faas_common/grpc/pb/data;data"; + +message PutRequest { + string objectId = 1; + bytes objectData = 2; + int32 writeMode = 3; // put param, default = 0 + int32 consistencyType = 4; // put param, default = 0 + repeated string nestedObjectIds = 5; + int32 cacheType = 6; +} + +message PutResponse { + int32 code = 1; + string message = 2; +} + +message GetRequest { + repeated string objectIds = 1; + int64 timeoutMs = 2; +} + +message GetResponse { + int32 code = 1; + string message = 2; + repeated bytes buffers = 3; +} + +message IncreaseRefRequest { + repeated string objectIds = 1; + string remoteClientId = 2; +} + +message IncreaseRefResponse { + int32 code = 1; + string message = 2; + repeated string failedObjectIds = 3; +} + +message DecreaseRefRequest { + repeated string objectIds = 1; + string remoteClientId = 2; +} + +message DecreaseRefResponse { + int32 code = 1; + string message = 2; + repeated string failedObjectIds = 3; +} + +message KvSetRequest { + string key = 1; + bytes value = 2; + int32 existence = 3; + int32 writeMode = 4; // set param, default = 0 + uint32 ttlSecond = 5; // set param, default = 0 + int32 cacheType = 6; +} + +message KvSetResponse { + int32 code = 1; + string message = 2; +} + +message KvMSetTxRequest { + repeated string keys = 1; + repeated bytes values = 2; + int32 existence = 3; + int32 writeMode = 4; + uint32 ttlSecond = 5; + int32 cacheType = 6; +} + +message KvMSetTxResponse { + int32 code = 1; + string message = 2; +} + +message KvGetRequest { + repeated string keys = 1; + uint32 timeoutMs = 2; // default = 0 +} + +message KvGetResponse { + int32 code = 1; + string message = 2; + repeated bytes values = 3; +} + +message KvDelRequest { + repeated string keys = 1; +} + +message KvDelResponse { + int32 code = 1; + string message = 2; + repeated string failedKeys = 3; +} \ No newline at end of file diff --git a/frontend/pkg/common/faas_common/protobuf/function_service.proto b/frontend/pkg/common/faas_common/protobuf/function_service.proto new file mode 100644 index 0000000000000000000000000000000000000000..78f2453304a1e73a2babe1abea932c357e7772f3 --- /dev/null +++ b/frontend/pkg/common/faas_common/protobuf/function_service.proto @@ -0,0 +1,174 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package function_service; + +import "common_args.proto"; + +option go_package = "frontend/pkg/common/faas_common/grpc/pb/function;function"; + +// Core service provides APIs to runtime, +service CoreService { + // Create an instance for specify function + rpc Create (CreateRequest) returns (CreateResponse) {} + // invoke the created instance + rpc Invoke (InvokeRequest) returns (InvokeResponse) {} + // terminate the created instance + rpc Terminate (TerminateRequest) returns (TerminateResponse) {} + // exit the created instance + rpc Exit (ExitRequest) returns (ExitResponse) {} + // save state of the created instance + rpc SaveState (StateSaveRequest) returns (StateSaveResponse) {} + // load state of the created instance + rpc LoadState (StateLoadRequest) returns (StateLoadResponse) {} + // Kill the signal to instance + rpc Kill (KillRequest) returns (KillResponse) {} +} + +enum AffinityType { + PreferredAffinity = 0; + PreferredAntiAffinity = 1; + RequiredAffinity = 2; + RequiredAntiAffinity = 3; +} + +message SchedulingOptions { + int32 priority = 1; + map resources = 2; + map extension = 3; + // will deprecate in future + map affinity = 4; + commonargs.Affinity scheduleAffinity = 5; +} + +message CreateRequest { + string function = 1; + repeated commonargs.Arg args = 2; + SchedulingOptions schedulingOps = 3; + string requestID = 4; + string traceID = 5; + repeated string labels = 6; // "key:value" or "key2" + // optional. if designated instanceID is not empty, the created instance id will be assigned designatedInstanceID + string designatedInstanceID = 7; + map createOptions = 8; +} + +message CreateResponse { + commonargs.ErrorCode code = 1; + string message = 2; + string instanceID = 3; +} + +// gang scheduling +message CreateRequests { + repeated CreateRequest requests = 1; +} + +// gang scheduling +message CreateResponses { + commonargs.ErrorCode code = 1; + string message = 2; + repeated string instanceIDs = 3; +} + +message InvokeRequest { + string function = 1; + repeated commonargs.Arg args = 2; + string instanceID = 3; + string requestID = 4; + string traceID = 5; + repeated string returnObjectIDs = 6; + string spanID = 7; +} + +message InvokeResponse { + commonargs.ErrorCode code = 1; + string message = 2; + string returnObjectID = 3; +} + +message NotifyRequest { + string requestID = 1; + commonargs.ErrorCode code = 2; + string message = 3; + repeated commonargs.SmallObject smallObjects = 4; +} + +message CallResult { + commonargs.ErrorCode code = 1; + string message = 2; + string instanceID = 3; + string requestID = 4; + repeated commonargs.SmallObject smallObjects = 5; +} + +message CallResultAck { + commonargs.ErrorCode code = 1; + string message = 2; +} + +message TerminateRequest { + string instanceID = 1; +} + +message TerminateResponse { + commonargs.ErrorCode code = 1; + string message = 2; +} + +message ExitRequest { + commonargs.ErrorCode code = 1; + string message = 2; +} + +message ExitResponse { + commonargs.ErrorCode code = 1; + string message = 2; +} + +message StateSaveRequest { + bytes state = 1; +} + +message StateSaveResponse { + commonargs.ErrorCode code = 1; + string message = 2; + string checkpointID = 3; +} + +message StateLoadRequest { + string checkpointID = 1; +} + +message StateLoadResponse { + commonargs.ErrorCode code = 1; + string message = 2; + bytes state = 3; +} + +message KillRequest { + string instanceID = 1; + int32 signal = 2; + bytes payload = 3; +} + +message KillResponse { + commonargs.ErrorCode code = 1; + string message = 2; +} + diff --git a/frontend/pkg/common/faas_common/protobuf/lease_service.proto b/frontend/pkg/common/faas_common/protobuf/lease_service.proto new file mode 100644 index 0000000000000000000000000000000000000000..d57da5db4d2ebc1a85028cf14b9314f8ce95cc37 --- /dev/null +++ b/frontend/pkg/common/faas_common/protobuf/lease_service.proto @@ -0,0 +1,31 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package lease_service; + +import "common_args.proto"; + +option go_package = "frontend/pkg/common/faas_common/grpc/pb/lease;lease"; + +message LeaseRequest { + string remoteClientId = 1; +} +message LeaseResponse { + commonargs.ErrorCode code = 1; + string message = 2; +} \ No newline at end of file diff --git a/frontend/pkg/common/faas_common/queue/fifoqueue.go b/frontend/pkg/common/faas_common/queue/fifoqueue.go new file mode 100644 index 0000000000000000000000000000000000000000..cf9010ef7f2b7ff5d159ddbc0020422e8da8269b --- /dev/null +++ b/frontend/pkg/common/faas_common/queue/fifoqueue.go @@ -0,0 +1,152 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package queue - +package queue + +import ( + "container/list" +) + +const ( + defaultMapSize = 16 +) + +// FifoQueue implements a fifo scheduling queue. +type FifoQueue struct { + queue *list.List + identityFunc IdentityFunc + elementRecord map[string]*list.Element +} + +// NewFifoQueue return fifo queue +func NewFifoQueue(identityFunc IdentityFunc) *FifoQueue { + return &FifoQueue{ + queue: list.New(), + identityFunc: identityFunc, + elementRecord: make(map[string]*list.Element, defaultMapSize), + } +} + +// Front return front item of queue +func (fq *FifoQueue) Front() interface{} { + if fq.queue.Len() == 0 { + return nil + } + obj := fq.queue.Front().Value + return obj +} + +// Back return rear item of queue +func (fq *FifoQueue) Back() interface{} { + if fq.queue.Len() == 0 { + return nil + } + obj := fq.queue.Back().Value + return obj +} + +// PopFront pops an object from front +func (fq *FifoQueue) PopFront() interface{} { + if fq.queue.Len() == 0 { + return nil + } + elem := fq.queue.Front() + if elem == nil { + return nil + } + obj := elem.Value + if fq.identityFunc != nil { + delete(fq.elementRecord, fq.identityFunc(obj)) + } + fq.queue.Remove(elem) + return obj +} + +// PopBack pops an object from back +func (fq *FifoQueue) PopBack() interface{} { + if fq.queue.Len() == 0 { + return nil + } + elem := fq.queue.Back() + if elem == nil { + return nil + } + obj := elem.Value + if fq.identityFunc != nil { + delete(fq.elementRecord, fq.identityFunc(obj)) + } + fq.queue.Remove(elem) + return obj +} + +// PushBack adds an object into queue +func (fq *FifoQueue) PushBack(obj interface{}) error { + if fq.identityFunc != nil { + fq.elementRecord[fq.identityFunc(obj)] = fq.queue.PushBack(obj) + } else { + fq.queue.PushBack(obj) + } + return nil +} + +// GetByID gets an object in queue by its ID +func (fq *FifoQueue) GetByID(objID string) interface{} { + elem, exist := fq.elementRecord[objID] + if !exist { + return nil + } + return elem.Value +} + +// DelByID deletes an object in queue by its ID +func (fq *FifoQueue) DelByID(objID string) error { + elem, exist := fq.elementRecord[objID] + if !exist { + return ErrObjectNotFound + } + delete(fq.elementRecord, fq.identityFunc(elem)) + fq.queue.Remove(elem) + return nil +} + +// Len returns length of queue +func (fq *FifoQueue) Len() int { + return fq.queue.Len() +} + +// UpdateObjByID will update an object in queue by its ID and fix the order +func (fq *FifoQueue) UpdateObjByID(objID string, obj interface{}) error { + return ErrMethodUnsupported +} + +// Range iterates item in queue and process item with given function +func (fq *FifoQueue) Range(f func(obj interface{}) bool) { + for item := fq.queue.Front(); item != nil; item = item.Next() { + if !f(item) { + break + } + } +} + +// SortedRange iterates item in queue and process item with given function in order +func (fq *FifoQueue) SortedRange(f func(obj interface{}) bool) { + for item := fq.queue.Front(); item != nil; item = item.Next() { + if !f(item) { + break + } + } +} diff --git a/frontend/pkg/common/faas_common/queue/fifoqueue_test.go b/frontend/pkg/common/faas_common/queue/fifoqueue_test.go new file mode 100644 index 0000000000000000000000000000000000000000..3a8f268253165623afa3adf23a4d4febe165f5a8 --- /dev/null +++ b/frontend/pkg/common/faas_common/queue/fifoqueue_test.go @@ -0,0 +1,62 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package queue - +package queue + +import ( + "testing" + + "github.com/smartystreets/goconvey/convey" +) + +func TestSetupLogger(t *testing.T) { + fq := NewFifoQueue(nil) + convey.Convey("front/back is nil", t, func() { + res := fq.Front() + convey.So(res, convey.ShouldBeNil) + res = fq.Back() + convey.So(res, convey.ShouldBeNil) + }) + convey.Convey("do support by id", t, func() { + res := fq.GetByID("test") + convey.So(res, convey.ShouldBeNil) + err := fq.UpdateObjByID("test", "test") + convey.So(err, convey.ShouldEqual, ErrMethodUnsupported) + err = fq.DelByID("test") + convey.So(err, convey.ShouldEqual, ErrObjectNotFound) + }) + convey.Convey("pushback one ele", t, func() { + res := fq.PushBack("obj1") + convey.So(res, convey.ShouldBeNil) + front := fq.Front() + back := fq.Back() + convey.So(front, convey.ShouldEqual, back) + len := fq.Len() + convey.So(len, convey.ShouldEqual, 1) + }) + convey.Convey("pushback other ele", t, func() { + res := fq.PushBack("obj2") + convey.So(res, convey.ShouldBeNil) + len := fq.Len() + convey.So(len, convey.ShouldEqual, 2) + + front := fq.PopFront() + convey.So(front, convey.ShouldEqual, "obj1") + back := fq.PopBack() + convey.So(back, convey.ShouldEqual, "obj2") + }) +} diff --git a/frontend/pkg/common/faas_common/queue/priorityqueue.go b/frontend/pkg/common/faas_common/queue/priorityqueue.go new file mode 100644 index 0000000000000000000000000000000000000000..5898e1349da6c3310d292a42cece469e9fc88033 --- /dev/null +++ b/frontend/pkg/common/faas_common/queue/priorityqueue.go @@ -0,0 +1,410 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package queue - +package queue + +import ( + "math/bits" + "math/rand" + "time" +) + +const ( + defaultQueueLength = 20 +) + +// Item is element stored in heap +type Item struct { + ObjID string + // Obj should be a pointer, otherwise UpdateObjByID will fail + Obj interface{} + Priority int +} + +// PriorityFunc returns priority of an object +type PriorityFunc func(interface{}) (int, error) + +// UpdateObjFunc updates object inside queue +type UpdateObjFunc func(interface{}) error + +// PriorityQueue is a two-ended priority queue which keeps item with max priority at front and item with min priority +// at rear using DeHeap +type PriorityQueue struct { + deHeap *DeHeap + identityFunc IdentityFunc + priorityFunc PriorityFunc +} + +// NewPriorityQueue creates priority queue +func NewPriorityQueue(idFunc IdentityFunc, priorityFunc PriorityFunc) *PriorityQueue { + return &PriorityQueue{ + deHeap: NewDeHeap(), + identityFunc: idFunc, + priorityFunc: priorityFunc, + } +} + +// Front returns the item with max priority +func (pq *PriorityQueue) Front() interface{} { + if item, ok := pq.deHeap.GetMax().(*Item); ok { + return item.Obj + } + return nil +} + +// Range iterates item in queue and process item with given function +func (pq *PriorityQueue) Range(f func(obj interface{}) bool) { + for _, item := range pq.deHeap.items { + if !f(item.Obj) { + break + } + } +} + +// SortedRange iterates item in queue and process item with given function in order +func (pq *PriorityQueue) SortedRange(f func(obj interface{}) bool) { + tmpHeap := pq.deHeap.Copy() + for { + item, ok := tmpHeap.PopMax().(*Item) + if !ok { + break + } + if !f(item.Obj) { + break + } + } +} + +// Back returns the item with min priority +func (pq *PriorityQueue) Back() interface{} { + if item, ok := pq.deHeap.GetMin().(*Item); ok { + return item.Obj + } + return nil +} + +// PopFront pops the item with max priority +func (pq *PriorityQueue) PopFront() interface{} { + if item, ok := pq.deHeap.PopMax().(*Item); ok { + return item.Obj + } + return nil +} + +// PopBack pops the item with min priority +func (pq *PriorityQueue) PopBack() interface{} { + if item, ok := pq.deHeap.PopMin().(*Item); ok { + return item.Obj + } + return nil +} + +// PushBack adds an object into queue +func (pq *PriorityQueue) PushBack(obj interface{}) error { + priority, err := pq.priorityFunc(obj) + if err != nil { + return err + } + pq.deHeap.Push(&Item{ObjID: pq.identityFunc(obj), Obj: obj, Priority: priority}) + return nil +} + +// GetByID gets an object in queue by its ID +func (pq *PriorityQueue) GetByID(objID string) interface{} { + index, item := pq.getIndexAndItemByObjID(objID) + if index == keyNotFoundIndex { + return nil + } + return item.Obj +} + +// DelByID deletes an object in queue by its ID +func (pq *PriorityQueue) DelByID(objID string) error { + index, _ := pq.getIndexAndItemByObjID(objID) + if index != keyNotFoundIndex { + pq.deHeap.Remove(index) + return nil + } + return ErrObjectNotFound +} + +// Len returns length of queue +func (pq *PriorityQueue) Len() int { + return pq.deHeap.Len() +} + +// UpdateObjByID will update an object in queue by its ID and fix the order +func (pq *PriorityQueue) UpdateObjByID(objID string, obj interface{}) error { + var err error + index, item := pq.getIndexAndItemByObjID(objID) + if index == keyNotFoundIndex { + return ErrObjectNotFound + } + item.Obj = obj + // update this object's priority and fix the heap + if item.Priority, err = pq.priorityFunc(obj); err != nil { + return err + } + pq.deHeap.Fix(index) + return nil +} + +// UpdatePriorityFunc - +func (pq *PriorityQueue) UpdatePriorityFunc(priorityFunc PriorityFunc) { + pq.priorityFunc = priorityFunc +} + +func (pq *PriorityQueue) getIndexAndItemByObjID(objID string) (int, *Item) { + for i := 0; i < pq.deHeap.Len(); i++ { + if pq.deHeap.items[i].ObjID == objID { + return i, pq.deHeap.items[i] + } + } + return keyNotFoundIndex, nil +} + +// DeHeap is a max-min heap which stores items in max and min levels, root contains the item with max value of all +// levels and one of root's children contains the item with min value of all levels +type DeHeap struct { + items []*Item + count int +} + +// NewDeHeap creates a DeHeap +func NewDeHeap() *DeHeap { + rand.Seed(time.Now().UnixNano()) + return &DeHeap{ + items: make([]*Item, 0, defaultQueueLength), + } +} + +// Copy creates a shallow copy of DeHeap +func (dh *DeHeap) Copy() *DeHeap { + copyItems := make([]*Item, len(dh.items)) + copy(copyItems, dh.items) + return &DeHeap{ + items: copyItems, + count: dh.count, + } +} + +// Len returns the number of deHeap in heap +func (dh *DeHeap) Len() int { return len(dh.items) } + +// Compare is used to compare two items in heap +func (dh *DeHeap) Compare(i, j int) bool { + if i >= len(dh.items) || j >= len(dh.items) { + return false + } + return dh.items[i].Priority > dh.items[j].Priority +} + +// Swap swaps two items in heap +func (dh *DeHeap) Swap(i, j int) { + if i >= len(dh.items) || j >= len(dh.items) { + return + } + dh.items[i], dh.items[j] = dh.items[j], dh.items[i] +} + +// Push pushes an item to heap +func (dh *DeHeap) Push(x interface{}) { + item, ok := x.(*Item) + if !ok { + return + } + dh.items = append(dh.items, item) + dh.shiftUp(dh.Len() - 1) +} + +// Fix fixes heap's order +func (dh *DeHeap) Fix(i int) { + if j := dh.shiftDown(i); j > 0 { + dh.shiftUp(j) + } +} + +// Remove removes an item from heap +func (dh *DeHeap) Remove(i int) { + n := dh.Len() - 1 + if i > n { + return + } + dh.Swap(i, n) + dh.items[n] = nil + dh.items = dh.items[0:n] + dh.shiftDown(i) +} + +// GetMax returns the item with max value +func (dh *DeHeap) GetMax() interface{} { + if dh.Len() < 1 { + return nil + } + return dh.items[0] +} + +// GetMin returns the item with min value +func (dh *DeHeap) GetMin() interface{} { + n := dh.Len() - 1 + if n < 0 { + return nil + } + lChd := 1 + if lChd > n { + return dh.items[0] + } + rChd := 2 + min := lChd + if rChd <= n && dh.Compare(lChd, rChd) { + min = rChd + } + return dh.items[min] +} + +// PopMax pops item with max value +func (dh *DeHeap) PopMax() interface{} { + n := dh.Len() - 1 + if n < 0 { + return nil + } + item := dh.items[0] + dh.Swap(0, n) + dh.items[n] = nil + dh.items = dh.items[0:n] + dh.shiftDown(0) + return item +} + +// PopMin pops item with min value +func (dh *DeHeap) PopMin() interface{} { + n := dh.Len() - 1 + if n < 0 { + return nil + } + lc := leftChild(0) + rc := rightChild(0) + if lc > n { + item := dh.items[0] + dh.items[0] = nil + dh.items = dh.items[0:n] + return item + } + t := lc + if rc <= n && dh.Compare(lc, rc) { + t = rc + } + if t >= len(dh.items) || n >= len(dh.items) { + return nil + } + item := dh.items[t] + dh.Swap(t, n) + dh.items[n] = nil + dh.items = dh.items[0:n] + dh.shiftDown(t) + return item +} + +func (dh *DeHeap) shiftUp(i int) int { + if i < 0 { + return i + } + isMax := isMaxLevel(i) + p := parent(i) + if p >= 0 { + if dh.Compare(p, i) == isMax { + dh.Swap(p, i) + i = p + isMax = !isMax + } + } + for g := grandparent(i); g >= 0; g = grandparent(i) { + if dh.Compare(g, i) == isMax { + break + } + dh.Swap(g, i) + i = g + } + return i +} + +func (dh *DeHeap) shiftDown(i int) int { + if i < 0 { + return i + } + n := dh.Len() + for i < n { + isMax := isMaxLevel(i) + t := i + // check i's children + lc, rc := leftChild(i), rightChild(i) + // no need to go further if lc reaches n but should handle rc reaches n and lc doesn't + if lc >= n { + break + } + if dh.Compare(lc, t) == isMax { + t = lc + } + if rc < n && dh.Compare(rc, t) == isMax { + t = rc + } + // check i's grandchildren + for gc := leftChild(lc); gc < n && gc <= rightChild(rc); gc++ { + if dh.Compare(gc, t) == isMax { + t = gc + } + } + if t == i { + break + } + dh.Swap(i, t) + i = t + // t is i's children, which means i has no conflict with its grandchildren who stand in the same max/min level + // with i, no need to go further + if t == lc || t == rc { + break + } + // t is i's grandchildren, need to check if t has conflict with t's parent + p := parent(t) + if dh.Compare(p, t) == isMax { + dh.Swap(p, t) + i = p + } + } + return i +} + +func isMaxLevel(i int) bool { + level := bits.Len(uint(i)+1) - 1 + return level%2 == 0 // whether the given integer i is at the maximum level +} + +func parent(i int) int { + return (i - 1) / 2 // find the parent node's index +} + +func grandparent(i int) int { + return ((i + 1) / 4) - 1 // find the grandparent node's index +} + +func leftChild(i int) int { + return i*2 + 1 // find the leftChild node's index +} + +func rightChild(i int) int { + return i*2 + 2 // find the rightChild node's index +} diff --git a/frontend/pkg/common/faas_common/queue/priorityqueue_test.go b/frontend/pkg/common/faas_common/queue/priorityqueue_test.go new file mode 100644 index 0000000000000000000000000000000000000000..64e2228491ae6796252120a142f71b5b9f9380da --- /dev/null +++ b/frontend/pkg/common/faas_common/queue/priorityqueue_test.go @@ -0,0 +1,274 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package queue - +package queue + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDeHeap(t *testing.T) { + items := []*Item{ + { + Obj: "apple", + Priority: 16, + }, + { + Obj: "banana", + Priority: 15, + }, + { + Obj: "berry", + Priority: 17, + }, + { + Obj: "cherry", + Priority: 14, + }, + { + Obj: "grape", + Priority: 18, + }, + { + Obj: "lemon", + Priority: 13, + }, + { + Obj: "haw", + Priority: 12, + }, + { + Obj: "mango", + Priority: 19, + }, + { + Obj: "orange", + Priority: 20, + }, + { + Obj: "watermelon", + Priority: 11, + }, + } + dh := NewDeHeap() + for _, item := range items { + dh.Push(item) + } + popMax1 := dh.PopMax().(*Item).Obj + popMin1 := dh.PopMin().(*Item).Obj + assert.Equal(t, "orange", popMax1.(string)) + assert.Equal(t, "watermelon", popMin1.(string)) + getMax1 := dh.GetMax().(*Item).Obj + getMin1 := dh.GetMin().(*Item).Obj + assert.Equal(t, "mango", getMax1.(string)) + assert.Equal(t, "haw", getMin1.(string)) + dh.items[3].Priority = 11 + dh.Fix(3) + getMin2 := dh.GetMin().(*Item).Obj + assert.Equal(t, "grape", getMin2.(string)) + dh.items[1].Priority = 20 + dh.Fix(1) + getMax2 := dh.GetMax().(*Item).Obj + assert.Equal(t, "grape", getMax2.(string)) + dh.Remove(2) + getMin3 := dh.GetMin().(*Item).Obj + assert.Equal(t, "lemon", getMin3.(string)) +} + +func TestPriorityQueue(t *testing.T) { + type testItem struct { + id string + priority int + } + identityFunc := func(obj interface{}) string { + if item, ok := obj.(*testItem); ok { + return item.id + } + return "" + } + priorityFunc := func(obj interface{}) (int, error) { + if item, ok := obj.(*testItem); ok { + return item.priority, nil + } + return -1, fmt.Errorf("failed to get priority") + } + item1 := &testItem{id: "1", priority: 50} + item2 := &testItem{id: "2", priority: 51} + item3 := &testItem{id: "3", priority: 51} + item4 := &testItem{id: "4", priority: 51} + item5 := &testItem{id: "5", priority: 60} + queue := NewPriorityQueue(identityFunc, priorityFunc) + frontItem1 := queue.Front() + backItem1 := queue.Back() + assert.Equal(t, nil, frontItem1) + assert.Equal(t, nil, backItem1) + + popBack1 := queue.PopBack() + assert.Equal(t, nil, popBack1) + popFront1 := queue.PopFront() + assert.Equal(t, nil, popFront1) + + queue.PushBack(item1) + frontItem2 := queue.Front().(*testItem) + backItem2 := queue.Back().(*testItem) + assert.Equal(t, "1", frontItem2.id) + assert.Equal(t, "1", backItem2.id) + + queue.PushBack(item2) + queue.PushBack(item3) + frontItem3 := queue.Front().(*testItem) + backItem3 := queue.Back().(*testItem) + assert.Equal(t, 51, frontItem3.priority) + assert.Equal(t, 50, backItem3.priority) + + queue.PushBack(item4) + queue.PushBack(item5) + frontItem4 := queue.Front().(*testItem) + backItem4 := queue.Back().(*testItem) + assert.Equal(t, 60, frontItem4.priority) + assert.Equal(t, 50, backItem4.priority) + + item2.priority = 40 + queue.UpdateObjByID("2", item2) + backItem5 := queue.Back().(*testItem) + assert.Equal(t, "2", backItem5.id) + item3.priority = 70 + queue.UpdateObjByID("3", item3) + frontItem5 := queue.Front().(*testItem) + assert.Equal(t, "3", frontItem5.id) + + queue.DelByID("4") + frontItem6 := queue.Front().(*testItem) + backItem6 := queue.Back().(*testItem) + assert.Equal(t, "3", frontItem6.id) + assert.Equal(t, "2", backItem6.id) + + item3.priority = 40 + queue.UpdateObjByID("3", item3) + frontItem7 := queue.Front().(*testItem) + assert.Equal(t, "5", frontItem7.id) + + item2.priority = 30 + queue.UpdateObjByID("2", item2) + backItem7 := queue.Back().(*testItem) + assert.Equal(t, "2", backItem7.id) + + getByID1 := queue.GetByID("qwe") + assert.Equal(t, getByID1, nil) + + getByID2 := queue.GetByID("2").(*testItem) + assert.Equal(t, "2", getByID2.id) + + popBack2 := queue.PopBack().(*testItem) + assert.Equal(t, "2", popBack2.id) + popFront2 := queue.PopFront().(*testItem) + assert.Equal(t, "5", popFront2.id) + + length := queue.Len() + assert.Equal(t, 2, length) +} + +func TestPriorityQueueUpdateFrontInSequence(t *testing.T) { + type testItem struct { + id string + priority int + } + identityFunc := func(obj interface{}) string { + if item, ok := obj.(*testItem); ok { + return item.id + } + return "" + } + priorityFunc := func(obj interface{}) (int, error) { + if item, ok := obj.(*testItem); ok { + return item.priority, nil + } + return -1, fmt.Errorf("failed to get priority") + } + items := []*testItem{ + &testItem{id: "1", priority: 2}, + &testItem{id: "2", priority: 2}, + &testItem{id: "3", priority: 2}, + &testItem{id: "4", priority: 2}, + } + queue := NewPriorityQueue(identityFunc, priorityFunc) + for _, item := range items { + queue.PushBack(item) + } + for { + front := queue.Front().(*testItem) + if front.priority == 0 { + break + } + front.priority -= 1 + queue.UpdateObjByID(front.id, front) + } + queue.Range(func(obj interface{}) bool { + item := obj.(*testItem) + if item.priority != 0 { + t.Errorf("item %s priority %d should be 0", item.id, item.priority) + } + return true + }) +} + +func TestPriorityQueue_SortedRange(t *testing.T) { + type testItem struct { + id string + priority int + } + identityFunc := func(obj interface{}) string { + if item, ok := obj.(*testItem); ok { + return item.id + } + return "" + } + priorityFunc := func(obj interface{}) (int, error) { + if item, ok := obj.(*testItem); ok { + return item.priority, nil + } + return -1, fmt.Errorf("failed to get priority") + } + items := []*testItem{ + &testItem{id: "1", priority: 1}, + &testItem{id: "2", priority: 2}, + &testItem{id: "3", priority: 3}, + &testItem{id: "4", priority: 4}, + } + queue := NewPriorityQueue(identityFunc, priorityFunc) + for _, item := range items { + queue.PushBack(item) + } + rangeItems := make([]*testItem, 0, 4) + queue.SortedRange(func(obj interface{}) bool { + item := obj.(*testItem) + rangeItems = append(rangeItems, item) + return true + }) + for i, item := range rangeItems { + if i+item.priority != 4 { + t.Errorf("range item %+v in wrong order %d\n", item, i) + } + } + front := queue.PopFront().(*testItem) + back := queue.PopBack().(*testItem) + assert.Equal(t, 4, front.priority) + assert.Equal(t, 1, back.priority) +} diff --git a/frontend/pkg/common/faas_common/queue/queue.go b/frontend/pkg/common/faas_common/queue/queue.go new file mode 100644 index 0000000000000000000000000000000000000000..4635a9d4e267a01a71e2d06fa2bfdc6c48a49417 --- /dev/null +++ b/frontend/pkg/common/faas_common/queue/queue.go @@ -0,0 +1,50 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package queue - +package queue + +import "errors" + +const ( + // keyNotFoundIndex stands for index of a non exist key + keyNotFoundIndex = -1 +) + +var ( + // ErrObjectNotFound is the error of object not found + ErrObjectNotFound = errors.New("object not found") + // ErrMethodUnsupported is the error of method unsupported + ErrMethodUnsupported = errors.New("method unsupported") +) + +// IdentityFunc will get ID from object in queue +type IdentityFunc func(interface{}) string + +// Queue is interface of queue used in faas pattern +type Queue interface { + Front() interface{} + Back() interface{} + PopFront() interface{} + PopBack() interface{} + PushBack(obj interface{}) error + GetByID(objID string) interface{} + DelByID(objID string) error + UpdateObjByID(objID string, obj interface{}) error + Len() int + Range(f func(obj interface{}) bool) + SortedRange(f func(obj interface{}) bool) +} diff --git a/frontend/pkg/common/faas_common/redisclient/redisclient.go b/frontend/pkg/common/faas_common/redisclient/redisclient.go new file mode 100644 index 0000000000000000000000000000000000000000..f5b8fd5bc4a44604c5973799ed4c7e7327723744 --- /dev/null +++ b/frontend/pkg/common/faas_common/redisclient/redisclient.go @@ -0,0 +1,510 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package redisclient new a redis client +package redisclient + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "strings" + "sync" + "time" + + "github.com/redis/go-redis/v9" + + "frontend/pkg/common/faas_common/logger/log" + commonTLS "frontend/pkg/common/faas_common/tls" + "frontend/pkg/common/faas_common/utils" +) + +const ( + // timeout : allow TCP reconnection for 3 times(1, 2, 4) + dialTimeout = 8 * time.Second + readTimeout = 8 * time.Second + writeTimeout = 8 * time.Second + idleTimeout = 300 * time.Second + defaultDialTimeout = 8 + defaultReadTimeout = 8 + defaultWriteTimeout = 8 + defaultIdleTimeout = 300 + defaultRedisConn = 20 + // TTL - + TTL = 1 * time.Minute + maxRetryTimes = 3 + // DefaultCAFile is the default ca file for tls client + DefaultCAFile = "/home/sn/resource/redis-secret/ca.pem" + // DefaultCertFile is the default cert file for tls client + DefaultCertFile = "/home/sn/resource/redis-secret/cert.pem" + // DefaultKeyFile is the default key file for tls client + DefaultKeyFile = "/home/sn/resource/redis-secret/key.pem" + // redisStringFile is the temp file to store string type data of redis + redisStringFile = "/tmp/redis-string" + // redisStringFile is the temp file to store slice type data of redis + redisSliceFile = "/tmp/redis-slice" + redisSeparator = "%WITH%" + + // the detection is performed every 5 seconds. + healthCheckIntervalTime = 5 + // 2 * 60min * 60s / 5 second, trigger every 5 minutes + twoHoursCount = 2 * 60 * 60 / healthCheckIntervalTime + success = 0 + fail = 1 + redisValueIndex = 2 + redisReconnectionInternal = 10 * time.Second + // DefaultRedisContextTimeout - + DefaultRedisContextTimeout = time.Second + + redisRetryTimes = 3 + redisRetryInterval = 100 * time.Millisecond +) + +var ( + errMode = errors.New("serverMode is not single or cluster") + // RedisClient - + redisClient = &Client{ + client: &redis.Client{}, + option: redisClientOption{}, + connected: false, + RWMutex: sync.RWMutex{}, + } + // defaultTimeoutConf is the default timeout conf + defaultTimeoutConf = TimeoutConf{ + DialTimeout: defaultDialTimeout, + ReadTimeout: defaultReadTimeout, + WriteTimeout: defaultWriteTimeout, + IdleTimeout: defaultIdleTimeout, + } +) + +var ( + mu sync.RWMutex + redisCmd *Client +) + +// Option - +type Option func(*redisClientOption) + +type redisClientOption struct { + tlsConfig *tls.Config + serverAddr string + dialTimeout time.Duration + readTimeout time.Duration + writeTimeout time.Duration + idleTimeout time.Duration + password string + serverMode string + enableTLS bool + hotloadConfFunc func() (string, TimeoutConf, error) + enableAlarm bool +} + +// RedisOperation - +type RedisOperation struct { + Key string + Value string + Method string + TTL time.Duration +} + +// Client - +type Client struct { + client redis.Cmdable + option redisClientOption + connected bool + sync.RWMutex +} + +// Config is the config of redis client +type Config struct { + ClusterID string `json:"clusterID,omitempty" valid:",optional"` + ServerAddr string `json:"serverAddr,omitempty" valid:",optional"` + ServerMode string `json:"serverMode,omitempty" valid:",optional"` + Password string `json:"password,omitempty" valid:",optional"` + EnableTLS bool `json:"enableTLS,omitempty" valid:",optional"` + TimeoutConf TimeoutConf `json:"timeoutConf,omitempty" valid:",optional"` +} + +// TimeoutConf A variety of timeout configurations +type TimeoutConf struct { + DialTimeout int `json:"dialTimeout,omitempty" valid:",optional"` + ReadTimeout int `json:"readTimeout,omitempty" valid:",optional"` + WriteTimeout int `json:"writeTimeout,omitempty" valid:",optional"` + IdleTimeout int `json:"idleTimeout,omitempty" valid:",optional"` +} + +// NewRedisClientParam parameters of a new redis client +type NewRedisClientParam struct { + ServerMode string + ServerAddr string + Password string + Timeout TimeoutConf + EnableTLS bool `json:"enableTLS,omitempty" valid:",optional"` + HotloadConfFunc func() (string, TimeoutConf, error) +} + +// GetRedisCmd - +func GetRedisCmd() *Client { + mu.Lock() + client := redisCmd + mu.Unlock() + return client +} + +// SetRedisCmd - +func SetRedisCmd(client *Client) { + mu.Lock() + redisCmd = client + mu.Unlock() +} + +// SetEnableTLS - +func SetEnableTLS(enableTLS bool) Option { + return func(c *redisClientOption) { + c.enableTLS = enableTLS + } +} + +// ZCard - +func (c *Client) ZCard(ctx context.Context, key string) *redis.IntCmd { + c.RLock() + cli := c.client + c.RUnlock() + return cli.ZCard(ctx, key) +} + +// ZRange - +func (c *Client) ZRange(ctx context.Context, key string, start, stop int64) *redis.StringSliceCmd { + c.RLock() + cli := c.client + c.RUnlock() + return cli.ZRange(ctx, key, start, stop) +} + +// ZRem - +func (c *Client) ZRem(ctx context.Context, key string, members ...interface{}) *redis.IntCmd { + c.RLock() + cli := c.client + c.RUnlock() + return cli.ZRem(ctx, key, members...) +} + +// ZAdd - +func (c *Client) ZAdd(ctx context.Context, key string, members ...redis.Z) *redis.IntCmd { + c.RLock() + cli := c.client + c.RUnlock() + return cli.ZAdd(ctx, key, members...) +} + +// Ping - +func (c *Client) Ping(ctx context.Context) *redis.StatusCmd { + c.RLock() + cli := c.client + c.RUnlock() + return cli.Ping(ctx) +} + +// Expire - +func (c *Client) Expire(ctx context.Context, key string, expiration time.Duration) *redis.BoolCmd { + c.RLock() + cli := c.client + c.RUnlock() + return cli.Expire(ctx, key, expiration) +} + +// Get - +func (c *Client) Get(ctx context.Context, key string) *redis.StringCmd { + c.RLock() + cli := c.client + c.RUnlock() + return cli.Get(ctx, key) +} + +// Del - +func (c *Client) Del(ctx context.Context, keys ...string) *redis.IntCmd { + c.RLock() + cli := c.client + c.RUnlock() + return cli.Del(ctx, keys...) +} + +// ZADDMetricsToRedis - +func ZADDMetricsToRedis(key string, metrics interface{}, limit int64, expireTime time.Duration) error { + redisCmd := GetRedisCmd() + if redisCmd == nil { + log.GetLogger().Errorf("redis client is nil") + return errors.New("redis client is nil") + } + count, err := redisCmd.ZCard(context.TODO(), key).Result() + if err != nil { + log.GetLogger().Errorf("failed to ZCard metrics key %s from redis, err: %s", key, err.Error()) + return err + } + // if count reach limit, delete the earliest metric + if count >= limit { + earliestValues, err := redisCmd.ZRange(context.TODO(), key, 0, 0).Result() + if err != nil { + log.GetLogger().Errorf("failed to ZRange metrics key %s from redis, err: %s", key, err.Error()) + return err + } + _, err = redisCmd.ZRem(context.TODO(), key, earliestValues[0]).Result() + if err != nil { + log.GetLogger().Errorf("failed to ZRem metrics key %s to redis, err: %s", key, err.Error()) + return err + } + } + // Add a new value to the sorted set, with the score being the current timestamp + score := time.Now().Unix() + err = redisCmd.ZAdd(context.TODO(), key, redis.Z{Score: float64(score), Member: metrics}).Err() + if err != nil { + log.GetLogger().Errorf("failed to ZAdd metrics key %s to redis, err: %s", key, err.Error()) + return err + } + redisCmd.Expire(context.TODO(), key, expireTime) + return nil +} + +// New create a redis client +func New(newClientParam NewRedisClientParam, stopCh <-chan struct{}, options ...Option) (*Client, error) { + o := getNewRedisOption(newClientParam) + for _, option := range options { + option(&o) + } + + var redisCMD redis.Cmdable + switch newClientParam.ServerMode { + case "single": + redisCMD = newSingleClient(o) + case "cluster": + redisCMD = newClusterClient(o) + default: + utils.ClearStringMemory(o.password) + return nil, errMode + } + + if redisCMD == nil { + return nil, errors.New("failed to new redis cmd") + } + finished := make(chan int) + go connectRedis(redisCMD, finished, o) + select { + case i, ok := <-finished: + if ok && i == fail { + return nil, errors.New("failed to connect redis server") + } + case <-time.After(o.dialTimeout): + log.GetLogger().Errorf("dialing redis server error with incorrect ip address:%s.", o.serverAddr) + return nil, errors.New("dialing redis server timeout") + } + redisClient = &Client{ + client: redisCMD, + option: o, + connected: true, + RWMutex: sync.RWMutex{}, + } + return redisClient, nil +} + +func getNewRedisOption(param NewRedisClientParam) redisClientOption { + o := redisClientOption{ + serverAddr: param.ServerAddr, + password: param.Password, + serverMode: param.ServerMode, + } + if param.Timeout.DialTimeout > 0 { + o.dialTimeout = time.Duration(param.Timeout.DialTimeout) * time.Second + log.GetLogger().Infof("new dialTimeout: %d", param.Timeout.DialTimeout) + } else { + o.dialTimeout = dialTimeout + } + if param.Timeout.ReadTimeout > 0 { + o.readTimeout = time.Duration(param.Timeout.ReadTimeout) * time.Second + log.GetLogger().Infof("new readTimeout: %d", param.Timeout.ReadTimeout) + } else { + o.readTimeout = readTimeout + } + if param.Timeout.WriteTimeout > 0 { + o.writeTimeout = time.Duration(param.Timeout.WriteTimeout) * time.Second + log.GetLogger().Infof("new writeTimeout: %d", param.Timeout.WriteTimeout) + } else { + o.writeTimeout = writeTimeout + } + if param.Timeout.IdleTimeout > 0 { + o.idleTimeout = time.Duration(param.Timeout.IdleTimeout) * time.Second + log.GetLogger().Infof("new idleTimeout: %d", param.Timeout.IdleTimeout) + } else { + o.idleTimeout = idleTimeout + } + return o +} + +func newSingleClient(o redisClientOption) redis.Cmdable { + options := &redis.Options{ + PoolSize: defaultRedisConn, + Addr: o.serverAddr, + Password: o.password, + DialTimeout: o.dialTimeout, + ReadTimeout: o.readTimeout, + WriteTimeout: o.writeTimeout, + ConnMaxIdleTime: o.idleTimeout, + MaxRetries: maxRetryTimes, + } + if o.enableTLS { + tlsConfig, err := buildCfg(DefaultCAFile, DefaultCertFile, DefaultKeyFile) + if err != nil { + utils.ClearStringMemory(options.Password) + log.GetLogger().Errorf("failed to build single client tls config: %s", err.Error()) + return nil + } + options.TLSConfig = tlsConfig + } + return redis.NewClient(options) +} + +func connectRedis(redisCmd redis.Cmdable, finished chan<- int, o redisClientOption) { + if finished == nil { + return + } + var err error + for i := 0; i < maxRetryTimes; i++ { + if redisCmd == nil { + log.GetLogger().Errorf("redis is not ready") + continue + } + _, err = redisCmd.Ping(context.Background()).Result() + if err == nil { + finished <- success + return + } + } + // The key relies on go's GC for memory cleanup + log.GetLogger().Errorf("dialing redis server error: %s", err.Error()) + finished <- fail + return +} + +func newClusterClient(o redisClientOption) redis.Cmdable { + options := &redis.ClusterOptions{ + PoolSize: defaultRedisConn, + Addrs: strings.Split(o.serverAddr, ","), + Password: o.password, + DialTimeout: o.dialTimeout, + ReadTimeout: o.readTimeout, + WriteTimeout: o.writeTimeout, + ConnMaxIdleTime: o.idleTimeout, + MaxRetries: maxRetryTimes, + } + if o.enableTLS { + tlsConfig, err := buildCfg(DefaultCAFile, DefaultCertFile, DefaultKeyFile) + if err != nil { + utils.ClearStringMemory(options.Password) + log.GetLogger().Errorf("failed to build redis ClusterClient tls config: %s", err.Error()) + return nil + } + options.TLSConfig = tlsConfig + } + return redis.NewClusterClient(options) +} + +func buildCfg(caFile string, certFile string, keyFile string) (*tls.Config, error) { + var pools *x509.CertPool + var err error + pools, err = commonTLS.GetX509CACertPool(caFile) + if err != nil { + log.GetLogger().Errorf("failed to get X509 CACert Pool: %s", err.Error()) + return nil, err + } + + var certs []tls.Certificate + if certs, err = commonTLS.LoadServerTLSCertificate(certFile, keyFile, "", "LOCAL", false); err != nil { + log.GetLogger().Errorf("failed to load Server TLS Certificate: %s", err.Error()) + return nil, err + } + + clientAuth := tls.NoClientCert + tlsConfig := &tls.Config{ + RootCAs: pools, + Certificates: certs, + ClientAuth: clientAuth, + } + return tlsConfig, nil +} + +// CheckRedisConnectivity - +func CheckRedisConnectivity(clientRedisConfig *NewRedisClientParam, client *Client, stopCh <-chan struct{}) { + if stopCh == nil { + log.GetLogger().Errorf("stopCh is nil") + return + } + ticker := time.NewTicker(redisReconnectionInternal) + for { + select { + case <-ticker.C: + if err := checkAndReconnectRedis(clientRedisConfig, client, stopCh); err != nil { + log.GetLogger().Errorf("failed to check or reconnect redis client, err:%s", err.Error()) + } + case <-stopCh: + log.GetLogger().Errorf("module process exit") + ticker.Stop() + return + } + } +} + +func checkAndReconnectRedis(clientRedisConfig *NewRedisClientParam, client *Client, stopCh <-chan struct{}) error { + log.GetLogger().Debug("redis check redis connection start") + if client != nil { + _, err := (*client).Ping(context.TODO()).Result() + if err == nil { + log.GetLogger().Debug("redis periodically checks availability") + return nil + } + } + newClient, err := initClient(clientRedisConfig, stopCh) + if err != nil { + return err + } + if client != nil { + client = newClient + } + SetRedisCmd(newClient) + return nil +} + +func initClient(clientRedisConfig *NewRedisClientParam, stopCh <-chan struct{}) (*Client, error) { + c, err := New(NewRedisClientParam{ + ServerMode: clientRedisConfig.ServerMode, + ServerAddr: clientRedisConfig.ServerAddr, + Password: clientRedisConfig.Password, + Timeout: clientRedisConfig.Timeout, + }, stopCh, SetEnableTLS(clientRedisConfig.EnableTLS), + SetGetRealTimeServerAddrFunc(clientRedisConfig.HotloadConfFunc)) + if err != nil { + log.GetLogger().Errorf("failed to new a redis Client, %s", err.Error()) + return nil, err + } + return c, nil +} + +// SetGetRealTimeServerAddrFunc hot update server address when disconnected +func SetGetRealTimeServerAddrFunc(getServerAddr func() (string, TimeoutConf, error)) Option { + return func(c *redisClientOption) { + c.hotloadConfFunc = getServerAddr + } +} diff --git a/frontend/pkg/common/faas_common/redisclient/redisclient_test.go b/frontend/pkg/common/faas_common/redisclient/redisclient_test.go new file mode 100644 index 0000000000000000000000000000000000000000..c7a20402b5a185ac38f48cf77031206211f757e8 --- /dev/null +++ b/frontend/pkg/common/faas_common/redisclient/redisclient_test.go @@ -0,0 +1,457 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package redisclient + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "reflect" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/redis/go-redis/v9" + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + + commonTLS "frontend/pkg/common/faas_common/tls" + "frontend/pkg/common/faas_common/utils" +) + +func TestZADDMetricsToRedis(t *testing.T) { + err := ZADDMetricsToRedis("mockKey", 1, 3, 5*time.Second) + assert.NotNil(t, err) + convey.Convey("TestZADDMetricsToRedis", t, func() { + convey.Convey("ZCard exception", func() { + redisCmd = &Client{client: &redis.Client{}} + patches := [...]*gomonkey.Patches{ + gomonkey.ApplyMethod(reflect.TypeOf(&redis.Client{}), "ZCard", + func(cli *redis.Client, ctx context.Context, key string) *redis.IntCmd { + return &redis.IntCmd{} + }), + gomonkey.ApplyMethod(reflect.TypeOf(&redis.IntCmd{}), "Result", + func(_ *redis.IntCmd) (int64, error) { + return 0, errors.New("mock ZCard error") + }), + } + defer func() { + for idx := range patches { + patches[idx].Reset() + } + }() + err = ZADDMetricsToRedis("mockKey", 1, 3, 5*time.Second) + assert.NotNil(t, err) + assert.Equal(t, "mock ZCard error", err.Error()) + }) + + convey.Convey("ZRange exception", func() { + redisCmd = &Client{client: &redis.Client{}} + patches := [...]*gomonkey.Patches{ + gomonkey.ApplyMethod(reflect.TypeOf(&redis.Client{}), "ZCard", + func(cli *redis.Client, ctx context.Context, key string) *redis.IntCmd { + return &redis.IntCmd{} + }), + gomonkey.ApplyMethod(reflect.TypeOf(&redis.IntCmd{}), "Result", + func(_ *redis.IntCmd) (int64, error) { + return 3, nil + }), + gomonkey.ApplyMethod(reflect.TypeOf(&redis.Client{}), "ZRange", + func(cli *redis.Client, ctx context.Context, key string, start, stop int64) *redis.StringSliceCmd { + return &redis.StringSliceCmd{} + }), + gomonkey.ApplyMethod(reflect.TypeOf(&redis.StringSliceCmd{}), "Result", + func(_ *redis.StringSliceCmd) ([]string, error) { + return nil, errors.New("mock ZRange error") + }), + } + defer func() { + for idx := range patches { + patches[idx].Reset() + } + }() + err = ZADDMetricsToRedis("mockKey", 1, 3, 5*time.Second) + assert.NotNil(t, err) + assert.Equal(t, "mock ZRange error", err.Error()) + }) + + convey.Convey("ZAdd success", func() { + redisCmd = &Client{client: &redis.Client{}} + patches := [...]*gomonkey.Patches{ + gomonkey.ApplyMethod(reflect.TypeOf(&redis.Client{}), "ZCard", + func(cli *redis.Client, ctx context.Context, key string) *redis.IntCmd { + return &redis.IntCmd{} + }), + gomonkey.ApplyMethod(reflect.TypeOf(&redis.IntCmd{}), "Result", + func(_ *redis.IntCmd) (int64, error) { + return 3, nil + }), + gomonkey.ApplyMethod(reflect.TypeOf(&redis.Client{}), "ZRange", + func(cli *redis.Client, ctx context.Context, key string, start, stop int64) *redis.StringSliceCmd { + return &redis.StringSliceCmd{} + }), + gomonkey.ApplyMethod(reflect.TypeOf(&redis.StringSliceCmd{}), "Result", + func(_ *redis.StringSliceCmd) ([]string, error) { + return []string{"0", "1", "2"}, nil + }), + gomonkey.ApplyMethod(reflect.TypeOf(&redis.Client{}), "ZRem", + func(cli *redis.Client, ctx context.Context, key string, members ...interface{}) *redis.IntCmd { + return &redis.IntCmd{} + }), + gomonkey.ApplyMethod(reflect.TypeOf(&redis.Client{}), "ZAdd", + func(cli *redis.Client, ctx context.Context, key string, members ...redis.Z) *redis.IntCmd { + return &redis.IntCmd{} + }), + gomonkey.ApplyMethod(reflect.TypeOf(&redis.Client{}), "Expire", + func(cli *redis.Client, ctx context.Context, key string, expiration time.Duration) *redis.BoolCmd { + return &redis.BoolCmd{} + }), + } + defer func() { + for idx := range patches { + patches[idx].Reset() + } + }() + err = ZADDMetricsToRedis("mockKey", 1, 3, 5*time.Second) + assert.Nil(t, err) + }) + }) +} + +func TestNew(t *testing.T) { + type args struct { + serverMode string + serverAddr string + password string + options []Option + } + var a args + var b args + option := SetEnableTLS(false) + b.serverMode = "single" + b.options = append(b.options, option) + var c args + c.serverMode = "cluster" + c.options = append(b.options, option) + c.options = append(b.options, SetGetRealTimeServerAddrFunc(func() (string, TimeoutConf, error) { + return "", TimeoutConf{}, nil + })) + tests := []struct { + name string + args args + want redis.Cmdable + wantErr bool + }{ + {"case1", a, nil, true}, + {"case2", b, nil, true}, + {"case3", c, nil, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := New(NewRedisClientParam{ + tt.args.serverMode, + tt.args.serverAddr, + tt.args.password, + TimeoutConf{}, + false, + nil, + }, nil, tt.args.options...) + if (err != nil) != tt.wantErr { + t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != nil { + t.Errorf("New() got = %v, want %v", got, tt.want) + } + }) + } + + patches := utils.InitPatchSlice() + statusCMD := &redis.StatusCmd{} + patches.Append(utils.PatchSlice{gomonkey.ApplyFunc((*redis.Client).Ping, + func(_ *redis.Client, _ context.Context) *redis.StatusCmd { + return statusCMD + })}) + defer patches.ResetAll() + _, err := New(NewRedisClientParam{ + "single", + "", + "", + TimeoutConf{}, + false, + nil, + }, nil) + if err != nil { + t.Errorf("failed to test new client with alarm switch on: %s", err.Error()) + } +} + +func Test_buildCfg(t *testing.T) { + type args struct { + caFile string + certFile string + keyFile string + } + var a args + tests := []struct { + name string + args args + want *tls.Config + wantErr bool + }{ + {"case1", a, nil, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, _ := buildCfg(tt.args.caFile, tt.args.certFile, tt.args.keyFile) + assert.Equalf(t, tt.want, got, "buildCfg(%v, %v, %v)", tt.args.caFile, tt.args.certFile, tt.args.keyFile) + }) + } +} + +func TestEmptyClients(t *testing.T) { + opt := redisClientOption{enableTLS: true} + redisCMD := newSingleClient(opt) + assert.Equal(t, redisCMD, nil) + redisCMD = newClusterClient(opt) + assert.Equal(t, redisCMD, nil) +} + +func TestBuildCfg(t *testing.T) { + patches := utils.InitPatchSlice() + patches.Append(utils.PatchSlice{gomonkey.ApplyFunc(commonTLS.GetX509CACertPool, + func(caCertFilePath string) (caCertPool *x509.CertPool, err error) { + return nil, nil + })}) + defer patches.ResetAll() + convey.Convey("Test build cfg error", t, func() { + tlsConfig, err := buildCfg(DefaultCAFile, DefaultCertFile, DefaultKeyFile) + convey.So(tlsConfig, convey.ShouldBeNil) + convey.So(err, convey.ShouldNotBeNil) + }) +} + +func Test_getNewRedisOption(t *testing.T) { + type args struct { + param NewRedisClientParam + } + tests := []struct { + name string + args args + want redisClientOption + }{ + { + name: "case1", + args: args{ + param: NewRedisClientParam{ + "single", + "127.0.0.1", + "aaa", + TimeoutConf{}, + false, + nil, + }, + }, + want: redisClientOption{ + serverAddr: "127.0.0.1", + serverMode: "single", + password: "aaa", + dialTimeout: dialTimeout, + readTimeout: readTimeout, + writeTimeout: writeTimeout, + idleTimeout: idleTimeout, + }, + }, + { + name: "case1", + args: args{ + param: NewRedisClientParam{ + "single", + "127.0.0.1", + "aaa", + TimeoutConf{ + DialTimeout: 1, + ReadTimeout: 1, + WriteTimeout: 1, + IdleTimeout: 1, + }, + false, + nil, + }, + }, + want: redisClientOption{ + serverAddr: "127.0.0.1", + serverMode: "single", + password: "aaa", + dialTimeout: 1 * time.Second, + readTimeout: 1 * time.Second, + writeTimeout: 1 * time.Second, + idleTimeout: 1 * time.Second, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getNewRedisOption(tt.args.param); !reflect.DeepEqual(got, tt.want) { + t.Errorf("getNewRedisOption() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_initClient(t *testing.T) { + convey.Convey("Test redis Client is success", t, func() { + param := NewRedisClientParam{ + ServerMode: "122", + ServerAddr: "333", + Password: "1222", + Timeout: TimeoutConf{}, + EnableTLS: false, + HotloadConfFunc: nil, + } + defer gomonkey.ApplyFunc(New, func(newClientParam NewRedisClientParam, stopCh <-chan struct{}, options ...Option) (*Client, error) { + return &Client{}, nil + }).Reset() + stopCh := make(chan struct{}) + redisClient, _ := initClient(¶m, stopCh) + convey.So(redisClient, convey.ShouldNotBeNil) + }) + convey.Convey("Test to not init redis client", t, func() { + param := NewRedisClientParam{ + ServerMode: "122", + ServerAddr: "333", + Password: "1222", + Timeout: TimeoutConf{}, + EnableTLS: false, + HotloadConfFunc: nil, + } + defer gomonkey.ApplyFunc(New, func(newClientParam NewRedisClientParam, stopCh <-chan struct{}, options ...Option) (*Client, error) { + return &Client{}, errors.New("redis is not ready") + }).Reset() + stopCh := make(chan struct{}) + _, err := initClient(¶m, stopCh) + convey.So(err, convey.ShouldNotBeNil) + }) +} + +func TestCheckRedisConnectivity(t *testing.T) { + var isCalled = 0 + convey.Convey("Test check redis to connect in cyclist", t, func() { + param := NewRedisClientParam{ + ServerMode: "122", + ServerAddr: "333", + Password: "1222", + Timeout: TimeoutConf{}, + EnableTLS: false, + HotloadConfFunc: nil, + } + patch1 := gomonkey.ApplyFunc(New, func(newClientParam NewRedisClientParam, stopCh <-chan struct{}, options ...Option) (*Client, error) { + isCalled++ + return &Client{}, nil + }) + patch := gomonkey.ApplyFunc((*redis.Client).Ping, + func(_ *redis.Client, _ context.Context) *redis.StatusCmd { + return &redis.StatusCmd{} + }) + defer patch.Reset() + stopCh := make(chan struct{}, 0) + tickerCh := make(chan time.Time) + patch.ApplyFunc(time.NewTicker, func(_ time.Duration) *time.Ticker { + return &time.Ticker{C: tickerCh} + }) + + go CheckRedisConnectivity(¶m, nil, stopCh) + tickerCh <- time.Time{} + stopCh <- struct{}{} + convey.So(isCalled, convey.ShouldEqual, 1) + patch1.Reset() + patch.ApplyFunc(New, func(newClientParam NewRedisClientParam, stopCh <-chan struct{}, options ...Option) (*Client, error) { + isCalled++ + return &Client{}, errors.New("state is not ready") + }) + stopCh = make(chan struct{}, 0) + go CheckRedisConnectivity(¶m, nil, stopCh) + tickerCh <- time.Time{} + tickerCh <- time.Time{} + stopCh <- struct{}{} + convey.So(isCalled, convey.ShouldEqual, 3) + + CheckRedisConnectivity(¶m, nil, nil) + convey.So(isCalled, convey.ShouldEqual, 3) + }) +} + +func TestClient_Del(t *testing.T) { + client := &Client{ + client: &redis.Client{}, + } + ctx := context.Background() + keys := []string{"key1", "key2"} + + mockResult := &redis.IntCmd{} + patches := gomonkey.ApplyMethod( + reflect.TypeOf(client.client), "Del", + func(_ redis.Cmdable, _ context.Context, _ ...string) *redis.IntCmd { + return mockResult + }, + ) + defer patches.Reset() + + result := client.Del(ctx, keys...) + + assert.Equal(t, mockResult, result) +} + +func TestClient_Get(t *testing.T) { + client := &Client{ + client: &redis.Client{}, + } + ctx := context.Background() + key := "key2" + + mockResult := &redis.StringCmd{} + patches := gomonkey.ApplyMethod( + reflect.TypeOf(client.client), "Get", + func(_ redis.Cmdable, _ context.Context, _ string) *redis.StringCmd { + return mockResult + }, + ) + defer patches.Reset() + + result := client.Get(ctx, key) + + assert.Equal(t, mockResult, result) +} +func TestClient_Ping(t *testing.T) { + client := &Client{ + client: &redis.Client{}, + } + ctx := context.Background() + + mockResult := &redis.StatusCmd{} + patches := gomonkey.ApplyMethod( + reflect.TypeOf(client.client), "Ping", + func(_ redis.Cmdable, _ context.Context) *redis.StatusCmd { + return mockResult + }, + ) + defer patches.Reset() + + result := client.Ping(ctx) + + assert.Equal(t, mockResult, result) +} diff --git a/frontend/pkg/common/faas_common/resspeckey/type.go b/frontend/pkg/common/faas_common/resspeckey/type.go new file mode 100644 index 0000000000000000000000000000000000000000..be9cdcdcd0ed8c9d6daf81db6a487e2afef23e6c --- /dev/null +++ b/frontend/pkg/common/faas_common/resspeckey/type.go @@ -0,0 +1,120 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package resspeckey - +package resspeckey + +import ( + "encoding/json" + "fmt" + "sort" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/log" +) + +// ResourceSpecification contains resource specification of a requested instance +type ResourceSpecification struct { + CPU int64 `json:"cpu"` + Memory int64 `json:"memory"` + InvokeLabel string `json:"invokeLabels"` + CustomResources map[string]int64 `json:"customResources"` + CustomResourcesSpec map[string]interface{} `json:"customResourcesSpec"` + EphemeralStorage int `json:"ephemeral_storage"` +} + +// DeepCopy return a ResourceSpecification Copy +func (rs *ResourceSpecification) DeepCopy() *ResourceSpecification { + customResource := map[string]int64{} + for k, v := range rs.CustomResources { + customResource[k] = v + } + customResourcesSpec := map[string]interface{}{} + for k, v := range rs.CustomResourcesSpec { + customResourcesSpec[k] = v + } + return &ResourceSpecification{ + CPU: rs.CPU, + Memory: rs.Memory, + CustomResources: customResource, + InvokeLabel: rs.InvokeLabel, + CustomResourcesSpec: customResourcesSpec, + EphemeralStorage: rs.EphemeralStorage, + } +} + +// String returns ResourceSpecification as string +func (rs *ResourceSpecification) String() string { + resourceExpression := fmt.Sprintf("cpu-%d-mem-%d", rs.CPU, rs.Memory) + for key, value := range rs.CustomResources { + if value <= constant.MinCustomResourcesSize { + continue + } + resourceExpression += fmt.Sprintf("-%s-%d", key, value) + } + keys := make([]string, 0, len(rs.CustomResourcesSpec)) + for k := range rs.CustomResourcesSpec { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + v := rs.CustomResourcesSpec[k] + resourceExpression += fmt.Sprintf("-%s-%v", k, v) + } + if rs.InvokeLabel != "" { + resourceExpression += fmt.Sprintf("-invoke-label-%s", rs.InvokeLabel) + } + resourceExpression += fmt.Sprintf("-ephemeral-storage-%v", rs.EphemeralStorage) + return resourceExpression +} + +// ResSpecKey is a representation of ResourceSpecification which can be used as key of map +type ResSpecKey struct { + CPU int64 + Memory int64 + EphemeralStorage int + CustomResources string + CustomResourcesSpec string + InvokeLabel string +} + +// String returns ResSpecKey as string +func (rsk *ResSpecKey) String() string { + return fmt.Sprintf("cpu-%d-mem-%d-storage-%d-cstRes-%s-cstResSpec-%s-invokeLabel-%s", rsk.CPU, rsk.Memory, + rsk.EphemeralStorage, rsk.CustomResources, rsk.CustomResourcesSpec, rsk.InvokeLabel) +} + +// ToResSpec convert ResSpecKey to ResourceSpecification +func (rsk *ResSpecKey) ToResSpec() *ResourceSpecification { + cstRes := map[string]int64{} + err := json.Unmarshal([]byte(rsk.CustomResources), &cstRes) + if err != nil { + log.GetLogger().Errorf("failed to unmarshal to customResources error %s", err.Error()) + } + cstResSpec := map[string]interface{}{} + err = json.Unmarshal([]byte(rsk.CustomResourcesSpec), &cstResSpec) + if err != nil { + log.GetLogger().Errorf("failed to unmarshal to customResourceSpec error %s", err.Error()) + } + return &ResourceSpecification{ + CPU: rsk.CPU, + Memory: rsk.Memory, + EphemeralStorage: rsk.EphemeralStorage, + CustomResources: cstRes, + CustomResourcesSpec: cstResSpec, + InvokeLabel: rsk.InvokeLabel, + } +} diff --git a/frontend/pkg/common/faas_common/resspeckey/util.go b/frontend/pkg/common/faas_common/resspeckey/util.go new file mode 100644 index 0000000000000000000000000000000000000000..8a010104d1b42aab9165054d02108766d02ccdfb --- /dev/null +++ b/frontend/pkg/common/faas_common/resspeckey/util.go @@ -0,0 +1,103 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package resspeckey - +package resspeckey + +import ( + "encoding/json" + + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/types" +) + +const ( + ascendResourceD910B = "huawei.com/ascend-1980" + ascendResourceD910BInstanceType = "instanceType" +) + +// ConvertToResSpecKey converts ResourceSpecification to ResSpecKey +func ConvertToResSpecKey(resSpec *ResourceSpecification) ResSpecKey { + // for Go 1.7+ version, json.Marshal sorts the keys of map, same kv pairs will get same serialization result + var ( + cstResExp string + cstResSpecExp string + ) + if resSpec.CustomResources != nil && len(resSpec.CustomResources) != 0 { + cstResBytes, err := json.Marshal(resSpec.CustomResources) + if err != nil { + log.GetLogger().Errorf("failed to marshal customResources %#v error %s", resSpec.CustomResources, err.Error()) + } + cstResExp = string(cstResBytes) + } + if len(cstResExp) != 0 && resSpec.CustomResourcesSpec != nil && len(resSpec.CustomResourcesSpec) != 0 { + cstResSpecBytes, err := json.Marshal(resSpec.CustomResourcesSpec) + if err != nil { + log.GetLogger().Errorf("failed to marshal customResourcesSpec %#v error %s", resSpec.CustomResourcesSpec, + err.Error()) + } + cstResSpecExp = string(cstResSpecBytes) + } + return ResSpecKey{ + CPU: resSpec.CPU, + Memory: resSpec.Memory, + EphemeralStorage: resSpec.EphemeralStorage, + CustomResources: cstResExp, + CustomResourcesSpec: cstResSpecExp, + InvokeLabel: resSpec.InvokeLabel, + } +} + +// GetResKeyFromStr - +func GetResKeyFromStr(note string) (ResSpecKey, error) { + resSpec := &ResourceSpecification{} + err := json.Unmarshal([]byte(note), resSpec) + if err != nil { + return ResSpecKey{}, err + } + return ConvertToResSpecKey(resSpec), nil +} + +// ConvertResourceMetaDataToResSpec will convert resource metadata +func ConvertResourceMetaDataToResSpec(resMeta types.ResourceMetaData) *ResourceSpecification { + customResources := map[string]int64{} + if resMeta.CustomResources != "" { + if err := json.Unmarshal([]byte(resMeta.CustomResources), &customResources); err != nil { + log.GetLogger().Warnf("failed to unmarshal custom resources %s, err: %s", + resMeta.CustomResources, err.Error()) + } + } + customResourcesSpec := make(map[string]interface{}) + // npu tag may be unspecified and be updated to 376T, default value is needed to be set, otherwise reserved instance + // will be recreated + err := json.Unmarshal([]byte(resMeta.CustomResourcesSpec), &customResourcesSpec) + if resMeta.CustomResourcesSpec != "" && err != nil { + log.GetLogger().Warnf("failed to unmarshal custom resourcesSpec: %s, err: %s", + resMeta.CustomResourcesSpec, err.Error()) + } + if _, ok := customResources[ascendResourceD910B]; ok { + if _, ok := customResourcesSpec[ascendResourceD910BInstanceType]; !ok { + customResourcesSpec[ascendResourceD910BInstanceType] = "376T" + } + } + return &ResourceSpecification{ + CPU: resMeta.CPU, + Memory: resMeta.Memory, + CustomResources: customResources, + CustomResourcesSpec: customResourcesSpec, + EphemeralStorage: resMeta.EphemeralStorage, + } +} diff --git a/frontend/pkg/common/faas_common/resspeckey/util_test.go b/frontend/pkg/common/faas_common/resspeckey/util_test.go new file mode 100644 index 0000000000000000000000000000000000000000..d986c48a1c2126cc9ee7945b1bdda51f626c487a --- /dev/null +++ b/frontend/pkg/common/faas_common/resspeckey/util_test.go @@ -0,0 +1,52 @@ +package resspeckey + +import ( + "encoding/json" + "testing" + + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + + "frontend/pkg/common/faas_common/types" +) + +func TestResSpecKey(t *testing.T) { + resSpec := &ResourceSpecification{ + CPU: 100, + Memory: 100, + CustomResources: map[string]int64{"NPU": 1}, + CustomResourcesSpec: map[string]interface{}{"Type": "type1"}, + InvokeLabel: "label1", + } + resKey := ConvertToResSpecKey(resSpec) + resKeyString := resKey.String() + assert.Equal(t, "cpu-100-mem-100-storage-0-cstRes-{\"NPU\":1}-cstResSpec-{\"Type\":\"type1\"}-invokeLabel-label1", resKeyString) + resSpec1 := resKey.ToResSpec() + assert.Equal(t, int64(100), resSpec1.CPU) + assert.Equal(t, int64(100), resSpec1.Memory) + assert.Equal(t, "label1", resSpec1.InvokeLabel) +} + +func TestConvertResourceMetaData(t *testing.T) { + convey.Convey("test ConvertResourceMetaData", t, func() { + convey.Convey("Unmarshal error", func() { + resMeta := types.ResourceMetaData{ + CustomResourcesSpec: "huawei.com/ascend-1980:D910B", + CustomResources: "", + } + resource := ConvertResourceMetaDataToResSpec(resMeta) + convey.So(len(resource.CustomResources), convey.ShouldEqual, 0) + }) + convey.Convey("Convert success", func() { + customResources := map[string]int64{"huawei.com/ascend-1980": 10} + data, _ := json.Marshal(customResources) + resMeta := types.ResourceMetaData{ + CustomResourcesSpec: "CustomResourcesSpec", + CustomResources: string(data), + } + resource := ConvertResourceMetaDataToResSpec(resMeta) + convey.So(resource.CustomResourcesSpec[ascendResourceD910BInstanceType], + convey.ShouldEqual, "376T") + }) + }) +} diff --git a/frontend/pkg/common/faas_common/signals/signal.go b/frontend/pkg/common/faas_common/signals/signal.go new file mode 100644 index 0000000000000000000000000000000000000000..f526b8f49a7fd423495e919be84d8177a2d3564f --- /dev/null +++ b/frontend/pkg/common/faas_common/signals/signal.go @@ -0,0 +1,55 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package signals - +package signals + +import ( + "os" + "os/signal" + "syscall" + + "frontend/pkg/common/faas_common/logger/log" +) + +var ( + shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM, syscall.SIGKILL} + onlyOneSignalHandler = make(chan struct{}) + shutdownHandler chan os.Signal + stopCh = make(chan struct{}) +) + +const channelCount = 2 + +func init() { + // 2 is the length of shutdown Handler channel + shutdownHandler = make(chan os.Signal, channelCount) + + signal.Notify(shutdownHandler, shutdownSignals...) + + go func() { + <-shutdownHandler + close(stopCh) + <-shutdownHandler + log.GetLogger().Sync() + os.Exit(1) + }() +} + +// WaitForSignal defines signal handler process. +func WaitForSignal() <-chan struct{} { + return stopCh +} diff --git a/frontend/pkg/common/faas_common/signals/signal_test.go b/frontend/pkg/common/faas_common/signals/signal_test.go new file mode 100644 index 0000000000000000000000000000000000000000..7314a2a6b9af61fe189e1b957e381de34386f478 --- /dev/null +++ b/frontend/pkg/common/faas_common/signals/signal_test.go @@ -0,0 +1,43 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package signals + +import ( + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestWaitForSignal(t *testing.T) { + stopCh := WaitForSignal() + + go func() { + time.Sleep(200 * time.Millisecond) + shutdownHandler <- syscall.SIGTERM + }() + select { + case <-stopCh: + t.Log("received termination signal") + case <-time.After(time.Second): + t.Fatal("failed to signal in 1s") + } + + _, ok := <-stopCh + assert.Equal(t, ok, false) +} diff --git a/frontend/pkg/common/faas_common/singleflight/singleflight.go b/frontend/pkg/common/faas_common/singleflight/singleflight.go new file mode 100644 index 0000000000000000000000000000000000000000..4baea53665e79f615e5ffc8e56fac2ab544fad6b --- /dev/null +++ b/frontend/pkg/common/faas_common/singleflight/singleflight.go @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2024 Huawei Technologies Co., Ltd + * + * This software is licensed under muxlan PSL v2. + * You can use this software according to the terms and conditions of the muxlan PSL v2. + * You may obtain a copy of muxlan PSL v2 at: + * + * http://license.coscl.org.cn/muxlanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the muxlan PSL v2 for more details. + */ + +// Package singleflight database query control to prevent cache breakdown +package singleflight + +import ( + "sync" +) + +type flightItem struct { + mux sync.Mutex + val interface{} + err error +} + +// SFCache - +type SFCache struct { + m map[string]*flightItem + mux sync.Mutex +} + +// NewSingleFlight - +func NewSingleFlight() *SFCache { + return &SFCache{ + m: make(map[string]*flightItem), + } +} + +// Do - +func (sf *SFCache) Do(key string, f func() (interface{}, error)) (interface{}, error) { + sf.mux.Lock() + if item, ok := sf.m[key]; ok { + sf.mux.Unlock() + item.mux.Lock() + val, err := item.val, item.err + item.mux.Unlock() + return val, err + } + item := new(flightItem) + sf.m[key] = item + item.mux.Lock() + sf.mux.Unlock() + val, err := f() + item.val, item.err = val, err + item.mux.Unlock() + return val, err +} + +// Remove - +func (sf *SFCache) Remove(key string) { + sf.mux.Lock() + delete(sf.m, key) + sf.mux.Unlock() +} diff --git a/frontend/pkg/common/faas_common/singleflight/singleflight_test.go b/frontend/pkg/common/faas_common/singleflight/singleflight_test.go new file mode 100644 index 0000000000000000000000000000000000000000..4bf57c6c570c8ceb3e09da9a2b87cd6771909cd6 --- /dev/null +++ b/frontend/pkg/common/faas_common/singleflight/singleflight_test.go @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2024 Huawei Technologies Co., Ltd + * + * This software is licensed under muxlan PSL v2. + * You can use this software according to the terms and conditions of the muxlan PSL v2. + * You may obtain a copy of muxlan PSL v2 at: + * + * http://license.coscl.org.cn/muxlanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the muxlan PSL v2 for more details. + */ + +// Package singleflight database query control to prevent cache breakdown +package singleflight + +import ( + "sync" + "testing" + + "github.com/smartystreets/goconvey/convey" +) + +// TestSingleFlight_Do - +func TestSingleFlight_Do(t *testing.T) { + // simulate 10 concurrent requests and query the key 'test' from the database, + // it is expected that the database will be accessed only once + convey.Convey("test: single flight do", t, func() { + count := 0 + sf := NewSingleFlight() + concurrentNum := 10 + var wg sync.WaitGroup + wg.Add(concurrentNum) + for i := 0; i < concurrentNum; i++ { + go func() { + defer wg.Done() + sf.Do("test", func() (interface{}, error) { + count++ + return nil, nil + }) + }() + } + wg.Wait() + convey.So(count, convey.ShouldEqual, 1) + }) +} + +// TestSingleFlight - +func TestNewSingleFlight_Remove(t *testing.T) { + convey.Convey("test: single flight remove", t, func() { + count := 0 + concurrentNum := 10 + sf := NewSingleFlight() + key := "test" + concurrentTestFunc := func() { + var wg sync.WaitGroup + wg.Add(concurrentNum) + for i := 0; i < concurrentNum; i++ { + go func() { + defer wg.Done() + sf.Do(key, func() (interface{}, error) { + count++ + return nil, nil + }) + }() + } + wg.Wait() + } + concurrentTestFunc() + convey.So(count, convey.ShouldEqual, 1) + sf.Remove(key) + concurrentTestFunc() + convey.So(count, convey.ShouldEqual, 2) + }) +} diff --git a/frontend/pkg/common/faas_common/snerror/snerror.go b/frontend/pkg/common/faas_common/snerror/snerror.go new file mode 100644 index 0000000000000000000000000000000000000000..1410d12b87a23bbaa78b2809a97ac2f43a2af656 --- /dev/null +++ b/frontend/pkg/common/faas_common/snerror/snerror.go @@ -0,0 +1,86 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package snerror is basic information contained in the SN error. +package snerror + +const ( + // UserErrorMax is maximum value of user error + UserErrorMax = 4999 + // UserErrorMin is minimal value of user error + UserErrorMin = 4000 + // ErrorSeparator split error codes and error information. + ErrorSeparator = "|" +) + +// BadResponse HTTP request message that does not return 200 +type BadResponse struct { + Code int `json:"code"` + Message string `json:"message"` +} + +// SNError defines the action contained in the SN error information. +type SNError interface { + // Code Returned error code + Code() int + + Error() string +} + +type snError struct { + code int + message string +} + +// New returns an error. +// message is a complete English sentence with punctuation. +func New(code int, message string) SNError { + return &snError{ + code: code, + message: message, + } +} + +// NewWithError err not nil. +func NewWithError(code int, err error) SNError { + var message = "" + if err != nil { + message = err.Error() + } + return &snError{ + code: code, + message: message, + } +} + +// Code Returned error code +func (s *snError) Code() int { + return s.code +} + +// Error Implement the native error interface. +func (s *snError) Error() string { + return s.message +} + +// IsUserError true if a user error occurs +func IsUserError(s SNError) bool { + // The user error is a four-digit integer. + if UserErrorMin <= s.Code() && s.Code() <= UserErrorMax { + return true + } + return false +} diff --git a/frontend/pkg/common/faas_common/snerror/snerror_test.go b/frontend/pkg/common/faas_common/snerror/snerror_test.go new file mode 100644 index 0000000000000000000000000000000000000000..514edd71c91e6546694d07f3a6f6c58b3bdcfcf8 --- /dev/null +++ b/frontend/pkg/common/faas_common/snerror/snerror_test.go @@ -0,0 +1,40 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package snerror - +package snerror + +import ( + "fmt" + "testing" + + "github.com/smartystreets/goconvey/convey" +) + +func TestSnError(t *testing.T) { + convey.Convey("New", t, func() { + snErr := New(1000, "test error") + convey.So(snErr.Code(), convey.ShouldEqual, 1000) + convey.So(snErr.Error(), convey.ShouldEqual, "test error") + res := IsUserError(snErr) + convey.So(res, convey.ShouldEqual, false) + }) + convey.Convey("NewWithError", t, func() { + snErr := NewWithError(1000, fmt.Errorf("test error")) + convey.So(snErr.Code(), convey.ShouldEqual, 1000) + convey.So(snErr.Error(), convey.ShouldEqual, "test error") + }) +} diff --git a/frontend/pkg/common/faas_common/state/observer.go b/frontend/pkg/common/faas_common/state/observer.go new file mode 100644 index 0000000000000000000000000000000000000000..3ec3f17acae010a257e2dd62f8de163ed8df4c3f --- /dev/null +++ b/frontend/pkg/common/faas_common/state/observer.go @@ -0,0 +1,109 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package state - +package state + +import ( + "context" + "fmt" + + "go.etcd.io/etcd/client/v3" + + "frontend/pkg/common/faas_common/etcd3" +) + +// Observer - +type Observer interface { + Update(value interface{}, tags ...string) // add & update state to datasystem +} + +// Queue is used to cache the state processing queue +type Queue struct { + client *etcd3.EtcdClient + queue chan stateData +} + +// stateData is a state input parameter structure +type stateData struct { + data interface{} + tags []string +} + +const ( + maxQueueSize = 10000 + defaultQueueSize = 1000 +) + +// NewStateQueue - +func NewStateQueue(size int) *Queue { + if size > maxQueueSize || size <= 0 { + size = defaultQueueSize + } + client := etcd3.GetRouterEtcdClient() + if client == nil { + return nil + } + return &Queue{ + queue: make(chan stateData, size), + client: client, + } +} + +// SaveState - +func (q *Queue) SaveState(state []byte, key string) error { + ctx := etcd3.CreateEtcdCtxInfoWithTimeout(context.Background(), etcd3.DurationContextTimeout) + return q.client.Put(ctx, key, string(state)) +} + +// GetState - get state from etcd with key +func (q *Queue) GetState(key string) ([]byte, error) { + ctx := etcd3.CreateEtcdCtxInfoWithTimeout(context.Background(), etcd3.DurationContextTimeout) + response, err := q.client.GetResponse(ctx, key, clientv3.WithSerializable()) + if err != nil { + return nil, err + } + if len(response.Kvs) == 0 { + return nil, fmt.Errorf("get empty state from etcd") + } + return response.Kvs[0].Value, nil +} + +// DeleteState - +func (q *Queue) DeleteState(key string) error { + ctx := etcd3.CreateEtcdCtxInfoWithTimeout(context.Background(), etcd3.DurationContextTimeout) + return q.client.Delete(ctx, key, clientv3.WithPrefix()) +} + +// Push - +func (q *Queue) Push(value interface{}, tags ...string) error { + select { + case q.queue <- stateData{ + data: value, + tags: tags, + }: + return nil + default: + return fmt.Errorf("state queue is full, can not write data") + } +} + +// Run - +func (q *Queue) Run(handler func(value interface{}, tags ...string)) { + for state := range q.queue { + handler(state.data, state.tags...) + } +} diff --git a/frontend/pkg/common/faas_common/state/observer_test.go b/frontend/pkg/common/faas_common/state/observer_test.go new file mode 100644 index 0000000000000000000000000000000000000000..81682dffbcf8b1113af22e0de00dd98cd1514d20 --- /dev/null +++ b/frontend/pkg/common/faas_common/state/observer_test.go @@ -0,0 +1,72 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package state - +package state + +import ( + "testing" + + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" + + "frontend/pkg/common/faas_common/etcd3" + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" +) + +func TestNewStateQueue(t *testing.T) { + defer gomonkey.ApplyFunc(etcd3.GetRouterEtcdClient, func() *etcd3.EtcdClient { + return &etcd3.EtcdClient{} + }).Reset() + convey.Convey("get queue", t, func() { + q := NewStateQueue(10) + q.queue <- stateData{} + convey.So(len(q.queue), convey.ShouldEqual, 1) + }) + convey.Convey("get queue", t, func() { + q := NewStateQueue(-1) + q.queue <- stateData{} + convey.So(len(q.queue), convey.ShouldEqual, 1) + }) +} + +func TestStateOperation(t *testing.T) { + defer gomonkey.ApplyFunc(etcd3.GetRouterEtcdClient, func() *etcd3.EtcdClient { + return &etcd3.EtcdClient{} + }).Reset() + q := NewStateQueue(10) + convey.Convey("save state", t, func() { + defer gomonkey.ApplyFunc((*etcd3.EtcdClient).Put, func(_ *etcd3.EtcdClient, ctxInfo etcd3.EtcdCtxInfo, + etcdKey string, value string, opts ...clientv3.OpOption) error { + return nil + }).Reset() + err := q.SaveState(nil, "testKey") + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("get state", t, func() { + defer gomonkey.ApplyFunc((*etcd3.EtcdClient).GetResponse, func(_ *etcd3.EtcdClient, ctxInfo etcd3.EtcdCtxInfo, + etcdKey string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + return &clientv3.GetResponse{Kvs: []*mvccpb.KeyValue{{Key: nil, Value: nil}}}, nil + }).Reset() + _, err := q.GetState("testKey") + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("get state", t, func() { + err := q.Push("someData", "someKey") + convey.So(err, convey.ShouldBeNil) + }) +} diff --git a/frontend/pkg/common/faas_common/statuscode/statuscode.go b/frontend/pkg/common/faas_common/statuscode/statuscode.go new file mode 100644 index 0000000000000000000000000000000000000000..7db7b41f586a16b79a66b08e1531ac4543b04893 --- /dev/null +++ b/frontend/pkg/common/faas_common/statuscode/statuscode.go @@ -0,0 +1,490 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package statuscode define status code of Frontend +package statuscode + +import ( + "errors" + "net/http" + "regexp" + "strconv" + "strings" + + "github.com/valyala/fasthttp" +) + +// system error code +const ( + // InnerResponseSuccessCode - + InnerResponseSuccessCode = 0 + // InternalErrorCode if the value is 331404, try again. + InternalErrorCode = 330404 + InternalRetryErrCode = 331404 + InternalErrorMessage = "internal system error" + + // InnerInstanceCircuitCode need retry + InnerInstanceCircuitCode = 4011 + + // BackpressureCode indicate that frontend should choose another proxy/worker and retry + BackpressureCode = 211429 +) + +// frontend error code +const ( + // FrontendStatusOk ok code + FrontendStatusOk = 200200 + // FrontendStatusAccepted - + FrontendStatusAccepted = 200202 + // FrontendStatusNoContent - + FrontendStatusNoContent = 200204 + + // FrontendStatusBadRequest - + FrontendStatusBadRequest = 200400 + // FrontendStatusUnAuthorized - + FrontendStatusUnAuthorized = 200401 + // FrontendStatusForbidden - + FrontendStatusForbidden = 200403 + // FrontendStatusNotFound - + FrontendStatusNotFound = 200404 + // FrontendStatusRequestEntityTooLarge - + FrontendStatusRequestEntityTooLarge = 200413 + // FrontendStatusTooManyRequests - + FrontendStatusTooManyRequests = 200429 + + // FrontendStatusInternalError - + FrontendStatusInternalError = 200500 + // HTTPStreamNOTEnableError - + HTTPStreamNOTEnableError = 200600 + // CreateStreamProducerError - + CreateStreamProducerError = 200601 + // QueryStreamCustomerError - + QueryStreamCustomerError = 200602 + // SendDataToStreamError - + SendDataToStreamError = 200603 + // WriteResponseError - + WriteResponseError = 200604 + + // DsUploadFailed - upload to data system failed + DsUploadFailed = 200701 + // DsDownloadFailed - download from data system failed + DsDownloadFailed = 200702 + // DsDeleteFailed - delete from data system failed + DsDeleteFailed = 200703 + // DsKeyNotFound - key not found on data system + DsKeyNotFound = 200704 + + // UserFunctionInvokeError - user function error + UserFunctionInvokeError = 200705 + + // FuncMetaNotFound function meta not found, this error occurs only when the internal service is abnormal. + FuncMetaNotFound = 150424 + // HeavyLoadCode indicate the server's memory usage reaches threshold + HeavyLoadCode = 214503 +) + +// User error code +const ( + // UserFuncEntryNotFoundErrCode - + UserFuncEntryNotFoundErrCode = 4001 + // UserFuncRunningExceptionErrCode - + UserFuncRunningExceptionErrCode = 4002 + // StateContentTooLargeErrCode state content is too large + StateContentTooLargeErrCode = 4003 + // UserFuncRspExceedLimitErrCode response of user function exceeds the platform limit + UserFuncRspExceedLimitErrCode = 4004 + // UndefinedStateErrCode state is undefined + UndefinedStateErrCode = 4005 + // HeartBeatFunctionInvalidErrCode heart beat function of user invalid + HeartBeatFunctionInvalidErrCode = 4006 + // FunctionResultInvalidErrCode user function result is invalid + FunctionResultInvalidErrCode = 4007 + // InitializeFunctionErrorErrCode user initialize function error + InitializeFunctionErrorErrCode = 4009 + // UserFuncInvokeTimeout - + UserFuncInvokeTimeout = 4010 + // FrontendStatusWorkerIoTimeout - + FrontendStatusWorkerIoTimeout = 4014 + // FrontendStatusTrafficLimitEffective is the error code for traffic limitation + FrontendStatusTrafficLimitEffective = 4021 + // FrontendStatusLabelUnavailable - + FrontendStatusLabelUnavailable = 4022 + // FrontendStatusFuncMetaNotFound is error code of function meta not found + FrontendStatusFuncMetaNotFound = 4024 + // FrontendStatusUnableSpecifyResource unable to specify resource in a scene where no resource specified + FrontendStatusUnableSpecifyResource = 4026 + // FrontendStatusMaxRequestBodySize - + FrontendStatusMaxRequestBodySize = 4140 + // UserFuncInitFailCode code of user function initialization failed + UserFuncInitFailCode = 4201 + // ErrSharedMemoryLimited - + ErrSharedMemoryLimited = 4202 + // ErrOperateDiskFailed - + ErrOperateDiskFailed = 4203 + // ErrInsufficientDiskSpace - + ErrInsufficientDiskSpace = 4204 + + // UserFuncInitTimeoutCode code of initialing runtime timed out + UserFuncInitTimeoutCode = 4211 + // StsConfigErrCode sts config set error code + StsConfigErrCode = 4036 + // InstanceSessionInvalidErrCode - + InstanceSessionInvalidErrCode = 4037 + // ErrFinalized - + ErrFinalized = 9000 + // ErrAllSchedulerUnavailable - + ErrAllSchedulerUnavailable = 9009 + // InnerUserErrBase - + InnerUserErrBase = 50_0000 + // InnerRuntimeInitTimeoutCode - + InnerRuntimeInitTimeoutCode = InnerUserErrBase + UserFuncInitTimeoutCode +) + +// proxy internal error codes which suggests to retry in cluster +const ( + // ClientExitErrCode function instance is exiting (proxy side) + ClientExitErrCode = 211503 + + // WorkerExitErrCode function instance is exiting (worker side) + WorkerExitErrCode = 211504 + + // UserFuncIsUpdatedCode - + UserFuncIsUpdatedCode = 211411 + // SendReqErrCode call request sending error + SendReqErrCode = 211406 +) + +// executor error code +const ( + // ExecutorErrCodeInitFail - + ExecutorErrCodeInitFail = 6001 +) + +// The kernel and faaspattern should maintain an appropriate set of error codes. +// Common, such as a unified understanding of whether retry is required. +// In addition, the current transmission involves various character string conversions, +// which increases transcoding and matching barriers and causes high overheads. +// These are important, otherwise it will cause a lot of unclear boundaries and rework :) +const ( + // ErrInstanceNotFound - + ErrInstanceNotFound = 1003 + // ErrInstanceExitedCode - + ErrInstanceExitedCode = 1007 + // ErrInstanceCircuitCode - + ErrInstanceCircuitCode = 1009 + // ErrInstanceEvicted - + ErrInstanceEvicted = 1013 + + // ErrRequestBetweenRuntimeBusCode - + ErrRequestBetweenRuntimeBusCode = 3001 + // ErrInnerCommunication - + ErrInnerCommunication = 3002 + // ErrRequestBetweenRuntimeFrontendCode - + ErrRequestBetweenRuntimeFrontendCode = 3008 + // ErrAcquireTimeoutCode - + ErrAcquireTimeoutCode = 3009 +) + +// errors comes from faas scheduler (FG worker manager error) +const ( + // StatusInternalServerError status internal server error + StatusInternalServerError = 150500 + // VIPClusterOverloadCode cluster has no available resource + VIPClusterOverloadCode = 150510 + // FuncMetaNotFoundErrCode function meta not found, this error occurs only when the internal service is abnormal. + FuncMetaNotFoundErrCode = 150424 + // FuncMetaNotFoundErrMsg is error message of function metadata not found + FuncMetaNotFoundErrMsg = "function metadata not found" + // InstanceNotFoundErrCode is error code of instance not found + InstanceNotFoundErrCode = 150425 + // InstanceNotFoundErrMsg is error message of instance not found + InstanceNotFoundErrMsg = "instance not exist" + // NoInstanceAvailableErrCode is error message of no available instance + NoInstanceAvailableErrCode = 150431 + // InstanceStatusAbnormalCode - + InstanceStatusAbnormalCode = 150427 + // InstanceStatusAbnormalMsg - + InstanceStatusAbnormalMsg = "instance status is abnormal" + // ReachMaxInstancesCode reach function max instances + ReachMaxInstancesCode = 150429 + // ReachMaxInstancesErrMsg is error message of reach max instance + ReachMaxInstancesErrMsg = "reach max instance num" + // InsThdReqTimeoutCode acquire instance lease timeout, FG: cluster is overload and unavailable now + InsThdReqTimeoutCode = 150430 + // InsThdReqTimeoutErrMsg acquire instance lease timeout + InsThdReqTimeoutErrMsg = "instance thread request timeout" + // ReachMaxInstancesPerTenantErrCode reach tenant max on-demand instances + ReachMaxInstancesPerTenantErrCode = 150432 + // GettingPodErrorCode getting pod error code + GettingPodErrorCode = 150431 + // ReachMaxOnDemandInstancesPerTenant reach tenant max on-demand instances + ReachMaxOnDemandInstancesPerTenant = 150432 + // ReachMaxInstancesPerTenantErrMsg reach tenant max on-demand instances + ReachMaxInstancesPerTenantErrMsg = "reach max instance number per tenant" + // ReachMaxReversedInstancesPerTenant reach tenant max reversed instances + ReachMaxReversedInstancesPerTenant = 150433 + // FunctionIsDisabled function is disabled + FunctionIsDisabled = 150434 + // RefreshSilentFunc waiting for silent function to refresh, retry required + RefreshSilentFunc = 150435 + // NotEnoughNIC marked that there were not enough network cards + NotEnoughNIC = 150436 + // InsufficientEphemeralStorage marked that ephemeral storage is insufficient + InsufficientEphemeralStorage = 150438 + // ClusterIsUpgrading - + ClusterIsUpgrading = 150439 + // DesignateInsNotAvailableErrCode - + DesignateInsNotAvailableErrCode = 150440 + // InstanceLabelNotFoundErrCode - + InstanceLabelNotFoundErrCode = 150444 + // InstanceLabelNotFoundErrMsg - + InstanceLabelNotFoundErrMsg = "instance label not found" + // CancelGeneralizePod user update function metadata to cancel generalize pod while generalizing is not finished + CancelGeneralizePod = 150439 + + // ScaleUpRequestErrCode failed to send scale up request to worker-manager + ScaleUpRequestErrCode = 214501 + // ScaleUpRequestErrMsg - + ScaleUpRequestErrMsg = "send scale up request to worker-manager error" + + // SpecificInstanceNotFound - + SpecificInstanceNotFound = 150460 + // InstanceExceedConcurrency - + InstanceExceedConcurrency = 150461 + + LeaseIDIllegalCode = 150462 + LeaseIDIllegalMsg = "lease id is illegal" + LeaseIDNotFoundCode = 150463 + LeaseIDNotFoundMsg = "lease id is not found" +) + +var ( + // ErrMap frontend code map to http code + // Only return 200 to the management interface if the execution is successful + ErrMap = map[int]int{ + // system error + InnerResponseSuccessCode: http.StatusOK, + InternalErrorCode: http.StatusInternalServerError, + // frontend error + FrontendStatusOk: http.StatusOK, + FrontendStatusAccepted: http.StatusAccepted, + FrontendStatusNoContent: http.StatusNoContent, + FrontendStatusBadRequest: http.StatusBadRequest, + FrontendStatusUnAuthorized: http.StatusUnauthorized, + FrontendStatusForbidden: http.StatusForbidden, + FrontendStatusNotFound: http.StatusNotFound, + FrontendStatusRequestEntityTooLarge: http.StatusRequestEntityTooLarge, + FrontendStatusTooManyRequests: http.StatusTooManyRequests, + FrontendStatusInternalError: http.StatusInternalServerError, + FuncMetaNotFound: http.StatusInternalServerError, + HeavyLoadCode: http.StatusInternalServerError, + FrontendStatusTrafficLimitEffective: http.StatusInternalServerError, + HTTPStreamNOTEnableError: http.StatusInternalServerError, + CreateStreamProducerError: http.StatusInternalServerError, + QueryStreamCustomerError: http.StatusInternalServerError, + SendDataToStreamError: http.StatusInternalServerError, + WriteResponseError: http.StatusInternalServerError, + // frontend caas / multidata error + // 500 + DsUploadFailed: http.StatusInternalServerError, + DsDownloadFailed: http.StatusInternalServerError, + DsDeleteFailed: http.StatusInternalServerError, + DsKeyNotFound: http.StatusInternalServerError, + UserFunctionInvokeError: http.StatusInternalServerError, + // user error + UserFuncEntryNotFoundErrCode: http.StatusInternalServerError, + UserFuncRunningExceptionErrCode: http.StatusInternalServerError, + UserFuncRspExceedLimitErrCode: http.StatusInternalServerError, + FrontendStatusMaxRequestBodySize: http.StatusInternalServerError, + FrontendStatusUnableSpecifyResource: http.StatusInternalServerError, + UserFuncInvokeTimeout: http.StatusInternalServerError, + UserFuncInitFailCode: http.StatusInternalServerError, + UserFuncInitTimeoutCode: http.StatusInternalServerError, + StsConfigErrCode: http.StatusInternalServerError, + // executor error + ExecutorErrCodeInitFail: http.StatusInternalServerError, + } +) + +const ( + // VpcNoOperationalPermissions vpc has no operational permissions + VpcNoOperationalPermissions = 4212 + // VPCNotFound error code of VPC not found + VPCNotFound = 4219 + // VPCXRoleNotFound vcp xrole not func + VPCXRoleNotFound = 4222 +) + +// vpc err comes from vpc controller +var ( + // ErrNoOperationalPermissionsVpc no operational permissions vpc + ErrNoOperationalPermissionsVpc = errors.New("no operational permissions vpc, check the func xrole permissions") + // ErrNoAvailableVpcPatInstance no available vpc pat instance + ErrNoAvailableVpcPatInstance = errors.New("no available vpc pat instance") + // ErrVPCNotFound VPC item not found error + ErrVPCNotFound = errors.New("vpc item not found") + // ErrVPCXRoleNotFound VPC xrole not found error + ErrVPCXRoleNotFound = errors.New("can't find xrole") + + vpcErrorMap = map[string]int{ + ErrNoOperationalPermissionsVpc.Error(): VpcNoOperationalPermissions, + ErrNoAvailableVpcPatInstance.Error(): NotEnoughNIC, + ErrVPCNotFound.Error(): VPCNotFound, + ErrVPCXRoleNotFound.Error(): VPCXRoleNotFound, + } + + vpcErrorCodeMsg = map[int]string{ + VpcNoOperationalPermissions: "no operational permissions vpc, check the func xrole permissions", + NotEnoughNIC: "not enough network cards", + VPCNotFound: "VPC item not found", + VPCXRoleNotFound: "VPC can't find xrole", + } +) + +const ( + // InvalidState - + InvalidState = 4040 + // InvalidStateErrMsg - + InvalidStateErrMsg = "invalid state, expect not blank" + // StateMismatch - + StateMismatch = 4006 + // StateMismatchErrMsg - + StateMismatchErrMsg = "invoke state id and function stateful flag are not matched" + // StateExistedErrCode - + StateExistedErrCode = 4027 + // StateExistedErrMsg - + StateExistedErrMsg = "state cannot be created repeatedly" + // StateNotExistedErrCode - + StateNotExistedErrCode = 4026 + // StateNotExistedErrMsg - + StateNotExistedErrMsg = "state not existed" + // StateInstanceNotExistedErrCode - + StateInstanceNotExistedErrCode = 4028 + // StateInstanceNotExistedErrMsg - + StateInstanceNotExistedErrMsg = "state instance not existed" + // StateInstanceNoLease - + StateInstanceNoLease = 4025 + // StateInstanceNoLeaseMsg - + StateInstanceNoLeaseMsg = "maximum number of leases reached" + // FaaSSchedulerInternalErrCode - + FaaSSchedulerInternalErrCode = 4029 + // FaaSSchedulerInternalErrMsg - + FaaSSchedulerInternalErrMsg = "internal system error" +) + +// worker error code +const ( + // WorkerInternalErrorCode code of unexpected error in worker + WorkerInternalErrorCode = 161900 + // ReadingCodeTimeoutCode reading code package timed out + ReadingCodeTimeoutCode = 161901 + // CallFunctionErrorCode code of calling other function error + CallFunctionErrorCode = 161902 + // FuncInsExceptionCode function instance exception + FuncInsExceptionCode = 161903 + // CheckSumErrorCode code of check sum error + CheckSumErrorCode = 161904 + // DownLoadCodeErrorCode code of download code error + DownLoadCodeErrorCode = 161905 + // RPCClientEmptyErrorCode code of when rpc client is nil + RPCClientEmptyErrorCode = 161906 + // RuntimeManagerProcessExited runtime-manager process exited code + RuntimeManagerProcessExited = 161907 + // WorkerPingVpcGatewayError code of worker ping vpc gateway error + WorkerPingVpcGatewayError = 161908 + // UploadSnapshotErrorCode code of worker upload snapshot error + UploadSnapshotErrorCode = 161909 + // RestoreDeadErrorCode code of restore is dead + RestoreDeadErrorCode = 161910 + // ContentInconsistentErrorCode code of worker content inconsistent error + ContentInconsistentErrorCode = 161911 + // CreateLimitErrorCode code of POSIX create limit error + CreateLimitErrorCode = 161912 + // KernelEtcdWriteFailedCode code of core write etcd failed or circuit + KernelEtcdWriteFailedCode = 161913 + // KernelResourceNotEnoughErrCode code of core resource not enough or schedule failure + KernelResourceNotEnoughErrCode = 161914 + // WiseCloudNuwaColdStartErrCode code of use nuwa cold start failed + WiseCloudNuwaColdStartErrCode = 161915 +) + +// Code trans frontend code to http code +func Code(frontendCode int) int { + httpCode, exist := ErrMap[frontendCode] + if !exist { + return http.StatusInternalServerError + } + return httpCode +} + +// Message trans frontend code to message +func Message(frontendCode int) string { + httpCode, exist := ErrMap[frontendCode] + if !exist { + return "" + } + + return fasthttp.StatusMessage(int(httpCode)) +} + +// VpcCode vpc controller err map to vpc err code +func VpcCode(errMsg string) int { + if errCode, ok := vpcErrorMap[errMsg]; ok { + return errCode + } + return 0 +} + +// VpcErMsg vpc err code map to err msg +func VpcErMsg(errCode int) string { + if errMsg, ok := vpcErrorCodeMsg[errCode]; ok { + return errMsg + } + return "" +} + +var ( + errCodeRegCompile = regexp.MustCompile("code:[ 0-9]+,") + errMsgRegCompile = regexp.MustCompile("message:.+") + codeRegCompile = regexp.MustCompile("[0-9]+") +) + +// GetKernelErrorCode will get kernel error code from error message +func GetKernelErrorCode(errMsg string) int { + res := errCodeRegCompile.FindStringSubmatch(errMsg) + if len(res) < 1 { + return InternalErrorCode + } + res = codeRegCompile.FindStringSubmatch(errMsg) + if len(res) != 1 { + return InternalErrorCode + } + code, err := strconv.Atoi(res[0]) + if err != nil { + return InternalErrorCode + } + return code +} + +// GetKernelErrorMessage will get kernel error message from error message +func GetKernelErrorMessage(errMsg string) string { + res := errMsgRegCompile.FindStringSubmatch(errMsg) + if len(res) < 1 { + return "" + } + trimRes := strings.TrimPrefix(res[0], "message: ") + return trimRes +} diff --git a/frontend/pkg/common/faas_common/statuscode/statuscode_test.go b/frontend/pkg/common/faas_common/statuscode/statuscode_test.go new file mode 100644 index 0000000000000000000000000000000000000000..64e5fcbed25a390e16eb90b98d006377d65129ca --- /dev/null +++ b/frontend/pkg/common/faas_common/statuscode/statuscode_test.go @@ -0,0 +1,86 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package statuscode define status code of Frontend +package statuscode + +import ( + "net/http" + "testing" + + "github.com/smartystreets/goconvey/convey" +) + +func TestStatusCode(t *testing.T) { + convey.Convey("get code", t, func() { + code := Code(InnerResponseSuccessCode) + convey.So(code, convey.ShouldEqual, http.StatusOK) + }) + convey.Convey("get message", t, func() { + msg := Message(InnerResponseSuccessCode) + convey.So(msg, convey.ShouldEqual, "OK") + }) + convey.Convey("error code get message", t, func() { + msg := Message(999999) + convey.So(msg, convey.ShouldEqual, "") + }) + convey.Convey("error code get message", t, func() { + code := Code(999999) + convey.So(code, convey.ShouldEqual, http.StatusInternalServerError) + }) +} + +func TestGetKernelErrorCode(t *testing.T) { + type args struct { + errMsg string + } + tests := []struct { + name string + args args + want int + }{ + {"case1 unknow error", args{errMsg: "unknown error"}, InternalErrorCode}, + {"case2 get code", args{errMsg: "code: 1007,"}, 1007}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GetKernelErrorCode(tt.args.errMsg); got != tt.want { + t.Errorf("GetKernelErrorCode() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGetKernelErrorMessage(t *testing.T) { + type args struct { + errMsg string + } + tests := []struct { + name string + args args + want string + }{ + {"case1 unknow message", args{errMsg: "unknown message"}, ""}, + {"case2 get message", args{errMsg: "message: yes"}, "yes"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GetKernelErrorMessage(tt.args.errMsg); got != tt.want { + t.Errorf("GetKernelErrorMessage() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/frontend/pkg/common/faas_common/sts/cert/cert.go b/frontend/pkg/common/faas_common/sts/cert/cert.go new file mode 100644 index 0000000000000000000000000000000000000000..5e92d43ad6bba559c1b33c7c3a77dcff81ad6eb5 --- /dev/null +++ b/frontend/pkg/common/faas_common/sts/cert/cert.go @@ -0,0 +1,106 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package cert parsing certificate +package cert + +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "os" + + "golang.org/x/crypto/pkcs12" + + "frontend/pkg/common/faas_common/utils" +) + +// LoadCerts - parsing certificate +func LoadCerts() (*x509.CertPool, *tls.Certificate, error) { + var keyStorePath string + var err error + if err != nil { + return nil, nil, err + } + caCertsPool := x509.NewCertPool() + var bytes []byte + if err != nil { + return nil, nil, err + } + fileContent, err := os.ReadFile(keyStorePath) + if err != nil { + return nil, nil, err + } + pemBlocks, err := pkcs12.ToPEM(fileContent, string(bytes)) + utils.ClearByteMemory(fileContent) + if err != nil { + return nil, nil, err + } + + caBytes, certByte, keyByte, err := parseSTSCerts(pemBlocks) + if err != nil { + return nil, nil, err + + } + for _, caByte := range caBytes { + caCertsPool.AppendCertsFromPEM(caByte) + } + tlsCert, err := tls.X509KeyPair(certByte, keyByte) + utils.ClearByteMemory(certByte) + utils.ClearByteMemory(keyByte) + if err != nil { + return nil, nil, err + + } + return caCertsPool, &tlsCert, nil +} + +func parseSTSCerts(pemBlocks []*pem.Block) ([][]byte, []byte, []byte, error) { + var certByte, keyByte []byte + var err error + var caBytes [][]byte + for _, pemBlock := range pemBlocks { + pemEncoded := pem.EncodeToMemory(pemBlock) + if pemBlock.Type == "PRIVATE KEY" { + keyByte = pemEncoded + } else { + var cert *x509.Certificate + if cert, err = x509.ParseCertificate(pemBlock.Bytes); err != nil { + return nil, nil, nil, err + } + if cert == nil { + return nil, nil, nil, fmt.Errorf("parse certificate err: cert is empty") + } + if cert.IsCA { + pemBlock.Headers = map[string]string{} + caBytes = append(caBytes, pem.EncodeToMemory(pemBlock)) + } else { + certByte = append(certByte, pemEncoded...) + } + } + } + if len(caBytes) == 0 { + return caBytes, certByte, keyByte, fmt.Errorf("ca certs not exists") + } + if len(certByte) == 0 { + return caBytes, certByte, keyByte, fmt.Errorf("certs not exists") + } + if len(keyByte) == 0 { + return caBytes, certByte, keyByte, fmt.Errorf("private key not exists") + } + return caBytes, certByte, keyByte, nil +} diff --git a/frontend/pkg/common/faas_common/sts/cert/cert_test.go b/frontend/pkg/common/faas_common/sts/cert/cert_test.go new file mode 100644 index 0000000000000000000000000000000000000000..b5c9420f0ad94d9c730f94f4decaebb922f7d9a3 --- /dev/null +++ b/frontend/pkg/common/faas_common/sts/cert/cert_test.go @@ -0,0 +1,74 @@ +package cert + +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "errors" + "os" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "golang.org/x/crypto/pkcs12" + + mockUtils "frontend/pkg/common/faas_common/utils" +) + +func Test_parseSTSCerts(t *testing.T) { + type args struct { + pemBlocks []*pem.Block + } + tests := []struct { + name string + args args + wantErr bool + patchesFunc mockUtils.PatchesFunc + }{ + {"case1 succeed to parse", args{pemBlocks: []*pem.Block{ + &pem.Block{Type: "PRIVATE KEY"}, &pem.Block{}, &pem.Block{Bytes: []byte("a")}}}, + false, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(pem.EncodeToMemory, func(b *pem.Block) []byte { + return []byte("a") + }), + gomonkey.ApplyFunc(x509.ParseCertificate, func(der []byte) (*x509.Certificate, error) { + if string(der) == "a" { + return &x509.Certificate{}, nil + } + return &x509.Certificate{IsCA: true}, nil + }), + }) + return patches + }}, + {"case2 failed to parse", args{pemBlocks: []*pem.Block{ + &pem.Block{Type: "PRIVATE KEY"}}}, + true, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(pem.EncodeToMemory, func(b *pem.Block) []byte { + return []byte("a") + }), + gomonkey.ApplyFunc(x509.ParseCertificate, func(der []byte) (*x509.Certificate, error) { + if string(der) == "a" { + return &x509.Certificate{}, nil + } + return &x509.Certificate{IsCA: true}, nil + }), + }) + return patches + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + patches := tt.patchesFunc() + _, _, _, err := parseSTSCerts(tt.args.pemBlocks) + if (err != nil) != tt.wantErr { + t.Errorf("parseSTSCerts() error = %v, wantErr %v", err, tt.wantErr) + return + } + patches.ResetAll() + }) + } +} diff --git a/frontend/pkg/common/faas_common/sts/common.go b/frontend/pkg/common/faas_common/sts/common.go new file mode 100644 index 0000000000000000000000000000000000000000..018a8fc31f2ec986f954b21a13c4133b05285d6b --- /dev/null +++ b/frontend/pkg/common/faas_common/sts/common.go @@ -0,0 +1,184 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package sts - +package sts + +import ( + "encoding/json" + "fmt" + + "k8s.io/api/core/v1" + + "frontend/pkg/common/faas_common/tls" + "frontend/pkg/common/faas_common/utils" +) + +// SecretConfig - +type SecretConfig struct{} + +const ( + // FaasfrontendName - + FaasfrontendName = "faasfrontend" + // FaaSSchedulerName - + FaaSSchedulerName = "faasscheduler" + mountPath = "/opt/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/" + faasSchedulerMountPath = "/opt/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorkerManager/" + // HTTPSMountPath mount https certs + HTTPSMountPath = "/home/sn/resource/https" + // LocalSecretMountPath mount local secrets + LocalSecretMountPath = "/home/sn/resource/cipher" +) + +var readOnlyVolumeMode int32 = 0440 + +// ConfigVolume - +func (u *SecretConfig) ConfigVolume(b *utils.VolumeBuilder) { + b.AddVolumeMount(utils.ContainerRuntimeManager, v1.VolumeMount{ + Name: "sts-config", + MountPath: mountPath + "HMSCaaSYuanRongWorker/apple/a", + SubPath: "a", + }) + b.AddVolumeMount(utils.ContainerRuntimeManager, v1.VolumeMount{ + Name: "sts-config", + MountPath: mountPath + "HMSCaaSYuanRongWorker/boy/b", + SubPath: "b", + }) + b.AddVolumeMount(utils.ContainerRuntimeManager, v1.VolumeMount{ + Name: "sts-config", + MountPath: mountPath + "HMSCaaSYuanRongWorker/cat/c", + SubPath: "c", + }) + b.AddVolumeMount(utils.ContainerRuntimeManager, v1.VolumeMount{ + Name: "sts-config", + MountPath: mountPath + "HMSCaaSYuanRongWorker/dog/d", + SubPath: "d", + }) + b.AddVolumeMount(utils.ContainerRuntimeManager, v1.VolumeMount{ + Name: "sts-config", + MountPath: mountPath + "HMSCaaSYuanRongWorker.ini", + SubPath: "HMSCaaSYuanRongWorker.ini", + }) + b.AddVolumeMount(utils.ContainerRuntimeManager, v1.VolumeMount{ + Name: "sts-config", + MountPath: mountPath + "HMSCaaSYuanRongWorker.sts.p12", + SubPath: "HMSCaaSYuanRongWorker.sts.p12", + }) +} + +// ConfigFaasSchedulerVolume - +func (u *SecretConfig) ConfigFaasSchedulerVolume(b *utils.VolumeBuilder) { + b.AddVolumeMount(utils.ContainerRuntimeManager, v1.VolumeMount{ + Name: "sts-workermanager-config", + MountPath: faasSchedulerMountPath + "HMSCaaSYuanRongWorkerManager/apple/a", + SubPath: "a", + }) + b.AddVolumeMount(utils.ContainerRuntimeManager, v1.VolumeMount{ + Name: "sts-workermanager-config", + MountPath: faasSchedulerMountPath + "HMSCaaSYuanRongWorkerManager/boy/b", + SubPath: "b", + }) + b.AddVolumeMount(utils.ContainerRuntimeManager, v1.VolumeMount{ + Name: "sts-workermanager-config", + MountPath: faasSchedulerMountPath + "HMSCaaSYuanRongWorkerManager/cat/c", + SubPath: "c", + }) + b.AddVolumeMount(utils.ContainerRuntimeManager, v1.VolumeMount{ + Name: "sts-workermanager-config", + MountPath: faasSchedulerMountPath + "HMSCaaSYuanRongWorkerManager/dog/d", + SubPath: "d", + }) + b.AddVolumeMount(utils.ContainerRuntimeManager, v1.VolumeMount{ + Name: "sts-workermanager-config", + MountPath: faasSchedulerMountPath + "HMSCaaSYuanRongWorkerManager.ini", + SubPath: "HMSCaaSYuanRongWorkerManager.ini", + }) + b.AddVolumeMount(utils.ContainerRuntimeManager, v1.VolumeMount{ + Name: "sts-workermanager-config", + MountPath: faasSchedulerMountPath + "HMSCaaSYuanRongWorkerManager.sts.p12", + SubPath: "HMSCaaSYuanRongWorkerManager.sts.p12", + }) +} + +// ConfigHTTPSAndLocalSecretVolume - +func (u *SecretConfig) ConfigHTTPSAndLocalSecretVolume(b *utils.VolumeBuilder, httpsConfig tls.InternalHTTPSConfig) { + b.AddVolume(buildVolumeOfSecretSource("https", httpsConfig.SecretName)) + b.AddVolumeMount(utils.ContainerRuntimeManager, v1.VolumeMount{ + Name: "https", + MountPath: httpsConfig.SSLBasePath, + }) +} + +func buildVolumeOfSecretSource(name string, secretName string) v1.Volume { + return v1.Volume{ + Name: name, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + DefaultMode: &readOnlyVolumeMode, + SecretName: secretName, + }, + }, + } +} + +// GenerateSecretVolumeMounts - +func GenerateSecretVolumeMounts(systemFunctionName string, builder *utils.VolumeBuilder) ([]byte, error) { + if builder == nil { + return nil, fmt.Errorf("sts volume builder is nil") + } + sc := &SecretConfig{} + if systemFunctionName == FaaSSchedulerName { + sc.ConfigFaasSchedulerVolume(builder) + } else { + sc.ConfigVolume(builder) + } + bytesData, err := json.Marshal(builder.Mounts[utils.ContainerRuntimeManager]) + if err != nil { + return nil, err + } + return bytesData, nil +} + +// CustomKeyProvider - +type CustomKeyProvider struct { + key []byte + tenantID string +} + +// NewCustomKeyProvider - +func NewCustomKeyProvider(tenantID string, key []byte) *CustomKeyProvider { + return &CustomKeyProvider{tenantID: tenantID, key: key} +} + +// GenerateHTTPSAndLocalSecretVolumeMounts - +func GenerateHTTPSAndLocalSecretVolumeMounts( + httpsConfig tls.InternalHTTPSConfig, builder *utils.VolumeBuilder) (string, string, error) { + if builder == nil { + return "", "", fmt.Errorf("https volume builder is nil") + } + sc := &SecretConfig{} + sc.ConfigHTTPSAndLocalSecretVolume(builder, httpsConfig) + + volumesData, err := json.Marshal(builder.Volumes) + if err != nil { + return "", "", err + } + volumesMountData, err := json.Marshal(builder.Mounts[utils.ContainerRuntimeManager]) + if err != nil { + return "", "", err + } + return string(volumesData), string(volumesMountData), nil +} diff --git a/frontend/pkg/common/faas_common/sts/common_test.go b/frontend/pkg/common/faas_common/sts/common_test.go new file mode 100644 index 0000000000000000000000000000000000000000..3bc112e95b2786da5708f28c1890943e8c9bfada --- /dev/null +++ b/frontend/pkg/common/faas_common/sts/common_test.go @@ -0,0 +1,68 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package sts - +package sts + +import ( + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/magiconair/properties" + "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/common/faas_common/sts/raw" + "frontend/pkg/common/faas_common/tls" + "frontend/pkg/common/faas_common/utils" + mockUtils "frontend/pkg/common/faas_common/utils" +) + +func TestGenerateSecretVolumeMounts(t *testing.T) { + type args struct { + systemFunctionName string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {"case1 faasshceduler generate", args{systemFunctionName: FaaSSchedulerName}, false}, + {"case2 faasfrontend generate", args{systemFunctionName: FaasfrontendName}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + build := utils.NewVolumeBuilder() + _, err := GenerateSecretVolumeMounts(tt.args.systemFunctionName, build) + if (err != nil) != tt.wantErr { + t.Errorf("GenerateSecretVolumeMounts() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} + +func TestGenerateHTTPSAndLocalSecretVolumeMounts(t *testing.T) { + convey.Convey("TestGenerateHTTPSAndLocalSecretVolumeMounts", t, func() { + httpsConfig := tls.InternalHTTPSConfig{} + volumeData, volumeMountData, err := GenerateHTTPSAndLocalSecretVolumeMounts(httpsConfig, nil) + convey.So(volumeData, convey.ShouldEqual, "") + convey.So(volumeMountData, convey.ShouldEqual, "") + convey.So(err, convey.ShouldNotBeNil) + + volumeData, volumeMountData, err = GenerateHTTPSAndLocalSecretVolumeMounts(httpsConfig, utils.NewVolumeBuilder()) + convey.So(err, convey.ShouldBeNil) + }) +} diff --git a/frontend/pkg/common/faas_common/sts/raw/crypto.go b/frontend/pkg/common/faas_common/sts/raw/crypto.go new file mode 100644 index 0000000000000000000000000000000000000000..404fd416056cc7932d8ad89fd8df3d77ec9a6f0f --- /dev/null +++ b/frontend/pkg/common/faas_common/sts/raw/crypto.go @@ -0,0 +1,84 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package raw use work key to encrypt and decrypt +package raw + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "fmt" +) + +const ( + defaultSaltSize = 12 +) + +// AesGCMDecrypt decrypt a cypher text using AES_GCM algorithm +func AesGCMDecrypt(secret, salt, cipherBytes []byte) ([]byte, error) { + defer postRecover() + block, err := aes.NewCipher(secret) + if err != nil { + return nil, err + } + // salt 长度和 nonceSize 保持一致 + // cipher.NewGCM(block) 使用的是默认12字节的nonceSize,也代表盐值长度必须是12Byte;为了适应性强,我们使用自定义的 nonceSize + gcm, err := cipher.NewGCMWithNonceSize(block, len(salt)) + if err != nil { + return nil, err + } + plainBytes, err := gcm.Open(nil, salt, cipherBytes, nil) + if err != nil { + return nil, err + } + return plainBytes, nil +} + +func postRecover() { + var err error + if r := recover(); r != nil { + switch value := r.(type) { + case string: + err = fmt.Errorf("%s", value) + case error: + err = value + default: + err = fmt.Errorf("unexpect panic error: %w", err) + } + err = fmt.Errorf("panic error: %w", err) + } +} + +// AesGCMEncrypt will encrypt plainBytes to cipherBytes +func AesGCMEncrypt(secret, plainBytes []byte) ([]byte, []byte, error) { + defer postRecover() + block, err := aes.NewCipher(secret) + if err != nil { + return nil, nil, err + } + gcm, err := cipher.NewGCMWithNonceSize(block, defaultSaltSize) + if err != nil { + return nil, nil, fmt.Errorf("failed NewGCM: %w", err) + } + salt := make([]byte, gcm.NonceSize()) + _, err = rand.Read(salt) + if err != nil { + return nil, nil, err + } + cipherBytes := gcm.Seal(nil, salt, plainBytes, nil) + return salt, cipherBytes, nil +} diff --git a/frontend/pkg/common/faas_common/sts/raw/crypto_test.go b/frontend/pkg/common/faas_common/sts/raw/crypto_test.go new file mode 100644 index 0000000000000000000000000000000000000000..8f6603dc0e5fc725f74bb4a52d3e57ff1f110478 --- /dev/null +++ b/frontend/pkg/common/faas_common/sts/raw/crypto_test.go @@ -0,0 +1,64 @@ +package raw + +import ( + "encoding/base64" + "encoding/hex" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + saltKeySep = ":" + + shareKey = "1752F862B5176946F18D45D67E256642F115D2D6A3D77773FAF1E5874AC5211D" + + plain = "{\"key1\":\"value1\",\"key2\":\"value2\"}" + + saltKey = "R8Mi3gSG3ou4X6eY:VIQASOEBJTQT3yd4qGrpqSbLrgemB5eTaD5KRefaOcXh/r18YSwhtv0j0A==" + plain2 = "{\"key1\":\"va1\",\"key2\":\"va2\"}" +) + +func TestAesGCMDecrypt(t *testing.T) { + shareKey2 := make([]byte, hex.DecodedLen(len(shareKey))) + _, err := hex.Decode(shareKey2, []byte(shareKey)) + if err != nil { + t.Errorf("%s", err) + } + + salt, cipherBytes, err := AesGCMEncrypt(shareKey2, []byte(plain)) + fmt.Println(string(salt), string(cipherBytes)) + saltBase64 := base64.StdEncoding.EncodeToString(salt) + cipherBase64 := base64.StdEncoding.EncodeToString(cipherBytes) + fmt.Println(saltBase64, cipherBase64) + + if err != nil { + t.Errorf("%s", err) + } + blocks1, err := AesGCMDecrypt(shareKey2, salt, cipherBytes) + + assert.Equal(t, string(blocks1), plain) +} + +func TestAesGCMDecrypt2(t *testing.T) { + shareKey2 := make([]byte, hex.DecodedLen(len(shareKey))) + _, err := hex.Decode(shareKey2, []byte(shareKey)) + if err != nil { + t.Errorf("%s", err) + } + + fields := strings.Split(saltKey, saltKeySep) + salt1, err := base64.StdEncoding.DecodeString(fields[0]) + if err != nil { + t.Errorf("%s", err) + } + cipher, err := base64.StdEncoding.DecodeString(fields[1]) + blocks, err := AesGCMDecrypt(shareKey2, salt1, cipher) + if err != nil { + t.Errorf("%s", err) + } + + assert.Equal(t, string(blocks), plain2) +} diff --git a/frontend/pkg/common/faas_common/sts/raw/raw.go b/frontend/pkg/common/faas_common/sts/raw/raw.go new file mode 100644 index 0000000000000000000000000000000000000000..d914b0b86dfbbcda79e0a8151323f75467b80b0f --- /dev/null +++ b/frontend/pkg/common/faas_common/sts/raw/raw.go @@ -0,0 +1,43 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package raw define the sts structure +package raw + +// StsConfig - +type StsConfig struct { + StsEnable bool `json:"stsEnable,omitempty"` + SensitiveConfigs SensitiveConfigs `json:"sensitiveConfigs,omitempty"` + ServerConfig ServerConfig `json:"serverConfig,omitempty"` + MgmtServerConfig MgmtServerConfig `json:"mgmtServerConfig"` + StsDomainForRuntime string `json:"stsDomainForRuntime"` +} + +// SensitiveConfigs - +type SensitiveConfigs struct { + ShareKeys map[string]string `json:"shareKeys"` +} + +// ServerConfig - +type ServerConfig struct { + Domain string `json:"domain,omitempty" validate:"max=255"` + Path string `json:"path,omitempty" validate:"max=255"` +} + +// MgmtServerConfig - +type MgmtServerConfig struct { + Domain string `json:"domain,omitempty"` +} diff --git a/frontend/pkg/common/faas_common/sts/sts.go b/frontend/pkg/common/faas_common/sts/sts.go new file mode 100644 index 0000000000000000000000000000000000000000..b62b158cc89387030500f83dd752c20e84a1581a --- /dev/null +++ b/frontend/pkg/common/faas_common/sts/sts.go @@ -0,0 +1,76 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package sts used for init sts +package sts + +import ( + "os" + "time" + + "frontend/pkg/common/faas_common/alarm" + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/config" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/sts/raw" +) + +// EnvSTSEnable flag +const EnvSTSEnable = "STS_ENABLE" +const fileMode = 0640 + +// InitStsSDK - Configure sts go sdk +func InitStsSDK(serverCfg raw.ServerConfig) error { + initStsSdkLog() + var err error + if err != nil { + reportStsAlarm(err.Error()) + } + return err +} + +func reportStsAlarm(errMsg string) { + alarmDetail := &alarm.Detail{ + SourceTag: os.Getenv(constant.PodNameEnvKey) + "|" + os.Getenv(constant.PodIPEnvKey) + + "|" + os.Getenv(constant.ClusterName), + OpType: alarm.GenerateAlarmLog, + Details: "Init sts err, " + errMsg, + StartTimestamp: int(time.Now().Unix()), + EndTimestamp: 0, + } + alarmInfo := &alarm.LogAlarmInfo{ + AlarmID: alarm.InitStsSdkErr00001, + AlarmName: "InitStsSdkErr", + AlarmLevel: alarm.Level3, + } + + alarm.ReportOrClearAlarm(alarmInfo, alarmDetail) +} + +func initStsSdkLog() { + coreInfo, err := config.GetCoreInfoFromEnv() + if err != nil { + coreInfo = config.GetDefaultCoreInfo() + } + stsSdkLogFilePath := coreInfo.FilePath + "/sts.sdk.log" + file, err := os.OpenFile(stsSdkLogFilePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fileMode) + if err != nil { + log.GetLogger().Errorf("failed to open stsSdkLogFile") + return + } + defer file.Close() + return +} diff --git a/frontend/pkg/common/faas_common/timeutil/conv.go b/frontend/pkg/common/faas_common/timeutil/conv.go new file mode 100644 index 0000000000000000000000000000000000000000..033dce438f38c105fefe00a4512c92e91084037f --- /dev/null +++ b/frontend/pkg/common/faas_common/timeutil/conv.go @@ -0,0 +1,41 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package timeutil provides some utils for time +package timeutil + +import "time" + +const ( + // NanosecondToMillisecond is the factor to convert nanosecond to millisecond, + // it should be equal to time.Millisecond/time.Nanosecond + NanosecondToMillisecond = 1000000 +) + +// UnixMillisecond convert time.Time to unix timestamp in millisecond +func UnixMillisecond(t time.Time) int64 { + return t.UnixNano() / NanosecondToMillisecond +} + +// NowUnixMillisecond get current unix timestamp in millisecond +func NowUnixMillisecond() int64 { + return UnixMillisecond(time.Now()) +} + +// NowUnixNanoseconds get current unix timestamp in nanoseconds +func NowUnixNanoseconds() int64 { + return time.Now().UnixNano() +} diff --git a/frontend/pkg/common/faas_common/timeutil/conv_test.go b/frontend/pkg/common/faas_common/timeutil/conv_test.go new file mode 100644 index 0000000000000000000000000000000000000000..29f25135c0e55cb41c936b74f7e35b8090fc995e --- /dev/null +++ b/frontend/pkg/common/faas_common/timeutil/conv_test.go @@ -0,0 +1,39 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package timeutil + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestUnixMillisecond(t *testing.T) { + assert.Equal(t, time.Second.Nanoseconds()/time.Second.Milliseconds(), int64(NanosecondToMillisecond)) + assert.Equal(t, int64(time.Millisecond/time.Nanosecond), int64(NanosecondToMillisecond)) +} + +func TestNowUnixMillisecond(t *testing.T) { + millisecond := NowUnixMillisecond() + assert.Equal(t, NowUnixMillisecond() >= millisecond, true) +} + +func TestNowUnixNanoseconds(t *testing.T) { + unixNanoseconds := NowUnixNanoseconds() + assert.Equal(t, NowUnixNanoseconds() >= unixNanoseconds, true) +} diff --git a/frontend/pkg/common/faas_common/timewheel/simpletimewheel.go b/frontend/pkg/common/faas_common/timewheel/simpletimewheel.go new file mode 100644 index 0000000000000000000000000000000000000000..ba994be4ae4aad636e34820a0fab867486a92ece --- /dev/null +++ b/frontend/pkg/common/faas_common/timewheel/simpletimewheel.go @@ -0,0 +1,242 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package timewheel - +package timewheel + +import ( + "errors" + "fmt" + "sync" + "time" +) + +const ( + minPace = 2 * time.Millisecond + minSlotNum = 1 + notifyChannelSize = 1000 +) + +var ( + timeTriggerPool = sync.Pool{New: func() interface{} { + return &timeTrigger{} + }} +) + +type timeTrigger struct { + taskID string + times int + index int64 + circle int64 + circleCount int64 + disable bool + ch chan struct{} + prev *timeTrigger + next *timeTrigger +} + +// SimpleTimeWheel will trigger task at given interval by given times, it contains a certain number of slots and moves +// from one slot to another with a pace which is also the granularity of time wheel, task interval will be measured +// with a number of slots and recorded in the slot arrays, each slot has a linked list to trigger a series of tasks +// when time wheel moves to this slot +type SimpleTimeWheel struct { + ticker *time.Ticker + pace time.Duration + perimeter int64 + slotNum int64 + curSlot int64 + pendingTask int + slots []*timeTrigger + readyList []string + record *sync.Map + notifyCh chan struct{} + readyCh chan struct{} + stopCh chan struct{} + sync.RWMutex +} + +// NewSimpleTimeWheel will create a SimpleTimeWheel +func NewSimpleTimeWheel(pace time.Duration, slotNum int64) TimeWheel { + if pace < minPace { + pace = minPace + } + if slotNum < minSlotNum { + slotNum = minSlotNum + } + timeWheel := &SimpleTimeWheel{ + ticker: time.NewTicker(pace), + pace: pace, + perimeter: slotNum * int64(pace), + slotNum: slotNum, + curSlot: 0, + slots: make([]*timeTrigger, slotNum, slotNum), + record: new(sync.Map), + notifyCh: make(chan struct{}, notifyChannelSize), + readyCh: make(chan struct{}, 1), + stopCh: make(chan struct{}), + } + go timeWheel.run() + return timeWheel +} + +func (gt *SimpleTimeWheel) run() { + for { + select { + case <-gt.ticker.C: + gt.Lock() + gt.curSlot = (gt.curSlot + 1) % int64(len(gt.slots)) + gt.Unlock() + gt.checkAndFireTrigger() + case <-gt.stopCh: + gt.ticker.Stop() + return + } + } +} + +func (gt *SimpleTimeWheel) checkAndFireTrigger() { + trigger := gt.slots[gt.curSlot] + var readyList []string + for trigger != nil { + if !trigger.disable && trigger.circleCount == trigger.circle { + trigger.circleCount = 0 + if trigger.times == 0 { + trigger.disable = true + gt.record.Delete(trigger.taskID) + gt.removeTrigger(trigger) + continue + } + readyList = append(readyList, trigger.taskID) + select { + case trigger.ch <- struct{}{}: + default: + } + if trigger.times > 0 { + trigger.times-- + } + } + trigger.circleCount++ + trigger = trigger.next + } + gt.Lock() + gt.readyList = readyList + gt.Unlock() + if len(readyList) != 0 { + gt.readyCh <- struct{}{} + } +} + +// Wait will block until tasks are triggered and returns triggered task list +func (gt *SimpleTimeWheel) Wait() []string { + select { + case _, ok := <-gt.readyCh: + if !ok { + return nil + } + } + gt.RLock() + readyList := gt.readyList + gt.RUnlock() + return readyList +} + +// AddTask will add a task which will be triggered periodically over an given interval with given times (-1 means to +// run endlessly), considering that pace has a reasonable size and the logic below won't cost more time than that, +// AddTask won't catch up with the curSlot, so we don't need a mutex. it's also worth noticing that interval can't be +// smaller than the circumference of this time wheel +func (gt *SimpleTimeWheel) AddTask(taskID string, interval time.Duration, times int) (<-chan struct{}, error) { + if interval < time.Duration(gt.perimeter) { + return nil, ErrInvalidTaskInterval + } + if _, exist := gt.record.Load(taskID); exist { + return nil, fmt.Errorf("%s, taskId: %s", ErrTaskAlreadyExist.Error(), taskID) + } + trigger, ok := timeTriggerPool.Get().(*timeTrigger) + if !ok { + return nil, errors.New("not a timeTrigger type") + } + gt.Lock() + curSlot := gt.curSlot + circle := (int64(interval)/int64(gt.pace) + curSlot + 1) / gt.slotNum + circleCount := int64(1) + index := (int64(interval)/int64(gt.pace) + curSlot + 1) % gt.slotNum + if index > curSlot { + circleCount-- + } + trigger.taskID = taskID + trigger.times = times + trigger.circle = circle + trigger.circleCount = circleCount + trigger.index = index + trigger.disable = false + trigger.ch = make(chan struct{}, 1) + trigger.prev = nil + trigger.next = gt.slots[index] + if gt.slots[index] != nil { + gt.slots[index].prev = trigger + } + gt.slots[index] = trigger + gt.Unlock() + gt.record.Store(taskID, trigger) + return trigger.ch, nil +} + +// DelTask will delete a task in SimpleTimeWheel and remove its trigger +func (gt *SimpleTimeWheel) DelTask(taskID string) error { + object, exist := gt.record.Load(taskID) + if !exist { + return nil + } + gt.record.Delete(taskID) + trigger, ok := object.(*timeTrigger) + if !ok { + return errors.New("not a timeTrigger type") + } + // since caller no longer need this task, it's ok that this trigger still fires + trigger.disable = true + gt.removeTrigger(trigger) + timeTriggerPool.Put(trigger) + return nil +} + +// Stop will stop time wheel +func (gt *SimpleTimeWheel) Stop() { + close(gt.stopCh) + close(gt.readyCh) +} + +// removeTrigger won't set trigger's prev and next to nil since checkAndFireTrigger may processing this trigger right +// now and we don't want to lose track of the next trigger +func (gt *SimpleTimeWheel) removeTrigger(trigger *timeTrigger) { + gt.Lock() + defer gt.Unlock() + // special treatment if this trigger is the head of linked list + if trigger.prev == nil { + if trigger.index >= int64(len(gt.slots)) { + fmt.Errorf("trigger.index is out of slots slice") + } else { + gt.slots[trigger.index] = trigger.next + } + if trigger.next != nil { + trigger.next.prev = nil + } + } else { + trigger.prev.next = trigger.next + if trigger.next != nil { + trigger.next.prev = trigger.prev + } + } +} diff --git a/frontend/pkg/common/faas_common/timewheel/simpletimewheel_test.go b/frontend/pkg/common/faas_common/timewheel/simpletimewheel_test.go new file mode 100644 index 0000000000000000000000000000000000000000..d78fdc2ea9a68e4ec1e5545befebd2ffb3adec3c --- /dev/null +++ b/frontend/pkg/common/faas_common/timewheel/simpletimewheel_test.go @@ -0,0 +1,192 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package timewheel - +package timewheel + +import ( + "math" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestSimpleTimeWheelBasic(t *testing.T) { + timeWheel := NewSimpleTimeWheel(5*time.Millisecond, 10) + defer timeWheel.Stop() + time.Sleep(11 * time.Millisecond) + taskName := "TestSimpleTimeWheelBasic_" + "task-1" + ch, err := timeWheel.AddTask(taskName, 500*time.Millisecond, -1) + addTime := time.Now() + if err != nil { + t.Errorf("failed to add task error %s", err) + } + var triggerTime time.Time + select { + case <-time.NewTimer(750 * time.Millisecond).C: + t.Errorf("timeout waiting for timeWheel to trigger after %d", time.Now().Sub(addTime).Milliseconds()) + case <-ch: + triggerTime = time.Now() + interval := int(math.Floor(float64(triggerTime.Sub(addTime).Milliseconds()))) + assert.Equal(t, true, interval >= 450 && interval <= 750) + } + + err = timeWheel.DelTask(taskName) + if err != nil { + t.Errorf("failed to delete task error %s", err) + } + select { + case <-time.NewTimer(200 * time.Millisecond).C: + case <-ch: + t.Errorf("trigger should not fire") + } +} + +func TestSimpleTimeWheel_Wait(t *testing.T) { + readyCh := make(chan struct{}) + readyList := []string{"TestSimpleTimeWheel_Wait_task1", "TestSimpleTimeWheel_Wait_task2"} + + wheel := &SimpleTimeWheel{ + readyCh: readyCh, + readyList: readyList, + } + + go func() { + readyCh <- struct{}{} + }() + + result := wheel.Wait() + assert.Equal(t, readyList, result, "The readyList should be returned") + close(readyCh) + result = wheel.Wait() + assert.Nil(t, result, "The result should be nil when channel is closed") +} + +func TestSimpleTimeWheelCombination(t *testing.T) { + timeWheel := NewSimpleTimeWheel(5*time.Millisecond, 10) + defer timeWheel.Stop() + var ( + err error + task1Ch <-chan struct{} + task2Ch <-chan struct{} + task3Ch <-chan struct{} + task2AddTime time.Time + task3AddTime time.Time + ) + wg := sync.WaitGroup{} + wg.Add(1) + task1Name := "TestSimpleTimeWheelCombination_" + "task-1" + task2Name := "TestSimpleTimeWheelCombination_" + "task-2" + task3Name := "TestSimpleTimeWheelCombination_" + "task-3" + go func() { + task1Ch, err = timeWheel.AddTask(task1Name, time.Duration(500)*time.Millisecond, -1) + if err != nil { + t.Errorf("failed to add task error %s", err) + } + wg.Done() + }() + wg.Add(1) + go func() { + task2Ch, err = timeWheel.AddTask(task2Name, time.Duration(500)*time.Millisecond, -1) + task2AddTime = time.Now() + if err != nil { + t.Errorf("failed to add task error %s", err) + } + wg.Done() + }() + wg.Add(1) + go func() { + task3Ch, err = timeWheel.AddTask(task3Name, time.Duration(500)*time.Millisecond, -1) + task3AddTime = time.Now() + if err != nil { + t.Errorf("failed to add task error %s", err) + } + wg.Done() + }() + wg.Wait() + err = timeWheel.DelTask(task1Name) + if err != nil { + t.Errorf("failed to delete task error %s", err) + } + done := 0 + timer := time.NewTimer(900 * time.Millisecond) + defer timer.Stop() + for done != 2 { + select { + case <-timer.C: + t.Errorf("timeout waiting for timeWheel to trigger") + case <-task1Ch: + t.Errorf("trigger should not fire") + case <-task2Ch: + interval := int(math.Floor(float64(time.Now().Sub(task2AddTime).Milliseconds()))) + if interval < 450 || interval > 800 { + t.Errorf("task2's trigger interval %d is out of range [450, 800]", interval) + } + done++ + case <-task3Ch: + interval := int(math.Floor(float64(time.Now().Sub(task3AddTime).Milliseconds()))) + if interval < 450 || interval > 800 { + t.Errorf("task3's trigger interval %d is out of range [450, 800]", interval) + } + done++ + } + } +} + +func TestSimpleTimeWheel_Stop(t *testing.T) { + timeWheel := NewSimpleTimeWheel(2*time.Millisecond, 10) + timeWheel.Stop() +} + +func TestNewSimpleTimeWheel(t *testing.T) { + timeWheel := NewSimpleTimeWheel(minPace-1, 0) + defer timeWheel.Stop() + assert.NotNil(t, timeWheel) +} + +func TestSimpleTimeWheelBasic1(t *testing.T) { + timeWheel := NewSimpleTimeWheel(10*time.Millisecond, 10) + defer timeWheel.Stop() + time.Sleep(11 * time.Millisecond) + task1Name := "TestSimpleTimeWheelBasic1_" + "task-1" + ch, err := timeWheel.AddTask(task1Name, 1000*time.Millisecond, -1) + addTime := time.Now() + if err != nil { + t.Errorf("failed to add task error %s", err) + } + var triggerTime time.Time + select { + case <-time.NewTimer(10000 * time.Millisecond).C: + t.Errorf("timeout waiting for timeWheel to trigger %s", time.Now().Format(time.RFC3339Nano)) + case <-ch: + triggerTime = time.Now() + interval := int(math.Floor(float64(triggerTime.Sub(addTime).Milliseconds()))) + t.Logf("show invterval %d\n", interval) + assert.Equal(t, true, interval >= 800 && interval <= 1400) + } + + err = timeWheel.DelTask(task1Name) + if err != nil { + t.Errorf("failed to delete task error %s", err) + } + select { + case <-time.NewTimer(1000 * time.Millisecond).C: + case <-ch: + t.Errorf("trigger should not fire") + } +} diff --git a/frontend/pkg/common/faas_common/timewheel/timewheel.go b/frontend/pkg/common/faas_common/timewheel/timewheel.go new file mode 100644 index 0000000000000000000000000000000000000000..5300d9ffdab208781f6c34db76f39ccbb8bd6480 --- /dev/null +++ b/frontend/pkg/common/faas_common/timewheel/timewheel.go @@ -0,0 +1,38 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package timewheel - +package timewheel + +import ( + "errors" + "time" +) + +var ( + // ErrTaskAlreadyExist is the error of task already exist + ErrTaskAlreadyExist = errors.New("task already exist") + // ErrInvalidTaskInterval is the error of invalid interval for TimeWheel task + ErrInvalidTaskInterval = errors.New("interval of task is invalid") +) + +// TimeWheel can trigger tasks periodically by given intervals +type TimeWheel interface { + Wait() []string + AddTask(taskID string, interval time.Duration, times int) (<-chan struct{}, error) + DelTask(taskID string) error + Stop() +} diff --git a/frontend/pkg/common/faas_common/tls/https.go b/frontend/pkg/common/faas_common/tls/https.go new file mode 100644 index 0000000000000000000000000000000000000000..3eacbc32eeac44cd0e62ad01568ff28b0d23d436 --- /dev/null +++ b/frontend/pkg/common/faas_common/tls/https.go @@ -0,0 +1,401 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package tls - +package tls + +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "io/ioutil" + "path/filepath" + "strings" + "sync" + + commonCrypto "frontend/pkg/common/crypto" + "frontend/pkg/common/faas_common/crypto" + "frontend/pkg/common/faas_common/localauth" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/utils" +) + +const urlIndex = 1 + +// HTTPSConfig is for needed HTTPS config +type HTTPSConfig struct { + CipherSuite []uint16 + MinVers uint16 + MaxVers uint16 + CACertFile string + CertFile string + SecretKeyFile string + PwdFilePath string + KeyPassPhase string + SecretName string + DecryptTool string + DisableClientCertVerify bool +} + +// InternalHTTPSConfig is for input config +type InternalHTTPSConfig struct { + HTTPSEnable bool `json:"httpsEnable" yaml:"httpsEnable" valid:"optional"` + TLSProtocol string `json:"tlsProtocol" yaml:"tlsProtocol" valid:"optional"` + TLSCiphers string `json:"tlsCiphers" yaml:"tlsCiphers" valid:"optional"` + SSLBasePath string `json:"sslBasePath" yaml:"sslBasePath" valid:"optional"` + RootCAFile string `json:"rootCAFile" yaml:"rootCAFile" valid:"optional"` + ModuleCertFile string `json:"moduleCertFile" yaml:"moduleCertFile" valid:"optional"` + ModuleKeyFile string `json:"moduleKeyFile" yaml:"moduleKeyFile" valid:"optional"` + PwdFile string `json:"pwdFile" yaml:"pwdFile" valid:"optional"` + SecretName string `json:"secretName" yaml:"secretName" valid:"optional"` + SSLDecryptTool string `json:"sslDecryptTool" yaml:"sslDecryptTool" valid:"optional"` + DisableClientCertVerify bool `json:"disEnableClientCertVerify" yaml:"disEnableClientCertVerify" valid:"optional"` +} + +var ( + // tlsVersionMap is a set of TLS versions + tlsVersionMap = map[string]uint16{ + "TLSv1.2": tls.VersionTLS12, + } + // httpsConfigs is a global variable of HTTPS config + httpsConfigs = &HTTPSConfig{} + // tlsConfig is a global variable of TLS config + tlsConfig *tls.Config + once sync.Once +) + +// GetURLScheme returns "http" or "https" +func GetURLScheme(https bool) string { + if https { + return "https" + } + return "http" +} + +// tlsCipherSuiteMap is a set of supported TLS algorithms +var tlsCipherSuiteMap = map[string]uint16{ + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256": tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384": tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256": tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384": tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +} + +// GetClientTLSConfig - +func GetClientTLSConfig() *tls.Config { + if tlsConfig == nil { + return nil + } + certs := make([]tls.Certificate, len(tlsConfig.Certificates)) + copy(certs, tlsConfig.Certificates) + suits := make([]uint16, len(tlsConfig.CipherSuites)) + copy(suits, tlsConfig.CipherSuites) + newCfg := &tls.Config{ + ClientCAs: tlsConfig.ClientCAs, + Certificates: certs, + CipherSuites: suits, + PreferServerCipherSuites: tlsConfig.PreferServerCipherSuites, + ClientAuth: tlsConfig.ClientAuth, + InsecureSkipVerify: tlsConfig.InsecureSkipVerify, + MinVersion: tlsConfig.MinVersion, + MaxVersion: tlsConfig.MaxVersion, + Renegotiation: tlsConfig.Renegotiation, + } + return newCfg +} + +func loadCerts(path string, filename string) string { + certPath, err := filepath.Abs(filepath.Join(path, filename)) + if err != nil { + log.GetLogger().Errorf("failed to return an absolute representation of filename: %s", filename) + return "" + } + ok := utils.FileExists(certPath) + if !ok { + log.GetLogger().Errorf("failed to load the cert file: %s", certPath) + return "" + } + return certPath +} + +func loadTLSConfig() error { + clientAuthMode := tls.RequireAndVerifyClientCert + if httpsConfigs.DisableClientCertVerify { + clientAuthMode = tls.NoClientCert + } + var pool *x509.CertPool + + pool, err := GetX509CACertPool(httpsConfigs.CACertFile) + if err != nil { + log.GetLogger().Errorf("failed to GetX509CACertPool: %s", err.Error()) + return err + } + + var certs []tls.Certificate + certs, err = LoadServerTLSCertificate(httpsConfigs.CertFile, httpsConfigs.SecretKeyFile, + httpsConfigs.KeyPassPhase, httpsConfigs.DecryptTool, true) + if err != nil { + log.GetLogger().Errorf("failed to loadServerTLSCertificate: %s", err.Error()) + return err + } + + tlsConfig = &tls.Config{ + ClientCAs: pool, + Certificates: certs, + CipherSuites: httpsConfigs.CipherSuite, + PreferServerCipherSuites: true, + ClientAuth: clientAuthMode, + InsecureSkipVerify: true, + MinVersion: httpsConfigs.MinVers, + MaxVersion: httpsConfigs.MaxVers, + Renegotiation: tls.RenegotiateNever, + } + + return nil +} + +// loadHTTPSConfig loads the protocol and ciphers of TLS +func loadHTTPSConfig(config InternalHTTPSConfig) error { + httpsConfigs = &HTTPSConfig{ + MinVers: tls.VersionTLS12, + MaxVers: tls.VersionTLS12, + CipherSuite: nil, + CACertFile: loadCerts(config.SSLBasePath, config.RootCAFile), + CertFile: loadCerts(config.SSLBasePath, config.ModuleCertFile), + SecretKeyFile: loadCerts(config.SSLBasePath, config.ModuleKeyFile), + PwdFilePath: loadCerts(config.SSLBasePath, config.PwdFile), + KeyPassPhase: "", + SecretName: config.SecretName, + DecryptTool: config.SSLDecryptTool, + DisableClientCertVerify: config.DisableClientCertVerify, + } + + minVersion := parseSSLProtocol(config.TLSProtocol) + if httpsConfigs.MinVers == 0 { + return errors.New("invalid TLS protocol") + } + if minVersion == 0 { + minVersion = tls.VersionTLS12 + } + httpsConfigs.MinVers = minVersion + cipherSuites := parseSSLCipherSuites(config.TLSCiphers) + if len(cipherSuites) == 0 { + return errors.New("invalid TLS ciphers") + } + httpsConfigs.CipherSuite = cipherSuites + + keyPassPhase, err := ioutil.ReadFile(httpsConfigs.PwdFilePath) + if err != nil { + log.GetLogger().Errorf("failed to read file cert_pwd: %s", err.Error()) + return err + } + httpsConfigs.KeyPassPhase = string(keyPassPhase) + utils.ClearByteMemory(keyPassPhase) + + return nil +} + +// InitTLSConfig inits config of HTTPS +func InitTLSConfig(config InternalHTTPSConfig) error { + var err error + once.Do(func() { + err = loadHTTPSConfig(config) + if err != nil { + err = fmt.Errorf("failed to load HTTPS config,err %s", err.Error()) + return + } + + err = loadTLSConfig() + if err != nil { + return + } + }) + return err +} + +// GetX509CACertPool generates CACertPool by CA certificate +func GetX509CACertPool(caCertFilePath string) (*x509.CertPool, error) { + pool := x509.NewCertPool() + caCertContent, err := loadCACertBytes(caCertFilePath) + if err != nil { + return nil, err + } + + pool.AppendCertsFromPEM(caCertContent) + return pool, nil + +} + +// LoadServerTLSCertificate generates tls certificate by certfile and keyfile +func LoadServerTLSCertificate(certFile, keyFile, passPhase, decryptTool string, + isHTTPS bool) ([]tls.Certificate, error) { + certContent, keyContent, err := loadCertAndKeyBytes(certFile, keyFile, passPhase, decryptTool, isHTTPS) + utils.ClearStringMemory(passPhase) + utils.ClearStringMemory(httpsConfigs.KeyPassPhase) + if err != nil { + utils.ClearByteMemory(certContent) + utils.ClearByteMemory(keyContent) + return nil, err + } + + cert, err := tls.X509KeyPair(certContent, keyContent) + utils.ClearByteMemory(certContent) + utils.ClearByteMemory(keyContent) + if err != nil { + log.GetLogger().Errorf("failed to load the X509 key pair from cert file with key file: %s", + err.Error()) + return nil, err + } + var certs []tls.Certificate + certs = append(certs, cert) + return certs, nil +} + +func containPassPhase(keyContent []byte, passPhase string, decryptTool string, + isHTTPS bool) (Content []byte, err error) { + if !isHTTPS { + plainkeyContent, err := localauth.Decrypt(string(keyContent)) + if err != nil { + log.GetLogger().Errorf("failed to decrypt keyContent: %s", err.Error()) + return nil, err + } + return plainkeyContent, nil + } + + keyBlock, _ := pem.Decode(keyContent) + if keyBlock == nil { + log.GetLogger().Errorf("failed to decode key file ") + return nil, errors.New("failed to decode key file") + } + + if commonCrypto.IsEncryptedPEMBlock(keyBlock) { + var plainPassPhase []byte + var err error + var decrypted string + if len(passPhase) > 0 { + if decryptTool == "SCC" { + decrypted, err = crypto.SCCDecrypt([]byte(passPhase)) + plainPassPhase = []byte(decrypted) + } else if decryptTool == "LOCAL" { + plainPassPhase, err = localauth.Decrypt(passPhase) + } + if err != nil { + log.GetLogger().Errorf("failed to decrypt the ssl passPhase(%d): %s", len(passPhase), + err.Error()) + return nil, err + } + } + + keyData, err := commonCrypto.DecryptPEMBlock(keyBlock, plainPassPhase) + clearByteMemory(plainPassPhase) + utils.ClearStringMemory(decrypted) + + if err != nil { + log.GetLogger().Errorf("failed to decrypt key file, error: %s", err.Error()) + return nil, err + } + + // The decryption is successful, then the file is re-encoded to a PEM file + plainKeyBlock := &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: keyData, + } + + keyContent = pem.EncodeToMemory(plainKeyBlock) + } + return keyContent, nil + +} + +func loadCertAndKeyBytes(certFilePath, keyFilePath, passPhase string, decryptTool string, isHTTPS bool) ( + certPEMBlock, keyPEMBlock []byte, err error) { + certContent, err := ioutil.ReadFile(certFilePath) + if err != nil { + log.GetLogger().Errorf("failed to read cert file %s: %s", certFilePath, err.Error()) + return nil, nil, err + } + + keyContent, err := ioutil.ReadFile(keyFilePath) + if err != nil { + log.GetLogger().Errorf("failed to read key file %s: %s", keyFilePath, err.Error()) + return nil, nil, err + } + keyContent, err = containPassPhase(keyContent, passPhase, decryptTool, isHTTPS) + if err != nil { + log.GetLogger().Errorf("failed to decode keyContent, error is %s", err.Error()) + return nil, nil, err + } + + return certContent, keyContent, nil + +} + +func clearByteMemory(src []byte) { + for idx := 0; idx < len(src)&32; idx++ { + src[idx] = 0 + } +} + +func loadCACertBytes(caCertFilePath string) ([]byte, error) { + caCertContent, err := ioutil.ReadFile(caCertFilePath) + if err != nil { + log.GetLogger().Errorf("failed to read ca cert file %s, err: %s", caCertFilePath, err.Error()) + return nil, err + } + + return caCertContent, nil +} + +func parseSSLProtocol(rawProtocol string) uint16 { + if protocol, ok := tlsVersionMap[rawProtocol]; ok { + return protocol + } + log.GetLogger().Errorf("invalid SSL version: %s, use the default protocol version", rawProtocol) + return 0 +} + +func parseSSLCipherSuites(ciphers string) []uint16 { + cipherSuiteNameList := strings.Split(ciphers, ",") + if len(cipherSuiteNameList) == 0 { + log.GetLogger().Errorf("input cipher suite is empty") + return nil + } + cipherSuites := make([]uint16, 0, len(cipherSuiteNameList)) + for _, cipherSuiteItem := range cipherSuiteNameList { + cipherSuiteItem = strings.TrimSpace(cipherSuiteItem) + if len(cipherSuiteItem) == 0 { + continue + } + + if cipherSuite, ok := tlsCipherSuiteMap[cipherSuiteItem]; ok { + cipherSuites = append(cipherSuites, cipherSuite) + } else { + log.GetLogger().Errorf("cipher %s does not exist", cipherSuiteItem) + } + } + + return cipherSuites +} + +// ParseURL URL may be: ip:port | http://ip:port | https://ip:port +func ParseURL(rawURL string) string { + urls := strings.Split(rawURL, "//") + if len(urls) > urlIndex { + return urls[urlIndex] + } + return rawURL +} diff --git a/frontend/pkg/common/faas_common/tls/https_test.go b/frontend/pkg/common/faas_common/tls/https_test.go new file mode 100644 index 0000000000000000000000000000000000000000..4a18e3a7c5fa58902633ebd2262a48f8de76a4d4 --- /dev/null +++ b/frontend/pkg/common/faas_common/tls/https_test.go @@ -0,0 +1,256 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package tls + +import ( + "crypto/tls" + "encoding/pem" + "errors" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + + "frontend/pkg/common/crypto" +) + +func TestGetURLScheme(t *testing.T) { + if "https" != GetURLScheme(true) { + t.Error("GetURLScheme failed") + } + if "http" != GetURLScheme(false) { + t.Error("GetURLScheme failed") + } +} + +func TestInitTLSConfig(t *testing.T) { + p := gomonkey.ApplyFunc(ioutil.ReadFile, func(filename string) ([]byte, error) { + return nil, nil + }) + p.ApplyFunc(containPassPhase, func(keyContent []byte, passPhase string, decryptTool string, isHttps bool) (Content []byte, err error) { + return nil, nil + }) + p.ApplyFunc(tls.X509KeyPair, func(certPEMBlock, keyPEMBlock []byte) (tls.Certificate, error) { + var cert tls.Certificate + return cert, nil + }) + defer p.Reset() + os.Setenv("SSL_ROOT", "/home/sn/resource/https") + var config InternalHTTPSConfig + config.TLSProtocol = "TLSv1.2" + config.TLSCiphers = "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, TLS_TEST" + err := InitTLSConfig(config) + assert.Equal(t, nil, err) +} + +func TestGetClientTLSConfig(t *testing.T) { + actual := GetClientTLSConfig() + assert.Equal(t, tlsConfig, actual) +} + +func TestContainPassPhase(t *testing.T) { + convey.Convey("ContainPassPhase", t, func() { + errCtrl := "" + patches := []*gomonkey.Patches{ + gomonkey.ApplyFunc(crypto.DecryptPEMBlock, func(b *pem.Block, password []byte) ([]byte, error) { + if errCtrl == "returnError" { + return nil, errors.New("some error") + } + return nil, nil + }), + } + defer func() { + for idx := range patches { + patches[idx].Reset() + } + }() + convey.Convey("http error case 1", func() { + keyContent := []byte{} + passPhase := "" + isHttps := false + content, err := containPassPhase(keyContent, passPhase, "LOCAL", isHttps) + convey.So(err, convey.ShouldNotBeNil) + convey.So(content, convey.ShouldBeNil) + }) + convey.Convey("https error case 1", func() { + keyContent := []byte{} + passPhase := "" + isHttps := true + content, err := containPassPhase(keyContent, passPhase, "LOCAL", isHttps) + convey.So(err, convey.ShouldNotBeNil) + convey.So(err.Error(), convey.ShouldEqual, "failed to decode key file") + convey.So(content, convey.ShouldBeNil) + }) + convey.Convey("Decrypt error", func() { + keyContent := pem.EncodeToMemory(&pem.Block{ + Type: "MESSAGE", + Headers: map[string]string{"DEK-Info": "test"}, + Bytes: []byte("test containPassPhase")}) + passPhase := "abc" + isHttps := true + errCtrl = "returnError" + content, err := containPassPhase(keyContent, passPhase, "LOCAL", isHttps) + convey.So(err, convey.ShouldNotBeNil) + convey.So(content, convey.ShouldBeNil) + }) + convey.Convey("Decrypt success", func() { + keyContent := pem.EncodeToMemory(&pem.Block{ + Type: "MESSAGE", + Headers: map[string]string{"DEK-Info": "test"}, + Bytes: []byte("test containPassPhase")}) + passPhase := "abc" + isHttps := true + errCtrl = "" + content, err := containPassPhase(keyContent, passPhase, "LOCAL", isHttps) + convey.So(err, convey.ShouldBeNil) + convey.So(content, convey.ShouldNotBeNil) + }) + }) +} + +func TestLoadCerts(t *testing.T) { + convey.Convey("https Load Certs 1", t, func() { + patch := gomonkey.ApplyFunc(os.Getenv, func(key string) string { + return "aaa" + }) + defer patch.Reset() + cert := loadCerts("./test", "trust.cer") + convey.So(cert, convey.ShouldNotBeNil) + }) + convey.Convey("https Load Certs 2", t, func() { + patch := gomonkey.ApplyFunc(os.Getenv, func(key string) string { + return "aaa" + }) + patch2 := gomonkey.ApplyFunc(filepath.Abs, func(path string) (string, error) { + return "a", errors.New("bbb") + }) + defer patch.Reset() + defer patch2.Reset() + cert := loadCerts("1", "trust.cer") + convey.So(cert, convey.ShouldNotBeNil) + }) + +} + +func Test_parseSSLProtocol(t *testing.T) { + convey.Convey("Test_parseSSLProtocol", t, func() { + convey.So(parseSSLProtocol("TLSv1.2"), convey.ShouldEqual, tls.VersionTLS12) + convey.So(parseSSLProtocol("abc"), convey.ShouldEqual, 0) + }) +} + +func Test_parseURL(t *testing.T) { + url := ParseURL("http://test.com") + assert.Equal(t, url, "test.com") + url1 := ParseURL("test.com") + assert.Equal(t, url1, "test.com") +} + +func TestGetClientTLSConfig_Multi(t *testing.T) { + old := tlsConfig + + tlsConfig = &tls.Config{} + + defer func() { + tlsConfig = old + }() + + a := GetClientTLSConfig() + a.CipherSuites = append(a.CipherSuites, 10) + + b := GetClientTLSConfig() + + assert.NotEqual(t, a, b) + assert.NotSame(t, a, b) + assert.Equal(t, 1, len(a.CipherSuites)) + assert.Equal(t, 0, len(b.CipherSuites)) +} + +func TestLoadServerTLSCertificate(t *testing.T) { + readFileCtrl := "" + readFileCtrlCount := 0 + patches := []*gomonkey.Patches{ + gomonkey.ApplyFunc(tls.X509KeyPair, func(certPEMBlock, keyPEMBlock []byte) (tls.Certificate, error) { + return tls.Certificate{}, errors.New("some error") + }), + gomonkey.ApplyFunc(ioutil.ReadFile, func(filename string) ([]byte, error) { + if readFileCtrl == "successOnce" { + if readFileCtrlCount == 0 { + readFileCtrlCount++ + return nil, nil + } + return nil, errors.New("some error") + } + readFileCtrlCount = 0 + return nil, nil + }), + } + defer func() { + for idx := range patches { + patches[idx].Reset() + } + }() + passLiteral := "testPassPhase" + passByteArray := []byte(passLiteral) + passPhase := string(passByteArray) + readFileCtrl = "successOnce" + certs, err := LoadServerTLSCertificate("testCertFile", "testKeyFile", passPhase, "LOCAL", true) + assert.NotNil(t, err) + assert.Empty(t, certs) + certs, err = LoadServerTLSCertificate("testCertFile", "testKeyFile", passPhase, "LOCAL", true) + assert.NotNil(t, err) + assert.Empty(t, certs) + readFileCtrl = "" + certs, err = LoadServerTLSCertificate("testCertFile", "testKeyFile", passPhase, "LOCAL", true) + assert.NotNil(t, err) + assert.Empty(t, certs) +} + +func Test_loadCertAndKeyBytes(t *testing.T) { + patches := [...]*gomonkey.Patches{ + gomonkey.ApplyFunc(ioutil.ReadFile, func(filename string) ([]byte, error) { + return []byte("abc"), nil + }), + gomonkey.ApplyFunc(pem.Decode, func(data []byte) (p *pem.Block, rest []byte) { + return &pem.Block{}, []byte{} + }), + gomonkey.ApplyFunc(crypto.IsEncryptedPEMBlock, func(b *pem.Block) bool { + return true + }), + gomonkey.ApplyFunc(crypto.DecryptPEMBlock, func(b *pem.Block, password []byte) ([]byte, error) { + return []byte{}, nil + }), + gomonkey.ApplyFunc(pem.EncodeToMemory, func(b *pem.Block) []byte { + return []byte("abc") + }), + } + defer func() { + for idx := range patches { + patches[idx].Reset() + } + }() + convey.Convey("loadCertAndKeyBytes", t, func() { + bytes, keyPEMBlock, err := loadCertAndKeyBytes("path1", "path2", "", "", true) + convey.So(err, convey.ShouldBeNil) + convey.So(string(bytes), convey.ShouldEqual, "abc") + convey.So(string(keyPEMBlock), convey.ShouldEqual, "abc") + }) +} diff --git a/frontend/pkg/common/faas_common/tls/option.go b/frontend/pkg/common/faas_common/tls/option.go new file mode 100644 index 0000000000000000000000000000000000000000..fea236898ebeae3eadbf6e8e4b44ce151310edce --- /dev/null +++ b/frontend/pkg/common/faas_common/tls/option.go @@ -0,0 +1,113 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package tls - +package tls + +import ( + "crypto/tls" + "crypto/x509" + + "frontend/pkg/common/faas_common/logger/log" +) + +const ( + // DefaultCAFile is the default file for tls client + DefaultCAFile = "/home/sn/resource/ca/ca.pem" +) + +// NewTLSConfig returns tls.Config with given options +func NewTLSConfig(opts ...Option) *tls.Config { + config := &tls.Config{ + MinVersion: tls.VersionTLS12, + CipherSuites: []uint16{ + // for TLS1.2 + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, + // for TLS1.3 + tls.TLS_AES_128_GCM_SHA256, + tls.TLS_AES_256_GCM_SHA384, + tls.TLS_CHACHA20_POLY1305_SHA256, + }, + PreferServerCipherSuites: true, + Renegotiation: tls.RenegotiateNever, + } + for _, opt := range opts { + opt.apply(config) + } + return config +} + +// Option is optional argument for tls.Config +type Option interface { + apply(*tls.Config) +} + +type rootCAOption struct { + cas *x509.CertPool +} + +func (r *rootCAOption) apply(config *tls.Config) { + config.RootCAs = r.cas +} + +// WithRootCAs returns Option that applies root CAs to tls.Config +func WithRootCAs(caFiles ...string) Option { + rootCAs, err := LoadRootCAs(caFiles...) + if err != nil { + log.GetLogger().Warnf("failed to load root ca, err: %s", err.Error()) + rootCAs = nil + } + return &rootCAOption{ + cas: rootCAs, + } +} + +type certsOption struct { + certs []tls.Certificate +} + +func (c *certsOption) apply(config *tls.Config) { + config.Certificates = c.certs +} + +// WithCerts returns Option that applies cert file and key file to tls.Config +func WithCerts(certFile, keyFile string) Option { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + log.GetLogger().Warnf("load cert.pem and key.pem error: %s", err) + cert = tls.Certificate{} + } + return &certsOption{ + certs: []tls.Certificate{cert}, + } +} + +type skipVerifyOption struct { +} + +func (s *skipVerifyOption) apply(config *tls.Config) { + config.InsecureSkipVerify = true +} + +// WithSkipVerify returns Option that skips to verify certificates +func WithSkipVerify() Option { + return &skipVerifyOption{} +} diff --git a/frontend/pkg/common/faas_common/tls/option_test.go b/frontend/pkg/common/faas_common/tls/option_test.go new file mode 100644 index 0000000000000000000000000000000000000000..3cf0a6483ae430775a764ea2f59ee7c41f9b3a77 --- /dev/null +++ b/frontend/pkg/common/faas_common/tls/option_test.go @@ -0,0 +1,146 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package tls + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "io/ioutil" + "net/http" + "os" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type TestSuite struct { + suite.Suite + server http.Server + rootKEY string + rootPEM string + rootSRL string + serverKEY string + serverPEM string + serverCSR string +} + +func (s *TestSuite) SetupSuite() { + certificatePath, err := os.Getwd() + if err != nil { + s.T().Errorf("failed to get current working dictionary: %s", err.Error()) + return + } + + certificatePath += "/../../../test/" + s.rootKEY = certificatePath + "ca.key" + s.rootPEM = certificatePath + "ca.crt" + s.rootSRL = certificatePath + "ca.srl" + s.serverKEY = certificatePath + "server.key" + s.serverPEM = certificatePath + "server.crt" + s.serverCSR = certificatePath + "server.csr" + + body := "Hello" + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "%s", body) + }) + + s.server = http.Server{ + Addr: "127.0.0.1:6061", + Handler: handler, + } +} + +func (s *TestSuite) TearDownSuite() { + s.server.Shutdown(context.Background()) + + os.Remove(s.serverKEY) + os.Remove(s.serverPEM) + os.Remove(s.serverCSR) + os.Remove(s.rootKEY) + os.Remove(s.rootPEM) + os.Remove(s.rootSRL) +} + +func TestOptionTestSuite(t *testing.T) { + suite.Run(t, new(TestSuite)) +} + +func TestVerifyCert(t *testing.T) { + var raw [][]byte + tlsConfig = &tls.Config{} + tlsConfig.ClientCAs = x509.NewCertPool() + err := VerifyCert(raw, nil) + assert.NotNil(t, err) + + raw = [][]byte{ + []byte("0"), + []byte("1"), + } + err = VerifyCert(raw, nil) + assert.NotNil(t, err) +} + +func TestNewTLSConfig(t *testing.T) { + defaultCertFile := "/home/sn/resource/secret/cert.pem" + defaultKeyFile := "/home/sn/resource/secret/key.pem" + p := gomonkey.ApplyFunc(ioutil.ReadFile, func(filename string) ([]byte, error) { + return nil, nil + }) + + p.ApplyFunc(tls.LoadX509KeyPair, func(certFile, keyFile string) (tls.Certificate, error) { + return tls.Certificate{}, nil + }) + defer p.Reset() + actual := NewTLSConfig(WithRootCAs(DefaultCAFile), + WithCerts(defaultCertFile, defaultKeyFile), WithSkipVerify()) + expect := &tls.Config{ + MinVersion: tls.VersionTLS12, + CipherSuites: []uint16{ + // for TLS1.2 + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, + // for TLS1.3 + tls.TLS_AES_128_GCM_SHA256, + tls.TLS_AES_256_GCM_SHA384, + tls.TLS_CHACHA20_POLY1305_SHA256, + }, + PreferServerCipherSuites: true, + Renegotiation: tls.RenegotiateNever, + InsecureSkipVerify: true, + RootCAs: nil, + Certificates: []tls.Certificate{{}}, + } + + assert.Equal(t, expect, actual) +} + +func TestWithCerts(t *testing.T) { + defer gomonkey.ApplyFunc(tls.LoadX509KeyPair, func(certFile, keyFile string) (tls.Certificate, error) { + return tls.Certificate{}, errors.New("LoadX509KeyPair error") + }).Reset() + certs := WithCerts("", "") + option := certs.(*certsOption) + assert.Nil(t, option.certs[0].Certificate) +} diff --git a/frontend/pkg/common/faas_common/tls/tls.go b/frontend/pkg/common/faas_common/tls/tls.go new file mode 100644 index 0000000000000000000000000000000000000000..a029adf810740451d5ce3a829bf04388b2eaf79d --- /dev/null +++ b/frontend/pkg/common/faas_common/tls/tls.go @@ -0,0 +1,74 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package tls provides tls utils +package tls + +import ( + "crypto/x509" + "errors" + "io/ioutil" + "time" + + "frontend/pkg/common/faas_common/logger/log" +) + +// LoadRootCAs returns system cert pool with caFiles added +func LoadRootCAs(caFiles ...string) (*x509.CertPool, error) { + rootCAs, err := x509.SystemCertPool() + if err != nil { + return nil, err + } + for _, file := range caFiles { + cert, err := ioutil.ReadFile(file) + if err != nil { + return nil, err + } + if !rootCAs.AppendCertsFromPEM(cert) { + return nil, err + } + } + return rootCAs, nil +} + +// VerifyCert Used to verity the server certificate +func VerifyCert(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { + certs := make([]*x509.Certificate, len(rawCerts)) + if len(certs) == 0 { + log.GetLogger().Errorf("cert number is 0") + return errors.New("cert number is 0") + } + opts := x509.VerifyOptions{ + Roots: tlsConfig.ClientCAs, + CurrentTime: time.Now(), + DNSName: "", + Intermediates: x509.NewCertPool(), + } + for i, asn1Data := range rawCerts { + cert, err := x509.ParseCertificate(asn1Data) + if err != nil { + log.GetLogger().Errorf("failed to parse certificate from server: %s", err.Error()) + return err + } + certs[i] = cert + if i == 0 { + continue + } + opts.Intermediates.AddCert(cert) + } + _, err := certs[0].Verify(opts) + return err +} diff --git a/frontend/pkg/common/faas_common/tls/tls_test.go b/frontend/pkg/common/faas_common/tls/tls_test.go new file mode 100644 index 0000000000000000000000000000000000000000..e6329ee36d90e7e40f609643f18dbc7d9e04a937 --- /dev/null +++ b/frontend/pkg/common/faas_common/tls/tls_test.go @@ -0,0 +1,79 @@ +package tls + +import ( + "crypto/x509" + "io/ioutil" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/require" +) + +// TestLoadRootCAs is used to test the root certificate loading error. +func TestLoadRootCAs(t *testing.T) { + convey.Convey("LoadRootCAs", t, func() { + convey.Convey("error case 1", func() { + caFiles := "" + _, err := LoadRootCAs(caFiles) + convey.So(err, convey.ShouldNotBeNil) + }) + + convey.Convey("error case 2", func() { + dir, err := ioutil.TempDir("", "*") + require.NoError(t, err) + cryptoFile, err := ioutil.TempFile(dir, "crypto") + require.NoError(t, err) + _, err = LoadRootCAs(cryptoFile.Name()) + convey.So(err, convey.ShouldBeNil) + }) + + convey.Convey("error case 3", func() { + dir, err := ioutil.TempDir("", "*") + require.NoError(t, err) + cryptoFile, err := ioutil.TempFile(dir, "test") + require.NoError(t, err) + err = ioutil.WriteFile(filepath.Join(dir, "test"), []byte("a"), os.ModePerm) + require.NoError(t, err) + _, err = LoadRootCAs(cryptoFile.Name()) + convey.So(err, convey.ShouldBeNil) + }) + }) +} + +// TestVerifyCert2 is used to test certificate modification errors. +func TestVerifyCert2(t *testing.T) { + convey.Convey("VerifyCert", t, func() { + convey.Convey("error case 1", func() { + rawCerts := [][]byte{} + verifiedChains := [][]*x509.Certificate{} + err := VerifyCert(rawCerts, verifiedChains) + convey.So(err, convey.ShouldNotBeNil) + convey.So(err.Error(), convey.ShouldEqual, "cert number is 0") + }) + + convey.Convey("error case 2", func() { + rawCerts := [][]byte{[]byte("test1"), []byte("test2")} + verifiedChains := [][]*x509.Certificate{} + err := VerifyCert(rawCerts, verifiedChains) + convey.So(err, convey.ShouldNotBeNil) + }) + + convey.Convey("success", func() { + defer gomonkey.ApplyFunc(x509.ParseCertificate, func(der []byte) (*x509.Certificate, error) { + return &x509.Certificate{}, nil + }).Reset() + defer gomonkey.ApplyMethod(reflect.TypeOf(&x509.Certificate{}), "Verify", + func(_ *x509.Certificate, opts x509.VerifyOptions) (chains [][]*x509.Certificate, err error) { + return nil, nil + }).Reset() + rawCerts := [][]byte{[]byte("test1"), []byte("test2")} + verifiedChains := [][]*x509.Certificate{} + err := VerifyCert(rawCerts, verifiedChains) + convey.So(err, convey.ShouldBeNil) + }) + }) +} diff --git a/frontend/pkg/common/faas_common/tracer/tracer.go b/frontend/pkg/common/faas_common/tracer/tracer.go new file mode 100644 index 0000000000000000000000000000000000000000..1b2d1d885ad5c092c4db3eee14901bb4537cc697 --- /dev/null +++ b/frontend/pkg/common/faas_common/tracer/tracer.go @@ -0,0 +1,175 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package tracer for init trace provider +package tracer + +import ( + "context" + "fmt" + "os" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/semconv/v1.4.0" + "google.golang.org/grpc" + + "frontend/pkg/common/faas_common/logger/log" +) + +const ( + // OtelGRPCEndpointEnvKey - + OtelGRPCEndpointEnvKey = "OTEL_GRPC_ENDPOINT" + // OtelGRPCTokenEnvKey - + OtelGRPCTokenEnvKey = "OTEL_GRPC_TOKEN" + // OtelServiceNameEnvKey - + OtelServiceNameEnvKey = "OTEL_SERVICE_NAME" + // OtelEnableSampleEnvKey - + OtelEnableSampleEnvKey = "OTEL_ENABLE_SAMPLE" +) + +var ( + hostIP = os.Getenv("HOST_IP") + hostName = os.Getenv("HOSTNAME") + otelGRPCEndpoint = os.Getenv(OtelGRPCEndpointEnvKey) + otelGRPCToken = os.Getenv(OtelGRPCTokenEnvKey) + otelServiceName = os.Getenv(OtelServiceNameEnvKey) + enableOTELTracer = os.Getenv(OtelEnableSampleEnvKey) == "true" +) + +// GetOtelGRPCEndpoint - +func GetOtelGRPCEndpoint() string { + return otelGRPCEndpoint +} + +// GetOtelGRPCToken - +func GetOtelGRPCToken() string { + return otelGRPCToken +} + +// GetOtelServiceName - +func GetOtelServiceName() string { + return otelServiceName +} + +// EnableOTELTracer - +func EnableOTELTracer() bool { + return enableOTELTracer +} + +// EnableCommonTracer - +func EnableCommonTracer() bool { + return enableOTELTracer && otelGRPCEndpoint != "" +} + +// InitCommonTracer init common tracer with service name +func InitCommonTracer(shutdown func(), serviceName string) { + var err error + shutdown, err = InitProvider(context.Background()) + if err != nil { + fmt.Printf("failed to init %s trace provider with error %s\n", serviceName, err.Error()) + log.GetLogger().Warnf("failed to init %s trace provider with error %s", serviceName, err.Error()) + return + } +} + +// InitProvider init provider for trace http request +func InitProvider(ctx context.Context) (func(), error) { + if !EnableCommonTracer() { + fmt.Println("otel tracer env is empty with ", hostName, otelGRPCEndpoint) + log.GetLogger().Warnf("otel tracer env is empty with %s, %s", hostName, otelGRPCEndpoint) + return func() {}, nil + } + start := time.Now() + fmt.Println("start to init provider for otel tracer with ", otelGRPCEndpoint) + log.GetLogger().Infof("start to init provider for otel tracer with %s", otelGRPCEndpoint) + traceExporter, err := makeTracerExporter(ctx) + if err != nil { + return func() {}, err + } + tracerProvider, err := makeTraceProvider(ctx, traceExporter) + if err != nil { + return func() {}, err + } + + // set global propagator to tracecontext (the default is no-op). + otel.SetTextMapPropagator(propagation.TraceContext{}) + otel.SetTracerProvider(tracerProvider) + fmt.Println("succeed to init provider for ", otelGRPCEndpoint, otelServiceName, time.Since(start).String()) + log.GetLogger().Infof("succeed to init provider for %s with %s cost %s", + otelGRPCEndpoint, otelServiceName, time.Since(start).String()) + + return func() { + cxt, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + if err := traceExporter.Shutdown(cxt); err != nil { + otel.Handle(err) + } + }, nil +} + +func makeTracerExporter(ctx context.Context) (*otlptrace.Exporter, error) { + headers := map[string]string{} + if otelGRPCToken != "" { + headers = map[string]string{"Authentication": otelGRPCToken} + } + traceGRPCClient := otlptracegrpc.NewClient( + otlptracegrpc.WithInsecure(), + otlptracegrpc.WithEndpoint(otelGRPCEndpoint), + otlptracegrpc.WithHeaders(headers), + otlptracegrpc.WithDialOption(grpc.WithBlock())) + + traceExporter, err := otlptrace.New(ctx, traceGRPCClient) + if err != nil { + log.GetLogger().Warnf("failed to create the collector trace exporter with %s", err.Error()) + return nil, err + } + return traceExporter, nil +} + +func makeTraceProvider(ctx context.Context, traceExporter *otlptrace.Exporter) (*trace.TracerProvider, error) { + res, err := resource.New(ctx, + resource.WithProcess(), + resource.WithTelemetrySDK(), + resource.WithHost(), + resource.WithAttributes( + semconv.ServiceNameKey.String(otelServiceName), + semconv.HostNameKey.String(hostName), + semconv.NetHostIPKey.String(hostIP), + ), + ) + if err != nil { + log.GetLogger().Warnf("failed to create otel resource with %s", err.Error()) + return nil, err + } + + bsp := trace.NewBatchSpanProcessor(traceExporter) + sample := trace.NeverSample() + if enableOTELTracer { + sample = trace.AlwaysSample() + } + tracerProvider := trace.NewTracerProvider( + trace.WithSampler(sample), + trace.WithResource(res), + trace.WithSpanProcessor(bsp), + ) + return tracerProvider, nil +} diff --git a/frontend/pkg/common/faas_common/tracer/tracer_test.go b/frontend/pkg/common/faas_common/tracer/tracer_test.go new file mode 100644 index 0000000000000000000000000000000000000000..47c2f8fc54f4854d9531df31329f4a9946d6143d --- /dev/null +++ b/frontend/pkg/common/faas_common/tracer/tracer_test.go @@ -0,0 +1,146 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package tracer for init trace provider +package tracer + +import ( + "context" + "errors" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/sdk/resource" + + mockUtils "frontend/pkg/common/faas_common/utils" +) + +func TestInitProvider(t *testing.T) { + type args struct { + ctx context.Context + } + otelGRPCEndpoint = "mockOtelGRPCEndpoint" + otelGRPCToken = "mockOtelGRPCToken" + otelServiceName = "mockOtelServiceName" + enableOTELTracer = true + tests := []struct { + name string + args args + patchesFunc mockUtils.PatchesFunc + wantErr bool + }{ + { + name: "test success", + args: args{ + ctx: context.Background(), + }, + patchesFunc: func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + gomonkey.ApplyFunc(otlptrace.New, func(ctx context.Context, + client otlptrace.Client) (*otlptrace.Exporter, error) { + return &otlptrace.Exporter{}, nil + }) + return patches + }, + wantErr: false, + }, // test success + { + name: "test error when new client", + args: args{ + ctx: context.Background(), + }, + patchesFunc: func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + gomonkey.ApplyFunc(otlptrace.New, func(ctx context.Context, + client otlptrace.Client) (*otlptrace.Exporter, error) { + return nil, errors.New("mock new client error") + }) + return patches + }, + wantErr: true, + }, // test error when new client + { + name: "test success", + args: args{ + ctx: context.Background(), + }, + patchesFunc: func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + gomonkey.ApplyFunc(otlptrace.New, func(ctx context.Context, + client otlptrace.Client) (*otlptrace.Exporter, error) { + return &otlptrace.Exporter{}, nil + }) + gomonkey.ApplyFunc(resource.New, func(ctx context.Context, + opts ...resource.Option) (*resource.Resource, error) { + return nil, errors.New("mock resource new error") + }) + return patches + }, + wantErr: true, + }, // test success + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + patches := tt.patchesFunc() + defer patches.ResetAll() + _, err := InitProvider(tt.args.ctx) + if (err != nil) != tt.wantErr { + t.Errorf("InitProvider() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} + +func TestInitCommonTracer(t *testing.T) { + type args struct { + shutdown func() + serviceName string + } + isMocked := false + tests := []struct { + name string + args args + patchesFunc mockUtils.PatchesFunc + isMocked bool + }{ + { + name: "test with error", + args: args{}, + patchesFunc: func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + gomonkey.ApplyFunc(InitProvider, func(ctx context.Context) (func(), error) { + isMocked = true + return nil, errors.New("mockInitProviderError") + }) + return patches + }, + isMocked: true, + }, + } + for _, tt := range tests { + isMocked = false + patches := tt.patchesFunc() + defer patches.ResetAll() + t.Run(tt.name, func(t *testing.T) { + InitCommonTracer(tt.args.shutdown, tt.args.serviceName) + }) + if tt.isMocked != isMocked { + t.Errorf("expect %v but found %v", tt.isMocked, isMocked) + } + } +} diff --git a/frontend/pkg/common/faas_common/tracer/wrap.go b/frontend/pkg/common/faas_common/tracer/wrap.go new file mode 100644 index 0000000000000000000000000000000000000000..dab73d8f6f405286f1deded8dce33583989c9227 --- /dev/null +++ b/frontend/pkg/common/faas_common/tracer/wrap.go @@ -0,0 +1,96 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package tracer for gin and fast http +package tracer + +import ( + "context" + + "github.com/gin-gonic/gin" + "github.com/valyala/fasthttp" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" + + "frontend/pkg/common/faas_common/constant" +) + +// WrapGinHandler wrap gin handler +func WrapGinHandler(originHandlerFunc func(c *gin.Context)) func(c *gin.Context) { + return func(c *gin.Context) { + if EnableCommonTracer() { + path := c.Request.URL.Path + tracerName := otelServiceName + method := c.Request.Method + tr := otel.Tracer(tracerName) + traceParent := c.Request.Header.Get(constant.HeaderTraceParent) + carrier := propagation.HeaderCarrier{} + carrier.Set(constant.HeaderTraceParent, traceParent) + propagattor := otel.GetTextMapPropagator() + // use request context as parent context for current parent span + pCtx := propagattor.Extract(c.Request.Context(), carrier) + + childCtx, span := tr.Start(pCtx, path, trace.WithSpanKind(trace.SpanKindServer)) + span.SetAttributes(attribute.Key("http.target").String(path)) + span.SetAttributes(attribute.Key("http.method").String(method)) + span.SetAttributes(attribute.Key("http.requestID").String(c.Request.Header.Get(constant.HeaderRequestID))) + span.SetAttributes(attribute.Key("http.traceID").String(c.Request.Header.Get(constant.HeaderTraceID))) + defer span.End() + // set child ctx to request header by carrier + c.Request = c.Request.WithContext(childCtx) + childCarrier := propagation.HeaderCarrier{} + propagattor.Inject(childCtx, childCarrier) + c.Request.Header.Set(constant.HeaderTraceParent, childCarrier.Get(constant.HeaderTraceParent)) + } + // call origin handler function + originHandlerFunc(c) + } +} + +// WrapFastHTTPHandler wrap fast http handler +func WrapFastHTTPHandler(originHandlerFunc func(ctx *fasthttp.RequestCtx)) func(ctx *fasthttp.RequestCtx) { + return func(ctx *fasthttp.RequestCtx) { + if EnableCommonTracer() { + path := ctx.Request.URI().String() + tracerName := otelServiceName + method := string(ctx.Method()) + tr := otel.Tracer(tracerName) + traceParent := string(ctx.Request.Header.Peek(constant.HeaderTraceParent)) + carrier := propagation.HeaderCarrier{} + carrier.Set(constant.HeaderTraceParent, traceParent) + propagattor := otel.GetTextMapPropagator() + // use request context as parent context for current parent span + pCtx := propagattor.Extract(context.Background(), carrier) + + childCtx, span := tr.Start(pCtx, path, trace.WithSpanKind(trace.SpanKindServer)) + span.SetAttributes(attribute.Key("http.target").String(path)) + span.SetAttributes(attribute.Key("http.method").String(method)) + requestID := string(ctx.Request.Header.Peek(constant.HeaderRequestID)) + traceID := string(ctx.Request.Header.Peek(constant.HeaderTraceID)) + span.SetAttributes(attribute.Key("http.requestID").String(requestID)) + span.SetAttributes(attribute.Key("http.traceID").String(traceID)) + defer span.End() + // set child ctx to request header by carrier + childCarrier := propagation.HeaderCarrier{} + propagattor.Inject(childCtx, childCarrier) + ctx.Request.Header.Set(constant.HeaderTraceParent, childCarrier.Get(constant.HeaderTraceParent)) + } + // call origin handler function + originHandlerFunc(ctx) + } +} diff --git a/frontend/pkg/common/faas_common/tracer/wrap_test.go b/frontend/pkg/common/faas_common/tracer/wrap_test.go new file mode 100644 index 0000000000000000000000000000000000000000..80477fbfaf6e14e56a732b899642033722e4dfc6 --- /dev/null +++ b/frontend/pkg/common/faas_common/tracer/wrap_test.go @@ -0,0 +1,134 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package tracer for init trace provider +package tracer + +import ( + "fmt" + "net/http" + "net/url" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/gin-gonic/gin" + "github.com/valyala/fasthttp" + + mockUtils "frontend/pkg/common/faas_common/utils" +) + +func TestWrapGinHandler(t *testing.T) { + type args struct { + originHandlerFunc func(c *gin.Context) + } + actualMocked := false + tests := []struct { + name string + args args + patchesFunc mockUtils.PatchesFunc + expectMocked bool + }{ + { + name: "test success", + args: args{ + originHandlerFunc: func(c *gin.Context) { + fmt.Println("mock gin origin handler func") + }, + }, + patchesFunc: func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + gomonkey.ApplyFunc(EnableCommonTracer, func() bool { + actualMocked = true + return true + }) + return patches + }, + expectMocked: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actualMocked = false + patches := tt.patchesFunc() + defer patches.ResetAll() + handlerFunc := WrapGinHandler(tt.args.originHandlerFunc) + if handlerFunc == nil { + t.Errorf("expect handler func is not nil") + return + } + handlerFunc(&gin.Context{ + Request: &http.Request{ + URL: &url.URL{ + Path: "mockURLPath", + }, + Header: make(http.Header), + }, + }) + if actualMocked != tt.expectMocked { + t.Errorf("expect %v but found %v", tt.expectMocked, actualMocked) + return + } + }) + } +} + +func TestWrapFastHTTPHandler(t *testing.T) { + type args struct { + originHandlerFunc func(ctx *fasthttp.RequestCtx) + } + actualMocked := false + tests := []struct { + name string + args args + patchesFunc mockUtils.PatchesFunc + expectMocked bool + }{ + { + name: "test success", + args: args{ + originHandlerFunc: func(ctx *fasthttp.RequestCtx) { + fmt.Println("mock fast http origin handler func") + }, + }, + patchesFunc: func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + gomonkey.ApplyFunc(EnableCommonTracer, func() bool { + actualMocked = true + return true + }) + return patches + }, + expectMocked: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actualMocked = false + patches := tt.patchesFunc() + defer patches.ResetAll() + handlerFunc := WrapFastHTTPHandler(tt.args.originHandlerFunc) + if handlerFunc == nil { + t.Errorf("expect handler func is not nil") + return + } + handlerFunc(&fasthttp.RequestCtx{}) + if actualMocked != tt.expectMocked { + t.Errorf("expect %v but found %v", tt.expectMocked, actualMocked) + return + } + }) + } +} diff --git a/frontend/pkg/common/faas_common/trafficlimit/trafficlimit.go b/frontend/pkg/common/faas_common/trafficlimit/trafficlimit.go new file mode 100644 index 0000000000000000000000000000000000000000..545610494bb20bdc14760e8020c2110d0a7d9d12 --- /dev/null +++ b/frontend/pkg/common/faas_common/trafficlimit/trafficlimit.go @@ -0,0 +1,107 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package trafficlimit - +package trafficlimit + +import ( + "math" + "sync" + + "golang.org/x/time/rate" +) + +const ( + // DefaultFunctionLimitRate default function limit rate for traffic limitation + DefaultFunctionLimitRate = 5000 + // TrafficRedundantRate limit redundancy rate for traffic limitation + TrafficRedundantRate = 1.1 + // DefaultAccessorInitCopies Initial number of copies + DefaultAccessorInitCopies = 3 +) + +var functionLimitRate int + +// LimiterContainer function key and function's Limiter +type LimiterContainer struct { + funcLimiterMap *sync.Map +} + +// FunctionLimiter - +type FunctionLimiter struct { + Quota int + Limiter *rate.Limiter +} + +var ( + // FunctionBuf - + funcLimiterContainer = &LimiterContainer{ + funcLimiterMap: &sync.Map{}, + } +) + +// RateLimiter rate limiter struct +type RateLimiter struct { + *rate.Limiter +} + +// Take return if a function request is allowed +func (r *RateLimiter) take() bool { + return r.Limiter.Allow() +} + +// SetFunctionLimitRate - +func SetFunctionLimitRate(limit int) { + if limit <= 0 { + limit = DefaultFunctionLimitRate + } + functionLimitRate = limit +} + +// FuncTrafficLimit is the main function of function traffic limitation +func FuncTrafficLimit(funcKey string) bool { + return funcLimiterContainer.funcTakeOneToken(funcKey) +} + +func (t *LimiterContainer) funcTakeOneToken(funcKey string) bool { + funcLimiter := t.getFunctionLimiter(funcKey) + if funcLimiter.Limiter == nil { + return true + } + return funcLimiter.Limiter.Allow() +} + +// getFunctionInfo to generator the function limiter +func (t *LimiterContainer) getFunctionLimiter(functionKey string) FunctionLimiter { + funcLimiter, ok := t.funcLimiterMap.Load(functionKey) + if !ok { + if functionLimitRate <= 0 { + functionLimitRate = DefaultFunctionLimitRate + } + limiter := FunctionLimiter{Limiter: t.getLimiter(functionLimitRate), Quota: DefaultFunctionLimitRate} + t.funcLimiterMap.Store(functionKey, limiter) + return limiter + } + return funcLimiter.(FunctionLimiter) +} + +func (t *LimiterContainer) getLimiter(quota int) *rate.Limiter { + limitRate := float64(quota) / DefaultAccessorInitCopies + limitBucketSize := int(math.Ceil(float64(quota)) / + DefaultAccessorInitCopies * TrafficRedundantRate) + tenantLimiter := rate.NewLimiter(rate.Limit(limitRate), limitBucketSize) + return tenantLimiter +} diff --git a/frontend/pkg/common/faas_common/trafficlimit/trafficlimit_test.go b/frontend/pkg/common/faas_common/trafficlimit/trafficlimit_test.go new file mode 100644 index 0000000000000000000000000000000000000000..41f9da3b886d5b72ac48dfe9844bc0cdf8956316 --- /dev/null +++ b/frontend/pkg/common/faas_common/trafficlimit/trafficlimit_test.go @@ -0,0 +1,42 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package trafficlimit - +package trafficlimit + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestFuncTrafficLimit(t *testing.T) { + SetFunctionLimitRate(0) + functionName := "funcTest1" + allow := FuncTrafficLimit(functionName) + assert.Equal(t, allow, true) + + for i := 0; i < DefaultFunctionLimitRate; i++ { + FuncTrafficLimit(functionName) + } + allow = FuncTrafficLimit(functionName) + assert.Equal(t, allow, false) + + time.Sleep(5 * time.Second) + allow = FuncTrafficLimit(functionName) + assert.Equal(t, allow, true) +} diff --git a/frontend/pkg/common/faas_common/trietree/trietree.go b/frontend/pkg/common/faas_common/trietree/trietree.go new file mode 100644 index 0000000000000000000000000000000000000000..3ffb5c3a2b302560b8da0e2cb1595af2fcc5d08a --- /dev/null +++ b/frontend/pkg/common/faas_common/trietree/trietree.go @@ -0,0 +1,129 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package trietree is for Prefix Matching +package trietree + +import ( + "strings" + "sync" + + "frontend/pkg/common/faas_common/constant" +) + +// Trie - +type Trie struct { + root *trieNode + sync.RWMutex +} + +type trieNode struct { + children map[string]*trieNode + isEnd bool +} + +// NewTrie - +func NewTrie() *Trie { + return &Trie{ + root: &trieNode{ + children: make(map[string]*trieNode), + }, + } +} + +// Insert - +func (t *Trie) Insert(word []string) { + t.Lock() + defer t.Unlock() + node := t.root + for _, char := range word { + if _, ok := node.children[char]; !ok { + node.children[char] = &trieNode{ + children: make(map[string]*trieNode), + } + } + node = node.children[char] + } + node.isEnd = true +} + +// Search - +func (t *Trie) Search(word []string) bool { + t.RLock() + defer t.RUnlock() + node := t.root + for _, char := range word { + if _, ok := node.children[char]; !ok { + return false + } + node = node.children[char] + } + return node.isEnd +} + +// Delete - +func (t *Trie) Delete(word []string) { + t.Lock() + t.delete(t.root, word, 0) + t.Unlock() +} + +func (t *Trie) delete(node *trieNode, word []string, depth int) bool { + if depth == len(word) { + if !node.isEnd { + return false + } + node.isEnd = false + // 如果删除后节点没有子节点了，可以删除这个节点 + return len(node.children) == 0 + } + + char := word[depth] + childNode, ok := node.children[char] + if !ok { + return false + } + + shouldDeleteChild := t.delete(childNode, word, depth+1) + if shouldDeleteChild { + delete(node.children, char) + // 删除子节点后如果当前节点也没有其他子节点了，并且不是其他单词的结尾，可以删除当前节点 + return !node.isEnd && len(node.children) == 0 + } + + return false +} + +// LongestMatch - +func (t *Trie) LongestMatch(s []string) string { + t.RLock() + defer t.RUnlock() + node := t.root + longestMatch := "" + var currentMatch []string + for _, char := range s { + if child, ok := node.children[char]; ok { + currentMatch = append(currentMatch, char) + node = child + if node.isEnd { + longestMatch = strings.Join(currentMatch, constant.URLSeparator) + } + } else { + break + } + } + return longestMatch +} diff --git a/frontend/pkg/common/faas_common/trietree/trietree_test.go b/frontend/pkg/common/faas_common/trietree/trietree_test.go new file mode 100644 index 0000000000000000000000000000000000000000..0d6cb62a2dbaf40e76c1730a64291ce305b60e93 --- /dev/null +++ b/frontend/pkg/common/faas_common/trietree/trietree_test.go @@ -0,0 +1,69 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package trietree + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "frontend/pkg/common/faas_common/constant" +) + +func TestTrie(t *testing.T) { + s1 := "/hello" + s2 := "/hello/server" + prefixTrie := NewTrie() + prefixTrie.Insert(strings.Split(s1, constant.URLSeparator)) + prefixTrie.Insert(strings.Split(s2, constant.URLSeparator)) + + input := "/hello" + longestMatch := prefixTrie.LongestMatch(strings.Split(input, constant.URLSeparator)) + assert.Equal(t, longestMatch, "/hello") + + input = "/helloe" + longestMatch = prefixTrie.LongestMatch(strings.Split(input, constant.URLSeparator)) + assert.Equal(t, longestMatch, "") + + input = "/hello/se" + longestMatch = prefixTrie.LongestMatch(strings.Split(input, constant.URLSeparator)) + assert.Equal(t, longestMatch, "/hello") + + input = "/hello/server" + longestMatch = prefixTrie.LongestMatch(strings.Split(input, constant.URLSeparator)) + assert.Equal(t, longestMatch, "/hello/server") + + input = "/hello/server/aaa" + longestMatch = prefixTrie.LongestMatch(strings.Split(input, constant.URLSeparator)) + assert.Equal(t, longestMatch, "/hello/server") + + prefixTrie.Delete(strings.Split(s2, constant.URLSeparator)) + + input = "/hello/server" + longestMatch = prefixTrie.LongestMatch(strings.Split(input, constant.URLSeparator)) + assert.Equal(t, longestMatch, "/hello") + + prefixTrie.Delete(strings.Split(s1, constant.URLSeparator)) + + ok := prefixTrie.Search(strings.Split(s1, constant.URLSeparator)) + assert.False(t, ok) + + input = "/hello" + longestMatch = prefixTrie.LongestMatch(strings.Split(input, constant.URLSeparator)) + assert.Equal(t, longestMatch, "") +} diff --git a/frontend/pkg/common/faas_common/types/serve.go b/frontend/pkg/common/faas_common/types/serve.go new file mode 100644 index 0000000000000000000000000000000000000000..96047dd170ed9121dcdd9be9e0d267a10832e9f9 --- /dev/null +++ b/frontend/pkg/common/faas_common/types/serve.go @@ -0,0 +1,236 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package types + +import ( + "fmt" + "regexp" + + "frontend/pkg/common/faas_common/constant" +) + +const ( + defaultServeAppRuntime = "python3.9" + defaultServeAppTimeout = 900 + defaultServeAppCpu = 1000 + defaultServeAppMemory = 1024 + defaultServeAppConcurrentNum = 1000 +) + +// ServeDeploySchema - +type ServeDeploySchema struct { + Applications []ServeApplicationSchema `json:"applications"` +} + +// ServeApplicationSchema - +type ServeApplicationSchema struct { + Name string `json:"name"` + RoutePrefix string `json:"route_prefix"` + ImportPath string `json:"import_path"` + RuntimeEnv ServeRuntimeEnvSchema `json:"runtime_env"` + Deployments []ServeDeploymentSchema `json:"deployments"` +} + +// ServeDeploymentSchema - +type ServeDeploymentSchema struct { + Name string `json:"name"` + NumReplicas int64 `json:"num_replicas"` + HealthCheckPeriodS int64 `json:"health_check_period_s"` + HealthCheckTimeoutS int64 `json:"health_check_timeout_s"` +} + +// ServeRuntimeEnvSchema - +type ServeRuntimeEnvSchema struct { + Pip []string `json:"pip"` + WorkingDir string `json:"working_dir"` + EnvVars map[string]any `json:"env_vars"` +} + +// ServeFuncWithKeysAndFunctionMetaInfo - +type ServeFuncWithKeysAndFunctionMetaInfo struct { + FuncMetaKey string + InstanceMetaKey string + FuncMetaInfo *FunctionMetaInfo +} + +// Validate serve deploy schema by set of rules +func (s *ServeDeploySchema) Validate() error { + // 1. app name unique + appNameSet := make(map[string]struct{}) + for _, app := range s.Applications { + if _, ok := appNameSet[app.Name]; ok { + return fmt.Errorf("duplicated application name: %s", app.Name) + } + appNameSet[app.Name] = struct{}{} + } + // 2. app routes unique + appRouteSet := make(map[string]struct{}) + for _, app := range s.Applications { + if _, ok := appRouteSet[app.RoutePrefix]; ok { + return fmt.Errorf("duplicated application route prefix: %s", app.RoutePrefix) + } + appRouteSet[app.RoutePrefix] = struct{}{} + } + // 3. app name non empty + for _, app := range s.Applications { + if app.Name == "" { + return fmt.Errorf("application names must be nonempty") + } + } + return nil +} + +// ToFaaSFuncMetas - +func (s *ServeDeploySchema) ToFaaSFuncMetas() []*ServeFuncWithKeysAndFunctionMetaInfo { + var allMetas []*ServeFuncWithKeysAndFunctionMetaInfo + for _, a := range s.Applications { + // we don't really check it there are some repeated part? and just assume translate won't fail + for _, deploymentFuncMeta := range a.ToFaaSFuncMetas() { + allMetas = append(allMetas, deploymentFuncMeta) + } + } + return allMetas +} + +// ToFaaSFuncMetas - +func (s *ServeApplicationSchema) ToFaaSFuncMetas() []*ServeFuncWithKeysAndFunctionMetaInfo { + var allMetas []*ServeFuncWithKeysAndFunctionMetaInfo + for _, d := range s.Deployments { + meta := d.ToFaaSFuncMeta(s) + allMetas = append(allMetas, meta) + } + return allMetas +} + +// ToFaaSFuncMeta - +func (s *ServeDeploymentSchema) ToFaaSFuncMeta( + belongedApp *ServeApplicationSchema) *ServeFuncWithKeysAndFunctionMetaInfo { + faasFuncUrn := NewServeFunctionKeyWithDefault() + faasFuncUrn.AppName = belongedApp.Name + faasFuncUrn.DeploymentName = s.Name + + // make a copied app to make it contains only this deployment info + copiedApp := *belongedApp + copiedApp.Deployments = []ServeDeploymentSchema{*s} + + return &ServeFuncWithKeysAndFunctionMetaInfo{ + FuncMetaKey: faasFuncUrn.ToFuncMetaKey(), + InstanceMetaKey: faasFuncUrn.ToInstancesMetaKey(), + FuncMetaInfo: &FunctionMetaInfo{ + FuncMetaData: FuncMetaData{ + Name: faasFuncUrn.DeploymentName, + Runtime: defaultServeAppRuntime, + Timeout: defaultServeAppTimeout, + Version: faasFuncUrn.Version, + FunctionURN: faasFuncUrn.ToFaasFunctionUrn(), + TenantID: faasFuncUrn.TenantID, + FunctionVersionURN: faasFuncUrn.ToFaasFunctionVersionUrn(), + FuncName: faasFuncUrn.DeploymentName, + BusinessType: constant.BusinessTypeServe, + }, + ResourceMetaData: ResourceMetaData{ + CPU: defaultServeAppCpu, + Memory: defaultServeAppMemory, + }, + InstanceMetaData: InstanceMetaData{ + MaxInstance: s.NumReplicas, + MinInstance: s.NumReplicas, + ConcurrentNum: defaultServeAppConcurrentNum, + IdleMode: false, + }, + ExtendedMetaData: ExtendedMetaData{ + ServeDeploySchema: ServeDeploySchema{ + Applications: []ServeApplicationSchema{ + copiedApp, + }, + }, + }, + }, + } +} + +const ( + defaultTenantID = "12345678901234561234567890123456" + defaultFuncVersion = "latest" + + faasMetaKey = constant.MetaFuncKey + instanceMetaKey = "/instances/business/yrk/cluster/cluster001/tenant/%s/function/%s/version/%s" + faasFuncURN6tuplePattern = "sn:cn:yrk:%s:function:%s" + faasFuncURN7tuplePattern = "sn:cn:yrk:%s:function:%s:%s" +) + +// ServeFunctionKey is a faas urn with necessary parts +type ServeFunctionKey struct { + TenantID string + AppName string + DeploymentName string + Version string +} + +// NewServeFunctionKeyWithDefault returns a struct with default values +func NewServeFunctionKeyWithDefault() *ServeFunctionKey { + return &ServeFunctionKey{ + TenantID: defaultTenantID, + Version: defaultFuncVersion, + } +} + +// ToFuncNameTriplet - 0@svc@func +func (f *ServeFunctionKey) ToFuncNameTriplet() string { + return fmt.Sprintf("0@%s@%s", f.AppName, f.DeploymentName) +} + +// ToFuncMetaKey - /sn/functions/business/yrk/tenant/12345678901234561234567890123456/function/0@svc@func/version/latest +func (f *ServeFunctionKey) ToFuncMetaKey() string { + return fmt.Sprintf(faasMetaKey, f.TenantID, f.ToFuncNameTriplet(), f.Version) +} + +// ToInstancesMetaKey - /instances/business/yrk/cluster/cluster001/tenant/125...346/function/0@svc@func/version/latest +func (f *ServeFunctionKey) ToInstancesMetaKey() string { + return fmt.Sprintf(instanceMetaKey, f.TenantID, f.ToFuncNameTriplet(), f.Version) +} + +// ToFaasFunctionUrn - sn:cn:yrk:12345678901234561234567890123456:function:0@service@function +func (f *ServeFunctionKey) ToFaasFunctionUrn() string { + return fmt.Sprintf(faasFuncURN6tuplePattern, f.TenantID, f.ToFuncNameTriplet()) +} + +// ToFaasFunctionVersionUrn - sn:cn:yrk:12345678901234561234567890123456:function:0@svc@func:latest +func (f *ServeFunctionKey) ToFaasFunctionVersionUrn() string { + return fmt.Sprintf(faasFuncURN7tuplePattern, f.TenantID, f.ToFuncNameTriplet(), f.Version) +} + +// FromFaasFunctionKey - 12345678901234561234567890123456/0@svc@func/latest +func (f *ServeFunctionKey) FromFaasFunctionKey(funcKey string) error { + const ( + serveFaasFuncKeyMatchesIdxTenantID = iota + 1 + serveFaasFuncKeyMatchesIdxAppName + serveFaasFuncKeyMatchesIdxDeploymentName + serveFaasFuncKeyMatchesIdxVersion + serveFaasFuncKeyMatchesIdxMax + ) + re := regexp.MustCompile(`^([a-zA-Z0-9]*)/.*@([^@]+)@([^/]+)/(.*)$`) + matches := re.FindStringSubmatch(funcKey) + if len(matches) < serveFaasFuncKeyMatchesIdxMax { + return fmt.Errorf("extract failed from %s", funcKey) + } + f.TenantID = matches[serveFaasFuncKeyMatchesIdxTenantID] + f.AppName = matches[serveFaasFuncKeyMatchesIdxAppName] + f.DeploymentName = matches[serveFaasFuncKeyMatchesIdxDeploymentName] + f.Version = matches[serveFaasFuncKeyMatchesIdxVersion] + return nil +} diff --git a/frontend/pkg/common/faas_common/types/serve_test.go b/frontend/pkg/common/faas_common/types/serve_test.go new file mode 100644 index 0000000000000000000000000000000000000000..3b91f50262e4c4a8926f1ebc62201368f1e3fafa --- /dev/null +++ b/frontend/pkg/common/faas_common/types/serve_test.go @@ -0,0 +1,198 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package types - +package types + +import ( + "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestServeFunctionKeyTrans(t *testing.T) { + k := NewServeFunctionKeyWithDefault() + k.AppName = "svc" + k.DeploymentName = "func" + convey.Convey("Given a serve function key", t, func() { + convey.Convey("When trans to a func name triplet", func() { + convey.So(k.ToFuncNameTriplet(), convey.ShouldEqual, "0@svc@func") + }) + convey.Convey("When trans to a func meta key", func() { + convey.So(k.ToFuncMetaKey(), convey.ShouldEqual, + "/sn/functions/business/yrk/tenant/12345678901234561234567890123456/function/0@svc@func/version/latest") + }) + convey.Convey("When trans to a instance meta key", func() { + convey.So(k.ToInstancesMetaKey(), convey.ShouldEqual, + "/instances/business/yrk/cluster/cluster001/tenant/12345678901234561234567890123456/function/0@svc@func/version/latest") + }) + convey.Convey("When trans to a FaasFunctionUrn", func() { + convey.So(k.ToFaasFunctionUrn(), convey.ShouldEqual, + "sn:cn:yrk:12345678901234561234567890123456:function:0@svc@func") + }) + convey.Convey("When trans ToFaasFunctionVersionUrn", func() { + convey.So(k.ToFaasFunctionVersionUrn(), convey.ShouldEqual, + "sn:cn:yrk:12345678901234561234567890123456:function:0@svc@func:latest") + }) + }) +} + +func TestServeDeploySchema_ToFaaSFuncMetas(t *testing.T) { + convey.Convey("Test ServeDeploySchema ToFaaSFuncMetas", t, func() { + // Setup mock data + app1 := ServeApplicationSchema{ + Name: "app1", + RoutePrefix: "/app1", + ImportPath: "path1", + RuntimeEnv: ServeRuntimeEnvSchema{ + Pip: []string{"package1", "package2"}, + WorkingDir: "/app1", + EnvVars: map[string]any{"key1": "value1"}, + }, + Deployments: []ServeDeploymentSchema{ + { + Name: "deployment1", + NumReplicas: 2, + HealthCheckPeriodS: 30, + HealthCheckTimeoutS: 10, + }, + }, + } + + serveDeploy := ServeDeploySchema{ + Applications: []ServeApplicationSchema{app1}, + } + + convey.Convey("It should return correct faas function metas", func() { + result := serveDeploy.ToFaaSFuncMetas() + convey.So(len(result), convey.ShouldBeGreaterThan, 0) + convey.So(result[0].FuncMetaKey, convey.ShouldNotBeEmpty) + }) + }) +} + +func TestServeFunctionKey(t *testing.T) { + convey.Convey("Test FromFaasFunctionKey", t, func() { + convey.Convey("It should return correct faas function metas", func() { + key := "12345678901234561234567890123456/0@svc@func/latest" + sfk := ServeFunctionKey{} + err := sfk.FromFaasFunctionKey(key) + convey.So(err, convey.ShouldBeNil) + convey.So(sfk.Version, convey.ShouldEqual, "latest") + convey.So(sfk.AppName, convey.ShouldEqual, "svc") + convey.So(sfk.DeploymentName, convey.ShouldEqual, "func") + convey.So(sfk.TenantID, convey.ShouldEqual, "12345678901234561234567890123456") + }) + convey.Convey("It should return incorrect faas function metas", func() { + key := "12345678901234561234567890123456/0@svc@func" + sfk := ServeFunctionKey{} + err := sfk.FromFaasFunctionKey(key) + convey.So(err, convey.ShouldNotBeNil) + }) + }) + + convey.Convey("Test FaasKey Test", t, func() { + convey.Convey("test default faas key", func() { + sfk := NewServeFunctionKeyWithDefault() + convey.So(sfk.TenantID, convey.ShouldEqual, defaultTenantID) + convey.So(sfk.Version, convey.ShouldEqual, defaultFuncVersion) + }) + + convey.Convey("test convert", func() { + sfk := NewServeFunctionKeyWithDefault() + sfk.AppName = "svc" + sfk.DeploymentName = "func" + + convey.So(sfk.ToFuncNameTriplet(), + convey.ShouldEqual, + "0@svc@func") + convey.So(sfk.ToFuncMetaKey(), + convey.ShouldEqual, + "/sn/functions/business/yrk/tenant/12345678901234561234567890123456/function/0@svc@func/version/latest") + convey.So(sfk.ToInstancesMetaKey(), + convey.ShouldEqual, + "/instances/business/yrk/cluster/cluster001/tenant/12345678901234561234567890123456/function/0@svc@func/version/latest") + convey.So(sfk.ToFaasFunctionUrn(), + convey.ShouldEqual, + "sn:cn:yrk:12345678901234561234567890123456:function:0@svc@func") + convey.So(sfk.ToFaasFunctionVersionUrn(), + convey.ShouldEqual, + "sn:cn:yrk:12345678901234561234567890123456:function:0@svc@func:latest") + }) + + convey.Convey("It should return incorrect faas function metas", func() { + sfk := NewServeFunctionKeyWithDefault() + convey.So(sfk.TenantID, convey.ShouldEqual, defaultTenantID) + convey.So(sfk.Version, convey.ShouldEqual, defaultFuncVersion) + }) + }) +} + +func TestServeDeploySchemaValidate(t *testing.T) { + convey.Convey("Test Validate", t, func() { + sds := ServeDeploySchema{ + Applications: []ServeApplicationSchema{ + { + Name: "app1", + RoutePrefix: "/app1", + ImportPath: "path1", + RuntimeEnv: ServeRuntimeEnvSchema{ + Pip: []string{"package1", "package2"}, + WorkingDir: "/app1", + EnvVars: map[string]any{"key1": "value1"}, + }, + Deployments: []ServeDeploymentSchema{ + { + Name: "deployment1", + NumReplicas: 2, + HealthCheckPeriodS: 30, + HealthCheckTimeoutS: 10, + }, + }, + }, + }} + convey.Convey("on repeated app name", func() { + sdsOther := sds + app0 := sdsOther.Applications[0] + sdsOther.Applications = append(sdsOther.Applications, app0) + + err := sdsOther.Validate() + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("on repeated route prefix", func() { + sdsOther := sds + app0 := sdsOther.Applications[0] + app0.Name = "othername" + sdsOther.Applications = append(sdsOther.Applications, app0) + + err := sdsOther.Validate() + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("on empty app name", func() { + sdsOther := sds + app0 := sdsOther.Applications[0] + app0.Name = "" + app0.RoutePrefix = "/other" + sdsOther.Applications = append(sdsOther.Applications, app0) + + err := sdsOther.Validate() + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("ok", func() { + err := sds.Validate() + convey.So(err, convey.ShouldBeNil) + }) + }) +} diff --git a/frontend/pkg/common/faas_common/types/types.go b/frontend/pkg/common/faas_common/types/types.go new file mode 100644 index 0000000000000000000000000000000000000000..554ddc9718b4dbf9543e8230850d8ab022ad5916 --- /dev/null +++ b/frontend/pkg/common/faas_common/types/types.go @@ -0,0 +1,894 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package types - +package types + +import ( + "yuanrong.org/kernel/runtime/libruntime/api" +) + +// HTTPResponse is general http response +type HTTPResponse struct { + Code int `json:"code"` + Message string `json:"message"` +} + +// InnerInstanceData is the function instance data stored in ETCD +type InnerInstanceData struct { + IP string `json:"ip"` + Port string `json:"port"` + Status string `json:"status"` + P2pPort string `json:"p2pPort"` + GrpcPort string `json:"grpcPort,omitempty"` + NodeIP string `json:"nodeIP,omitempty"` + NodePort string `json:"nodePort,omitempty"` + NodeName string `json:"nodeName,omitempty"` + NodeID string `json:"nodeID,omitempty"` + Applier string `json:"applier,omitempty"` // silimar to OwnerIP + OwnerIP string `json:"ownerIP,omitempty"` + FuncSig string `json:"functionSignature,omitempty"` + Reserved bool `json:"reserved,omitempty"` + CPU int64 `json:"cpu,omitempty"` + Memory int64 `json:"memory,omitempty"` + GroupID string `json:"groupID,omitempty"` + StackID string `json:"stackID,omitempty"` + CustomResources map[string]int64 `json:"customResources,omitempty" valid:"optional"` +} + +// LogTankService - +type LogTankService struct { + GroupID string `json:"logGroupId" valid:",optional"` + StreamID string `json:"logStreamId" valid:",optional"` +} + +// TraceService - +type TraceService struct { + TraceAK string `json:"tracing_ak" valid:",optional"` + TraceSK string `json:"tracing_sk" valid:",optional"` + ProjectName string `json:"project_name" valid:",optional"` +} + +// Initializer include initializer handler and timeout +type Initializer struct { + Handler string `json:"initializer_handler" valid:",optional"` + Timeout int64 `json:"initializer_timeout" valid:",optional"` +} + +// FuncMountConfig function mount config +type FuncMountConfig struct { + FuncMountUser FuncMountUser `json:"mount_user" valid:",optional"` + FuncMounts []FuncMount `json:"func_mounts" valid:",optional"` +} + +// FuncMountUser function mount user +type FuncMountUser struct { + UserID int `json:"user_id" valid:",optional"` + GroupID int `json:"user_group_id" valid:",optional"` +} + +// FuncMount function mount +type FuncMount struct { + MountType string `json:"mount_type" valid:",optional"` + MountResource string `json:"mount_resource" valid:",optional"` + MountSharePath string `json:"mount_share_path" valid:",optional"` + LocalMountPath string `json:"local_mount_path" valid:",optional"` + Status string `json:"status" valid:",optional"` +} + +// Role include x_role and app_x_role +type Role struct { + XRole string `json:"xrole" valid:",optional"` + AppXRole string `json:"app_xrole" valid:",optional"` +} + +// FunctionDeploymentSpec define function deployment spec +type FunctionDeploymentSpec struct { + BucketID string `json:"bucket_id"` + ObjectID string `json:"object_id"` + Layers string `json:"layers"` + DeployDir string `json:"deploydir"` +} + +// InstanceResource describes the cpu and memory info of an instance +type InstanceResource struct { + CPU string `json:"cpu"` + Memory string `json:"memory"` + CustomResources map[string]int64 `json:"customresources"` +} + +// Worker define a worker +type Worker struct { + Instances []*Instance `json:"instances"` + FunctionName string `json:"functionname"` + FunctionVersion string `json:"functionversion"` + Tenant string `json:"tenant"` + Business string `json:"business"` +} + +// Instance define a instance +type Instance struct { + IP string `json:"ip"` + Port string `json:"port"` + GrpcPort string `json:"grpcPort"` + InstanceID string `json:"instanceID,omitempty"` + DeployedIP string `json:"deployed_ip"` + DeployedNode string `json:"deployed_node"` + DeployedNodeID string `json:"deployed_node_id"` + TenantID string `json:"tenant_id"` +} + +// InstanceCreationRequest is used to create instance +type InstanceCreationRequest struct { + LogicInstanceID string `json:"logicInstanceID"` + FuncName string `json:"functionName"` + Applier string `json:"applier"` + DeployNode string `json:"deployNode"` + Business string `json:"business"` + TenantID string `json:"tenantID"` + Version string `json:"version"` + OwnerIP string `json:"ownerIP"` + TraceID string `json:"traceID"` + TriggerFlag string `json:"triggerFlag"` + VersionUrn string `json:"versionUrn"` + CPU int64 `json:"cpu"` + Memory int64 `json:"memory"` + GroupID string `json:"groupID"` + StackID string `json:"stackID"` + CustomResources map[string]int64 `json:"customResources,omitempty" valid:"optional"` +} + +// InstanceCreationSuccessResponse is the struct returned by workermanager upon successful instance creation +type InstanceCreationSuccessResponse struct { + HTTPResponse + Worker *Worker `json:"worker"` + Instance *Instance `json:"instance"` +} + +// InstanceDeletionRequest is used to delete instance +type InstanceDeletionRequest struct { + InstanceID string `json:"instanceID"` + FuncName string `json:"functionName"` + FuncVersion string `json:"functionVersion"` + TenantID string `json:"tenantID"` + BusinessID string `json:"businessID"` + Applier string `json:"applier"` + Force bool `json:"force"` +} + +// InstanceDeletionResponse is the struct returned by workermanager upon successful instance deletion +type InstanceDeletionResponse struct { + HTTPResponse + Reserved bool `json:"reserved"` +} + +// HookArgs keeps args of hook +type HookArgs struct { + FuncArgs []byte // Call() request in worker + SrcTenant string + DstTenant string + StateID string + LogType string + StateKey string // for trigger state call + FunctionVersion string // for trigger state call + ExternalRequest bool // for trigger state call + ServiceID string + TraceID string + InvokeType string +} + +// ResourceStack stores properties of resource stack +type ResourceStack struct { + StackID string `json:"id" valid:"required"` + CPU int64 `json:"cpu" valid:"required"` + Mem int64 `json:"mem" valid:"required"` + CustomResources map[string]int64 `json:"customResources,omitempty" valid:"optional"` +} + +// ResourceGroup stores properties of resource group +type ResourceGroup struct { + GroupID string `json:"id" valid:"required"` + DeployOption string `json:"deployOption" valid:"required"` + GroupState string `json:"groupState" valid:"required"` + ResourceStacks []ResourceStack `json:"resourceStacks" valid:"required"` + ScheduledStacks map[string][]ResourceStack `json:"scheduledStacks,omitempty" valid:"optional"` +} + +// AffinityInfo is data affinity information +type AffinityInfo struct { + AffinityRequest AffinityRequest + AffinityNode string // if AffinityNode is not empty, the affinity node has been calculated + NeedToForward bool +} + +// AffinityRequest is affinity request parameter +type AffinityRequest struct { + Strategy string `json:"strategy"` + ObjectIDs []string `json:"object_ids"` +} + +// GroupInfo stores groupID and stackID +type GroupInfo struct { + GroupID string `json:"groupID"` + StackID string `json:"stackID"` +} + +// InvokeOption contains invoke options +type InvokeOption struct { + AffinityRequest AffinityRequest + GroupInfo GroupInfo + ResourceMetaData map[string]float32 +} + +// ScheduleConfig defines schedule config +type ScheduleConfig struct { + Policy int `json:"policy" valid:"optional"` + ForwardScheduleFirst bool `json:"forwardScheduleResourceNotEnough" valid:"optional"` + SleepingMemThreshold float32 `json:"sleepingMemoryThreshold" valid:"optional"` + SelectInstanceToSleepingPolicy string `json:"selectInstanceToSleepingPolicy" valid:"optional"` +} + +// MetricsData shows the quantities of a specific resource +type MetricsData struct { + TotalResource float32 `json:"totalResource"` + InUseResource float32 `json:"inUseResource"` +} + +// ResourceMetrics contains several resources' MetricsData +type ResourceMetrics map[string]MetricsData + +// WorkerMetrics stores metrics used for scheduler +type WorkerMetrics struct { + SystemResources ResourceMetrics + // key levels: functionUrn instanceID + FunctionResources map[string]map[string]ResourceMetrics +} + +// InnerWorkerData is the worker data stored in ETCD +type InnerWorkerData struct { + IP string `json:"ip"` + Port string `json:"port"` + NodeIP string `json:"nodeIP"` + P2pPort string `json:"p2pPort"` + NodeName string `json:"nodeName"` + NodeID string `json:"nodeID"` + WorkerAgentID string `json:"workerAgentID"` + AllocatableCPU int64 `json:"allocatableCPU"` + AllocatableMemory int64 `json:"allocatableMemory"` + AllocatableCustomResource map[string]int64 `json:"allocatableCustomResource"` +} + +// TerminateRequest sent from worker manager to worker to delete function instance +type TerminateRequest struct { + RuntimeID string `json:"runtime_id"` + FuncName string `json:"function_name"` + FuncVersion string `json:"function_version"` + TenantID string `json:"tenant_id"` + BusinessID string `json:"business_id" valid:"optional"` +} + +// UserAgency define AK/SK of user's agency +type UserAgency struct { + AccessKey string `json:"accessKey"` + SecretKey string `json:"secretKey"` + Token string `json:"token"` + SecurityAk string `json:"securityAk"` + SecuritySk string `json:"securitySk"` + SecurityToken string `json:"securityToken"` +} + +// CustomHealthCheck custom health check +type CustomHealthCheck struct { + TimeoutSeconds int `json:"timeoutSeconds" valid:",optional"` + PeriodSeconds int `json:"periodSeconds" valid:",optional"` + FailureThreshold int `json:"failureThreshold" valid:",optional"` +} + +// FuncCode include function code file and link info +type FuncCode struct { + File string `json:"file" valid:",optional"` + Link string `json:"link" valid:",optional"` +} + +// StrategyConfig - +type StrategyConfig struct { + Concurrency int `json:"concurrency" valid:",optional"` +} + +// FuncSpec contains specifications of a function +type FuncSpec struct { + ETCDType string `json:"-"` + FunctionKey string `json:"-"` + FuncMetaSignature string `json:"-"` + FuncMetaData FuncMetaData `json:"funcMetaData" valid:",optional"` + S3MetaData S3MetaData `json:"s3MetaData" valid:",optional"` + CodeMetaData CodeMetaData `json:"codeMetaData" valid:",optional"` + EnvMetaData EnvMetaData `json:"envMetaData" valid:",optional"` + StsMetaData StsMetaData `json:"stsMetaData" valid:",optional"` + ResourceMetaData ResourceMetaData `json:"resourceMetaData" valid:",optional"` + InstanceMetaData InstanceMetaData `json:"instanceMetaData" valid:",optional"` + ExtendedMetaData ExtendedMetaData `json:"extendedMetaData" valid:",optional"` +} + +// FunctionMetaInfo define function meta info for FunctionGraph +type FunctionMetaInfo struct { + FuncMetaData FuncMetaData `json:"funcMetaData" valid:",optional"` + S3MetaData S3MetaData `json:"s3MetaData" valid:",optional"` + CodeMetaData CodeMetaData `json:"codeMetaData" valid:",optional"` + EnvMetaData EnvMetaData `json:"envMetaData" valid:",optional"` + StsMetaData StsMetaData `json:"stsMetaData" valid:",optional"` + ResourceMetaData ResourceMetaData `json:"resourceMetaData" valid:",optional"` + InstanceMetaData InstanceMetaData `json:"instanceMetaData" valid:",optional"` + ExtendedMetaData ExtendedMetaData `json:"extendedMetaData" valid:",optional"` +} + +// FuncMetaData define meta data of functions +type FuncMetaData struct { + Layers []*Layer `json:"layers" valid:",optional"` + Name string `json:"name"` + FunctionDescription string `json:"description" valid:"stringlength(1|1024)"` + FunctionURN string `json:"functionUrn"` + TenantID string `json:"tenantId"` + Tags map[string]string `json:"tags" valid:",optional"` + FunctionUpdateTime string `json:"functionUpdateTime" valid:",optional"` + FunctionVersionURN string `json:"functionVersionUrn"` + RevisionID string `json:"revisionId" valid:"stringlength(1|20),optional"` + CodeSize int `json:"codeSize" valid:"int"` + CodeSha512 string `json:"codeSha512" valid:"stringlength(1|128),optional"` + Handler string `json:"handler" valid:"stringlength(1|255)"` + Runtime string `json:"runtime" valid:"stringlength(1|63)"` + Timeout int64 `json:"timeout" valid:"required"` + Version string `json:"version" valid:"stringlength(1|32)"` + DeadLetterConfig string `json:"deadLetterConfig" valid:"stringlength(1|255)"` + BusinessID string `json:"businessId" valid:"stringlength(1|32)"` + FunctionType string `json:"functionType" valid:",optional"` + FuncID string `json:"func_id" valid:",optional"` + FuncName string `json:"func_name" valid:",optional"` + DomainID string `json:"domain_id" valid:",optional"` + ProjectName string `json:"project_name" valid:",optional"` + Service string `json:"service" valid:",optional"` + Dependencies string `json:"dependencies" valid:",optional"` + EnableCloudDebug string `json:"enable_cloud_debug" valid:",optional"` + IsStatefulFunction bool `json:"isStatefulFunction" valid:"optional"` + IsBridgeFunction bool `json:"isBridgeFunction" valid:"optional"` + IsStreamEnable bool `json:"isStreamEnable" valid:"optional"` + Type string `json:"type" valid:"optional"` + EnableAuthInHeader bool `json:"enable_auth_in_header" valid:"optional"` + DNSDomainCfg []DNSDomainInfo `json:"dns_domain_cfg" valid:",optional"` + VPCTriggerImage string `json:"vpcTriggerImage" valid:",optional"` + StateConfig StateConfig `json:"stateConfig" valid:",optional"` + BusinessType string `json:"businessType" valid:"optional"` +} + +// StateConfig ConsistentWithInstance- The lifecycle is consistent with that of the instance. +// Independent - The lifecycle is independent of instances. +type StateConfig struct { + LifeCycle string `json:"lifeCycle"` +} + +// S3MetaData define meta function info for OBS +type S3MetaData struct { + AppID string `json:"appId" valid:"stringlength(1|128),optional"` + BucketID string `json:"bucketId" valid:"stringlength(1|255),optional"` + ObjectID string `json:"objectId" valid:"stringlength(1|255),optional"` + BucketURL string `json:"bucketUrl" valid:"url,optional"` + CodeType string `json:"code_type" valid:",optional"` + CodeURL string `json:"code_url" valid:",optional"` + CodeFileName string `json:"code_filename" valid:",optional"` + FuncCode FuncCode `json:"func_code" valid:",optional"` +} + +// LocalMetaData - +type LocalMetaData struct { + StorageType string `json:"storage_type" valid:",optional"` + CodePath string `json:"code_path" valid:",optional"` +} + +// CodeMetaData - +type CodeMetaData struct { + Sha512 string `json:"sha512" valid:",optional"` + LocalMetaData + S3MetaData +} + +// EnvMetaData - +type EnvMetaData struct { + Environment string `json:"environment"` + EncryptedUserData string `json:"encrypted_user_data"` + EnvKey string `json:"envKey" valid:",optional"` + CryptoAlgorithm string `json:"cryptoAlgorithm" valid:",optional"` +} + +// StsMetaData define sts info of functions +type StsMetaData struct { + EnableSts bool `json:"enableSts"` + ServiceName string `json:"serviceName,omitempty"` + MicroService string `json:"microService,omitempty"` + SensitiveConfigs map[string]string `json:"sensitiveConfigs,omitempty"` + StsCertConfig map[string]string `json:"stsCertConfig,omitempty"` +} + +// ResourceMetaData include resource data such as cpu and memory +type ResourceMetaData struct { + CPU int64 `json:"cpu"` + Memory int64 `json:"memory"` + GpuMemory int64 `json:"gpu_memory"` + EnableDynamicMemory bool `json:"enable_dynamic_memory" valid:",optional"` + CustomResources string `json:"customResources" valid:",optional"` + EnableTmpExpansion bool `json:"enable_tmp_expansion" valid:",optional"` + EphemeralStorage int `json:"ephemeral_storage" valid:"int,optional"` + CustomResourcesSpec string `json:"CustomResourcesSpec" valid:",optional"` +} + +// InstanceMetaData define instance meta data of FG functions +type InstanceMetaData struct { + MaxInstance int64 `json:"maxInstance" valid:",optional"` + MinInstance int64 `json:"minInstance" valid:",optional"` + ConcurrentNum int `json:"concurrentNum" valid:",optional"` + DiskLimit int64 `json:"diskLimit" valid:",optional"` + InstanceType string `json:"instanceType" valid:",optional"` + SchedulePolicy string `json:"schedulePolicy" valid:",optional"` + ScalePolicy string `json:"scalePolicy" valid:",optional"` + IdleMode bool `json:"idleMode" valid:",optional"` + PoolLabel string `json:"poolLabel"` + PoolID string `json:"poolId" valid:",optional"` +} + +// ExtendedMetaData define external meta data of functions +type ExtendedMetaData struct { + ImageName string `json:"image_name" valid:",optional"` + Role Role `json:"role" valid:",optional"` + VpcConfig *VpcConfig `json:"func_vpc" valid:",optional"` + EndpointTenantVpc *VpcConfig `json:"endpoint_tenant_vpc" valid:",optional"` + FuncMountConfig *FuncMountConfig `json:"mount_config" valid:",optional"` + StrategyConfig StrategyConfig `json:"strategy_config" valid:",optional"` + ExtendConfig string `json:"extend_config" valid:",optional"` + Initializer Initializer `json:"initializer" valid:",optional"` + Heartbeat Heartbeat `json:"heartbeat" valid:",optional"` + EnterpriseProjectID string `json:"enterprise_project_id" valid:",optional"` + LogTankService LogTankService `json:"log_tank_service" valid:",optional"` + TraceService TraceService `json:"tracing_config" valid:",optional"` + CustomContainerConfig CustomContainerConfig `json:"custom_container_config" valid:",optional"` + AsyncConfigLoaded bool `json:"async_config_loaded" valid:",optional"` + RestoreHook RestoreHook `json:"restore_hook,omitempty" valid:",optional"` + NetworkController NetworkController `json:"network_controller" valid:",optional"` + UserAgency UserAgency `json:"user_agency" valid:",optional"` + CustomFilebeatConfig CustomFilebeatConfig `json:"custom_filebeat_config"` + CustomHealthCheck CustomHealthCheck `json:"custom_health_check" valid:",optional"` + DynamicConfig DynamicConfigEvent `json:"dynamic_config" valid:",optional"` + CustomGracefulShutdown CustomGracefulShutdown `json:"runtime_graceful_shutdown"` + PreStop PreStop `json:"pre_stop"` + RaspConfig RaspConfig `json:"rasp_config"` + ServeDeploySchema ServeDeploySchema `json:"serveDeploySchema" valid:"optional"` +} + +// CustomGracefulShutdown define the option of custom container's runtime graceful shutdown +type CustomGracefulShutdown struct { + MaxShutdownTimeout int `json:"maxShutdownTimeout"` +} + +// PreStop include pre_stop handler and timeout +type PreStop struct { + Handler string `json:"pre_stop_handler" valid:",optional"` + Timeout int `json:"pre_stop_timeout" valid:",optional"` +} + +// DynamicConfigEvent dynamic config etcd event +type DynamicConfigEvent struct { + Enabled bool `json:"enabled"` // use for signature + UpdateTime string `json:"update_time"` + ConfigContent []KV `json:"config_content"` +} + +// KV config key and value +type KV struct { + Name string `json:"name"` + Value string `json:"value"` +} + +// Heartbeat define user custom heartbeat function config +type Heartbeat struct { + // Handler define heartbeat function entry + Handler string `json:"heartbeat_handler" valid:",optional"` +} + +// CustomContainerConfig contains the metadata for custom container +type CustomContainerConfig struct { + ControlPath string `json:"control_path" valid:",optional"` + Image string `json:"image" valid:",optional"` + Command []string `json:"command" valid:",optional"` + Args []string `json:"args" valid:",optional"` + WorkingDir string `json:"working_dir" valid:",optional"` + UID int `json:"uid" valid:",optional"` + GID int `json:"gid" valid:",optional"` +} + +// CustomFilebeatConfig custom filebeat config +type CustomFilebeatConfig struct { + SidecarConfigInfo *SidecarConfigInfo `json:"sidecarConfigInfo"` + CPU int64 `json:"cpu"` + Memory int64 `json:"memory"` + Version string `json:"version"` + ImageAddress string `json:"imageAddress"` +} + +// RaspConfig rasp config key and value +type RaspConfig struct { + InitImage string `json:"init-image"` + RaspImage string `json:"rasp-image"` + RaspServerIP string `json:"rasp-server-ip"` + RaspServerPort string `json:"rasp-server-port"` + Envs []KV `json:"envs"` +} + +// SidecarConfigInfo sidecat config info +type SidecarConfigInfo struct { + ConfigFiles []CustomLogConfigFile `json:"configFiles"` + LiveNessShell string `json:"livenessShell"` + ReadNessShell string `json:"readnessShell"` + PreStopCommands string `json:"preStopCommands"` +} + +// CustomLogConfigFile custom log config file +type CustomLogConfigFile struct { + Path string `json:"path"` + Data string `json:"data"` + Secret bool `json:"secret"` +} + +// RestoreHook include restorehook handler and timeout +type RestoreHook struct { + Handler string `json:"restore_hook_handler,omitempty" valid:",optional"` + Timeout int64 `json:"restore_hook_timeout,omitempty" valid:",optional"` +} + +// NetworkController contains some special network settings +type NetworkController struct { + DisablePublicNetwork bool `json:"disable_public_network" valid:",optional"` + TriggerAccessVpcs []VpcInfo `json:"trigger_access_vpcs" valid:",optional"` +} + +// VpcInfo contains the information of VPC access restriction +type VpcInfo struct { + VpcName string `json:"vpc_name,omitempty"` + VpcID string `json:"vpc_id,omitempty"` +} + +// VpcConfig include info of function vpc +type VpcConfig struct { + ID string `json:"id,omitempty"` + DomainID string `json:"domain_id,omitempty"` + Namespace string `json:"namespace,omitempty"` + VpcName string `json:"vpc_name,omitempty"` + VpcID string `json:"vpc_id,omitempty"` + SubnetName string `json:"subnet_name,omitempty"` + SubnetID string `json:"subnet_id,omitempty"` + TenantCidr string `json:"tenant_cidr,omitempty"` + HostVMCidr string `json:"host_vm_cidr,omitempty"` + Gateway string `json:"gateway,omitempty"` + Xrole string `json:"xrole,omitempty"` +} + +// Layer define layer info +type Layer struct { + BucketURL string `json:"bucketUrl" valid:"url,optional"` + ObjectID string `json:"objectId" valid:"stringlength(1|255),optional"` + BucketID string `json:"bucketId" valid:"stringlength(1|255),optional"` + AppID string `json:"appId" valid:"stringlength(1|128),optional"` + ETag string `json:"etag" valid:"optional"` + Link string `json:"link" valid:"optional"` + Name string `json:"name" valid:",optional"` + Sha256 string `json:"sha256" valid:"optional"` + DependencyType string `json:"dependencyType" valid:",optional"` +} + +// DNSDomainInfo dns domain info +type DNSDomainInfo struct { + ID string `json:"id"` + DomainName string `json:"domain_name"` + Type string `json:"type" valid:",optional"` + ZoneType string `json:"zone_type" valid:",optional"` +} + +// DataSystemConfig data system client config +type DataSystemConfig struct { + TimeoutMs int `json:"timeoutMs" validate:"required"` + Clusters []string `json:"clusters"` +} + +// XiangYunFourConfig - +type XiangYunFourConfig struct { + Site string `json:"site"` + TenantID string `json:"tenantID"` + ApplicationID string `json:"applicationID"` + ServiceID string `json:"serviceID"` +} + +// MemoryControlConfig Memory use control config +type MemoryControlConfig struct { + LowerMemoryPercent float64 `json:"lowerMemoryPercent" valid:",optional"` + HighMemoryPercent float64 `json:"highMemoryPercent" valid:",optional"` + StatefulHighMemPercent float64 `json:"statefulHighMemoryPercent" valid:",optional"` + BodyThreshold uint64 `json:"bodyThreshold" valid:",optional"` + MemDetectIntervalMs int `json:"memDetectIntervalMs" valid:",optional"` +} + +// InstanceStatus Instance status, controlled by the kernel +type InstanceStatus struct { + Code int32 `json:"code" validate:"required"` + Msg string `json:"msg" validate:"required"` + Type int32 `json:"type" validate:"optional"` + ExitCode int32 `json:"exitCode" validate:"optional"` + ErrorCode int32 `json:"errCode" validate:"optional"` +} + +// PodResourceInfo describe actual resource info of pod +type PodResourceInfo struct { + Worker ResourceConfig `json:"worker,omitempty"` + Runtime ResourceConfig `json:"runtime,omitempty"` +} + +// ResourceConfig sub-struct of FuncInstanceInfo +type ResourceConfig struct { + CPULimit int64 `json:"cpuLimit" valid:",optional"` // unit: milli-cores(m) + CPURequest int64 `json:"cpuRequest" valid:",optional"` + MemoryLimit int64 `json:"memoryLimit" valid:",optional"` // unit: byte + MemoryRequest int64 `json:"memoryRequest" valid:",optional"` +} + +// Extensions - +type Extensions struct { + Source string `json:"source"` + CreateTimestamp string `json:"createTimestamp"` + UpdateTimestamp string `json:"updateTimestamp"` + PID string `json:"pid"` + PodName string `json:"podName"` + PodNamespace string `json:"podNamespace"` + PodDeploymentName string `json:"podDeploymentName"` +} + +// InstanceSpecification contains specification of a instance in etcd +type InstanceSpecification struct { + InstanceID string `json:"instanceID" validate:"required"` + DataSystemHost string `json:"dataSystemHost" validate:"required"` + RequestID string `json:"requestID" valid:",optional"` + RuntimeID string `json:"runtimeID" valid:",optional"` + RuntimeAddress string `json:"runtimeAddress" valid:",optional"` + FunctionAgentID string `json:"functionAgentID" valid:",optional"` + FunctionProxyID string `json:"functionProxyID" valid:",optional"` + Function string `json:"function"` + RestartPolicy string `json:"restartPolicy" valid:",optional"` + Resources Resources `json:"resources"` + ActualUse Resources `json:"actualUse" valid:",optional"` + ScheduleOption ScheduleOption `json:"scheduleOption"` + CreateOptions map[string]string `json:"createOptions"` + Labels []string `json:"labels"` + StartTime string `json:"startTime"` + InstanceStatus InstanceStatus `json:"instanceStatus"` + JobID string `json:"jobID"` + SchedulerChain []string `json:"schedulerChain" valid:",optional"` + ParentID string `json:"parentID"` + DeployTimes int32 `json:"deployTimes"` + Extensions Extensions `json:"extensions" valid:",optional"` +} + +// InstanceSpecificationFG contains specification of instance in etcd for functionGraph +type InstanceSpecificationFG struct { + OwnerIP string `json:"ownerIP"` + CreationTime int `json:"creationTime"` + Applier string `json:"applier"` + NodeIP string `json:"nodeIP"` + NodePort string `json:"nodePort"` + InstanceIP string `json:"ip"` + InstancePort string `json:"port"` + CPU int `json:"cpu"` + Memory int `json:"memory"` + BusinessType string `json:"businessType"` + Resource PodResourceInfo `json:"resource,omitempty"` +} + +// Resources - +type Resources struct { + Resources map[string]Resource `json:"resources"` +} + +// Resource - +type Resource struct { + Name string `json:"name"` + Type ValueType `json:"type"` + Scalar ValueScalar `json:"scalar"` + Ranges ValueRanges `json:"ranges"` + Set ValueSet `json:"set"` + Runtime string `json:"runtime"` + Driver string `json:"driver"` + Disk DiskInfo `json:"disk"` +} + +// ValueType - +type ValueType int32 + +// ValueScalar - +type ValueScalar struct { + Value float64 `json:"value"` + Limit float64 `json:"limit"` +} + +// ValueRanges - +type ValueRanges struct { + Range []ValueRange `protobuf:"bytes,1,rep,name=range,proto3" json:"range,omitempty"` +} + +// ValueSet - +type ValueSet struct { + Items string `json:"items"` +} + +// ValueRange - +type ValueRange struct { + Begin uint64 `json:"begin"` + End uint64 `json:"end"` +} + +// DiskInfo - +type DiskInfo struct { + Volume Volume `json:"volume"` + Type string `json:"type"` + DevPath string `json:"devPath"` + MountPath string `json:"mountPath"` +} + +// Volume - +type Volume struct { + Mode int32 `json:"mode"` + SourceType int32 `json:"sourceType"` + HostPaths string `json:"hostPaths"` + ContainerPath string `json:"containerPath"` + ConfigMapPath string `json:"configMapPath"` + EmptyDir string `json:"emptyDir"` + ElaraPath string `json:"elaraPath"` +} + +// ScheduleOption - +type ScheduleOption struct { + SchedPolicyName string `json:"schedPolicyName"` + Priority int32 `json:"priority"` + Affinity Affinity `json:"affinity"` +} + +// Affinity - +type Affinity struct { + NodeAffinity NodeAffinity `json:"nodeAffinity"` + InstanceAffinity InstanceAffinity `json:"instanceAffinity"` + InstanceAntiAffinity InstanceAffinity `json:"instanceAntiAffinity"` +} + +// NodeAffinity - +type NodeAffinity struct { + Affinity map[string]string `json:"affinity"` +} + +// InstanceAffinity - +type InstanceAffinity struct { + Affinity map[string]string `json:"affinity"` +} + +// InstanceInfo the instance info which can be parsed from the etcd path, instanceName is used to hold a place in the +// hash ring while instanceID is used to invoke this instance +type InstanceInfo struct { + TenantID string + FunctionName string + Version string + InstanceName string `json:"instanceName"` + InstanceID string `json:"instanceId"` + Exclusivity string + Address string +} + +// InstanceResponse is the response returned by faas scheduler's CallHandler +type InstanceResponse struct { + InstanceAllocationInfo + ErrorCode int `json:"errorCode"` + ErrorMessage string `json:"errorMessage"` + SchedulerTime float64 `json:"schedulerTime"` +} + +// BatchInstanceResponse is the batch response returned by faas scheduler's CallHandler +type BatchInstanceResponse struct { + InstanceAllocSucceed map[string]InstanceAllocationSucceedInfo `json:"instanceAllocSucceed"` + InstanceAllocFailed map[string]InstanceAllocationFailedInfo `json:"instanceAllocFailed"` + LeaseInterval int64 `json:"leaseInterval"` + SchedulerTime float64 `json:"schedulerTime"` +} + +// RolloutResponse - +type RolloutResponse struct { + AllocRecord map[string][]string `json:"allocRecord"` + RegisterKey string `json:"registerKey"` + ErrorCode int `json:"errorCode"` + ErrorMessage string `json:"errorMessage"` +} + +// InstanceAllocationSucceedInfo is the response returned by faas scheduler's CallHandler +type InstanceAllocationSucceedInfo struct { + FuncKey string `json:"funcKey"` + FuncSig string `json:"funcSig"` + InstanceID string `json:"instanceID"` + ThreadID string `json:"threadID"` +} + +// InstanceAllocationFailedInfo contains err info for allocation failed info +type InstanceAllocationFailedInfo struct { + ErrorCode int `json:"errorCode"` + ErrorMessage string `json:"errorMessage"` +} + +// InstanceAllocationInfo contains instance router info and lease returned to function accessor +type InstanceAllocationInfo struct { + FuncKey string `json:"funcKey"` + FuncSig string `json:"funcSig"` + InstanceID string `json:"instanceID"` + ThreadID string `json:"threadID"` + InstanceIP string `json:"instanceIP"` + InstancePort string `json:"instancePort"` + NodeIP string `json:"nodeIP"` + NodePort string `json:"nodePort"` + LeaseInterval int64 `json:"leaseInterval"` + CPU int64 `json:"cpu"` + Memory int64 `json:"memory"` +} + +// ExtraParams for interface CreateInstance +type ExtraParams struct { + DesignatedInstanceID string + Label []string + Resources map[string]float64 + CustomResources map[string]float64 + CreateOpt map[string]string + CustomExtensions map[string]string + ScheduleAffinities []api.Affinity +} + +// NuwaRuntimeInfo contains ers workload info for function +type NuwaRuntimeInfo struct { + WisecloudRuntimeId string `json:"wisecloudRuntimeId"` + WisecloudSite string `json:"wisecloudSite"` + WisecloudTenantId string `json:"wisecloudTenantId"` + WisecloudApplicationId string `json:"wisecloudApplicationId"` + WisecloudServiceId string `json:"wisecloudServiceId"` + WisecloudEnvironmentId string `json:"wisecloudEnvironmentId"` + EnvLabel string `json:"envLabel"` +} + +// InstanceSessionConfig - +type InstanceSessionConfig struct { + SessionID string `json:"sessionID"` + SessionTTL int `json:"sessionTTL"` + Concurrency int `json:"concurrency"` +} + +// CallHandlerResponse is the response returned by faas manager's CallHandler +type CallHandlerResponse struct { + Code int `json:"code"` + Message string `json:"message"` +} + +// LeaseEvent - +type LeaseEvent struct { + Type string `json:"type"` + RemoteClientID string `json:"remoteClientId"` + Timestamp int64 `json:"timestamp"` + TraceID string `json:"traceId"` +} diff --git a/frontend/pkg/common/faas_common/urnutils/gadgets.go b/frontend/pkg/common/faas_common/urnutils/gadgets.go new file mode 100644 index 0000000000000000000000000000000000000000..a7dfb7362cd501419b4514c7bb742c4d0dd5366f --- /dev/null +++ b/frontend/pkg/common/faas_common/urnutils/gadgets.go @@ -0,0 +1,98 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package urnutils contains URN element definitions and tools +package urnutils + +import ( + "strings" +) + +var ( + separator = "-" +) + +const ( + // ServiceIDPrefix is the prefix of the function with serviceID. + ServiceIDPrefix = "0" + + // DefaultSeparator is a character that separates functions and services. + DefaultSeparator = "-" + + // ServicePrefix is the prefix of the function with serviceID. + ServicePrefix = "0@" + + // TenantProductSplitStr separator between a tenant and a product + TenantProductSplitStr = "@" + + minEleSize = 3 +) + +// ComplexFuncName contains service ID and raw function name +type ComplexFuncName struct { + prefix string + ServiceID string + FuncName string +} + +// NewComplexFuncName - +func NewComplexFuncName(svcID, funcName string) *ComplexFuncName { + return &ComplexFuncName{ + prefix: ServiceIDPrefix, + ServiceID: svcID, + FuncName: funcName, + } +} + +// IsComplexFuncName - +func IsComplexFuncName(funcName string) bool { + return strings.Contains(funcName, separator) +} + +// ParseFrom parse ComplexFuncName from string +func (c *ComplexFuncName) ParseFrom(name string) *ComplexFuncName { + fields := strings.Split(name, separator) + if len(fields) < minEleSize || fields[0] != ServiceIDPrefix { + c.prefix = "" + c.ServiceID = "" + c.FuncName = name + return c + } + idx := 0 + c.prefix = fields[idx] + idx++ + c.ServiceID = fields[idx] + // $prefix$separator$ServiceID$separator$FuncName equals name + c.FuncName = name[(len(c.prefix) + len(separator) + len(c.ServiceID) + len(separator)):] + return c +} + +// String - +func (c *ComplexFuncName) String() string { + return strings.Join([]string{c.prefix, c.ServiceID, c.FuncName}, separator) +} + +// GetSvcIDWithPrefix get serviceID with prefix from function name +func (c *ComplexFuncName) GetSvcIDWithPrefix() string { + return c.prefix + separator + c.ServiceID +} + +// SetSeparator - +func SetSeparator(sep string) { + if sep != "" { + separator = sep + } +} diff --git a/frontend/pkg/common/faas_common/urnutils/gadgets_test.go b/frontend/pkg/common/faas_common/urnutils/gadgets_test.go new file mode 100644 index 0000000000000000000000000000000000000000..338653148f330ec27262f630eaef6ed7eded4178 --- /dev/null +++ b/frontend/pkg/common/faas_common/urnutils/gadgets_test.go @@ -0,0 +1,152 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package urnutils contains URN element definitions and tools +package urnutils + +import ( + "github.com/stretchr/testify/assert" + "reflect" + "testing" +) + +func TestComplexFuncName_GetSvcIDWithPrefix(t *testing.T) { + tests := []struct { + name string + fields ComplexFuncName + want string + }{ + { + name: "normal", + fields: ComplexFuncName{ + prefix: ServiceIDPrefix, + ServiceID: "absserviceid", + FuncName: "absFuncName", + }, + want: "0-absserviceid", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &ComplexFuncName{ + prefix: tt.fields.prefix, + ServiceID: tt.fields.ServiceID, + FuncName: tt.fields.FuncName, + } + if got := c.GetSvcIDWithPrefix(); got != tt.want { + t.Errorf("GetSvcIDWithPrefix() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestComplexFuncName_ParseFrom(t *testing.T) { + type args struct { + name string + } + tests := []struct { + name string + args args + want *ComplexFuncName + }{ + { + name: "normal", + args: args{ + name: "0-absserviceid-absFunc-Name", + }, + want: &ComplexFuncName{ + prefix: ServiceIDPrefix, + ServiceID: "absserviceid", + FuncName: "absFunc-Name", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &ComplexFuncName{} + if got := c.ParseFrom(tt.args.name); !reflect.DeepEqual(got, tt.want) { + t.Errorf("ParseFrom() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestComplexFuncName_String(t *testing.T) { + tests := []struct { + name string + fields ComplexFuncName + want string + }{ + { + name: "normal", + fields: ComplexFuncName{ + prefix: ServiceIDPrefix, + ServiceID: "absserviceid", + FuncName: "absFunc-Name", + }, + want: "0-absserviceid-absFunc-Name", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &ComplexFuncName{ + prefix: tt.fields.prefix, + ServiceID: tt.fields.ServiceID, + FuncName: tt.fields.FuncName, + } + if got := c.String(); got != tt.want { + t.Errorf("String() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNewComplexFuncName(t *testing.T) { + type args struct { + svcID string + funcName string + } + tests := []struct { + name string + args args + want *ComplexFuncName + }{ + { + name: "normal", + args: args{ + svcID: "absserviceid", + funcName: "absFunc-Name", + }, + want: &ComplexFuncName{ + prefix: ServiceIDPrefix, + ServiceID: "absserviceid", + FuncName: "absFunc-Name", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := NewComplexFuncName(tt.args.svcID, tt.args.funcName); !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewComplexFuncName() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestSetSeparator(t *testing.T) { + SetSeparator("@") + assert.Equal(t, "@", separator) +} diff --git a/frontend/pkg/common/faas_common/urnutils/urn_utils.go b/frontend/pkg/common/faas_common/urnutils/urn_utils.go new file mode 100644 index 0000000000000000000000000000000000000000..84b80a67486b788a05c3577d9c912cadd0027b87 --- /dev/null +++ b/frontend/pkg/common/faas_common/urnutils/urn_utils.go @@ -0,0 +1,561 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package urnutils contains URN element definitions and tools +package urnutils + +import ( + "errors" + "fmt" + "net" + "os" + "regexp" + "strconv" + "strings" + "sync" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/utils" +) + +var ( + once sync.Once + serverIP = "" +) + +const ( + funcNamePrefix = "0@default@" + shortFuncNameSplit = 1 + standardFuncNameSplit = 3 +) + +// example of function URN: :::::: +// Indices of elements in FunctionURN +const ( + // ProductIDIndex is the index of the product ID in a URN + ProductIDIndex = iota + // RegionIDIndex is the index of the region ID in a URN + RegionIDIndex + // BusinessIDIndex is the index of the business ID in a URN + BusinessIDIndex + // TenantIDIndex is the index of the tenant ID in a URN + TenantIDIndex + // FunctionSignIndex is the index of the product ID in a URN + FunctionSignIndex + // FunctionNameIndex is the index of the product name in a URN + FunctionNameIndex + // VersionIndex is the index of the version in a URN + VersionIndex + // URNLenWithVersion is the normal URN length with a version + URNLenWithVersion +) + +// An example of a function functionkey: // +const ( + // TenantIDIndexKey is the index of the tenant ID in a functionkey + TenantIDIndexKey = iota + // FunctionNameIndexKey is the index of the function name in a functionkey + FunctionNameIndexKey + // VersionIndexKey is the index of the version in a functionkey + VersionIndexKey +) + +const ( + // TenantMetadataTenantIndex is the index of the tenant ID in a tenantMetadataEtcdKey + TenantMetadataTenantIndex = 6 +) + +const ( + urnLenWithoutVersion = URNLenWithVersion - 1 + // URNSep is a URN separator of functions + URNSep = ":" + // FunctionKeySep is a functionkey separator of functions + FunctionKeySep = "/" + // DefaultURNProductID is the default product ID of a URN + DefaultURNProductID = "sn" + // DefaultURNRegion is the default region of a URN + DefaultURNRegion = "cn" + // DefaultURNFuncSign is the default function sign of a URN + DefaultURNFuncSign = "function" + defaultURNLayerSign = "layer" + anonymization = "****" + anonymizeLen = 3 + + // BranchAliasPrefix is used to remove "!" from aliasing rules at the begining of "!" + BranchAliasPrefix = 1 + // BranchAliasRule is an aliased rule that begins with an "!" + BranchAliasRule = "!" + functionNameStartIndex = 2 + // ServiceNameIndex is index of service name in urn + ServiceNameIndex = 1 + funcNameMinLen = 3 + // defaultFunctionMaxLen is max length of function name + defaultFunctionMaxLen = 128 +) + +// An example of a worker-manager URN: +// +// /sn/workers/business/iot/tenant/j0f4413f7b4b4c33be576d432f7ee085/function/functest/version/$latest +// /cn-north-1a/cn-north-1a-#-ws-j0f4413f7b-functest-faaslatest-deployment-55b5f9dcb7-r2dsv +const ( + // URNIndexZero URN index 0 + URNIndexZero = iota + // URNIndexOne URN index 1 + URNIndexOne + // URNIndexTwo URN index 2 + URNIndexTwo + // URNIndexThree URN index 3 + URNIndexThree + // URNIndexFour URN index 4 + URNIndexFour + // URNIndexFive URN index 5 + URNIndexFive + // URNIndexSix URN index 6 + URNIndexSix + // URNIndexSeven URN index 7 + URNIndexSeven + // URNIndexEight URN index 8 + URNIndexEight + // URNIndexNine URN index 9 + URNIndexNine + // URNIndexTen URN index 10 + URNIndexTen + // URNIndexEleven URN index 11 + URNIndexEleven + // URNIndexTwelve URN index 12 + URNIndexTwelve + // URNIndexThirteen URN index 13 + URNIndexThirteen +) +const ( + k8sLabelLen = 63 + otherStrLen = 4 + crHashMaxLen = 10 + versionManLen = 30 +) + +const ( + // OwnerReadWrite - + OwnerReadWrite = 416 // 640:rw- r-- --- + // DefaultMode - + DefaultMode = 420 // 644:rw- r-- r-- + // CertMode - + CertMode = 384 // 600:rw- --- --- +) + +var ( + functionGraphFuncNameRegexp = regexp.MustCompile("^[a-zA-Z]([a-zA-Z0-9_-]*[a-zA-Z0-9])?$") +) + +// FunctionURN contains elements of a product URN. It can expand to FunctionURN, LayerURN and WorkerURN +type FunctionURN struct { + ProductID string + RegionID string + BusinessID string + TenantID string + TypeSign string + FuncName string + FuncVersion string +} + +// String serializes elements of function URN struct to string +func (p *FunctionURN) String() string { + urn := fmt.Sprintf("%s:%s:%s:%s:%s:%s", p.ProductID, p.RegionID, + p.BusinessID, p.TenantID, p.TypeSign, p.FuncName) + if p.FuncVersion != "" { + return fmt.Sprintf("%s:%s", urn, p.FuncVersion) + } + return urn +} + +// ParseFrom parses elements from a function URN +func (p *FunctionURN) ParseFrom(urn string) error { + elements := strings.Split(urn, URNSep) + urnLen := len(elements) + if urnLen < urnLenWithoutVersion || urnLen > URNLenWithVersion { + return fmt.Errorf("failed to parse urn from: %s, invalid length: %d", urn, urnLen) + } + p.ProductID = elements[ProductIDIndex] + p.RegionID = elements[RegionIDIndex] + p.BusinessID = elements[BusinessIDIndex] + p.TenantID = elements[TenantIDIndex] + p.TypeSign = elements[FunctionSignIndex] + p.FuncName = elements[FunctionNameIndex] + if urnLen == URNLenWithVersion { + p.FuncVersion = elements[VersionIndex] + } + return nil +} + +// StringWithoutVersion return string without version +func (p *FunctionURN) StringWithoutVersion() string { + return fmt.Sprintf("%s:%s:%s:%s:%s:%s", p.ProductID, p.RegionID, + p.BusinessID, p.TenantID, p.TypeSign, p.FuncName) +} + +// GetFunctionInfo collects function information from a URN +func GetFunctionInfo(urn string) (FunctionURN, error) { + var parsedURN FunctionURN + if err := parsedURN.ParseFrom(urn); err != nil { + log.GetLogger().Errorf("error while parsing an URN: %s", err.Error()) + return FunctionURN{}, fmt.Errorf("parsing an URN error: %s", err) + } + return parsedURN, nil +} + +// GetFuncInfoWithVersion collects function information and distinguishes if the URN contains a version +func GetFuncInfoWithVersion(urn string) (FunctionURN, error) { + parsedURN, err := GetFunctionInfo(urn) + if err != nil { + return parsedURN, err + } + if parsedURN.FuncVersion == "" { + log.GetLogger().Errorf("incorrect URN length: %s", Anonymize(urn)) + return parsedURN, errors.New("incorrect URN length, no version") + } + return parsedURN, nil +} + +// ParseAliasURN is used to remove "!" from the beginning of the alias +func ParseAliasURN(aliasURN string) string { + elements := strings.Split(aliasURN, URNSep) + if len(elements) == URNLenWithVersion { + if strings.HasPrefix(elements[VersionIndex], BranchAliasRule) { + elements[VersionIndex] = elements[VersionIndex][BranchAliasPrefix:] + } + return strings.Join(elements, ":") + } + return aliasURN +} + +// GetAlias returns an alias +func (p *FunctionURN) GetAlias() string { + if p.FuncVersion == constant.DefaultURNVersion { + return "" + } + if _, err := strconv.Atoi(p.FuncVersion); err == nil { + return "" + } + return p.FuncVersion +} + +// GetAliasForFuncBranch returns an alias for function branch +func (p *FunctionURN) GetAliasForFuncBranch() string { + if strings.HasPrefix(p.FuncVersion, BranchAliasRule) { + // remove "!" from the beginning of the alias + return p.FuncVersion[BranchAliasPrefix:] + } + return "" +} + +// Valid check whether the self-verification function name complies with the specifications. +func (p *FunctionURN) Valid() error { + serviceID, functionName, err := GetFunctionNameAndServiceName(p.FuncName) + if err != nil { + log.GetLogger().Errorf("failed to get serviceID and functionName") + return err + } + if !(functionGraphFuncNameRegexp.MatchString(serviceID) || + functionGraphFuncNameRegexp.MatchString(functionName)) { + errmsg := "failed to match reg%s" + log.GetLogger().Errorf(errmsg, functionGraphFuncNameRegexp) + return fmt.Errorf(errmsg, functionGraphFuncNameRegexp) + } + if len(serviceID) > defaultFunctionMaxLen || len(functionName) > defaultFunctionMaxLen { + errmsg := "serviceID or functionName's len is out of range %d" + log.GetLogger().Errorf(errmsg, defaultFunctionMaxLen) + return fmt.Errorf(errmsg, defaultFunctionMaxLen) + } + return nil +} + +// GetFunctionNameAndServiceName returns serviceName and FunctionName +func GetFunctionNameAndServiceName(funcName string) (string, string, error) { + if strings.HasPrefix(funcName, ServiceIDPrefix) { + split := strings.Split(funcName, separator) + if len(split) < funcNameMinLen { + log.GetLogger().Errorf("incorrect function name length: %s", len(split)) + return "", "", errors.New("parsing a function name error") + } + return split[ServiceNameIndex], strings.Join(split[functionNameStartIndex:], separator), nil + } + log.GetLogger().Errorf("incorrect function name: %s", funcName) + return "", "", errors.New("parsing a function name error") +} + +// Anonymize anonymize input str to xxx****xxx +func Anonymize(str string) string { + if len(str) < anonymizeLen+1+anonymizeLen { + return anonymization + } + return str[:anonymizeLen] + anonymization + str[len(str)-anonymizeLen:] +} + +// AnonymizeTenantURN Anonymize tenant info in urn +func AnonymizeTenantURN(urn string) string { + elements := strings.Split(urn, URNSep) + urnLen := len(elements) + if urnLen < urnLenWithoutVersion || urnLen > URNLenWithVersion { + return urn + } + elements[TenantIDIndex] = Anonymize(elements[TenantIDIndex]) + return strings.Join(elements, URNSep) +} + +// AnonymizeTenantKey Anonymize tenant info in functionkey +func AnonymizeTenantKey(functionKey string) string { + elements := strings.Split(functionKey, FunctionKeySep) + keyLen := len(elements) + if TenantIDIndexKey >= keyLen { + return functionKey + } + elements[TenantIDIndexKey] = Anonymize(elements[TenantIDIndexKey]) + return strings.Join(elements, FunctionKeySep) +} + +// AnonymizeTenantURNSlice Anonymize tenant info in urn slice +func AnonymizeTenantURNSlice(urns []string) []string { + var anonymizeUrns []string + for i := 0; i < len(urns); i++ { + anonymizeUrn := AnonymizeTenantURN(urns[i]) + anonymizeUrns = append(anonymizeUrns, anonymizeUrn) + } + return anonymizeUrns +} + +// AnonymizeTenantMetadataEtcdKey Anonymize tenant info in tenant metadata etcd key +// /sn/quota/cluster/cluster001/tenant/7e1ad6a6-cc5c-44fa-bd54-25873f72a86a/instancemetadata +func AnonymizeTenantMetadataEtcdKey(etcdKey string) string { + elements := strings.Split(etcdKey, "/") + if len(elements) <= TenantMetadataTenantIndex { + return etcdKey + } + elements[TenantMetadataTenantIndex] = Anonymize(elements[TenantMetadataTenantIndex]) + return strings.Join(elements, "/") +} + +// AnonymizeKeys - anonymize the input slice of string to slice of xxx****xxx +// data system key example: 638cf733-a625-4850-9f23-9ef49873f5a3;2ba6f9cd-c8d3-4655-a9d0-e67d7abcfb3f +func AnonymizeKeys(keys []string) []string { + res := make([]string, len(keys)) + for i, str := range keys { + res[i] = Anonymize(str) + } + return res +} + +// BuildURNOrAliasURNTemp - build urn format +func BuildURNOrAliasURNTemp(business, tenant, function, versionOrAlias string) string { + if business == "" || tenant == "" || function == "" || versionOrAlias == "" { + return "" + } + return fmt.Sprintf("%s:%s:%s:%s:%s:%s:%s", DefaultURNProductID, DefaultURNRegion, + business, tenant, DefaultURNFuncSign, function, versionOrAlias) +} + +// GetServerIP - +func GetServerIP() (string, error) { + var err error + once.Do(func() { + addr, errMsg := GetHostAddr() + if errMsg != nil { + err = errMsg + return + } + serverIP = addr[0] + }) + return serverIP, err +} + +// GetHostAddr - +func GetHostAddr() ([]string, error) { + name, err := os.Hostname() + if err != nil { + log.GetLogger().Errorf("get hostname failed: %v", err) + return nil, err + } + + addrs, err := net.LookupHost(name) + if err != nil || len(addrs) == 0 { + log.GetLogger().Errorf("look up host by name failed") + return nil, fmt.Errorf("look up host by name failed") + } + return addrs, nil +} + +// CrNameByURN returns a CR name by URN +func CrNameByURN(urn string) string { + if len(urn) == URNIndexZero { + return "" + } + baseUrn, err := GetFunctionInfo(urn) + if err != nil { + return "" + } + return CrName(baseUrn.BusinessID, baseUrn.TenantID, baseUrn.FuncName, baseUrn.FuncVersion) +} + +// CrName CR Name +// [y/z]brief-functionname-version-hash +func CrName(business, tenant, funcName, version string) string { + hashStr := genFunctionCRStr(business, tenant, funcName, version) + crHash := utils.FnvHash(hashStr) + if len(crHash) > crHashMaxLen { + crHash = crHash[:crHashMaxLen] + } + brief := acquireBrief(business, tenant) + ver := VersionConvForBranch(version) + // cannot contain (urnutils.separator, ususually @) or _. If contains, replace it with -. + funcName = strings.ReplaceAll(funcName, "@", "-") + funcName = strings.ReplaceAll(funcName, "_", "-") + + // otherStrLen is 4 contains three - and a z or y. + shortFunctionNameLen := k8sLabelLen - len(brief) - len(ver) - len(crHash) - otherStrLen + // funcName prefix is 0- means funcName has joint sn service id + // k8s label max length is 63, so cr name need to delete sn service id + // otherwise, cr name length more than 63 characters, error + if strings.HasPrefix(funcName, ServiceIDPrefix) && len(funcName) > shortFunctionNameLen { + funcName = acquireShorter(funcName, shortFunctionNameLen) + } + + crName := brief + "-" + funcName + "-" + ver + "-" + crHash + crNameLower := strings.ToLower(crName) + if crName == crNameLower { + return "y" + crNameLower + } + + return "z" + crNameLower +} + +func genFunctionCRStr(business string, tenant string, funcName string, version string) string { + return business + "-" + tenant + "-" + funcName + "-" + version +} + +func acquireBrief(business, tenant string) string { + if len(business) > URNIndexFour { + business = business[:URNIndexFour] + } + product, tenant := splitTenant(tenant) + if len(tenant) > URNIndexFour { + tenant = tenant[:URNIndexFour] + } + + if len(product) > URNIndexFour { + product = product[:URNIndexFour] + } + + return business + tenant + product +} + +func splitTenant(tenant string) (string, string) { + var product string + t := strings.Split(tenant, TenantProductSplitStr) + l := len(t) + if l == URNIndexOne { + return product, tenant + } + if l == URNIndexTwo { + tenant = t[URNIndexZero] + product = t[URNIndexOne] + return product, tenant + } + return "", product +} + +// VersionConvForBranch return version Conv for branch +func VersionConvForBranch(v string) string { + // cannot contain _. If the version cr contains _, replace it with -. + version := strings.ReplaceAll(v, "_", "-") + if len(version) > versionManLen { + version = version[:versionManLen] + } + return version +} + +// if funcName contains sn service id, this method can acquire +// first 4 character of sn id and real function name with split _ +// return shorter serviceID and shorter funcName +func acquireShorter(funcName string, functionNameLen int) string { + shorterFuncName := []rune(funcName) + return string(shorterFuncName[len(shorterFuncName)-functionNameLen : len(shorterFuncName)-1]) +} + +// GetTenantFromFuncKey - +func GetTenantFromFuncKey(funcKey string) string { + elements := strings.Split(funcKey, FunctionKeySep) + keyLen := len(elements) + if keyLen != URNIndexThree { + return "" + } + return elements[TenantIDIndexKey] +} + +// GetFuncNameFromFuncKey - +func GetFuncNameFromFuncKey(funcKey string) string { + elements := strings.Split(funcKey, FunctionKeySep) + keyLen := len(elements) + if keyLen != URNIndexThree { + return "" + } + return elements[TenantIDIndexKey] + FunctionKeySep + elements[FunctionNameIndexKey] +} + +// GetTenantFromAliasUrn - +func GetTenantFromAliasUrn(aliasUrn string) string { + elements := strings.Split(aliasUrn, URNSep) + keyLen := len(elements) + if keyLen != URNIndexSeven { + return "" + } + return elements[URNIndexThree] +} + +// CheckAliasUrnTenant - +func CheckAliasUrnTenant(tenantID string, aliasUrn string) bool { + if GetTenantFromAliasUrn(aliasUrn) != "" && + GetTenantFromAliasUrn(aliasUrn) == tenantID { + return true + } + return false +} + +// CombineFunctionKey will generate funcKey from three IDs +func CombineFunctionKey(tenantID, funcName, version string) string { + return fmt.Sprintf("%s/%s/%s", tenantID, funcName, version) +} + +// GetShortFuncName - +func GetShortFuncName(funcName string) string { + if len(funcName) > k8sLabelLen { + // labels must begin and end with an alphanumeric character, so set first character always X + funcName = "X" + funcName[len(funcName)-k8sLabelLen+1:] + } + return funcName +} + +// BuildStandardFunctionName - 将不带版本、别名的方法名拼接成0@default@开头的完整方法名 +func BuildStandardFunctionName(functionName string) string { + splits := strings.Split(functionName, "@") + if len(splits) != shortFuncNameSplit && len(splits) != standardFuncNameSplit { + return "" + } + standardFunctionName := functionName + if len(splits) == shortFuncNameSplit { + standardFunctionName = funcNamePrefix + standardFunctionName + } + return standardFunctionName +} diff --git a/frontend/pkg/common/faas_common/urnutils/urn_utils_test.go b/frontend/pkg/common/faas_common/urnutils/urn_utils_test.go new file mode 100644 index 0000000000000000000000000000000000000000..c6d532f1c33bc0d0eeb633a6958a92ec06660bd4 --- /dev/null +++ b/frontend/pkg/common/faas_common/urnutils/urn_utils_test.go @@ -0,0 +1,475 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package urnutils + +import ( + "net" + "os" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/stretchr/testify/assert" + + "frontend/pkg/common/faas_common/constant" + mockUtils "frontend/pkg/common/faas_common/utils" +) + +func TestProductUrn_ParseFrom(t *testing.T) { + absURN := FunctionURN{ + "absPrefix", + "absZone", + "absBusinessID", + "absTenantID", + "absProductID", + "absName", + "latest", + } + absURNStr := "absPrefix:absZone:absBusinessID:absTenantID:absProductID:absName:latest" + type args struct { + urn string + } + tests := []struct { + name string + fields FunctionURN + args args + want FunctionURN + }{ + { + name: "normal test", + args: args{ + absURNStr, + }, + want: absURN, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &FunctionURN{} + if _ = p.ParseFrom(tt.args.urn); !reflect.DeepEqual(*p, tt.want) { + t.Errorf("ParseFrom() p = %v, want %v", *p, tt.want) + } + }) + } +} + +func TestProductUrn_String(t *testing.T) { + tests := []struct { + name string + fields FunctionURN + want string + }{ + { + "stringify with version", + FunctionURN{ + "absPrefix", + "absZone", + "absBusinessID", + "absTenantID", + "absProductID", + "absName", + "latest", + }, + "absPrefix:absZone:absBusinessID:absTenantID:absProductID:absName:latest", + }, + { + "stringify without version", + FunctionURN{ + ProductID: "absPrefix", + RegionID: "absZone", + BusinessID: "absBusinessID", + TenantID: "absTenantID", + TypeSign: "absProductID", + FuncName: "absName", + }, + "absPrefix:absZone:absBusinessID:absTenantID:absProductID:absName", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &FunctionURN{ + ProductID: tt.fields.ProductID, + RegionID: tt.fields.RegionID, + BusinessID: tt.fields.BusinessID, + TenantID: tt.fields.TenantID, + TypeSign: tt.fields.TypeSign, + FuncName: tt.fields.FuncName, + FuncVersion: tt.fields.FuncVersion, + } + if got := p.String(); got != tt.want { + t.Errorf("String() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestProductUrn_StringWithoutVersion(t *testing.T) { + tests := []struct { + name string + fields FunctionURN + want string + }{ + { + "stringify without version", + FunctionURN{ + "absPrefix", + "absZone", + "absBusinessID", + "absTenantID", + "absProductID", + "absName", + "latest", + }, + "absPrefix:absZone:absBusinessID:absTenantID:absProductID:absName", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &FunctionURN{ + ProductID: tt.fields.ProductID, + RegionID: tt.fields.RegionID, + BusinessID: tt.fields.BusinessID, + TenantID: tt.fields.TenantID, + TypeSign: tt.fields.TypeSign, + FuncName: tt.fields.FuncName, + FuncVersion: tt.fields.FuncVersion, + } + if got := p.StringWithoutVersion(); got != tt.want { + t.Errorf("StringWithoutVersion() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestAnonymize(t *testing.T) { + tests := []struct { + input string + want string + }{ + {"0", anonymization}, + {"123", anonymization}, + {"123456", anonymization}, + {"1234567", "123****567"}, + {"12345678901234546", "123****546"}, + } + for _, tt := range tests { + assert.Equal(t, tt.want, Anonymize(tt.input)) + } +} + +func TestAnonymizeTenantURN(t *testing.T) { + tests := []struct { + input string + want string + }{ + {"absPrefix:absZone:absBusinessID:absTenantID:absProductID:absName", "absPrefix:absZone:absBusinessID:abs****tID:absProductID:absName"}, + {"absPrefix:absZone:absBusinessID:absTenantID:absProductID:absName:latest", "absPrefix:absZone:absBusinessID:abs****tID:absProductID:absName:latest"}, + {"a:b:c", "a:b:c"}, + } + for _, tt := range tests { + assert.Equal(t, tt.want, AnonymizeTenantURN(tt.input)) + } +} + +func TestBaseURN_Valid(t *testing.T) { + separator = "@" + urn := FunctionURN{ + ProductID: "", + RegionID: "", + BusinessID: "", + TenantID: "", + TypeSign: "", + FuncName: "0@a_-9AA@AA", + FuncVersion: "", + } + success := urn.Valid() + assert.Equal(t, nil, success) + + urn.FuncName = "0@a_-9AA@tttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt" + success = urn.Valid() + assert.Equal(t, nil, success) + + urn.FuncName = "0@a_-9AA@ttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt" + err := urn.Valid() + assert.NotEqual(t, nil, err) + + urn.FuncName = "@func" + err = urn.Valid() + assert.NotEqual(t, nil, err) + + urn.FuncName = "0@func" + err = urn.Valid() + assert.NotEqual(t, nil, err) + + urn.FuncName = "0@^@^" + err = urn.Valid() + assert.NotEqual(t, nil, err) + + separator = "-" +} + +func TestBaseURN_GetAlias(t *testing.T) { + urn := FunctionURN{ + ProductID: "", + RegionID: "", + BusinessID: "", + TenantID: "", + TypeSign: "", + FuncName: "0@a_-9AA@AA", + FuncVersion: constant.DefaultURNVersion, + } + + alias := urn.GetAlias() + assert.Equal(t, "", alias) + + urn.FuncVersion = "old" + alias = urn.GetAlias() + assert.Equal(t, "old", alias) +} + +func TestGetFuncInfoWithVersion(t *testing.T) { + urn := "urn" + _, err := GetFuncInfoWithVersion(urn) + assert.NotEqual(t, nil, err) + + urn = "absPrefix:absZone:absBusinessID:absTenantID:absProductID:absName" + _, err = GetFuncInfoWithVersion(urn) + assert.NotEqual(t, nil, err) + + urn = "absPrefix:absZone:absBusinessID:absTenantID:absProductID:absName:latest" + parsedURN, err := GetFuncInfoWithVersion(urn) + assert.Equal(t, "absName", parsedURN.FuncName) +} + +func TestAnonymizeTenantKey(t *testing.T) { + inputKey := "" + outputKey := AnonymizeTenantKey(inputKey) + assert.Equal(t, "****", outputKey) + + inputKey = "input/key" + outputKey = AnonymizeTenantKey(inputKey) + assert.Equal(t, "****/key", outputKey) +} + +func TestParseAliasURN(t *testing.T) { + urn := "" + alias := ParseAliasURN(urn) + assert.Equal(t, urn, alias) + + urn = "absPrefix:absZone:absBusinessID:absTenantID:absProductID:absName:!latest" + alias = ParseAliasURN(urn) + assert.Equal(t, "absPrefix:absZone:absBusinessID:absTenantID:absProductID:absName:latest", alias) +} + +func TestAnonymizeTenantURNSlice(t *testing.T) { + inUrn := []string{"in", "in/urn"} + outUrn := AnonymizeTenantURNSlice(inUrn) + assert.Equal(t, "in", outUrn[0]) + assert.Equal(t, "in/urn", outUrn[1]) +} + +func TestBaseURN_GetAliasForFuncBranch(t *testing.T) { + urn := FunctionURN{ + ProductID: "", + RegionID: "", + BusinessID: "", + TenantID: "", + TypeSign: "", + FuncName: "0@a_-9AA@AA", + FuncVersion: "!latest", + } + + alias := urn.GetAliasForFuncBranch() + assert.Equal(t, "latest", alias) + + urn.FuncVersion = "latest" + alias = urn.GetAliasForFuncBranch() + assert.Equal(t, "", alias) +} + +func TestAnonymizeKeys(t *testing.T) { + type args struct { + keys []string + } + tests := []struct { + name string + args args + want []string + }{ + {"case", args{keys: []string{"123", "1234567"}}, []string{"****", "123****567"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, AnonymizeKeys(tt.args.keys), "AnonymizeKeys(%v)", tt.args.keys) + }) + } +} + +func TestBuildURNOrAliasURNTemp(t *testing.T) { + type args struct { + business string + tenant string + function string + versionOrAlias string + } + tests := []struct { + name string + args args + want string + }{ + {"empty", args{}, ""}, + {"empty", args{"1", "2", "3", "4"}, "sn:cn:1:2:function:3:4"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, BuildURNOrAliasURNTemp(tt.args.business, tt.args.tenant, tt.args.function, tt.args.versionOrAlias), "BuildURNOrAliasURNTemp(%v, %v, %v, %v)", tt.args.business, tt.args.tenant, tt.args.function, tt.args.versionOrAlias) + }) + } +} + +func TestCrNameByUrn(t *testing.T) { + type args struct { + args string + } + var a args + a.args = "sn:cn:yrk:12345678901234561234567890123456:function:0@yrservice@test_func:v1" + var b args + b.args = "" + tests := []struct { + name string + args args + want string + }{ + {"case1", a, "yyrk1234-0-yrservice-test-func-v1-2966683772"}, + {"case2", b, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := CrNameByURN(tt.args.args); got != tt.want { + t.Errorf("CrNameByURN() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGetServerIP(t *testing.T) { + tests := []struct { + name string + want string + wantErr bool + patchesFunc mockUtils.PatchesFunc + }{ + {"case1 succeed to get ip", "127.0.0.1", false, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(os.Hostname, func() (name string, err error) { return "127.0.0.1", nil })}) + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(net.LookupHost, + func(host string) (addrs []string, err error) { return []string{"127.0.0.1", "0"}, nil })}) + return patches + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + patches := tt.patchesFunc() + got, err := GetServerIP() + if (err != nil) != tt.wantErr { + t.Errorf("GetServerIP() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("GetServerIP() got = %v, want %v", got, tt.want) + } + patches.ResetAll() + }) + } +} + +func TestCheckAliasUrnTenant(t *testing.T) { + type args struct { + tenantID string + aliasUrn string + } + tests := []struct { + name string + args args + want bool + }{ + {"case1", args{tenantID: "12345678901234561234567890123456", + aliasUrn: "sn:cn:yrk:12345678901234561234567890123456:function:helloworld:myaliasv1"}, true}, + {"case2 error", args{tenantID: "12345678901234561234567890123456", + aliasUrn: "sn:cn:yrk:12345678901234561234567890123456:function:helloworld"}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := CheckAliasUrnTenant(tt.args.tenantID, tt.args.aliasUrn); got != tt.want { + t.Errorf("CheckAliasUrnTenant() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGetTenantFormFuncKey(t *testing.T) { + type args struct { + funcKey string + } + tests := []struct { + name string + args args + want string + }{ + {"case1", args{funcKey: "12345678901234561234567890123456/0-system-faasscheduler/$latest"}, + "12345678901234561234567890123456"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GetTenantFromFuncKey(tt.args.funcKey); got != tt.want { + t.Errorf("GetTenantFromFuncKey() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGetShortFuncName(t *testing.T) { + funcName := "testFunc1111111111111111111111111111111111111111111111111111111" + shortFuncName := GetShortFuncName(funcName) + assert.Equal(t, "testFunc1111111111111111111111111111111111111111111111111111111", shortFuncName) + + funcName = "testFunc1111111111111111111111111111111111111111111111111111111111111111111111111111111" + shortFuncName = GetShortFuncName(funcName) + assert.Equal(t, "X11111111111111111111111111111111111111111111111111111111111111", shortFuncName) +} + +func TestGetFuncNameFromFuncKey(t *testing.T) { + funcKey := "12345/test_func/latest/1" + funcName := GetFuncNameFromFuncKey(funcKey) + assert.Equal(t, "", funcName) + + funcKey = "12345/test_func/latest" + funcName = GetFuncNameFromFuncKey(funcKey) + assert.Equal(t, "12345/test_func", funcName) +} + +func TestAnonymizeTenantMetadataEtcdKey(t *testing.T) { + etcdKey := "/sn/quota/cluster/cluster001/tenant/7e1ad6a6-cc5c-44fa-bd54-25873f72a86a" + AnonymizedKey := AnonymizeTenantMetadataEtcdKey(etcdKey) + assert.Equal(t, "/sn/quota/cluster/cluster001/tenant/7e1****86a", AnonymizedKey) + + etcdKey = "/sn/quota/cluster/cluster001/tenant/7e1ad6a6-cc5c-44fa-bd54-25873f72a86a/instancemetadata" + AnonymizedKey = GetFuncNameFromFuncKey(etcdKey) + assert.Equal(t, "", AnonymizedKey) +} diff --git a/frontend/pkg/common/faas_common/urnutils/urnconv.go b/frontend/pkg/common/faas_common/urnutils/urnconv.go new file mode 100644 index 0000000000000000000000000000000000000000..30bacf88bb5a66ecb9af28837e45750559c34b13 --- /dev/null +++ b/frontend/pkg/common/faas_common/urnutils/urnconv.go @@ -0,0 +1,56 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package urnutils - +package urnutils + +import ( + "strings" +) + +// FunctionInfo defines Function Info +type FunctionInfo struct { + Business string + Tenant string + FuncName string + Version string +} + +// CrNameByKey return Cr Name By function key +func CrNameByKey(funcKey string) string { + functionInfo := GetFunctionInfoByKey(funcKey) + business, tenant, funcName, version := functionInfo.Business, functionInfo.Tenant, + functionInfo.FuncName, functionInfo.Version + + return CrName(business, tenant, funcName, version) +} + +// GetFunctionInfoByKey - +func GetFunctionInfoByKey(key string) FunctionInfo { + var functionInfo FunctionInfo + keyFields := strings.Split(key, "/") + + if len(keyFields) != URNIndexEleven && len(keyFields) != URNIndexThirteen { + return functionInfo + } + + functionInfo.Business = keyFields[URNIndexFour] + functionInfo.Tenant = keyFields[URNIndexSix] + functionInfo.FuncName = keyFields[URNIndexEight] + functionInfo.Version = keyFields[URNIndexTen] + + return functionInfo +} diff --git a/frontend/pkg/common/faas_common/urnutils/urnconv_test.go b/frontend/pkg/common/faas_common/urnutils/urnconv_test.go new file mode 100644 index 0000000000000000000000000000000000000000..4181f0ca8216df5d40100e7f767876fdc22fff48 --- /dev/null +++ b/frontend/pkg/common/faas_common/urnutils/urnconv_test.go @@ -0,0 +1,48 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package urnutils - +package urnutils + +import "testing" + +func TestCrNameByKey(t *testing.T) { + type args struct { + funcKey string + } + tests := []struct { + name string + args args + want string + }{ + {"case1 succeed to get CrNameByKey", args{funcKey: "/sn/functions/business/yrk/tenant" + + "/172120022624850603/function/0@default@testurpccustomoom002/version/latest"}, + "yyrk1721-0-default-testurpccustomoom002-latest-1257561201"}, + {"case2 long funcName", args{funcKey: "/sn/functions/business/yrk/tenant/12345678901234561234567890123456/" + + "function/0-actordemo-test-actor-support-version-publish-delete-version/version/$latest"}, + "yyrk1234-port-version-publish-delete-versio-$latest-4279038269"}, + {"case3 long version", args{funcKey: "/sn/functions/business/yrk/tenant/12345678901234561234567890123456/function" + + "/0-actordemo-test-actor-support-version-publish-delete-version/version/123456789123456789123456789123456789123456789123456789123456"}, + "yyrk1234-lete-versio-123456789123456789123456789123-3816641367"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := CrNameByKey(tt.args.funcKey); got != tt.want { + t.Errorf("CrNameByKey() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/frontend/pkg/common/faas_common/utils/component_util.go b/frontend/pkg/common/faas_common/utils/component_util.go new file mode 100644 index 0000000000000000000000000000000000000000..b89c712d5e6285fc1d30c88d4f2250c5ffbea22c --- /dev/null +++ b/frontend/pkg/common/faas_common/utils/component_util.go @@ -0,0 +1,52 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package utils - +package utils + +import ( + "k8s.io/api/core/v1" +) + +type container string + +const ( + // ContainerRuntimeManager - + ContainerRuntimeManager container = "runtime-manager" +) + +// VolumeBuilder - +type VolumeBuilder struct { + Volumes []v1.Volume + Mounts map[container][]v1.VolumeMount +} + +// AddVolume - +func (vc *VolumeBuilder) AddVolume(volume v1.Volume) { + vc.Volumes = append(vc.Volumes, volume) +} + +// AddVolumeMount - +func (vc *VolumeBuilder) AddVolumeMount(name container, mount v1.VolumeMount) { + vc.Mounts[name] = append(vc.Mounts[name], mount) +} + +// NewVolumeBuilder - +func NewVolumeBuilder() *VolumeBuilder { + return &VolumeBuilder{ + Mounts: make(map[container][]v1.VolumeMount), + } +} diff --git a/frontend/pkg/common/faas_common/utils/file_test.go b/frontend/pkg/common/faas_common/utils/file_test.go new file mode 100644 index 0000000000000000000000000000000000000000..5c9ec263bbd36d4117da03d58fbaa5d2f0570e6a --- /dev/null +++ b/frontend/pkg/common/faas_common/utils/file_test.go @@ -0,0 +1,53 @@ +package utils + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestFileExists(t *testing.T) { + Convey("Given a temp file", t, func() { + file, err := ioutil.TempFile("", "test-file") + So(err, ShouldBeNil) + filename := file.Name() + + Convey("When it is created", func() { + Convey("Then it should return true", func() { + So(FileExists(filename), ShouldBeTrue) + }) + }) + + Convey("When we delete the file", func() { + err := file.Close() + So(err, ShouldBeNil) + err = os.Remove(filename) + So(err, ShouldBeNil) + + Convey("Then it should return false", func() { + So(FileExists(filename), ShouldBeFalse) + }) + }) + }) +} + +func TestValidateFilePath(t *testing.T) { + Convey("Given a abs file path and a rel file path", t, func() { + relPath := "a/b" + absPath, err := filepath.Abs(relPath) + So(err, ShouldBeNil) + + Convey("The abs path should not return an error", func() { + err = ValidateFilePath(absPath) + So(err, ShouldBeNil) + }) + + Convey("The rel path should return an error", func() { + err := ValidateFilePath(relPath) + So(err, ShouldNotBeNil) + }) + }) +} diff --git a/frontend/pkg/common/faas_common/utils/func_meta_util.go b/frontend/pkg/common/faas_common/utils/func_meta_util.go new file mode 100644 index 0000000000000000000000000000000000000000..13528e9dcce9825ed92c8f792e6fafee6a631989 --- /dev/null +++ b/frontend/pkg/common/faas_common/utils/func_meta_util.go @@ -0,0 +1,193 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package utils - +package utils + +import ( + "encoding/json" + "fmt" + "hash/fnv" + "strconv" + "strings" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/types" +) + +const ( + funcInfoMinLen = 3 + // InstanceScalePolicyStaticFunction is the schedule policy for static function + InstanceScalePolicyStaticFunction = "staticFunction" +) + +// GetFuncMetaSignature will calculate function signature based on essentials +func GetFuncMetaSignature(metaInfo *types.FunctionMetaInfo, filterFlag bool) string { + // static function set revisionID as signature + if metaInfo.InstanceMetaData.ScalePolicy == InstanceScalePolicyStaticFunction { + return metaInfo.FuncMetaData.RevisionID + } + metaInfoCopy := &types.FunctionMetaInfo{} + if err := DeepCopyObj(metaInfo, metaInfoCopy); err != nil { + return "invalid function meta info" + } + if filterFlag { + metaInfoFieldFilter(metaInfoCopy) + } + metaInfoCopy.FuncMetaData.FuncID = "" + metaInfoCopy.FuncMetaData.Type = "" + metaInfoCopy.FuncMetaData.EnableCloudDebug = "" + metaInfoCopy.FuncMetaData.Dependencies = "" + metaInfoCopy.FuncMetaData.CodeSize = 0 + metaInfoCopy.FuncMetaData.CodeSha512 = "" + metaInfoCopy.FuncMetaData.FunctionType = "" + metaInfoCopy.FuncMetaData.Tags = nil + metaInfoCopy.FuncMetaData.FunctionDescription = "" + metaInfoCopy.FuncMetaData.FunctionUpdateTime = "" + metaInfoCopy.InstanceMetaData.ScalePolicy = "" + metaInfoCopy.InstanceMetaData.MaxInstance = 0 + metaInfoCopy.InstanceMetaData.MinInstance = 0 + metaInfoCopy.ExtendedMetaData.DynamicConfig.UpdateTime = "" + metaInfoCopy.ExtendedMetaData.DynamicConfig.ConfigContent = []types.KV{} + metaInfoCopy.ExtendedMetaData.StrategyConfig = types.StrategyConfig{} + metaInfoCopy.ExtendedMetaData.ExtendConfig = "" + metaInfoCopy.ExtendedMetaData.EnterpriseProjectID = "" + metaInfoCopy.ExtendedMetaData.AsyncConfigLoaded = false + metaInfoCopy.ExtendedMetaData.NetworkController = types.NetworkController{} + metaInfoCopy.ResourceMetaData.CustomResourcesSpec = + getCustomResourceSpec(metaInfo.ResourceMetaData.CustomResources, metaInfo.ResourceMetaData.CustomResourcesSpec) + data, err := json.Marshal(metaInfoCopy) + if err != nil { + return "invalid function meta info" + } + return FnvHash(string(data)) +} +func getCustomResourceSpec(customResources string, customResourceSpec string) string { + // customResources为空，customResourceSpec必然为空 + if customResources == "" { + return "" + } + customResourcesJSON := make(map[string]int64) + customResourcesSpecJSON := make(map[string]interface{}) + err1 := json.Unmarshal([]byte(customResources), &customResourcesJSON) + + err2 := json.Unmarshal([]byte(customResourceSpec), &customResourcesSpecJSON) + if err1 != nil || (err2 != nil && customResourceSpec != "") { + return "" + } + for k := range customResourcesJSON { + if k == "huawei.com/ascend-1980" { + _, ok := customResourcesSpecJSON["instanceType"] + if !ok { + customResourcesSpecJSON["instanceType"] = "376T" + } + break + } + } + v, err3 := json.Marshal(customResourcesSpecJSON) + if err3 != nil { + return "" + } + return string(v) +} + +func metaInfoFieldFilter(metaInfoCopy *types.FunctionMetaInfo) { + metaInfoCopy.FuncMetaData.Service = "" + metaInfoCopy.S3MetaData = types.S3MetaData{} + + metaInfoCopy.EnvMetaData = types.EnvMetaData{} + + metaInfoCopy.ResourceMetaData.EnableDynamicMemory = false + metaInfoCopy.ResourceMetaData.EnableTmpExpansion = false + metaInfoCopy.ResourceMetaData.GpuMemory = 0 + metaInfoCopy.ResourceMetaData.EphemeralStorage = 0 + + metaInfoCopy.ExtendedMetaData.ImageName = "" + if metaInfoCopy.ExtendedMetaData.VpcConfig != nil { + metaInfoCopy.ExtendedMetaData.VpcConfig.Xrole = "" + } + metaInfoCopy.ExtendedMetaData.UserAgency = types.UserAgency{} +} + +// FnvHash a hash function +func FnvHash(s string) string { + h := fnv.New32a() + _, err := h.Write([]byte(s)) + if err != nil { + return "" + } + + // for 2 <= base <= 36. The result uses the lower-case letters 'a' to 'z' + return strconv.FormatUint(uint64(h.Sum32()), 10) +} + +// DeepCopyObj deal with src and dst +func DeepCopyObj(src interface{}, dst interface{}) error { + if dst == nil { + return fmt.Errorf("dst cannot be nil") + } + if src == nil { + return fmt.Errorf("src cannot be nil") + } + + bytes, err := json.Marshal(src) + if err != nil { + return fmt.Errorf("unable to marshal src: %s", err) + } + + err = json.Unmarshal(bytes, dst) + if err != nil { + return fmt.Errorf("unable to unmarshal into dst: %s", err) + } + return nil +} + +// SetFuncMetaDynamicConfEnable will calculate DynamicConfig and set DynamicConfig.Enabled +func SetFuncMetaDynamicConfEnable(metaInfo *types.FunctionMetaInfo) { + // The DynamicConfig.Enabled will use for calculate function signature. + // When DynamicConfig.Enabled changes, the instance will be restarted. + // If function version is not latest,DynamicConfig.Enabled will never change + if len(metaInfo.ExtendedMetaData.DynamicConfig.UpdateTime) == 0 { + metaInfo.ExtendedMetaData.DynamicConfig.Enabled = false + return + } + // + if metaInfo.FuncMetaData.Version == constant.DefaultURNVersion && + len(metaInfo.ExtendedMetaData.DynamicConfig.ConfigContent) == 0 { + metaInfo.ExtendedMetaData.DynamicConfig.Enabled = false + return + } + metaInfo.ExtendedMetaData.DynamicConfig.Enabled = true +} + +// ParseFuncKey parse funcKey with format "tenantID/funcName/funcVersion" or "tenantID/funcName/funcVersion/CPU-memory" +func ParseFuncKey(funcKey string) (string, string, string) { + funcInfo := strings.Split(funcKey, "/") + if len(funcInfo) < funcInfoMinLen { + return "", "", "" + } + return funcInfo[0], funcInfo[1], funcInfo[2] +} + +// GetAPIType - +func GetAPIType(BusinessType string) api.ApiType { + if BusinessType == constant.BusinessTypeServe { + return api.ServeApi + } + return api.FaaSApi +} diff --git a/frontend/pkg/common/faas_common/utils/func_meta_util_test.go b/frontend/pkg/common/faas_common/utils/func_meta_util_test.go new file mode 100644 index 0000000000000000000000000000000000000000..12e924e93688e40b2a67ae91077b2bdd01e71fa3 --- /dev/null +++ b/frontend/pkg/common/faas_common/utils/func_meta_util_test.go @@ -0,0 +1,87 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package utils - +package utils + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/types" +) + +func TestGetFuncMetaSignature(t *testing.T) { + convey.Convey("success", t, func() { + signature := GetFuncMetaSignature(&types.FunctionMetaInfo{}, true) + convey.So(signature, convey.ShouldEqual, "2778597263") + }) + convey.Convey("marshal error", t, func() { + defer gomonkey.ApplyFunc(json.Marshal, func(v interface{}) ([]byte, error) { + return nil, fmt.Errorf("marshal error") + }).Reset() + str := GetFuncMetaSignature(&types.FunctionMetaInfo{}, true) + convey.So(str, convey.ShouldContainSubstring, "invalid function meta info") + }) + convey.Convey("unmarshal error", t, func() { + defer gomonkey.ApplyFunc(json.Unmarshal, func(data []byte, v interface{}) error { + return fmt.Errorf("unmarshal error") + }).Reset() + str := GetFuncMetaSignature(&types.FunctionMetaInfo{}, true) + convey.So(str, convey.ShouldContainSubstring, "invalid function meta info") + }) +} + +func TestSetFuncMetaDynamicConfEnable(t *testing.T) { + type args struct { + metaInfo *types.FunctionMetaInfo + } + tests := []struct { + name string + args args + }{ + {"case1", args{metaInfo: &types.FunctionMetaInfo{}}}, + {"case2", args{metaInfo: &types.FunctionMetaInfo{FuncMetaData: types.FuncMetaData{Version: constant.DefaultURNVersion}, + ExtendedMetaData: types.ExtendedMetaData{DynamicConfig: types.DynamicConfigEvent{UpdateTime: "1"}}}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + SetFuncMetaDynamicConfEnable(tt.args.metaInfo) + }) + } +} + +func TestGetCustomResource(t *testing.T) { + convey.Convey("success", t, func() { + customResources := getCustomResourceSpec("{\"huawei.com/ascend-1980\":8}", "") + convey.So(customResources, convey.ShouldEqual, "{\"instanceType\":\"376T\"}") + }) + + convey.Convey("success", t, func() { + customResources := getCustomResourceSpec("{\"huawei.com/ascend-1980\": 8}", "{\"instanceType\": \"376T\"}") + convey.So(customResources, convey.ShouldEqual, "{\"instanceType\":\"376T\"}") + }) + + convey.Convey("success", t, func() { + customResources := getCustomResourceSpec("{\"huawei.com/ascend-1980\":8}", "{ \"instanceType\": \"280T\"}") + convey.So(customResources, convey.ShouldEqual, "{\"instanceType\":\"280T\"}") + }) +} diff --git a/frontend/pkg/common/faas_common/utils/helper.go b/frontend/pkg/common/faas_common/utils/helper.go new file mode 100644 index 0000000000000000000000000000000000000000..957806326d75771747b27925d0ab7c85cf51a77a --- /dev/null +++ b/frontend/pkg/common/faas_common/utils/helper.go @@ -0,0 +1,170 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package utils for common functions +package utils + +import ( + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "path" + "path/filepath" + "strings" +) + +const ( + envPathSeparators = ":" +) + +// IsFile returns true if the path is a file +func IsFile(path string) bool { + file, err := os.Stat(path) + if err != nil { + return false + } + return file.Mode().IsRegular() +} + +// IsDir returns true if the path is a dir +func IsDir(path string) bool { + dir, err := os.Stat(path) + if err != nil { + return false + } + + return dir.IsDir() +} + +// FileExists returns true if the path exists +func FileExists(path string) bool { + _, err := os.Stat(path) + if err != nil { + return false + } + return true +} + +// IsHexString judge If Hex String +func IsHexString(str string) bool { + + str = strings.ToLower(str) + + for _, c := range str { + if c < '0' || (c > '9' && c < 'a') || c > 'f' { + return false + } + } + + return true +} + +// ValidateFilePath verify the legitimacy of the file path +func ValidateFilePath(path string) error { + absPath, err := filepath.Abs(path) + if err != nil || !strings.HasPrefix(path, absPath) { + return errors.New("invalid file path, expect to be configured as an absolute path") + } + return nil +} + +// ValidEnvValuePath verify the legitimacy of the env path +func ValidEnvValuePath(envValues string) error { + if envValues == "" { + return nil + } + envByte := strings.Split(envValues, envPathSeparators) + for _, envValue := range envByte { + if err := ValidateFilePath(envValue); err != nil { + return err + } + } + return nil +} + +// copyFile copies a single file from src to dst +func copyFile(srcPath, dstPath string) error { + var err error + var fromFd *os.File + var toFd *os.File + var fromFdInfo os.FileInfo + + if fromFd, err = os.Open(srcPath); err != nil { + return err + } + defer func(fromFd *os.File) { + if fromFd != nil { + err = fromFd.Close() + } + }(fromFd) + + if fromFdInfo, err = os.Stat(srcPath); err != nil { + return err + } + + toFd, err = os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, fromFdInfo.Mode()) + defer func(toFd *os.File) { + if toFd != nil { + err = toFd.Close() + } + }(toFd) + + if err != nil { + return err + } + + if _, err = io.Copy(toFd, fromFd); err != nil { + return err + } + + return err +} + +// CopyDir copies a whole directory recursively +func CopyDir(srcPath string, dstPath string) error { + var err error + var dirFds []os.FileInfo + var fromInfo os.FileInfo + + if fromInfo, err = os.Stat(srcPath); err != nil { + return err + } + + if err = os.MkdirAll(dstPath, fromInfo.Mode()); err != nil { + return err + } + + if dirFds, err = ioutil.ReadDir(srcPath); err != nil { + return err + } + for _, fd := range dirFds { + fromPath := path.Join(srcPath, fd.Name()) + toPath := path.Join(dstPath, fd.Name()) + + if fd.IsDir() { + if err = CopyDir(fromPath, toPath); err != nil { + fmt.Println(err) + } + } else { + if err = copyFile(fromPath, toPath); err != nil { + fmt.Println(err) + } + } + } + return nil +} diff --git a/frontend/pkg/common/faas_common/utils/helper_test.go b/frontend/pkg/common/faas_common/utils/helper_test.go new file mode 100644 index 0000000000000000000000000000000000000000..0478f9f2d3461501e675a9d42dcc1ccdd9a4c122 --- /dev/null +++ b/frontend/pkg/common/faas_common/utils/helper_test.go @@ -0,0 +1,196 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type IsFileTestSuite struct { + suite.Suite + tempDir string +} + +// SetupSuite Setup Suite +func (suite *IsFileTestSuite) SetupSuite() { + var err error + + // Create temp dir for IsFileTestSuite + suite.tempDir, err = ioutil.TempDir("", "isfile-test") + suite.Require().NoError(err) +} + +// TearDownSuite TearDown Suite +func (suite *IsDirTestSuite) TearDownSuite() { + defer os.RemoveAll(suite.tempDir) +} + +// TestPositive Test Positive +func (suite *IsFileTestSuite) TestPositive() { + + // Create temp file + tempFile, err := ioutil.TempFile(suite.tempDir, "temp_file") + suite.Require().NoError(err) + defer os.Remove(tempFile.Name()) + + // Verify that function isFile() returns true when file is created + suite.Require().True(IsFile(tempFile.Name())) + +} + +// TestFileIsNotExist Test File Is Not Exist +func (suite *IsFileTestSuite) TestFileIsNotExist() { + + // Set path to unexisted file + tempFile := filepath.Join(suite.tempDir, "somePath.txt") + + // Verify that function isFile() returns false when file doesn't exist in the system + suite.Require().False(IsFile(tempFile)) +} + +// TestFileIsADirectory Test File Is A Directory +func (suite *IsFileTestSuite) TestFileIsADirectory() { + suite.Require().False(IsFile(suite.tempDir)) +} + +type IsDirTestSuite struct { + suite.Suite + tempDir string +} + +// SetupSuite Setup Suite +func (suite *IsDirTestSuite) SetupSuite() { + var err error + + // Create temp dir for IsDirTestSuite + suite.tempDir, err = ioutil.TempDir("", "isdir-test") + suite.Require().NoError(err) +} + +// TearDownSuite TearDown Suite +func (suite *IsFileTestSuite) TearDownSuite() { + defer os.RemoveAll(suite.tempDir) +} + +// TestPositive Test Positive +func (suite *IsDirTestSuite) TestPositive() { + + // Verify that function IsDir() returns true when directory exists in the system + suite.Require().True(IsDir(suite.tempDir)) +} + +// TestNegative Test Negative +func (suite *IsDirTestSuite) TestNegative() { + + // Create temp file + tempFile, err := ioutil.TempFile(suite.tempDir, "temp_file") + suite.Require().NoError(err) + defer os.Remove(tempFile.Name()) + + // Verify that function IsDir( returns false when file instead of directory is function argument + suite.Require().False(IsDir(tempFile.Name())) +} + +type FileExistTestSuite struct { + suite.Suite + tempDir string +} + +// SetupSuite Setup Suite +func (suite *FileExistTestSuite) SetupSuite() { + var err error + + // Create temp dir for FileExistTestSuite + suite.tempDir, err = ioutil.TempDir("", "file_exists-test") + suite.Require().NoError(err) +} + +// TearDownSuite TearDown Suite +func (suite *FileExistTestSuite) TearDownSuite() { + defer os.RemoveAll(suite.tempDir) +} + +// TestPositive Test Positive +func (suite *FileExistTestSuite) TestPositive() { + + // Create temp file + tempFile, err := ioutil.TempFile(suite.tempDir, "temp_file") + suite.Require().NoError(err) + defer os.Remove(tempFile.Name()) + + // Verify that function FileExists() returns true when file is exist + suite.Require().True(FileExists(tempFile.Name())) +} + +// TestFileNotExist Test File Not Exist +func (suite *FileExistTestSuite) TestFileNotExist() { + + // Set path to unexisted file + tempFile := filepath.Join(suite.tempDir, "somePath.txt") + + // Verify that function FileExists() returns false when file doesn't exist + suite.Require().False(FileExists(tempFile)) +} + +// TestFileIsNotAFile Test File Is Not A File +func (suite *FileExistTestSuite) TestFileIsNotAFile() { + + // Verify that function returns true when folder is exist in the system + suite.Require().True(FileExists(suite.tempDir)) +} + +// TestHelperTestSuite Test Helper Test Suite +func TestHelperTestSuite(t *testing.T) { + suite.Run(t, new(FileExistTestSuite)) + suite.Run(t, new(IsDirTestSuite)) + suite.Run(t, new(IsFileTestSuite)) +} + +func TestValidEnvValuePath(t *testing.T) { + tests := []struct { + input string + want bool + }{ + {"", true}, + {"/home/sn/test", true}, + {"../../home/sn", false}, + {"/home/sn:/home/test:/opt", true}, + } + for _, tt := range tests { + assert.Equal(t, tt.want, ValidEnvValuePath(tt.input) == nil) + } +} + +func TestCopyDir(t *testing.T) { + convey.Convey("CopyDir", t, func() { + convey.Convey("CopyDir case 1", func() { + srcPath, _ := ioutil.TempDir("", "src") + dstPath, _ := ioutil.TempDir("", "dst") + fileName := "fastfreeze.log" + _, err := ioutil.TempFile(srcPath, fileName) + err = CopyDir(srcPath, dstPath) + convey.So(err, convey.ShouldBeNil) + }) + }) +} diff --git a/frontend/pkg/common/faas_common/utils/libruntimeapi_mock.go b/frontend/pkg/common/faas_common/utils/libruntimeapi_mock.go new file mode 100644 index 0000000000000000000000000000000000000000..7949f71c8f6e214a717e0f1882934f29b50af78b --- /dev/null +++ b/frontend/pkg/common/faas_common/utils/libruntimeapi_mock.go @@ -0,0 +1,300 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package utils is sdk +package utils + +import ( + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/uuid" +) + +// FakeLibruntimeSdkClient - +type FakeLibruntimeSdkClient struct{} + +// CreateInstance - +func (f *FakeLibruntimeSdkClient) CreateInstance(funcMeta api.FunctionMeta, args []api.Arg, + invokeOpt api.InvokeOptions) (string, error) { + + InstanceID := uuid.New().String() + return InstanceID, nil +} + +// InvokeByInstanceId - +func (f *FakeLibruntimeSdkClient) InvokeByInstanceId(funcMeta api.FunctionMeta, + instanceID string, args []api.Arg, invokeOpt api.InvokeOptions) (string, error) { + return "", nil +} + +// InvokeByFunctionName - +func (f *FakeLibruntimeSdkClient) InvokeByFunctionName(funcMeta api.FunctionMeta, + args []api.Arg, invokeOpt api.InvokeOptions) (string, error) { + return "", nil +} + +// AcquireInstance - +func (f *FakeLibruntimeSdkClient) AcquireInstance(state string, funcMeta api.FunctionMeta, + acquireOpt api.InvokeOptions) (api.InstanceAllocation, error) { + return api.InstanceAllocation{}, nil +} + +// ReleaseInstance - +func (f *FakeLibruntimeSdkClient) ReleaseInstance(allocation api.InstanceAllocation, + stateID string, abnormal bool, option api.InvokeOptions) { + return +} + +// Kill - +func (f *FakeLibruntimeSdkClient) Kill(instanceID string, signal int, payload []byte) error { + return nil +} + +// CreateInstanceRaw - +func (f *FakeLibruntimeSdkClient) CreateInstanceRaw(createReqRaw []byte) ([]byte, error) { + return nil, nil +} + +// InvokeByInstanceIdRaw - +func (f *FakeLibruntimeSdkClient) InvokeByInstanceIdRaw(invokeReqRaw []byte) ([]byte, error) { + return nil, nil +} + +// KillRaw - +func (f *FakeLibruntimeSdkClient) KillRaw(killReqRaw []byte) ([]byte, error) { + return nil, nil +} + +// SaveState - +func (f *FakeLibruntimeSdkClient) SaveState(state []byte) (string, error) { + return "", nil +} + +// LoadState - +func (f *FakeLibruntimeSdkClient) LoadState(checkpointID string) ([]byte, error) { + return nil, nil +} + +// Exit - +func (f *FakeLibruntimeSdkClient) Exit(code int, message string) { + return +} + +// Finalize - +func (f *FakeLibruntimeSdkClient) Finalize() { + return +} + +// KVSet - +func (f *FakeLibruntimeSdkClient) KVSet(key string, value []byte, param api.SetParam) error { + return nil +} + +// KVSetWithoutKey - +func (f *FakeLibruntimeSdkClient) KVSetWithoutKey(value []byte, param api.SetParam) (string, error) { + return "", nil +} + +// KVMSetTx - +func (f *FakeLibruntimeSdkClient) KVMSetTx(keys []string, values [][]byte, param api.MSetParam) error { + return nil +} + +// KVGet - +func (f *FakeLibruntimeSdkClient) KVGet(key string, timeoutms uint) ([]byte, error) { + return nil, nil +} + +// KVGetMulti - +func (f *FakeLibruntimeSdkClient) KVGetMulti(keys []string, timeoutms uint) ([][]byte, error) { + return nil, nil +} + +// KVDel - +func (f *FakeLibruntimeSdkClient) KVDel(key string) error { + return nil +} + +// KVDelMulti - +func (f *FakeLibruntimeSdkClient) KVDelMulti(keys []string) ([]string, error) { + return []string{}, nil +} + +// CreateProducer - +func (f *FakeLibruntimeSdkClient) CreateProducer(streamName string, + producerConf api.ProducerConf) (api.StreamProducer, error) { + return &FakeStreamProducer{}, nil +} + +// Subscribe - +func (f *FakeLibruntimeSdkClient) Subscribe(streamName string, + config api.SubscriptionConfig) (api.StreamConsumer, error) { + return &FakeStreamConsumer{}, nil +} + +// DeleteStream - +func (f *FakeLibruntimeSdkClient) DeleteStream(streamName string) error { + return nil +} + +// QueryGlobalProducersNum - +func (f *FakeLibruntimeSdkClient) QueryGlobalProducersNum(streamName string) (uint64, error) { + return 0, nil +} + +// QueryGlobalConsumersNum - +func (f *FakeLibruntimeSdkClient) QueryGlobalConsumersNum(streamName string) (uint64, error) { + return 0, nil +} + +// SetTraceID - +func (f *FakeLibruntimeSdkClient) SetTraceID(traceID string) { + return +} + +// SetTenantID - +func (f *FakeLibruntimeSdkClient) SetTenantID(tenantID string) error { + return nil +} + +// Put - +func (f *FakeLibruntimeSdkClient) Put(objectID string, value []byte, + param api.PutParam, nestedObjectIDs ...string) error { + return nil +} + +// PutRaw - +func (f *FakeLibruntimeSdkClient) PutRaw(objectID string, value []byte, + param api.PutParam, nestedObjectIDs ...string) error { + return nil +} + +// Get - +func (f *FakeLibruntimeSdkClient) Get(objectIDs []string, timeoutMs int) ([][]byte, error) { + return nil, nil +} + +// GetRaw - +func (f *FakeLibruntimeSdkClient) GetRaw(objectIDs []string, timeoutMs int) ([][]byte, error) { + return nil, nil +} + +// Wait - +func (f *FakeLibruntimeSdkClient) Wait(objectIDs []string, + waitNum uint64, timeoutMs int) ([]string, []string, map[string]error) { + return nil, nil, nil +} + +// GIncreaseRef - +func (f *FakeLibruntimeSdkClient) GIncreaseRef(objectIDs []string, remoteClientID ...string) ([]string, error) { + return nil, nil +} + +// GIncreaseRefRaw - +func (f *FakeLibruntimeSdkClient) GIncreaseRefRaw(objectIDs []string, remoteClientID ...string) ([]string, error) { + return nil, nil +} + +// GDecreaseRef - +func (f *FakeLibruntimeSdkClient) GDecreaseRef(objectIDs []string, remoteClientID ...string) ([]string, error) { + return nil, nil +} + +// GDecreaseRefRaw - +func (f *FakeLibruntimeSdkClient) GDecreaseRefRaw(objectIDs []string, remoteClientID ...string) ([]string, error) { + return nil, nil +} + +// GetAsync - +func (f *FakeLibruntimeSdkClient) GetAsync(objectID string, cb api.GetAsyncCallback) { + return +} + +// GetFormatLogger - +func (f *FakeLibruntimeSdkClient) GetFormatLogger() api.FormatLogger { + return nil +} + +// CreateClient - +func (f *FakeLibruntimeSdkClient) CreateClient(config api.ConnectArguments) (api.KvClient, error) { + return nil, nil +} + +// ReleaseGRefs - +func (f *FakeLibruntimeSdkClient) ReleaseGRefs(remoteClientID string) error { + return nil +} + +// GetCredential - +func (f *FakeLibruntimeSdkClient) GetCredential() api.Credential { + return api.Credential{} +} + +// UpdateSchdulerInfo - +func (f *FakeLibruntimeSdkClient) UpdateSchdulerInfo(schedulerName string, schedulerId string, option string) { + return +} + +// IsHealth - +func (f *FakeLibruntimeSdkClient) IsHealth() bool { + return true +} + +// IsDsHealth - +func (f *FakeLibruntimeSdkClient) IsDsHealth() bool { + return true +} + +// FakeStreamProducer - +type FakeStreamProducer struct{} + +// Send - +func (fsp *FakeStreamProducer) Send(element api.Element) error { + return nil +} + +// SendWithTimeout - +func (fsp *FakeStreamProducer) SendWithTimeout(element api.Element, timeoutMs int64) error { + return nil +} + +// Close - +func (fsp *FakeStreamProducer) Close() error { + return nil +} + +// FakeStreamConsumer - +type FakeStreamConsumer struct{} + +// ReceiveExpectNum - +func (fsc *FakeStreamConsumer) ReceiveExpectNum(expectNum uint32, timeoutMs uint32) ([]api.Element, error) { + return nil, nil +} + +// Receive - +func (fsc *FakeStreamConsumer) Receive(timeoutMs uint32) ([]api.Element, error) { + return nil, nil +} + +// Ack - +func (fsc *FakeStreamConsumer) Ack(elementId uint64) error { + return nil +} + +// Close - +func (fsc *FakeStreamConsumer) Close() error { + return nil +} diff --git a/frontend/pkg/common/faas_common/utils/libruntimeapi_mock_test.go b/frontend/pkg/common/faas_common/utils/libruntimeapi_mock_test.go new file mode 100644 index 0000000000000000000000000000000000000000..431f66a8d6b7a8422911f929506f04f8fad899b9 --- /dev/null +++ b/frontend/pkg/common/faas_common/utils/libruntimeapi_mock_test.go @@ -0,0 +1,140 @@ +package utils + +import ( + "github.com/stretchr/testify/assert" + "testing" + "yuanrong.org/kernel/runtime/libruntime/api" +) + +func TestFakeLibruntimeSdkClient(t *testing.T) { + fakeLibruntimeSdkClient := FakeLibruntimeSdkClient{} + instanceID, err := fakeLibruntimeSdkClient.CreateInstance(api.FunctionMeta{}, []api.Arg{}, api.InvokeOptions{}) + assert.NotEqual(t, 0, len(instanceID)) + assert.Equal(t, nil, err) + + _, err = fakeLibruntimeSdkClient.InvokeByInstanceId(api.FunctionMeta{}, "", []api.Arg{}, api.InvokeOptions{}) + assert.Equal(t, nil, err) + + _, err = fakeLibruntimeSdkClient.InvokeByFunctionName(api.FunctionMeta{}, []api.Arg{}, api.InvokeOptions{}) + assert.Equal(t, nil, err) + + _, err = fakeLibruntimeSdkClient.AcquireInstance("", api.FunctionMeta{}, api.InvokeOptions{}) + assert.Equal(t, nil, err) + + err = fakeLibruntimeSdkClient.Kill("", 0, []byte{}) + assert.Equal(t, nil, err) + + _, err = fakeLibruntimeSdkClient.CreateInstanceRaw([]byte{}) + assert.Equal(t, nil, err) + + _, err = fakeLibruntimeSdkClient.InvokeByInstanceIdRaw([]byte{}) + assert.Equal(t, nil, err) + + _, err = fakeLibruntimeSdkClient.KillRaw([]byte{}) + assert.Equal(t, nil, err) + + _, err = fakeLibruntimeSdkClient.SaveState([]byte{}) + assert.Equal(t, nil, err) + + _, err = fakeLibruntimeSdkClient.LoadState("") + assert.Equal(t, nil, err) + + err = fakeLibruntimeSdkClient.KVSet("", []byte{}, api.SetParam{}) + assert.Equal(t, nil, err) + + _, err = fakeLibruntimeSdkClient.KVSetWithoutKey([]byte{}, api.SetParam{}) + assert.Equal(t, nil, err) + + err = fakeLibruntimeSdkClient.KVMSetTx([]string{}, [][]byte{}, api.MSetParam{}) + assert.Equal(t, nil, err) + + _, err = fakeLibruntimeSdkClient.KVGet("", 1) + assert.Equal(t, nil, err) + + _, err = fakeLibruntimeSdkClient.KVGetMulti([]string{}, 1) + assert.Equal(t, nil, err) + + err = fakeLibruntimeSdkClient.KVDel("") + assert.Equal(t, nil, err) + + _, err = fakeLibruntimeSdkClient.KVDelMulti([]string{}) + assert.Equal(t, nil, err) + + _, err = fakeLibruntimeSdkClient.CreateProducer("", api.ProducerConf{}) + assert.Equal(t, nil, err) + + _, err = fakeLibruntimeSdkClient.Subscribe("", api.SubscriptionConfig{}) + assert.Equal(t, nil, err) + + err = fakeLibruntimeSdkClient.DeleteStream("") + assert.Equal(t, nil, err) + + _, err = fakeLibruntimeSdkClient.QueryGlobalProducersNum("") + assert.Equal(t, nil, err) + + _, err = fakeLibruntimeSdkClient.QueryGlobalConsumersNum("") + assert.Equal(t, nil, err) + + err = fakeLibruntimeSdkClient.SetTenantID("") + assert.Equal(t, nil, err) + + err = fakeLibruntimeSdkClient.Put("", []byte{}, api.PutParam{}, "") + assert.Equal(t, nil, err) + + err = fakeLibruntimeSdkClient.PutRaw("", []byte{}, api.PutParam{}, "") + assert.Equal(t, nil, err) + + _, err = fakeLibruntimeSdkClient.Get([]string{}, 1) + assert.Equal(t, nil, err) + + _, err = fakeLibruntimeSdkClient.GetRaw([]string{}, 1) + assert.Equal(t, nil, err) + + _, _, aa := fakeLibruntimeSdkClient.Wait([]string{}, 1, 1) + assert.Equal(t, map[string]error(map[string]error(nil)), aa) + + _, err = fakeLibruntimeSdkClient.GIncreaseRef([]string{}, "") + assert.Equal(t, nil, err) + + _, err = fakeLibruntimeSdkClient.GDecreaseRefRaw([]string{}, "") + assert.Equal(t, nil, err) + + bb := fakeLibruntimeSdkClient.GetFormatLogger() + assert.Equal(t, nil, bb) + + _, err = fakeLibruntimeSdkClient.CreateClient(api.ConnectArguments{}) + assert.Equal(t, nil, err) + + err = fakeLibruntimeSdkClient.ReleaseGRefs("") + assert.Equal(t, nil, err) + + credential := fakeLibruntimeSdkClient.GetCredential() + assert.NotEqual(t, nil, credential) +} + +func TestFakeStreamProducer(t *testing.T) { + fakeStreamProducer := FakeStreamProducer{} + err := fakeStreamProducer.Send(api.Element{}) + assert.Equal(t, nil, err) + + err = fakeStreamProducer.SendWithTimeout(api.Element{}, 1) + assert.Equal(t, nil, err) + + err = fakeStreamProducer.Close() + assert.Equal(t, nil, err) +} + +func TestFakeStreamConsumer(t *testing.T) { + fakeStreamConsumer := FakeStreamConsumer{} + _, err := fakeStreamConsumer.ReceiveExpectNum(1, 1) + assert.Equal(t, nil, err) + + _, err = fakeStreamConsumer.Receive(1) + assert.Equal(t, nil, err) + + err = fakeStreamConsumer.Ack(1) + assert.Equal(t, nil, err) + + err = fakeStreamConsumer.Close() + assert.Equal(t, nil, err) +} diff --git a/frontend/pkg/common/faas_common/utils/memory_test.go b/frontend/pkg/common/faas_common/utils/memory_test.go new file mode 100644 index 0000000000000000000000000000000000000000..611b18fd36ade0f188b0eed6b46c6e4de8f69927 --- /dev/null +++ b/frontend/pkg/common/faas_common/utils/memory_test.go @@ -0,0 +1,22 @@ +package utils + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestClearStringMemory(t *testing.T) { + Convey("Given a string", t, func() { + testStr := "helloworld" + + b := []byte(testStr) + s := string(b) + Convey("When we clear the string", func() { + ClearStringMemory(s) + Convey("The string should be empty", func() { + So(s, ShouldEqual, string([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0})) + }) + }) + }) +} diff --git a/frontend/pkg/common/faas_common/utils/mock_utils.go b/frontend/pkg/common/faas_common/utils/mock_utils.go new file mode 100644 index 0000000000000000000000000000000000000000..ff3714b07da58e718c93823b9b42a4a965909b1e --- /dev/null +++ b/frontend/pkg/common/faas_common/utils/mock_utils.go @@ -0,0 +1,138 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package utils - +package utils + +import ( + "context" + "errors" + + "github.com/agiledragon/gomonkey/v2" + "go.etcd.io/etcd/client/v3" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "yuanrong.org/kernel/runtime/libruntime/api" +) + +// PatchSlice - +type PatchSlice []*gomonkey.Patches + +// PatchesFunc - +type PatchesFunc func() PatchSlice + +// InitPatchSlice - +func InitPatchSlice() PatchSlice { + return make([]*gomonkey.Patches, 0) +} + +// Append - +func (p *PatchSlice) Append(patches PatchSlice) { + if len(patches) > 0 { + *p = append(*p, patches...) + } +} + +// ResetAll - +func (p PatchSlice) ResetAll() { + for _, item := range p { + item.Reset() + } +} + +// FakeLogger - +type FakeLogger struct{} + +// With - +func (f *FakeLogger) With(fields ...zapcore.Field) api.FormatLogger { + return f +} + +// Infof - +func (f *FakeLogger) Infof(format string, paras ...interface{}) {} + +// Errorf - +func (f *FakeLogger) Errorf(format string, paras ...interface{}) {} + +// Warnf - +func (f *FakeLogger) Warnf(format string, paras ...interface{}) {} + +// Debugf - +func (f *FakeLogger) Debugf(format string, paras ...interface{}) {} + +// Fatalf - +func (f *FakeLogger) Fatalf(format string, paras ...interface{}) {} + +// Info - +func (f *FakeLogger) Info(msg string, fields ...zap.Field) {} + +// Error - +func (f *FakeLogger) Error(msg string, fields ...zap.Field) {} + +// Warn - +func (f *FakeLogger) Warn(msg string, fields ...zap.Field) {} + +// Debug - +func (f *FakeLogger) Debug(msg string, fields ...zap.Field) {} + +// Fatal - +func (f *FakeLogger) Fatal(msg string, fields ...zap.Field) {} + +// Sync - +func (f *FakeLogger) Sync() {} + +// FakeEtcdLease - +type FakeEtcdLease struct { +} + +// Grant - +func (m FakeEtcdLease) Grant(_ context.Context, _ int64) (*clientv3.LeaseGrantResponse, error) { + return &clientv3.LeaseGrantResponse{ID: 1}, nil +} + +// Revoke - +func (m FakeEtcdLease) Revoke(_ context.Context, _ clientv3.LeaseID) (*clientv3.LeaseRevokeResponse, error) { + return nil, nil +} + +// TimeToLive - +func (m FakeEtcdLease) TimeToLive(_ context.Context, _ clientv3.LeaseID, + _ ...clientv3.LeaseOption) (*clientv3.LeaseTimeToLiveResponse, error) { + return nil, nil +} + +// Leases - +func (m FakeEtcdLease) Leases(_ context.Context) (*clientv3.LeaseLeasesResponse, error) { + return nil, nil +} + +// KeepAlive - +func (m FakeEtcdLease) KeepAlive(_ context.Context, _ clientv3.LeaseID) ( + <-chan *clientv3.LeaseKeepAliveResponse, error) { + return nil, nil +} + +// KeepAliveOnce - +func (m FakeEtcdLease) KeepAliveOnce(_ context.Context, _ clientv3.LeaseID) ( + *clientv3.LeaseKeepAliveResponse, error) { + return nil, nil +} + +// Close - +func (m FakeEtcdLease) Close() error { + return errors.New("close error") +} diff --git a/frontend/pkg/common/faas_common/utils/scheduler_option.go b/frontend/pkg/common/faas_common/utils/scheduler_option.go new file mode 100644 index 0000000000000000000000000000000000000000..2285d6f6ead8866f5537295fedfaa495b3f55aed --- /dev/null +++ b/frontend/pkg/common/faas_common/utils/scheduler_option.go @@ -0,0 +1,108 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package utils - +package utils + +import ( + "fmt" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/types" +) + +const ( + schedulePolicyKey = "schedule_policy" + scheduleCPU = "CPU" + scheduleMemory = "Memory" +) + +const ( + // NodeSelectorKey - + NodeSelectorKey = "node_selector" + // MonopolyPolicyValue - + MonopolyPolicyValue = "monopoly" + // SharedPolicyValue - + SharedPolicyValue = "shared" +) + +// CreateCustomExtensions create customExtensions +func CreateCustomExtensions(customExtensions map[string]string, schedulePolicy string) map[string]string { + if customExtensions == nil { + customExtensions = make(map[string]string, 1) + } + customExtensions[schedulePolicyKey] = schedulePolicy + return customExtensions +} + +// CreatePodAffinity - create pod affinity +func CreatePodAffinity(key, label string, affinityType api.AffinityType) []api.Affinity { + var ( + operators []api.LabelOperator + affinity []api.Affinity + ) + if label != "" { + operators = append(operators, api.LabelOperator{ + Type: api.LabelOpIn, + LabelKey: key, + LabelValues: []string{label}, + }) + } else { + operators = append(operators, api.LabelOperator{ + Type: api.LabelOpExists, + LabelKey: key, + LabelValues: []string{}, + }) + } + affinity = append(affinity, api.Affinity{ + Kind: api.AffinityKindInstance, + Affinity: affinityType, + PreferredPriority: false, + PreferredAntiOtherLabels: false, + LabelOps: operators, + }) + return affinity +} + +// CreateCreateOptions create CreateOptions +func CreateCreateOptions(createOptions map[string]string, key, value string) map[string]string { + if createOptions == nil { + return make(map[string]string) + } + createOptions[key] = value + return createOptions +} + +// GenerateResourcesMap - +func GenerateResourcesMap(cpu, memory float64) map[string]float64 { + resourcesMap := make(map[string]float64) + resourcesMap[scheduleCPU] = cpu + resourcesMap[scheduleMemory] = memory + return resourcesMap +} + +// AddNodeSelector - +func AddNodeSelector(nodeSelectorMap map[string]string, extraParams *types.ExtraParams) { + if extraParams.CustomExtensions == nil { + extraParams.CustomExtensions = make(map[string]string, 1) + } + if nodeSelectorMap != nil && len(nodeSelectorMap) != 0 { + for k, v := range nodeSelectorMap { + extraParams.CustomExtensions[NodeSelectorKey] = fmt.Sprintf(`{"%s": "%s"}`, k, v) + } + } +} diff --git a/frontend/pkg/common/faas_common/utils/scheduler_option_test.go b/frontend/pkg/common/faas_common/utils/scheduler_option_test.go new file mode 100644 index 0000000000000000000000000000000000000000..2de0e3597de54981bca7a0255e51ac6664c57e34 --- /dev/null +++ b/frontend/pkg/common/faas_common/utils/scheduler_option_test.go @@ -0,0 +1,97 @@ +package utils + +import ( + "reflect" + "testing" + + . "github.com/smartystreets/goconvey/convey" + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/types" +) + +func TestCreateCustomExtensions(t *testing.T) { + Convey("Test CreateCustomExtensions", t, func() { + got := CreateCustomExtensions(nil, MonopolyPolicyValue) + So(got, ShouldNotBeNil) + }) +} + +func TestCreateCreateOptions(t *testing.T) { + Convey("Test CreateSchedulingOptions", t, func() { + expectValue := "test" + createOptions := make(map[string]string, 20) + got := CreateCreateOptions(createOptions, "test", "test") + So(got["test"], ShouldEqual, expectValue) + }) +} + +func TestGenerateResourcesMap(t *testing.T) { + Convey("Test GenerateResourcesMap", t, func() { + res := GenerateResourcesMap(300, 128) + So(res, ShouldResemble, map[string]float64{ + scheduleCPU: 300, + scheduleMemory: 128, + }) + }) +} + +func TestCreatePodAffinity(t *testing.T) { + type args struct { + key string + label string + affinityType api.AffinityType + } + tests := []struct { + name string + args args + want []api.Affinity + }{ + {"case1", args{ + key: "faasfrontend", + label: "faasfrontend", + affinityType: api.PreferredAntiAffinity, + }, []api.Affinity{ + api.Affinity{ + Kind: api.AffinityKindInstance, + Affinity: api.PreferredAntiAffinity, + PreferredPriority: false, + PreferredAntiOtherLabels: false, + LabelOps: []api.LabelOperator{{ + Type: api.LabelOpIn, + LabelKey: "faasfrontend", + LabelValues: []string{"faasfrontend"}, + }, + }, + }}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := CreatePodAffinity(tt.args.key, tt.args.label, tt.args.affinityType); !reflect.DeepEqual(got, tt.want) { + t.Errorf("CreatePodAffinity() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestAddNodeSelector(t *testing.T) { + type args struct { + nodeSelectorMap map[string]string + extraParams *types.ExtraParams + } + tests := []struct { + name string + args args + }{ + {"case1", args{ + nodeSelectorMap: map[string]string{"k": "v"}, + extraParams: &types.ExtraParams{CustomExtensions: make(map[string]string)}, + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + AddNodeSelector(tt.args.nodeSelectorMap, tt.args.extraParams) + }) + } +} diff --git a/frontend/pkg/common/faas_common/utils/tools.go b/frontend/pkg/common/faas_common/utils/tools.go new file mode 100644 index 0000000000000000000000000000000000000000..d4b6fa795aeadcdd7ce2abb1adef4d268e6ffb59 --- /dev/null +++ b/frontend/pkg/common/faas_common/utils/tools.go @@ -0,0 +1,656 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package utils for common functions +package utils + +import ( + "bufio" + "crypto/md5" + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "encoding/hex" + "fmt" + "hash/fnv" + "io" + "io/ioutil" + "math" + "math/rand" + "net" + "os" + "path" + "path/filepath" + "reflect" + "strings" + "sync" + "syscall" + "time" + "unsafe" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/snerror" + "frontend/pkg/common/faas_common/types" + "frontend/pkg/common/uuid" +) + +const ( + // OriginDefaultTimeout is 900 + OriginDefaultTimeout = 900 + // maxTimeout is 100 days + maxTimeout = 100 * 24 * 3600 + bytesToMb = 1024 * 1024 + uint64ArrayLength = 8 + uint32Len = 4 + // DirMode dir mode + DirMode = 0700 + // FileMode file mode + FileMode = 0600 + readSize = 32 * 1024 + // ObsMaxRetry obs max retry times 0 + ObsMaxRetry = 0 + // ObsDefaultTimeout 30 seconds + ObsDefaultTimeout = 30 + // ObsDefaultConnectTimeout 10 seconds + ObsDefaultConnectTimeout = 5 + // LayerListSep define the LayerList separation character + LayerListSep = "-#-" + instanceIDLength = 2 + dnsPairLength = 2 + hostFilePath = "/etc/hosts" + defaultMessageLen = 256 +) + +const ( + minimumMemoryUnit = 128 + minimumCPUUnit = 100 + minimumReservedCPUUnit = 200 +) + +const ( + tenantValueIndex = 6 + funcNameValueIndex = 8 + versionValueIndex = 10 + instanceIDValueIndex = 13 + functionSchedulerKeyLen = 14 + moduleSchedulerKeyLen = 7 + functionNameIndex = 6 + defaultVersion = "latest" + defaultTenant = "0" + defaultFunctionName = "faas-scheduler" +) + +type hostFileInfo struct { + Sha256 string + Content []byte + Mutex sync.Mutex +} + +// HostFile /etc/hosts file info +var HostFile hostFileInfo + +// SetClusterNameEnv - +func SetClusterNameEnv(clusterName string) error { + if err := os.Setenv(constant.ClusterNameEnvKey, clusterName); err != nil { + return fmt.Errorf("failed to set env of %s, err: %s", constant.ClusterNameEnvKey, err.Error()) + } + return nil +} + +// CalculateCPUByMemory CPU and memory calculation methods presented by fg: cpu=memory/128*100+200 +func CalculateCPUByMemory(memory int) int { + return memory/minimumMemoryUnit*minimumCPUUnit + minimumReservedCPUUnit +} + +var azEnv = parseAzEnv() + +func parseAzEnv() string { + az := os.Getenv(constant.ZoneKey) + if az == "" { + az = constant.DefaultAZ + } + if len(az) > constant.ZoneNameLen { + az = az[0 : constant.ZoneNameLen-1] + } + return az +} + +// AzEnv set defaultaz env +func AzEnv() string { + return azEnv +} + +// GenerateInstanceID - +func GenerateInstanceID(podName string) string { + return AzEnv() + "-#-" + podName +} + +// GetPodNameByInstanceID - +func GetPodNameByInstanceID(instanceID string) string { + elements := strings.Split(instanceID, LayerListSep) + if len(elements) < instanceIDLength { + return "" + } + return elements[1] +} + +// Domain2IP convert domain to ip +func Domain2IP(endpoint string) (string, error) { + var host, port string + var err error + host = endpoint + if strings.Contains(endpoint, ":") { + host, port, err = net.SplitHostPort(endpoint) + if err != nil { + return "", err + } + } + if net.ParseIP(host) != nil { + return endpoint, nil + } + ips, err := net.LookupHost(host) + if err != nil { + return "", err + } + if port == "" { + return ips[0], nil + } + return net.JoinHostPort(ips[0], port), nil +} + +// DeepCopy will generate a new copy of original collection type +// currently this function is not recursive so elements will not be deep copied +func DeepCopy(origin interface{}) interface{} { + oriTyp := reflect.TypeOf(origin) + oriVal := reflect.ValueOf(origin) + switch oriTyp.Kind() { + case reflect.Slice: + elemType := oriTyp.Elem() + length := oriVal.Len() + capacity := oriVal.Cap() + newObj := reflect.MakeSlice(reflect.SliceOf(elemType), length, capacity) + reflect.Copy(newObj, oriVal) + return newObj.Interface() + case reflect.Map: + newObj := reflect.MakeMapWithSize(oriTyp, len(oriVal.MapKeys())) + for _, key := range oriVal.MapKeys() { + value := oriVal.MapIndex(key) + newObj.SetMapIndex(key, value) + } + return newObj.Interface() + default: + return nil + } +} + +// ValidateTimeout check timeout +func ValidateTimeout(timeout *int64, defaultTimeout int64) { + if *timeout <= 0 { + *timeout = defaultTimeout + return + } + if *timeout > maxTimeout { + *timeout = maxTimeout + } +} + +// ClearStringMemory - +func ClearStringMemory(s string) { + if len(s) == 0 { + return + } + bs := *(*[]byte)(unsafe.Pointer(&s)) + ClearByteMemory(bs) +} + +// ClearByteMemory - +func ClearByteMemory(b []byte) { + for i := 0; i < len(b); i++ { + b[i] = 0 + } +} + +// Float64ToByte - +func Float64ToByte(float float64) []byte { + bits := math.Float64bits(float) + bytes := make([]byte, 8) + binary.LittleEndian.PutUint64(bytes, bits) + return bytes +} + +// ByteToFloat64 - +func ByteToFloat64(bytes []byte) float64 { + // bounds check to guarantee safety of function Uint64 + if len(bytes) != uint64ArrayLength { + return 0 + } + bits := binary.LittleEndian.Uint64(bytes) + return math.Float64frombits(bits) +} + +// ExistPath whether path exists +func ExistPath(path string) bool { + _, err := os.Stat(path) + if err != nil && os.IsNotExist(err) { + return false + } + return true +} + +// IsInputParameterValid check if input parameter is valid +func IsInputParameterValid(cmdName string) bool { + if strings.Contains(cmdName, "&") || + strings.Contains(cmdName, "|") || + strings.Contains(cmdName, ";") || + strings.Contains(cmdName, "$") || + strings.Contains(cmdName, "'") || + strings.Contains(cmdName, "`") || + strings.Contains(cmdName, "(") || + strings.Contains(cmdName, ")") || + strings.Contains(cmdName, "\"") { + return false + } + return true +} + +// UniqueID get unique ID +func UniqueID() string { + return uuid.New().String() +} + +// ShortUUID return short uuid encode by base64 +func ShortUUID() string { + id := uuid.New() + buf := make([]byte, base64.StdEncoding.EncodedLen(len(id))) + base64.StdEncoding.Encode(buf, id[:]) + for i := range buf { + if buf[i] == '=' || buf[i] == '+' || buf[i] == '/' { + buf[i] = '-' + } + } + return strings.ToLower(strings.Trim(string(buf), "-")) +} + +// WriteFileToPath write file to path +func WriteFileToPath(writePath string, buffer []byte) error { + baseDir := path.Dir(writePath) + err := os.MkdirAll(baseDir, DirMode) + if err != nil { + return err + } + if err = ioutil.WriteFile(writePath, buffer, FileMode); err != nil { + return err + } + return nil +} + +// IsConnRefusedErr - +func IsConnRefusedErr(err error) bool { + netErr, ok := err.(net.Error) + if !ok { + return false + } + opErr, ok := netErr.(*net.OpError) + if !ok { + return false + } + syscallErr, ok := opErr.Err.(*os.SyscallError) + if !ok { + return false + } + if errno, ok := syscallErr.Err.(syscall.Errno); ok { + if errno == syscall.ECONNREFUSED { + return true + } + } + return false +} + +// ContainsConnRefusedErr - +func ContainsConnRefusedErr(err error) bool { + const connRefusedStr = "connection refused" + return strings.Contains(err.Error(), connRefusedStr) +} + +// DefaultStringEnv return environment variable named by key and return val when not exist +func DefaultStringEnv(key string, val string) string { + if env := os.Getenv(key); env != "" { + return env + } + return val +} + +// ReplaceByDNS update /etc/hosts +func ReplaceByDNS(filePath string, domainNames map[string]string) error { + lines, err := ReadLines(filePath) + if err != nil { + return err + } + checkedDNSNames := make(map[string]bool, len(domainNames)) + var hasChange bool + for i := range lines { + arr := strings.Fields(lines[i]) + if len(arr) != dnsPairLength { + continue + } + for name, ipAddress := range domainNames { + if arr[0] == name || arr[1] == name { + originLine := lines[i] + lines[i] = ipAddress + " " + name + checkedDNSNames[name] = true + if lines[i] != originLine { + hasChange = true + } + break + } + } + } + for name, ipAddress := range domainNames { + // domain name is not in hosts file will append to hosts file + if !checkedDNSNames[name] { + lines = append(lines, ipAddress+" "+name) + hasChange = true + } + } + if !hasChange { + return nil + } + HostFile.Mutex.Lock() + defer HostFile.Mutex.Unlock() + if err := WriteLines(filePath, lines); err != nil { + return err + } + if err := HostFile.SaveHostFileInfo(); err != nil { + return err + } + return nil +} + +func (hostFileInfo) SaveHostFileInfo() error { + _, sha, err := GetFileHashInfo(hostFilePath) + if err != nil { + return err + } + HostFile.Sha256 = sha + content, err := ioutil.ReadFile(hostFilePath) + if err != nil { + return err + } + HostFile.Content = content + return nil +} + +// ReadLines read the lines of the given file. +func ReadLines(path string) ([]string, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + + var lines []string + scanner := bufio.NewScanner(file) + for scanner.Scan() { + lines = append(lines, scanner.Text()) + } + return lines, scanner.Err() +} + +// WriteLines writes the lines to the given file. +func WriteLines(path string, lines []string) error { + file, err := os.Create(path) + if err != nil { + return err + } + defer file.Close() + + w := bufio.NewWriter(file) + for _, line := range lines { + fmt.Fprintln(w, line) + } + return w.Flush() +} + +// GenStateIDByKey returns stateID by serviceID, functionName and key +func GenStateIDByKey(tenantID, serviceID, funcName, key string) string { + // if stateKey is empty, stateID is generated by default. + if len(key) == 0 { + return uuid.New().String() + } + preAllocationSlice := make([]byte, 0, len(tenantID)+len(serviceID)+len(funcName)+len(key)) + preAllocationSlice = append(preAllocationSlice, tenantID...) + preAllocationSlice = append(preAllocationSlice, serviceID...) + preAllocationSlice = append(preAllocationSlice, funcName...) + preAllocationSlice = append(preAllocationSlice, key...) + stateID := uuid.NewSHA1(uuid.NameSpaceURL, preAllocationSlice) + return stateID.String() +} + +// GetFileHashInfo get file hash info +func GetFileHashInfo(path string) (int64, string, error) { + var fileSize int64 + realPath, err := filepath.Abs(path) + if err != nil { + return 0, "", err + } + file, err := os.Open(realPath) + if err != nil { + return 0, "", err + } + defer file.Close() + stat, err := file.Stat() + if err != nil { + return 0, "", err + } + fileSize = stat.Size() + fileHash := sha256.New() + if _, err := io.Copy(fileHash, file); err != nil { + return 0, "", err + } + hashValue := hex.EncodeToString(fileHash.Sum(nil)) + return fileSize, hashValue, nil +} + +// IsNetworkError judge whether it is a network error +func IsNetworkError(err error) bool { + if err == nil { + return false + } + _, ok := err.(net.Error) + if !ok { + return false + } + return true +} + +// IsUserError - +func IsUserError(err error) bool { + newErr, ok := err.(snerror.SNError) + if !ok { + return false + } + return snerror.IsUserError(newErr) +} + +// FnvHashInt a hash function +func FnvHashInt(s string) int { + h := fnv.New32a() + _, err := h.Write([]byte(s)) + if err != nil { + return 0 + } + + // for 2 <= base <= 36. The result uses the lower-case letters 'a' to 'z' + return int(h.Sum32()) +} + +// FileMD5 calculate the md5 of file +func FileMD5(filePath string) (string, error) { + file, err := os.Open(filePath) + defer file.Close() + if err != nil { + return "", err + } + hash := md5.New() + _, err = io.Copy(hash, file) + if err != nil { + return "", err + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + +// ShuffleOneArray - +func ShuffleOneArray(arr []string) []string { + arrLength := len(arr) + if arrLength <= 1 { + return arr + } + copyArr := make([]string, arrLength) + copy(copyArr, arr) + rand.Seed(time.Now().UnixNano()) + rand.Shuffle(arrLength, func(i, j int) { copyArr[i], copyArr[j] = copyArr[j], copyArr[i] }) + return copyArr +} + +// IsCAEFunc judge whether it is a CAE function +func IsCAEFunc(businessType string) bool { + return businessType == constant.BusinessTypeCAE +} + +// IsWebSocketFunc return true if the business type is websocket or cae with enable remote debug +func IsWebSocketFunc(businessType string, enableRemoteDebug bool) bool { + return businessType == constant.BusinessTypeWebSocket || + (businessType == constant.BusinessTypeCAE && enableRemoteDebug) +} + +var directFunctions = map[string]struct{}{ + "javax": {}, +} + +// IsDirectFunc check whether it if a direct function (runtime connect to bus directly) +func IsDirectFunc(language string) bool { + _, ok := directFunctions[language] + return ok +} + +// IsStringInArray - +func IsStringInArray(str string, arr []string) bool { + for _, s := range arr { + if s == str { + return true + } + } + return false +} + +// GetFunctionInstanceInfoFromEtcdKey parses the instance info from the etcd path +// e.g. /sn/instance/business/yrk/tenant/0/function/xxx/version/lastest/defaultaz/ +// job-9e54951c-task-77156757-fb16-4b4a-ad61-6646c7d1c57c-d4ad6c74-0/3f079541-15fc-4009-8c41-50b2b2936772 +func GetFunctionInstanceInfoFromEtcdKey(path string) (*types.InstanceInfo, error) { + elements := strings.Split(path, "/") + if len(elements) != functionSchedulerKeyLen { + return nil, fmt.Errorf("unexpected etcd path format: %s", path) + } + return &types.InstanceInfo{ + TenantID: elements[tenantValueIndex], + FunctionName: elements[funcNameValueIndex], + Version: elements[versionValueIndex], + InstanceName: elements[instanceIDValueIndex], + InstanceID: elements[instanceIDValueIndex], + }, nil +} + +// GetModuleSchedulerInfoFromEtcdKey /sn/faas-scheduler/instances/cluster001/7.xx.xx.25/faas-scheduler-xxxx-8xdjf +func GetModuleSchedulerInfoFromEtcdKey(path string) (*types.InstanceInfo, error) { + elements := strings.Split(path, "/") + if len(elements) != moduleSchedulerKeyLen { + return nil, fmt.Errorf("unexpected etcd path format: %s", path) + } + return &types.InstanceInfo{ + TenantID: defaultTenant, + FunctionName: defaultFunctionName, + Version: defaultVersion, + InstanceName: elements[functionNameIndex], + }, nil +} + +// CheckFaaSSchedulerInstanceFault - +func CheckFaaSSchedulerInstanceFault(status types.InstanceStatus) bool { + faultInstanceStatusMap := map[constant.InstanceStatus]struct{}{ + constant.KernelInstanceStatusFatal: {}, + constant.KernelInstanceStatusScheduleFailed: {}, + constant.KernelInstanceStatusEvicting: {}, + constant.KernelInstanceStatusEvicted: {}, + constant.KernelInstanceStatusExiting: {}, + constant.KernelInstanceStatusExited: {}, + } + + _, ok := faultInstanceStatusMap[constant.InstanceStatus(status.Code)] + return ok +} + +// IsNil checks if an object (could be an interface) is nil +func IsNil(i interface{}) bool { + return i == nil || (reflect.ValueOf(i).Kind() == reflect.Ptr && reflect.ValueOf(i).IsNil()) +} + +// CalcFileMD5 calculates file MD5 +func CalcFileMD5(filepath string) string { + file, err := os.Open(filepath) + if err != nil { + return "" + } + defer file.Close() + hash := md5.New() + _, err = io.Copy(hash, file) + if err != nil { + return "" + } + return hex.EncodeToString(hash.Sum(nil)) +} + +// ReceiveWithinTimeout first element is the chan, second element is the timeout +func ReceiveWithinTimeout[T any](ch <-chan T, timeout time.Duration) (T, bool) { + var val T + select { + case val, ok := <-ch: + return val, ok + case <-time.After(timeout): + return val, false + } +} + +func MessageTruncation(message string) string { + if len(message) > defaultMessageLen { + return message[:defaultMessageLen] + } + return message +} + +// SafeCloseChannel will close channel in a safe way +func SafeCloseChannel(stopCh chan struct{}) { + if stopCh == nil { + return + } + select { + case _, ok := <-stopCh: + if ok { + close(stopCh) + } + default: + close(stopCh) + } +} diff --git a/frontend/pkg/common/faas_common/utils/tools_test.go b/frontend/pkg/common/faas_common/utils/tools_test.go new file mode 100644 index 0000000000000000000000000000000000000000..d79548c850f448cf2bc01982c0aed900496f9f02 --- /dev/null +++ b/frontend/pkg/common/faas_common/utils/tools_test.go @@ -0,0 +1,649 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "errors" + "fmt" + "io/ioutil" + "net" + "os" + "strings" + "syscall" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/snerror" + "frontend/pkg/common/faas_common/types" +) + +// TestDomain2IP convert domain to ip +func TestDomain2IP(t *testing.T) { + patches := [...]*gomonkey.Patches{ + gomonkey.ApplyFunc(net.LookupHost, func(_ string) ([]string, error) { + return []string{"1.1.1.1"}, nil + }), + } + defer func() { + for idx := range patches { + patches[idx].Reset() + } + }() + + type args struct { + endpoint string + } + tests := []struct { + args args + want string + wantErr bool + }{ + { + args{endpoint: "1.1.1.1:9000"}, + "1.1.1.1:9000", + false, + }, + { + args{endpoint: "1.1.1.1"}, + "1.1.1.1", + false, + }, + { + args{endpoint: "test:9000"}, + "1.1.1.1:9000", + false, + }, + { + args{endpoint: "test"}, + "1.1.1.1", + false, + }, + } + for _, tt := range tests { + got, err := Domain2IP(tt.args.endpoint) + if (err != nil) != tt.wantErr { + t.Errorf("Domain2IP() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("Domain2IP() got = %v, want %v", got, tt.want) + } + } +} + +func TestGenStateIDByKey(t *testing.T) { + convey.Convey("Test gen stateID by UUID", t, func() { + stateID := GenStateIDByKey("tenantID", "serviceID", "funcName", "") + convey.So(stateID, convey.ShouldNotBeNil) + }) + convey.Convey("Test gen stateID by params", t, func() { + stateID := GenStateIDByKey("tenantID", "serviceID", "funcName", "key") + convey.So(stateID, convey.ShouldEqual, "993e96b4-0550-523f-a412-a4b58682cb2e") + }) +} + +func TestGetFileHashInfo(t *testing.T) { + convey.Convey("Test get file hashInfo failed", t, func() { + _, _, err := GetFileHashInfo("/xyz") + convey.So(err, convey.ShouldNotBeNil) + }) +} + +func TestFloat64ToByte(t *testing.T) { + value := 123.45 + bytesValue := Float64ToByte(value) + if ByteToFloat64(bytesValue) != value { + t.Errorf("Float64ToByte and ByteToFloat64 failed") + } +} + +func TestExistPath(t *testing.T) { + path := os.Args[0] + if !ExistPath(path) { + t.Errorf("test path exist true failed, path: %s", path) + } + if ExistPath(path + "abc") { + t.Errorf("test path exist false failed, path: %s", path+"abc") + } +} + +func TestUniqueID(t *testing.T) { + uuid1 := UniqueID() + uuid2 := UniqueID() + assert.NotEqual(t, uuid1, uuid2) +} + +func Test_parseAzEnv(t *testing.T) { + assert.Equal(t, constant.DefaultAZ, AzEnv()) + tests := []struct { + name string + zoneValue string + want string + }{ + { + name: "empty zoneValue", + zoneValue: "", + want: constant.DefaultAZ, + }, + { + name: fmt.Sprintf("ZoneName > %d", constant.ZoneNameLen), + zoneValue: "12345678901234567890123456789012345678901234567890" + + "12345678901234567890123456789012345678901234567890" + + "12345678901234567890123456789012345678901234567890" + + "12345678901234567890123456789012345678901234567890" + + "12345678901234567890123456789012345678901234567890" + + "123456", + want: "12345678901234567890123456789012345678901234567890" + + "12345678901234567890123456789012345678901234567890" + + "12345678901234567890123456789012345678901234567890" + + "12345678901234567890123456789012345678901234567890" + + "12345678901234567890123456789012345678901234567890" + + "1234", + }, + { + name: "Normal", + zoneValue: "1234567890", + want: "1234567890", + }, + } + for _, tt := range tests { + if err := os.Setenv(constant.ZoneKey, tt.zoneValue); err != nil { + t.Errorf("failed to set Zone env, %s", err) + } + actual := parseAzEnv() + assert.Equal(t, tt.want, actual) + } +} + +func TestIsConnRefusedErr(t *testing.T) { + err := errors.New("abc") + assert.False(t, IsConnRefusedErr(err)) + err = syscall.EADDRINUSE + assert.False(t, IsConnRefusedErr(err)) + _, err = net.Dial("tcp", "127.0.0.1:33334") + assert.True(t, IsConnRefusedErr(err)) +} + +func TestContainsConnRefusedErr(t *testing.T) { + err := errors.New("dial tcp 10.249.0.54:22668: connect: connection refused") + assert.True(t, ContainsConnRefusedErr(err)) +} + +// TestWriteFileToPath is used to test the function of writing a file to a specified path. +func TestWriteFileToPath(t *testing.T) { + dir, err := ioutil.TempDir("", "test") + assert.Equal(t, err, nil) + addFile, err := ioutil.TempFile(dir, "test") + err = WriteFileToPath(addFile.Name(), []byte("test")) + assert.Equal(t, err, nil) +} + +// TestIsHexString is used to test whether the character string meets the requirements. +func TestIsHexString(t *testing.T) { + flag := IsHexString("2345") + assert.True(t, flag) + flag = IsHexString("test") + assert.False(t, flag) +} + +// TestValidateTimeout: indicates whether the timeout interval exceeds the maximum value or is the default value. +func TestValidateTimeout(t *testing.T) { + var timeout int64 = -1 + var defaultTimeout int64 = 1 + ValidateTimeout(&timeout, defaultTimeout) + assert.Equal(t, timeout, int64(1)) + timeout = 100*24*3600 + 1 + ValidateTimeout(&timeout, defaultTimeout) + assert.Equal(t, timeout, int64(100*24*3600)) +} + +// TestDeepCopy is used to test the deep copy of maps and slices. +func TestDeepCopy(t *testing.T) { + str := []string{"test1", "test2"} + cpyStr := DeepCopy(str) + curStr, ok := cpyStr.([]string) + assert.True(t, ok) + assert.Equal(t, len(curStr), 2) + assert.Equal(t, curStr[0], "test1") + assert.Equal(t, curStr[1], "test2") + + tmpMap := make(map[string]string) + tmpMap["test1"] = "test1" + tmpMap["test2"] = "test2" + cpyMap := DeepCopy(tmpMap) + curMap, ok := cpyMap.(map[string]string) + assert.True(t, ok) + assert.Equal(t, len(curMap), 2) + assert.Equal(t, curMap["test1"], "test1") + assert.Equal(t, curMap["test2"], "test2") +} + +func TestIsInputParameterValid(t *testing.T) { + res1 := IsInputParameterValid("|") + assert.Equal(t, res1, false) + res2 := IsInputParameterValid("ddd") + assert.Equal(t, res2, true) + res3 := IsInputParameterValid("ab(d)e") + assert.Equal(t, res3, false) + res4 := IsInputParameterValid("abde;") + assert.Equal(t, res4, false) + res5 := IsInputParameterValid("&abde") + assert.Equal(t, res5, false) +} + +func TestDefaultString(t *testing.T) { + convey.Convey("TestDefaultString", t, func() { + convey.So(DefaultStringEnv("abc", "def"), convey.ShouldEqual, "def") + }) +} + +func Test_replaceByDNS(t *testing.T) { + convey.Convey("Test_replaceByDNSError", t, func() { + patches := []*gomonkey.Patches{ + gomonkey.ApplyFunc(ReadLines, func(path string) ([]string, error) { + return nil, errors.New("mock error") + }), + } + defer func() { + for idx := range patches { + patches[idx].Reset() + } + }() + err := ReplaceByDNS("", nil) + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("Test_replaceByDNS", t, func() { + patches := []*gomonkey.Patches{ + gomonkey.ApplyFunc(ReadLines, func(path string) ([]string, error) { + return []string{"192.168.1.1 www.example.com"}, nil + }), + gomonkey.ApplyFunc(WriteLines, func(path string, lines []string) error { + return nil + }), + } + defer func() { + for idx := range patches { + patches[idx].Reset() + } + }() + err := ReplaceByDNS("", map[string]string{"www.example.com": "192.168.1.2"}) + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("Test_replaceByDNSFileReWrite", t, func() { + testLine := []string{"192.168.1.1 www.example.com"} + err1 := WriteLines("/tmp/dnsTestFile", testLine) + convey.So(err1, convey.ShouldBeNil) + + // 能够将第一次的文件内容覆盖 + testLine = []string{"192.168.1.1 www.example.com", + "192.168.1.2 www.example2.com", + "192.168.1.3 www.example3.com", + "192.168.1.4 www.example4.com", + "192.168.1.5 www.example5.com", + "192.168.1.6 www.example6.com", + "192.168.1.7 www.example7.com", + "192.168.1.8 www.example8.com", + "192.168.1.9 www.example9.com", + "192.168.1.10 www.example10.com", + "192.168.1.11 www.example11.com", + "192.168.1.12 www.example12.com", + "192.168.1.13 www.example13.com test-array-len-3-exception"} + err1 = WriteLines("/tmp/dnsTestFile", testLine) + lineContext, err1 := ReadLines("/tmp/dnsTestFile") + convey.So(err1, convey.ShouldBeNil) + convey.So(len(lineContext), convey.ShouldEqual, 13) + var rst = 1 + for i := range lineContext { + if strings.Contains(lineContext[i], "www.example.com") { + if strings.Contains(lineContext[i], "192.168.1.1") { + rst = 0 + } + } + } + convey.So(rst, convey.ShouldEqual, 0) + + // 修改其中的一个条，其他内容不变 + err := ReplaceByDNS("/tmp/dnsTestFile", map[string]string{"www.example.com": "192.168.1.4"}) + err = ReplaceByDNS("/tmp/dnsTestFile", map[string]string{"www.example.com": "192.168.1.4"}) + convey.So(err, convey.ShouldBeNil) + lineContext, err1 = ReadLines("/tmp/dnsTestFile") + fmt.Println(lineContext) + convey.So(err1, convey.ShouldBeNil) + convey.So(len(lineContext), convey.ShouldEqual, 13) + lineContext, err = ReadLines("/tmp/dnsTestFile") + rst = 1 + for i := range lineContext { + if strings.Contains(lineContext[i], "www.example.com") { + if strings.Contains(lineContext[i], "192.168.1.4") { + rst = 0 + } + } + } + convey.So(rst, convey.ShouldEqual, 0) + rst = 1 + for i := range lineContext { + if strings.Contains(lineContext[i], "www.example2.com") { + if strings.Contains(lineContext[i], "192.168.1.2") { + rst = 0 + } + } + } + convey.So(rst, convey.ShouldEqual, 0) + + // 新增一条,修改一条，一条不变，能够保持成功 + testLine = []string{"192.168.1.1 www.example.com", + "192.168.1.2 www.example2.com", + "192.168.1.14 www.example14.com"} + err = ReplaceByDNS("/tmp/dnsTestFile", map[string]string{"www.example.com": "192.168.1.1", + "www.example14.com": "192.168.1.14", + "www.example2.com": "192.168.1.2"}) + convey.So(err, convey.ShouldBeNil) + lineContext, err1 = ReadLines("/tmp/dnsTestFile") + convey.So(err1, convey.ShouldBeNil) + convey.So(len(lineContext), convey.ShouldEqual, 14) + lineContext, err = ReadLines("/tmp/dnsTestFile") + rst = 1 + for i := range lineContext { + if strings.Contains(lineContext[i], "www.example.com") { + if strings.Contains(lineContext[i], "192.168.1.1") { + rst = 0 + } + } + } + convey.So(rst, convey.ShouldEqual, 0) + rst = 1 + for i := range lineContext { + if strings.Contains(lineContext[i], "www.example2.com") { + if strings.Contains(lineContext[i], "192.168.1.2") { + rst = 0 + } + } + } + convey.So(rst, convey.ShouldEqual, 0) + rst = 1 + for i := range lineContext { + if strings.Contains(lineContext[i], "www.example14.com") { + if strings.Contains(lineContext[i], "192.168.1.14") { + rst = 0 + } + } + } + convey.So(rst, convey.ShouldEqual, 0) + }) +} + +func TestReadLines(t *testing.T) { + convey.Convey("Test_replaceByDNSFileReWrite", t, func() { + defer gomonkey.ApplyFunc(os.Open, func(name string) (*os.File, error) { + return nil, fmt.Errorf("os.Open error") + }).Reset() + _, err := ReadLines("/test") + convey.So(err, convey.ShouldNotBeNil) + }) +} + +func TestShortUUID(t *testing.T) { + assert.Greater(t, 30, len(ShortUUID())) +} + +func TestIsNetworkError(t *testing.T) { + assert.Equal(t, false, IsNetworkError(nil)) + assert.Equal(t, true, IsNetworkError(syscall.EHOSTUNREACH)) + assert.Equal(t, true, IsNetworkError(os.ErrDeadlineExceeded)) + assert.Equal(t, false, IsNetworkError(errors.New("test error"))) +} + +func Test_IsUserError(t *testing.T) { + flag := IsUserError(errors.New("test")) + assert.Equal(t, false, flag) + + snErr := snerror.New(100, "test") + flag = IsUserError(snErr) + assert.Equal(t, false, flag) + + snErr = snerror.New(10500, "test") + flag = IsUserError(snErr) + assert.Equal(t, false, flag) + + snErr = snerror.New(4001, "test") + flag = IsUserError(snErr) + assert.Equal(t, true, flag) +} + +func TestCalculateCPUByMemory(t *testing.T) { + cpuInfo := CalculateCPUByMemory(10) + assert.Equal(t, cpuInfo, 200) +} + +func TestGenerateInstanceID(t *testing.T) { + instanceID := GenerateInstanceID("podName") + assert.Equal(t, instanceID, "defaultaz-#-podName") +} + +func TestGetPodNameByInstanceID(t *testing.T) { + podName := GetPodNameByInstanceID("defaultaz-#-podName") + assert.Equal(t, podName, "podName") +} + +func TestShuffleOneArray(t *testing.T) { + arr1 := []string{"1"} + arr2 := ShuffleOneArray(arr1) + assert.Equal(t, arr1, arr2) + + arr3 := []string{"1", "2", "5", "6", "7"} + arr4 := ShuffleOneArray(arr3) + assert.NotEqual(t, arr3, arr4) + + arr5 := make([]string, 0) + arr6 := ShuffleOneArray(arr5) + assert.Equal(t, len(arr6), 0) +} + +func TestIsCAEFunc(t *testing.T) { + assert.Equal(t, true, IsCAEFunc(constant.BusinessTypeCAE)) + assert.Equal(t, false, IsCAEFunc(constant.WorkerManagerApplier)) +} + +func TestIsDirectFunc(t *testing.T) { + tests := []struct { + name string + expected bool + }{ + { + name: "python3.6", + expected: false, + }, + { + name: "java8", + expected: false, + }, + { + name: "javax", + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, IsDirectFunc(tt.name), tt.name) + }) + } +} + +func TestIsNil(t *testing.T) { + var obj *os.File + assert.Equal(t, true, IsNil(obj)) + getObjFunc := func() interface{} { + return obj + } + assert.Equal(t, true, IsNil(getObjFunc())) +} + +func TestCalcFileMD5(t *testing.T) { + os.Remove("./testFile") + assert.Equal(t, "", CalcFileMD5("invalidPath")) + os.WriteFile("./testFile", + []byte("/sn/function/123/hello/latest|100|{\"name\":\"hello\",\"version\":\"latest\"}\n"), 0600) + assert.Equal(t, "4fca8f1c736ca30135ed16538f4aebfc", CalcFileMD5("./testFile")) + os.Remove("./testFile") +} + +func TestFileMD5(t *testing.T) { + os.Remove("./testFile") + md5, err := FileMD5("invalidPath") + assert.NotNil(t, err) + assert.Equal(t, "", md5) + os.WriteFile("./testFile", + []byte("/sn/function/123/hello/latest|100|{\"name\":\"hello\",\"version\":\"latest\"}\n"), 0600) + md5, err = FileMD5("./testFile") + assert.Nil(t, err) + assert.Equal(t, "4fca8f1c736ca30135ed16538f4aebfc", md5) + os.Remove("./testFile") +} + +func TestFnvHashInt(t *testing.T) { + hashInt := FnvHashInt("123") + assert.Equal(t, 1916298011, hashInt) +} + +func TestSafeCloseStopCh(t *testing.T) { + convey.Convey("stopCh", t, func() { + stopCh := make(chan struct{}, 1) + stopCh <- struct{}{} + SafeCloseChannel(stopCh) + _, ok := <-stopCh + assert.Equal(t, false, ok) + }) + convey.Convey("default", t, func() { + stopCh := make(chan struct{}, 1) + SafeCloseChannel(stopCh) + _, ok := <-stopCh + assert.Equal(t, false, ok) + }) + convey.Convey("chan is nil", t, func() { + SafeCloseChannel(nil) + }) +} + +func TestMessageTruncation(t *testing.T) { + message := "aaaaaaaaaaaaaaaaaaa" + truncationMessage := MessageTruncation(message) + assert.Equal(t, message, truncationMessage) + rawMessage := "" + for i := 0; i < 300; i++ { + rawMessage = rawMessage + "a" + } + truncationMessage = MessageTruncation(rawMessage) + assert.Equal(t, len(truncationMessage), 256) +} + +func TestGetFunctionInstanceInfoFromEtcdKey(t *testing.T) { + convey.Convey("Test GetFunctionInstanceInfoFromEtcdKey", t, func() { + key := "/sn/instance/business/yrk/tenant/0/function/faasscheduler/version/latest/defaultaz/falseParam/requestID/3f079541-15fc-4009-8c41-50b2b2936772" + _, err := GetFunctionInstanceInfoFromEtcdKey(key) + convey.So(err, convey.ShouldNotBeNil) + + key = "/sn/instance/business/yrk/tenant/0/function/faasscheduler/version/latest/defaultaz/requestID/3f079541-15fc-4009-8c41-50b2b2936772" + info, err := GetFunctionInstanceInfoFromEtcdKey(key) + convey.So(err, convey.ShouldBeNil) + convey.So(info.FunctionName, convey.ShouldEqual, "faasscheduler") + convey.So(info.TenantID, convey.ShouldEqual, "0") + convey.So(info.Version, convey.ShouldEqual, "latest") + convey.So(info.InstanceName, convey.ShouldEqual, "3f079541-15fc-4009-8c41-50b2b2936772") + + key = "/sn/instance/business/yrk/tenant/0/function/faasscheduler/version/$latest/defaultaz/requestID/876a3352-44ea-4f0f-83b2-851c50aa89e1" + info, err = GetFunctionInstanceInfoFromEtcdKey(key) + convey.So(err, convey.ShouldBeNil) + convey.So(info.FunctionName, convey.ShouldEqual, "faasscheduler") + convey.So(info.TenantID, convey.ShouldEqual, "0") + convey.So(info.Version, convey.ShouldEqual, "$latest") + convey.So(info.InstanceName, convey.ShouldEqual, "876a3352-44ea-4f0f-83b2-851c50aa89e1") + }) +} + +func TestGetModuleSchedulerInfoFromEtcdKey(t *testing.T) { + convey.Convey("Test GetModuleSchedulerInfoFromEtcdKey", t, func() { + key := "/sn/faas-scheduler/instances/cluster1/node1/falseParam/faas-scheduler-123" + _, err := GetModuleSchedulerInfoFromEtcdKey(key) + convey.So(err, convey.ShouldNotBeNil) + + key = "/sn/faas-scheduler/instances/cluster1/node1/faas-scheduler-123" + info, err := GetModuleSchedulerInfoFromEtcdKey(key) + convey.So(err, convey.ShouldBeNil) + convey.So(info.FunctionName, convey.ShouldEqual, defaultFunctionName) + convey.So(info.TenantID, convey.ShouldEqual, defaultTenant) + convey.So(info.Version, convey.ShouldEqual, defaultVersion) + convey.So(info.InstanceName, convey.ShouldEqual, "faas-scheduler-123") + }) +} + +func TestCheckFaaSSchedulerInstanceFault(t *testing.T) { + convey.Convey("Test CheckFaaSSchedulerInstanceFault", t, func() { + testCases := []struct { + name string + input types.InstanceStatus + expected bool + }{ + { + name: "should return true for KernelInstanceStatusFatal", + input: types.InstanceStatus{Code: int32(constant.KernelInstanceStatusFatal)}, + expected: true, + }, + { + name: "should return true for KernelInstanceStatusScheduleFailed", + input: types.InstanceStatus{Code: int32(constant.KernelInstanceStatusScheduleFailed)}, + expected: true, + }, + { + name: "should return true for KernelInstanceStatusEvicting", + input: types.InstanceStatus{Code: int32(constant.KernelInstanceStatusEvicting)}, + expected: true, + }, + { + name: "should return true for KernelInstanceStatusEvicted", + input: types.InstanceStatus{Code: int32(constant.KernelInstanceStatusEvicted)}, + expected: true, + }, + { + name: "should return true for KernelInstanceStatusExiting", + input: types.InstanceStatus{Code: int32(constant.KernelInstanceStatusExiting)}, + expected: true, + }, + { + name: "should return true for KernelInstanceStatusExited", + input: types.InstanceStatus{Code: int32(constant.KernelInstanceStatusExited)}, + expected: true, + }, + { + name: "should return false for unknown status", + input: types.InstanceStatus{Code: 999}, + expected: false, + }, + } + + for _, tc := range testCases { + convey.Convey(tc.name, func() { + result := CheckFaaSSchedulerInstanceFault(tc.input) + convey.So(result, convey.ShouldEqual, tc.expected) + }) + } + }) +} diff --git a/frontend/pkg/common/faas_common/wisecloudtool/pod_operator.go b/frontend/pkg/common/faas_common/wisecloudtool/pod_operator.go new file mode 100644 index 0000000000000000000000000000000000000000..26ee416d9eb7dd3e592dae513e14fa8ca0daeb23 --- /dev/null +++ b/frontend/pkg/common/faas_common/wisecloudtool/pod_operator.go @@ -0,0 +1,185 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package wisecloudtool - +package wisecloudtool + +import ( + "crypto/tls" + "fmt" + "time" + + "github.com/json-iterator/go" + "github.com/valyala/fasthttp" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/wait" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/resspeckey" + "frontend/pkg/common/faas_common/wisecloudtool/serviceaccount" + "frontend/pkg/common/faas_common/wisecloudtool/types" +) + +const ( + queryRetryTime = 10 + queryRetryDuration = 200 * time.Millisecond // 初始等待时间 + queryRetryFactor = 4 // 倍数因子（每次翻4倍） + queryRetryJitter = 0.5 // 随机抖动系数 + queryRetryCap = 20 * time.Second // 最大等待时间上限 +) + +var ( + coldStartBackoff = wait.Backoff{ + Duration: queryRetryDuration, + Factor: queryRetryFactor, + Jitter: queryRetryJitter, + Steps: queryRetryTime, + Cap: queryRetryCap, + } +) + +// PodOperator - +type PodOperator struct { + nuwaConsoleAddr string // + nuwaGatewayAddr string + *types.ServiceAccountJwt + *fasthttp.Client + logger api.FormatLogger +} + +// NewColdStarter - +func NewColdStarter(serviceAccountJwt *types.ServiceAccountJwt, logger api.FormatLogger) *PodOperator { + return &PodOperator{ + nuwaConsoleAddr: serviceAccountJwt.NuwaRuntimeAddr, + nuwaGatewayAddr: serviceAccountJwt.NuwaGatewayAddr, + ServiceAccountJwt: serviceAccountJwt, + Client: &fasthttp.Client{ + TLSConfig: &tls.Config{ + InsecureSkipVerify: serviceAccountJwt.TlsConfig.HttpsInsecureSkipVerify, + CipherSuites: serviceAccountJwt.TlsConfig.TlsCipherSuites, + MinVersion: tls.VersionTLS12, + }, + MaxIdemponentCallAttempts: 3, + }, + logger: logger, + } +} + +// ColdStart - +func (p *PodOperator) ColdStart(funcKeyWithRes string, resSpec resspeckey.ResSpecKey, + info *types.NuwaRuntimeInfo) error { + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + req.SetRequestURI(fmt.Sprintf("%s/activator/coldstart", p.nuwaConsoleAddr)) + req.Header.SetMethod(fasthttp.MethodPost) + createInstanceReq := types.NuwaColdCreateInstanceReq{ + RuntimeId: info.WisecloudRuntimeId, + RuntimeType: "Function", + PoolType: "noPool", + Memory: resSpec.Memory, + CPU: resSpec.CPU, + EnvLabel: info.EnvLabel, + } + logger := p.logger.With(zap.Any("funcKeyWithRes", funcKeyWithRes), zap.Any("resKey", resSpec.String())) + + body, err := jsoniter.Marshal(createInstanceReq) + if err != nil { + return err + } + err = serviceaccount.GenerateJwtSignedHeaders(req, body, *info, p.ServiceAccountJwt) + if err != nil { + return err + } + req.SetBodyRaw(body) + + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(resp) + backoffErr := wait.ExponentialBackoff( + coldStartBackoff, func() (bool, error) { + err = p.Client.Do(req, resp) + if err != nil { + return false, nil + } + return true, nil + }) + if backoffErr != nil { + logger.Warnf("cold start error, backoffErr: %s", backoffErr.Error()) + return backoffErr + } + if err != nil { + logger.Warnf("cold start error, backoffErr: %s", err.Error()) + return err + } + if resp.StatusCode()/100 != 2 { // resp http code != 2xx + logger.Warnf("cold start error, code: %d, body: %s", resp.StatusCode(), string(resp.Body())) + return fmt.Errorf("failed to cold start") + } + logger.Infof("cold start %s succeed", info.WisecloudRuntimeId) + return nil +} + +// DelPod will send a req to erase runtime pod +func (p *PodOperator) DelPod(nuwaRuntimeInfo *types.NuwaRuntimeInfo, deploymentName string, + podId string) error { + p.logger.Infof("delete nuwa pod %s", podId) + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + req.SetRequestURI(fmt.Sprintf("%s/runtime/instance", p.NuwaGatewayAddr)) + req.Header.SetMethod(fasthttp.MethodDelete) + destroyInsReq := types.NuwaDestroyInstanceReq{ + RuntimeType: "Function", + RuntimeId: nuwaRuntimeInfo.WisecloudRuntimeId, + InstanceId: podId, + WorkLoadName: deploymentName, + } + + reqBody, err := jsoniter.Marshal(destroyInsReq) + if err != nil { + return err + } + err = serviceaccount.GenerateJwtSignedHeaders(req, reqBody, *nuwaRuntimeInfo, p.ServiceAccountJwt) + if err != nil { + return err + } + req.SetBodyRaw(reqBody) + + logger := p.logger.With(zap.Any("deployment", deploymentName), zap.Any("podId", podId)) + rsp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(rsp) + backoffError := wait.ExponentialBackoff( + coldStartBackoff, func() (bool, error) { + err = p.Client.Do(req, rsp) + if err != nil { + return false, nil + } + return true, nil + }) + if backoffError != nil { + logger.Warnf("delete runtime pod error, backoffErr: %s", backoffError.Error()) + return backoffError + } + if err != nil { + logger.Warnf("delete runtime pod error, err: %s", err.Error()) + return err + } + if rsp.StatusCode()/100 != 2 { // resp http code != 2xx + logger.Warnf("delete runtime pod error, code: %d, body: %s", rsp.StatusCode(), string(rsp.Body())) + return fmt.Errorf("failed to delete runtime pod") + } + logger.Infof("succeed to delete runtime pod %s", podId) + return nil +} diff --git a/frontend/pkg/common/faas_common/wisecloudtool/pod_operator_test.go b/frontend/pkg/common/faas_common/wisecloudtool/pod_operator_test.go new file mode 100644 index 0000000000000000000000000000000000000000..9d17c6885cb4d631d4c77211b877be3774400643 --- /dev/null +++ b/frontend/pkg/common/faas_common/wisecloudtool/pod_operator_test.go @@ -0,0 +1,116 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wisecloudtool + +import ( + "crypto/tls" + "errors" + "net" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/valyala/fasthttp" + + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/resspeckey" + "frontend/pkg/common/faas_common/wisecloudtool/serviceaccount" + "frontend/pkg/common/faas_common/wisecloudtool/types" +) + +func TestNewColdStarter(t *testing.T) { + saJwt := &types.ServiceAccountJwt{ + NuwaRuntimeAddr: "http://test-addr", + NuwaGatewayAddr: "http://gateway-addr", + TlsConfig: &types.TLSConfig{ + HttpsInsecureSkipVerify: true, + TlsCipherSuites: []uint16{ + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + }, + }, + } + + po := NewColdStarter(saJwt, log.GetLogger()) + + if po.nuwaConsoleAddr != saJwt.NuwaRuntimeAddr { + t.Errorf("expected nuwaConsoleAddr %s, got %s", saJwt.NuwaRuntimeAddr, po.nuwaConsoleAddr) + } + if po.Client == nil { + t.Error("expected non-nil client") + } +} + +func TestColdStart_Success(t *testing.T) { + po := NewColdStarter(&types.ServiceAccountJwt{ + ServiceAccount: &types.ServiceAccount{}, + TlsConfig: &types.TLSConfig{}, + }, log.GetLogger()) + patches := gomonkey.NewPatches() + defer patches.Reset() + + patches.ApplyFunc(serviceaccount.GenerateJwtSignedHeaders, func(*fasthttp.Request, []byte, types.NuwaRuntimeInfo, *types.ServiceAccountJwt) error { + return nil + }) + patches.ApplyMethodFunc(&fasthttp.Client{}, "Do", func(*fasthttp.Request, *fasthttp.Response) error { + return nil + }) + + err := po.ColdStart("funcKey", resspeckey.ResSpecKey{}, &types.NuwaRuntimeInfo{}) + if err != nil { + t.Errorf("expected nil error, got %v", err) + } +} + +func TestDelPod_Success(t *testing.T) { + po := NewColdStarter(&types.ServiceAccountJwt{ + ServiceAccount: &types.ServiceAccount{}, + TlsConfig: &types.TLSConfig{}, + }, log.GetLogger()) + patches := gomonkey.NewPatches() + defer patches.Reset() + + patches.ApplyFunc(serviceaccount.GenerateJwtSignedHeaders, func(*fasthttp.Request, []byte, types.NuwaRuntimeInfo, *types.ServiceAccountJwt) error { + return nil + }) + patches.ApplyMethodFunc(&fasthttp.Client{}, "Do", func(*fasthttp.Request, *fasthttp.Response) error { + return nil + }) + runtimeInfo := &types.NuwaRuntimeInfo{ + WisecloudRuntimeId: "test-runtime", + } + err := po.DelPod(runtimeInfo, "deploy1", "pod1") + if err != nil { + t.Errorf("expected nil error, got %v", err) + } +} + +func TestDelPod_Error(t *testing.T) { + po := NewColdStarter(&types.ServiceAccountJwt{ + ServiceAccount: &types.ServiceAccount{}, + TlsConfig: &types.TLSConfig{}, + }, log.GetLogger()) + patches := gomonkey.NewPatches() + defer patches.Reset() + + patches.ApplyFunc(net.Listen, func(string, string) (net.Listener, error) { + return nil, errors.New("test error") + }) + + err := po.DelPod(&types.NuwaRuntimeInfo{}, "deploy1", "pod1") + if err == nil { + t.Error("expected error, got nil") + } +} diff --git a/frontend/pkg/common/faas_common/wisecloudtool/prometheus_metrics.go b/frontend/pkg/common/faas_common/wisecloudtool/prometheus_metrics.go new file mode 100644 index 0000000000000000000000000000000000000000..5478dd9b84c6a4da96350819db50306aa1b98640 --- /dev/null +++ b/frontend/pkg/common/faas_common/wisecloudtool/prometheus_metrics.go @@ -0,0 +1,285 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package wisecloudtool - +package wisecloudtool + +import ( + "fmt" + "strings" + "sync" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + k8stype "k8s.io/apimachinery/pkg/types" + + "frontend/pkg/common/faas_common/types" + "frontend/pkg/common/faas_common/urnutils" +) + +const defaultLabel = "UNKNOWN_LABEL" +const labelLen = 8 + +var ( + concurrencyGauge = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "yuanrong_concurrency_num", + Help: "The current concurrency number of the application.", + }, + []string{"businessid", "tenantid", "funcname", "version", "label", "namespace", "deployment_name", "pod_name"}, + ) + + leaseRequestTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "yuanrong_lease_total", + Help: "The lease total number of the application.", + }, + []string{"businessid", "tenantid", "funcname", "version", "label", "namespace", "deployment_name", "pod_name"}, + ) +) + +// GetLeaseRequestTotal - +func GetLeaseRequestTotal() *prometheus.CounterVec { + return leaseRequestTotal +} + +// GetConcurrencyGauge - +func GetConcurrencyGauge() *prometheus.GaugeVec { + return concurrencyGauge +} + +// MetricProvider - +type MetricProvider struct { + sync.RWMutex + // key is {funcKey}#{invokeLabel}, subKey namespace value is {namespace, podName} + WorkLoadMap map[string]map[string]*k8stype.NamespacedName +} + +// NewMetricProvider - +func NewMetricProvider() *MetricProvider { + return &MetricProvider{ + RWMutex: sync.RWMutex{}, + WorkLoadMap: make(map[string]map[string]*k8stype.NamespacedName), + } +} + +// AddWorkLoad - +func (m *MetricProvider) AddWorkLoad(funcKey string, invokeLabel string, namespaceName *k8stype.NamespacedName) { + workload := getWorkloadName(funcKey, invokeLabel) + m.Lock() + defer m.Unlock() + + deployments, ok := m.WorkLoadMap[workload] + if !ok { + deployments = make(map[string]*k8stype.NamespacedName) + m.WorkLoadMap[workload] = deployments + } + if _, ok = deployments[namespaceName.String()]; !ok { + deployments[namespaceName.String()] = namespaceName + } +} + +// EnsureConcurrencyGaugeWithLabel - +func (m *MetricProvider) EnsureConcurrencyGaugeWithLabel(labels []string) error { + if len(labels) != labelLen { + return fmt.Errorf("labels len must be 8") + } + + m.RLock() + defer m.RUnlock() + _, err := concurrencyGauge.GetMetricWithLabelValues(labels...) + return err +} + +// EnsureLeaseRequestTotalWithLabel - +func (m *MetricProvider) EnsureLeaseRequestTotalWithLabel(labels []string) error { + if len(labels) != labelLen { + return fmt.Errorf("labels len must be 8") + } + + m.RLock() + defer m.RUnlock() + _, err := leaseRequestTotal.GetMetricWithLabelValues(labels...) + return err +} + +// Exist - +func (m *MetricProvider) Exist(funcKey string, invokeLabel string) bool { + return m.GetRandomDeployment(funcKey, invokeLabel) != nil +} + +// GetRandomDeployment - +func (m *MetricProvider) GetRandomDeployment(funcKey string, invokeLabel string) *k8stype.NamespacedName { + workName := getWorkloadName(funcKey, invokeLabel) + m.RLock() + defer m.RUnlock() + deployments, ok := m.WorkLoadMap[workName] + if !ok { + return nil + } + if len(deployments) == 0 { + return nil + } + for _, namespaceName := range deployments { + return namespaceName + } + return nil +} + +// IncLeaseRequestTotalWithLabel - +func (m *MetricProvider) IncLeaseRequestTotalWithLabel(labels []string) error { + if len(labels) != labelLen { + return fmt.Errorf("labels len must be 8") + } + counter, err := leaseRequestTotal.GetMetricWithLabelValues(labels...) + if err != nil { + return err + } + counter.Inc() + return nil +} + +// IncConcurrencyGaugeWithLabel - +func (m *MetricProvider) IncConcurrencyGaugeWithLabel(labels []string) error { + if len(labels) != labelLen { + return fmt.Errorf("labels len must be 8") + } + gauge, err := concurrencyGauge.GetMetricWithLabelValues(labels...) + if err != nil { + return err + } + gauge.Inc() + return nil +} + +// DecConcurrencyGaugeWithLabel - +func (m *MetricProvider) DecConcurrencyGaugeWithLabel(labels []string) error { + if len(labels) != labelLen { + return fmt.Errorf("labels len must be 8") + } + gauge, err := concurrencyGauge.GetMetricWithLabelValues(labels...) + if err != nil { + return err + } + gauge.Dec() + return nil +} + +// ClearConcurrencyGaugeWithLabel - +func (m *MetricProvider) ClearConcurrencyGaugeWithLabel(labels []string) error { + if len(labels) != labelLen { + return fmt.Errorf("labels len must be 8") + } + concurrencyGauge.DeleteLabelValues(labels...) + return nil +} + +// ClearLeaseRequestTotalWithLabel - +func (m *MetricProvider) ClearLeaseRequestTotalWithLabel(labels []string) error { + if len(labels) != labelLen { + return fmt.Errorf("labels len must be 8") + } + leaseRequestTotal.DeleteLabelValues(labels...) + return nil +} + +// ClearMetricsForFunction - +func (m *MetricProvider) ClearMetricsForFunction(funcMetaData *types.FuncMetaData) { + funcKey0 := urnutils.CombineFunctionKey(funcMetaData.TenantID, funcMetaData.FuncName, funcMetaData.Version) + m.Lock() + defer m.Unlock() + for workload, _ := range m.WorkLoadMap { + funcKey1, invokeLabel := GetFuncKeyAndLabelFromWorkload(workload) + if funcKey0 == funcKey1 { + m.clearMetricsForInsConfigWithoutLock(funcMetaData, invokeLabel) + } + } +} + +// ClearMetricsForInsConfig - +func (m *MetricProvider) ClearMetricsForInsConfig(funcMetaData *types.FuncMetaData, invokeLabel string) { + m.Lock() + m.clearMetricsForInsConfigWithoutLock(funcMetaData, invokeLabel) + m.Unlock() +} + +func (m *MetricProvider) clearMetricsForInsConfigWithoutLock(funcMetaData *types.FuncMetaData, invokeLabel string) { + // 得看下和FunctionVersion有啥区别 + funcKey := urnutils.CombineFunctionKey(funcMetaData.TenantID, funcMetaData.FuncName, funcMetaData.Version) + workload := getWorkloadName(funcKey, invokeLabel) + deployments, ok := m.WorkLoadMap[workload] + if !ok { + return + } + delete(m.WorkLoadMap, workload) + + if invokeLabel == "" { + invokeLabel = defaultLabel + } + + for _, deployment := range deployments { + labels := map[string]string{ + "businessid": funcMetaData.BusinessID, + "tenantid": funcMetaData.TenantID, + "funcname": funcMetaData.FuncName, + "version": funcMetaData.Version, + "label": invokeLabel, + "namespace": deployment.Namespace, + "deployment_name": deployment.Name, + } + concurrencyGauge.DeletePartialMatch(labels) + leaseRequestTotal.DeletePartialMatch(labels) + } +} + +// GetMetricLabels - +// 判断label是否符合预期 +func GetMetricLabels(funcMetaData *types.FuncMetaData, invokeLabel string, + namespace string, deploymentName string, podName string) []string { + var metricLabelValue []string + if namespace != "" && deploymentName != "" && podName != "" && funcMetaData != nil { + if invokeLabel == "" { + invokeLabel = defaultLabel + } + metricLabelValue = []string{ + funcMetaData.BusinessID, + funcMetaData.TenantID, + funcMetaData.FuncName, + funcMetaData.Version, + invokeLabel, + namespace, + deploymentName, + podName} + } + return metricLabelValue +} + +// GetFuncKeyAndLabelFromWorkload - +func GetFuncKeyAndLabelFromWorkload(workload string) (string, string) { + strs := strings.Split(workload, "#") + if len(strs) == 2 { // deployment key must be 2 + return strs[0], strs[1] + } + return "", "" +} + +// getWorkloadName - get deploymentforfunckey +func getWorkloadName(funcKey, invokeLabel string) string { + if invokeLabel == "" { + invokeLabel = defaultLabel + } + return fmt.Sprintf("%s#%s", funcKey, invokeLabel) +} diff --git a/frontend/pkg/common/faas_common/wisecloudtool/prometheus_metrics_test.go b/frontend/pkg/common/faas_common/wisecloudtool/prometheus_metrics_test.go new file mode 100644 index 0000000000000000000000000000000000000000..ba8d4d887c1e718b88f535137715bfc9003e05a5 --- /dev/null +++ b/frontend/pkg/common/faas_common/wisecloudtool/prometheus_metrics_test.go @@ -0,0 +1,313 @@ +package wisecloudtool + +import ( + "fmt" + "github.com/agiledragon/gomonkey/v2" + "github.com/prometheus/client_golang/prometheus" + "github.com/smartystreets/goconvey/convey" + "testing" + + "github.com/stretchr/testify/assert" + k8stype "k8s.io/apimachinery/pkg/types" + + "frontend/pkg/common/faas_common/types" + "frontend/pkg/common/faas_common/urnutils" +) + +func TestNewMetricProvider(t *testing.T) { + provider := NewMetricProvider() + assert.NotNil(t, provider) + assert.NotNil(t, provider.WorkLoadMap) + assert.Equal(t, 0, len(provider.WorkLoadMap)) +} + +func TestMetricProvider_AddWorkLoad(t *testing.T) { + t.Run("Add new workload", func(t *testing.T) { + provider := NewMetricProvider() + funcKey := "test-func" + invokeLabel := "test-label" + namespaceName := &k8stype.NamespacedName{ + Namespace: "test-ns", + Name: "test-name", + } + + provider.AddWorkLoad(funcKey, invokeLabel, namespaceName) + + assert.Equal(t, 1, len(provider.WorkLoadMap)) + assert.Equal(t, 1, len(provider.WorkLoadMap[getWorkloadName(funcKey, invokeLabel)])) + }) + + t.Run("Add duplicate workload", func(t *testing.T) { + provider := NewMetricProvider() + funcKey := "test-func" + invokeLabel := "test-label" + namespaceName := &k8stype.NamespacedName{ + Namespace: "test-ns", + Name: "test-name", + } + + // Add twice + provider.AddWorkLoad(funcKey, invokeLabel, namespaceName) + provider.AddWorkLoad(funcKey, invokeLabel, namespaceName) + + assert.Equal(t, 1, len(provider.WorkLoadMap)) + assert.Equal(t, 1, len(provider.WorkLoadMap[getWorkloadName(funcKey, invokeLabel)])) + }) +} + +func TestMetricProvider_Exist(t *testing.T) { + provider := NewMetricProvider() + funcKey := "test-func" + invokeLabel := "test-label" + + t.Run("Workload does not exist", func(t *testing.T) { + assert.False(t, provider.Exist(funcKey, invokeLabel)) + }) + + t.Run("Workload exists", func(t *testing.T) { + provider.AddWorkLoad(funcKey, invokeLabel, &k8stype.NamespacedName{ + Namespace: "test-ns", + Name: "test-name", + }) + assert.True(t, provider.Exist(funcKey, invokeLabel)) + }) +} + +func TestMetricProvider_GetRandomDeployment(t *testing.T) { + provider := NewMetricProvider() + funcKey := "test-func" + invokeLabel := "test-label" + testDeployment0 := &k8stype.NamespacedName{ + Namespace: "test-ns-0", + Name: "test-name-0", + } + + testDeployment1 := &k8stype.NamespacedName{ + Namespace: "test-ns-1", + Name: "test-name-1", + } + + t.Run("Get non-existent deployment", func(t *testing.T) { + assert.Nil(t, provider.GetRandomDeployment(funcKey, invokeLabel)) + }) + + t.Run("Get existing deployment", func(t *testing.T) { + provider.AddWorkLoad(funcKey, invokeLabel, testDeployment0) + provider.AddWorkLoad(funcKey, invokeLabel, testDeployment1) + flag0 := false + flag1 := false + for i := 0; i < 100; i++ { + result := provider.GetRandomDeployment(funcKey, invokeLabel) + switch result.Name { + case "test-name-0": + flag0 = true + case "test-name-1": + flag1 = true + } + if flag1 && flag0 { + break + } + } + assert.True(t, flag0 && flag1) + }) +} + +func TestMetricProvider_ClearMetrics(t *testing.T) { + provider := NewMetricProvider() + funcMeta := &types.FuncMetaData{ + TenantID: "tenant1", + FuncName: "func1", + Version: "v1", + BusinessID: "biz1", + } + invokeLabel := "test-label" + workload := getWorkloadName(urnutils.CombineFunctionKey(funcMeta.TenantID, funcMeta.FuncName, funcMeta.Version), invokeLabel) + + // Add test data + provider.AddWorkLoad( + urnutils.CombineFunctionKey(funcMeta.TenantID, funcMeta.FuncName, funcMeta.Version), + invokeLabel, + &k8stype.NamespacedName{ + Namespace: "test-ns", + Name: "test-name", + }, + ) + + t.Run("Clear function metrics", func(t *testing.T) { + provider.ClearMetricsForFunction(funcMeta) + assert.Equal(t, 0, len(provider.WorkLoadMap)) + }) + + t.Run("Clear instance config metrics", func(t *testing.T) { + // Re-add data + provider.AddWorkLoad( + urnutils.CombineFunctionKey(funcMeta.TenantID, funcMeta.FuncName, funcMeta.Version), + invokeLabel, + &k8stype.NamespacedName{ + Namespace: "test-ns", + Name: "test-name", + }, + ) + + provider.ClearMetricsForInsConfig(funcMeta, invokeLabel) + assert.Nil(t, provider.WorkLoadMap[workload]) + }) +} + +func TestGetMetricLabels(t *testing.T) { + funcMeta := &types.FuncMetaData{ + BusinessID: "biz1", + TenantID: "tenant1", + FuncName: "func1", + Version: "v1", + } + + t.Run("Generate complete labels", func(t *testing.T) { + labels := GetMetricLabels(funcMeta, "label1", "ns1", "deploy1", "pod1") + assert.Equal(t, []string{"biz1", "tenant1", "func1", "v1", "label1", "ns1", "deploy1", "pod1"}, labels) + }) + + t.Run("Use default label", func(t *testing.T) { + labels := GetMetricLabels(funcMeta, "", "ns1", "deploy1", "pod1") + assert.Equal(t, "UNKNOWN_LABEL", labels[4]) + }) + + t.Run("Return nil when missing required parameters", func(t *testing.T) { + assert.Nil(t, GetMetricLabels(nil, "label1", "ns1", "deploy1", "pod1")) + assert.Nil(t, GetMetricLabels(funcMeta, "label1", "", "deploy1", "pod1")) + }) +} + +func TestWorkloadHelpers(t *testing.T) { + t.Run("Get workload name", func(t *testing.T) { + name := getWorkloadName("func1", "label1") + assert.Equal(t, "func1#label1", name) + assert.Equal(t, "func1#UNKNOWN_LABEL", getWorkloadName("func1", "")) + }) + + t.Run("Parse from workload name", func(t *testing.T) { + funcKey, label := GetFuncKeyAndLabelFromWorkload("func1#label1") + assert.Equal(t, "func1", funcKey) + assert.Equal(t, "label1", label) + + funcKey, label = GetFuncKeyAndLabelFromWorkload("invalid") + assert.Equal(t, "", funcKey) + assert.Equal(t, "", label) + }) +} + +func TestMetricProvider(t *testing.T) { + convey.Convey("Test MetricProvider Functions", t, func() { + m := &MetricProvider{} + validLabels := make([]string, labelLen) + invalidLabels := make([]string, labelLen-1) + + convey.Convey("Test IncLeaseRequestTotalWithLabel", func() { + convey.Convey("should return error for invalid label length", func() { + err := m.IncLeaseRequestTotalWithLabel(invalidLabels) + convey.So(err, convey.ShouldNotBeNil) + convey.So(err.Error(), convey.ShouldContainSubstring, "labels len must be 8") + }) + + convey.Convey("should handle GetMetricWithLabelValues error", func() { + patches := gomonkey.ApplyMethodFunc(leaseRequestTotal, "GetMetricWithLabelValues", func(...string) (prometheus.Counter, error) { + return nil, fmt.Errorf("mock error") + }) + defer patches.Reset() + + err := m.IncLeaseRequestTotalWithLabel(validLabels) + convey.So(err, convey.ShouldNotBeNil) + }) + + convey.Convey("should increment counter successfully", func() { + patches := gomonkey.ApplyMethodFunc(leaseRequestTotal, "GetMetricWithLabelValues", func(...string) (prometheus.Counter, error) { + counter := &fakeCounter{} + return counter, nil + }) + defer patches.Reset() + + err := m.IncLeaseRequestTotalWithLabel(validLabels) + convey.So(err, convey.ShouldBeNil) + }) + }) + + convey.Convey("Test IncConcurrencyGaugeWithLabel", func() { + convey.Convey("should return error for invalid label length", func() { + err := m.IncConcurrencyGaugeWithLabel(invalidLabels) + convey.So(err, convey.ShouldNotBeNil) + convey.So(err.Error(), convey.ShouldContainSubstring, "labels len must be 8") + }) + + convey.Convey("should handle GetMetricWithLabelValues error", func() { + patches := gomonkey.ApplyMethodFunc(concurrencyGauge, "GetMetricWithLabelValues", func(...string) (prometheus.Gauge, error) { + return nil, fmt.Errorf("mock error") + }) + defer patches.Reset() + + err := m.IncConcurrencyGaugeWithLabel(validLabels) + convey.So(err, convey.ShouldNotBeNil) + }) + + convey.Convey("should increment gauge successfully", func() { + patches := gomonkey.ApplyMethodFunc(concurrencyGauge, "GetMetricWithLabelValues", func(...string) (prometheus.Gauge, error) { + gauge := &fakeGauge{} + return gauge, nil + }) + defer patches.Reset() + + err := m.IncConcurrencyGaugeWithLabel(validLabels) + convey.So(err, convey.ShouldBeNil) + }) + }) + + convey.Convey("Test DecConcurrencyGaugeWithLabel", func() { + convey.Convey("should decrement gauge successfully", func() { + patches := gomonkey.ApplyMethodFunc(concurrencyGauge, "GetMetricWithLabelValues", func(...string) (prometheus.Gauge, error) { + gauge := &fakeGauge{} + return gauge, nil + }) + defer patches.Reset() + + err := m.DecConcurrencyGaugeWithLabel(validLabels) + convey.So(err, convey.ShouldBeNil) + }) + }) + + convey.Convey("Test ClearConcurrencyGaugeWithLabel", func() { + convey.Convey("should clear gauge successfully", func() { + patches := gomonkey.ApplyMethodFunc(concurrencyGauge, "DeleteLabelValues", func(...string) bool { + return true + }) + defer patches.Reset() + + err := m.ClearConcurrencyGaugeWithLabel(validLabels) + convey.So(err, convey.ShouldBeNil) + }) + }) + + convey.Convey("Test ClearLeaseRequestTotalWithLabel", func() { + convey.Convey("should clear counter successfully", func() { + patches := gomonkey.ApplyMethodFunc(leaseRequestTotal, "DeleteLabelValues", func(...string) bool { + return true + }) + defer patches.Reset() + + err := m.ClearLeaseRequestTotalWithLabel(validLabels) + convey.So(err, convey.ShouldBeNil) + }) + }) + }) +} + +type fakeCounter struct { + prometheus.Counter +} + +func (f *fakeCounter) Inc() {} + +type fakeGauge struct { + prometheus.Gauge +} + +func (f *fakeGauge) Inc() {} +func (f *fakeGauge) Dec() {} diff --git a/frontend/pkg/common/faas_common/wisecloudtool/serviceaccount/jwtsign.go b/frontend/pkg/common/faas_common/wisecloudtool/serviceaccount/jwtsign.go new file mode 100644 index 0000000000000000000000000000000000000000..e45500caf3b188ae1945c84a2fa146067fa901df --- /dev/null +++ b/frontend/pkg/common/faas_common/wisecloudtool/serviceaccount/jwtsign.go @@ -0,0 +1,203 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package serviceaccount sign http request by jwttoken +package serviceaccount + +import ( + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/hex" + "encoding/pem" + "fmt" + "sort" + "strconv" + "strings" + "time" + + "github.com/valyala/fasthttp" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/wisecloudtool/types" +) + +const defaultExp = 300 * time.Second + +// GenerateJwtSignedHeaders put header authorization to request header +func GenerateJwtSignedHeaders(req *fasthttp.Request, body []byte, wiseCloudCtx types.NuwaRuntimeInfo, + serviceAccountJwt *types.ServiceAccountJwt) error { + headers := map[string]string{} + req.Header.Set("x-wisecloud-site", wiseCloudCtx.WisecloudSite) + req.Header.Set("x-wisecloud-service-id", wiseCloudCtx.WisecloudServiceId) + req.Header.Set("x-wisecloud-environment-id", wiseCloudCtx.WisecloudEnvironmentId) + headers = map[string]string{ + "x-wisecloud-site": wiseCloudCtx.WisecloudSite, + "x-wisecloud-service-id": wiseCloudCtx.WisecloudServiceId, + "x-wisecloud-environment-id": wiseCloudCtx.WisecloudEnvironmentId, + } + + jwtToken, err := generateJWTToken(req, string(body), headers, serviceAccountJwt) + if err != nil { + return err + } + // Set headers + req.Header.Set(constant.HeaderAuthorization, jwtToken) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("x-client-id", strconv.FormatInt(serviceAccountJwt.ClientId, 10)) // decimal notation + return nil +} + +func generateJWTToken(req *fasthttp.Request, body string, headers map[string]string, + serviceAccountJwt *types.ServiceAccountJwt) (string, error) { + return generateJWTTokenGeneric(&types.ServiceAccount{ + PrivateKey: serviceAccountJwt.PrivateKey, + ClientId: serviceAccountJwt.ClientId, + KeyId: serviceAccountJwt.KeyId, + }, headers, buildQueryPayload(string(req.URI().Path()), string(req.Header.Method()), + string(req.URI().QueryString()), body), serviceAccountJwt.OauthTokenUrl, "JWT-PRO2") +} + +func buildQueryPayload(queryPath, method, queryString, body string) string { + var payloadBuilder strings.Builder + payloadBuilder.WriteString(body) + if queryPath != "" { + payloadBuilder.WriteString("\n") + payloadBuilder.WriteString(queryPath) + } + if method != "" { + payloadBuilder.WriteString("\n") + payloadBuilder.WriteString(method) + } + if queryString != "" { + payloadBuilder.WriteString("\n") + payloadBuilder.WriteString(queryString) + } + return payloadBuilder.String() +} + +func generateJWTTokenGeneric(sa *types.ServiceAccount, headers map[string]string, + body string, aud string, jwtTokenType string) (string, error) { + requestSign, err := getRequestSignature(headers, body) + if err != nil { + return "", err + } + iat := time.Now() + exp := iat.Add(defaultExp) + token := &Token{ + Header: map[string]interface{}{ + "typ": jwtTokenType, + "sdkVersion": 20200, + "clientVersion": 2, + "alg": "RS256", + "kid": sa.KeyId, + }, + Claims: map[string]interface{}{ + "aud": aud, + "iss": strconv.FormatInt(sa.ClientId, 10), + "exp": exp.Unix(), + "iat": iat.Unix(), + "signedHeaders": getSignedHeaders1(headers), + "requestSignature": requestSign, + }, + } + rsaPrikey, err := getRSAPrivateKey(sa.PrivateKey) + if err != nil { + return "", err + } + signToken, err := token.Sign(rsaPrikey) + if err != nil { + return "", err + } + return "Bearer " + signToken, nil +} + +func getRequestSignature(headers map[string]string, payload string) (string, error) { + canonicalHeaders := "" + if headers != nil && len(headers) != 0 { + var keys []string + for k := range headers { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + canonicalHeaders += strings.ToLower(k) + canonicalHeaders += ":" + canonicalHeaders += strings.TrimSpace(headers[k]) + canonicalHeaders += "\n" + } + } + + if len(canonicalHeaders) == 0 { + canonicalHeaders += "\n" + } + if len(payload) != 0 { + ch, err := sha256String(payload) + if err != nil { + return "", err + } + canonicalHeaders += hex.EncodeToString(ch) + } + + ch, err := sha256String(canonicalHeaders) + if err != nil { + return "", err + } + + return hex.EncodeToString(ch), nil +} + +func getSignedHeaders1(headMap map[string]string) string { + if headMap != nil && len(headMap) != 0 { + var keyArray []string + for key := range headMap { + keyArray = append(keyArray, key) + } + + sort.Strings(keyArray) + return strings.Join(keyArray, ";") + } + + return "" +} + +func sha256String(input string) ([]byte, error) { + h := sha256.New() + _, err := h.Write([]byte(input)) + if err != nil { + return nil, err + } + + output := h.Sum(nil) + return output, nil +} + +func getRSAPrivateKey(privateKey string) (interface{}, error) { + priKeyByte, err := hex.DecodeString(privateKey) + if err != nil { + return nil, err + } + private := []byte(fmt.Sprintf("-----BEGIN PRIVATE KEY-----\n%s\n-----END PRIVATE KEY-----", + base64.StdEncoding.EncodeToString(priKeyByte))) + pkPem, _ := pem.Decode(private) + + privateRsa, err := x509.ParsePKCS8PrivateKey(pkPem.Bytes) + if err != nil { + return nil, err + } + + return privateRsa, nil +} diff --git a/frontend/pkg/common/faas_common/wisecloudtool/serviceaccount/parse.go b/frontend/pkg/common/faas_common/wisecloudtool/serviceaccount/parse.go new file mode 100644 index 0000000000000000000000000000000000000000..86809bfbc43051939d1daffe6985acf6c059271f --- /dev/null +++ b/frontend/pkg/common/faas_common/wisecloudtool/serviceaccount/parse.go @@ -0,0 +1,55 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package serviceaccount sign http request by jwttoken +package serviceaccount + +import ( + "crypto/tls" + "fmt" +) + +// ParseTlsCipherSuites - +func ParseTlsCipherSuites(tlsCipherSuitesStrs []string) ([]uint16, error) { + if len(tlsCipherSuitesStrs) <= 0 { + return nil, fmt.Errorf("tlsCipherSuitesStr is empty") + } + + return cipherSuitesID(cipherSuitesFromName(tlsCipherSuitesStrs)), nil +} + +func cipherSuitesFromName(names []string) []*tls.CipherSuite { + m := make(map[string]*tls.CipherSuite, len(tls.CipherSuites())) + for _, cipher := range tls.CipherSuites() { + m[cipher.Name] = cipher + } + + r := make([]*tls.CipherSuite, 0) + for _, n := range names { + if _, ok := m[n]; ok { + r = append(r, m[n]) + } + } + return r +} + +func cipherSuitesID(cs []*tls.CipherSuite) []uint16 { + ids := make([]uint16, 0) + for _, value := range cs { + ids = append(ids, value.ID) + } + return ids +} diff --git a/frontend/pkg/common/faas_common/wisecloudtool/serviceaccount/parse_test.go b/frontend/pkg/common/faas_common/wisecloudtool/serviceaccount/parse_test.go new file mode 100644 index 0000000000000000000000000000000000000000..61fe12d7ba0fb3828be93c34a9ab6d999255c3c4 --- /dev/null +++ b/frontend/pkg/common/faas_common/wisecloudtool/serviceaccount/parse_test.go @@ -0,0 +1,22 @@ +package serviceaccount + +import ( + "fmt" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/json-iterator/go" + "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/common/faas_common/wisecloudtool/types" +) + +func TestCipherSuitesFromName(t *testing.T) { + convey.Convey("Test cipherSuitesFromName", t, func() { + convey.Convey("success", func() { + cipherSuitesArr := []string{"TLS_AES_128_GCM_SHA256", "TLS_AES_256_GCM_SHA384"} + tlsSuite := cipherSuitesID(cipherSuitesFromName(cipherSuitesArr)) + convey.So(len(tlsSuite), convey.ShouldEqual, 2) + }) + }) +} diff --git a/frontend/pkg/common/faas_common/wisecloudtool/serviceaccount/token.go b/frontend/pkg/common/faas_common/wisecloudtool/serviceaccount/token.go new file mode 100644 index 0000000000000000000000000000000000000000..5992f7b87800848a14bb6e6e5e7842cb2aa9e6d3 --- /dev/null +++ b/frontend/pkg/common/faas_common/wisecloudtool/serviceaccount/token.go @@ -0,0 +1,80 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package serviceaccount sign http request by jwttoken +package serviceaccount + +import ( + "crypto" + "crypto/rand" + "crypto/rsa" + "encoding/base64" + "encoding/json" + "errors" + "strings" +) + +var methodHash = crypto.SHA256 + +// Token - +type Token struct { + Header map[string]interface{} + Claims map[string]interface{} +} + +// Sign sign jwt token string +func (t *Token) Sign(key interface{}) (string, error) { + jsonHeader, err := json.Marshal(t.Header) + if err != nil { + return "", err + } + header := base64.RawURLEncoding.EncodeToString(jsonHeader) + + jsonClaims, err := json.Marshal(t.Claims) + if err != nil { + return "", err + } + claim := base64.RawURLEncoding.EncodeToString(jsonClaims) + + stringToBeSign := strings.Join([]string{header, claim}, ".") + + sig, err := t.getSig(stringToBeSign, key) + if err != nil { + return "", err + } + return strings.Join([]string{stringToBeSign, sig}, "."), nil +} + +func (t *Token) getSig(signingString string, key interface{}) (string, error) { + rsaKey, ok := key.(*rsa.PrivateKey) + if !ok { + return "", errors.New("key is invalid") + } + if !methodHash.Available() { + return "", errors.New("the requested hash function is unavailable") + } + hasher := methodHash.New() + _, err := hasher.Write([]byte(signingString)) + if err != nil { + return "", errors.New("hash write failed") + } + + sigBytes, err := rsa.SignPKCS1v15(rand.Reader, rsaKey, methodHash, hasher.Sum(nil)) + if err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(sigBytes), nil +} diff --git a/frontend/pkg/common/faas_common/wisecloudtool/types/types.go b/frontend/pkg/common/faas_common/wisecloudtool/types/types.go new file mode 100644 index 0000000000000000000000000000000000000000..06286a5fd7eae50eb0d09d7bf107f6740b9dfa12 --- /dev/null +++ b/frontend/pkg/common/faas_common/wisecloudtool/types/types.go @@ -0,0 +1,74 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package types - +package types + +// ServiceAccountJwt service account config +type ServiceAccountJwt struct { + NuwaRuntimeAddr string `json:"nuwaRuntimeAddr,omitempty"` + NuwaGatewayAddr string `json:"nuwaGatewayAddr,omitempty"` + OauthTokenUrl string `json:"oauthTokenUrl"` + ServiceAccountKeyStr string `json:"serviceAccountKey"` + *ServiceAccount `json:"-"` + TlsConfig *TLSConfig `json:"tlsConfig"` +} + +// TLSConfig tls config +type TLSConfig struct { + HttpsInsecureSkipVerify bool `json:"httpsInsecureSkipVerify"` + TlsCipherSuitesStr []string `json:"tlsCipherSuites"` + TlsCipherSuites []uint16 `json:"-"` +} + +// ServiceAccount service account config +type ServiceAccount struct { + PrivateKey string `json:"privateKey"` + ClientId int64 `json:"clientId"` + KeyId string `json:"keyId"` + PublicKey string `json:"publicKey"` + UserId int64 `json:"userId"` + Version int32 `json:"version"` +} + +// NuwaRuntimeInfo contains ers workload info for function +type NuwaRuntimeInfo struct { + WisecloudRuntimeId string `json:"wisecloudRuntimeId"` + WisecloudSite string `json:"wisecloudSite"` + WisecloudTenantId string `json:"wisecloudTenantId"` + WisecloudApplicationId string `json:"wisecloudApplicationId"` + WisecloudServiceId string `json:"wisecloudServiceId"` + WisecloudEnvironmentId string `json:"wisecloudEnvironmentId"` + EnvLabel string `json:"envLabel"` +} + +// NuwaColdCreateInstanceReq request to nuwa +type NuwaColdCreateInstanceReq struct { + RuntimeId string `json:"runtimeId"` + RuntimeType string `json:"type"` // function/microservice + PoolType string `json:"poolType"` // java1.8/nodejs/python3 + EnvLabel string `json:"envLabel"` + Memory int64 `json:"memory"` + CPU int64 `json:"cpu"` +} + +// NuwaDestroyInstanceReq request to nuwa +type NuwaDestroyInstanceReq struct { + RuntimeId string `json:"runtimeId"` + RuntimeType string `json:"type"` + InstanceId string `json:"instanceId"` // podNamespace:podName + WorkLoadName string `json:"workLoadName"` +} diff --git a/frontend/pkg/common/go.mod b/frontend/pkg/common/go.mod new file mode 100644 index 0000000000000000000000000000000000000000..55060b98d6e0c75bd3b7326c9ba4a1b74c105d3b --- /dev/null +++ b/frontend/pkg/common/go.mod @@ -0,0 +1,78 @@ +module frontend/pkg/common + +go 1.24.1 + +require ( + github.com/agiledragon/gomonkey v2.0.1+incompatible + github.com/agiledragon/gomonkey/v2 v2.9.0 + github.com/asaskevich/govalidator/v11 v11.0.1-0.20250122183457-e11347878e23 + github.com/fsnotify/fsnotify v1.7.0 + github.com/gin-gonic/gin v1.10.0 + github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.12+incompatible + github.com/magiconair/properties v1.8.7 + github.com/panjf2000/ants/v2 v2.10.0 + github.com/redis/go-redis/v9 v9.0.5 + github.com/smartystreets/goconvey v1.8.1 + github.com/prometheus/client_golang v1.16.0 + github.com/stretchr/testify v1.9.0 + github.com/valyala/fasthttp v1.58.0 + go.etcd.io/etcd/api/v3 v3.5.11 + go.etcd.io/etcd/client/v3 v3.5.11 + go.opentelemetry.io/otel v1.24.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 + go.opentelemetry.io/otel/sdk v1.24.0 + go.opentelemetry.io/otel/trace v1.24.0 + go.uber.org/zap v1.27.0 + golang.org/x/crypto v0.24.0 + golang.org/x/net v0.26.0 + golang.org/x/time v0.10.0 + google.golang.org/grpc v1.67.0 + google.golang.org/protobuf v1.36.6 + gopkg.in/yaml.v3 v3.0.1 + gotest.tools v2.3.0+incompatible + yuanrong.org/kernel/runtime v1.0.0 + k8s.io/api v0.31.2 + k8s.io/apimachinery v0.31.2 + k8s.io/client-go v0.31.2 +) + +replace ( + github.com/agiledragon/gomonkey => github.com/agiledragon/gomonkey v2.0.1+incompatible + github.com/fsnotify/fsnotify => github.com/fsnotify/fsnotify v1.7.0 + // for test or internal use + github.com/gin-gonic/gin => github.com/gin-gonic/gin v1.10.0 + github.com/olekukonko/tablewriter => github.com/olekukonko/tablewriter v0.0.5 + github.com/operator-framework/operator-lib => github.com/operator-framework/operator-lib v0.4.0 + github.com/prashantv/gostub => github.com/prashantv/gostub v1.0.0 + github.com/robfig/cron/v3 => github.com/robfig/cron/v3 v3.0.1 + github.com/smartystreets/goconvey => github.com/smartystreets/goconvey v1.6.4 + github.com/spf13/cobra => github.com/spf13/cobra v1.8.1 + github.com/stretchr/testify => github.com/stretchr/testify v1.5.1 + github.com/valyala/fasthttp => github.com/valyala/fasthttp v1.58.0 + go.etcd.io/etcd/api/v3 => go.etcd.io/etcd/api/v3 v3.5.11 + go.etcd.io/etcd/client/v3 => go.etcd.io/etcd/client/v3 v3.5.11 + go.opentelemetry.io/otel => go.opentelemetry.io/otel v1.24.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace => go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc => go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 + go.opentelemetry.io/otel/sdk => go.opentelemetry.io/otel/sdk v1.24.0 + go.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace v1.24.0 + go.uber.org/zap => go.uber.org/zap v1.27.0 + golang.org/x/crypto => golang.org/x/crypto v0.24.0 + // affects VPC plugin building, will cause error if not pinned + golang.org/x/net => golang.org/x/net v0.26.0 + golang.org/x/sync => golang.org/x/sync v0.0.0-20190423024810-112230192c58 + golang.org/x/sys => golang.org/x/sys v0.21.0 + golang.org/x/text => golang.org/x/text v0.16.0 + golang.org/x/time => golang.org/x/time v0.10.0 + google.golang.org/genproto => google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e + google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d + google.golang.org/grpc => google.golang.org/grpc v1.67.0 + google.golang.org/protobuf => google.golang.org/protobuf v1.36.6 + gopkg.in/yaml.v3 => gopkg.in/yaml.v3 v3.0.1 + yuanrong.org/kernel/runtime => ../../../runtime/api/go + k8s.io/api => k8s.io/api v0.31.2 + k8s.io/apimachinery => k8s.io/apimachinery v0.31.2 + k8s.io/client-go => k8s.io/client-go v0.31.2 + github.com/asaskevich/govalidator/v11 => github.com/asaskevich/govalidator/v11 v11.0.1-0.20250122183457-e11347878e23 +) diff --git a/frontend/pkg/common/httputil/config/adminconfig.go b/frontend/pkg/common/httputil/config/adminconfig.go new file mode 100644 index 0000000000000000000000000000000000000000..5f6d837c44d46b35c6c658fd1c7d21f705004684 --- /dev/null +++ b/frontend/pkg/common/httputil/config/adminconfig.go @@ -0,0 +1,32 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package admin config +package config + +var AdminConf *CliConfig + +// CliConfig config parameter structure +type CliConfig struct { + AdminHost string `json:"adminHost"` +} + +func InitAdminConf(adminHost string) error { + AdminConf = &CliConfig{ + AdminHost: adminHost, + } + return nil +} diff --git a/frontend/pkg/common/httputil/http/client/client.go b/frontend/pkg/common/httputil/http/client/client.go new file mode 100644 index 0000000000000000000000000000000000000000..34187318301a2d3faefd4460e097af61dfefec9c --- /dev/null +++ b/frontend/pkg/common/httputil/http/client/client.go @@ -0,0 +1,116 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package client is define interface of client +package client + +import ( + "crypto/tls" + "net" + "net/http" + "sync" + "time" + + fhttp "github.com/valyala/fasthttp" + + "frontend/pkg/common/faas_common/logger/log" + shttp "frontend/pkg/common/httputil/http" + "frontend/pkg/common/httputil/http/client/fast" +) + +const ( + // 默认最大重试次数 + defaultMaxRetryTimes = 3 + + // MaxClientConcurrency is the max concurrency of fast http client + MaxClientConcurrency = 1000 + + // DialTimeOut - + DialTimeOut = 10 + + // TCPKeepAlivePeriod - + TCPKeepAlivePeriod = 10 +) + +var tcpDialer = fhttp.TCPDialer{Concurrency: MaxClientConcurrency} + +var globalTLSConf *tls.Config + +// Client 客户端接口 +type Client interface { + PostMultipart(url string, params map[string]string, + headers map[string]string, filePath string) (*shttp.SuccessResponse, error) + Get(url string, headers map[string]string) (*shttp.SuccessResponse, error) + PutMultipart(url string, params map[string]string, + headers map[string]string, filePath string) (*shttp.SuccessResponse, error) +} + +func adminDial(addr string) (net.Conn, error) { + conn, err := tcpDialer.DialTimeout(addr, DialTimeOut*time.Second) + if err != nil { + log.GetLogger().Errorf("failed to dial %s, error: %s ", addr, err.Error()) + return nil, err + } + + tcpConn, ok := conn.(*net.TCPConn) + if !ok { + log.GetLogger().Errorf("failed to dial %s", addr) + return nil, nil + } + err = tcpConn.SetKeepAlive(true) + if err != nil { + log.GetLogger().Errorf("failed to set connection keepalive %s, error: %s", addr, err.Error()) + return nil, err + } + + err = tcpConn.SetKeepAlivePeriod(TCPKeepAlivePeriod * time.Second) + if err != nil { + log.GetLogger().Errorf("failed to set connection keepalive period %s, error: %s", + addr, err.Error()) + return nil, err + } + + return tcpConn, nil +} + +// newClient 创建client +func newClient(tlsConf *tls.Config) Client { + cli := &fast.FastClient{ + Client: &fhttp.Client{ + TLSConfig: tlsConf, + MaxIdemponentCallAttempts: defaultMaxRetryTimes, + ReadBufferSize: http.DefaultMaxHeaderBytes, + Dial: adminDial, + }} + return cli +} + +var once sync.Once +var client Client + +// GetInstance get client instance +func GetInstance() Client { + once.Do(func() { + client = newClient(globalTLSConf) + }) + + return client +} + +// InitTlsConf init tls conf +func InitTlsConf(tlsConf *tls.Config) { + globalTLSConf = tlsConf +} diff --git a/frontend/pkg/common/httputil/http/client/client_test.go b/frontend/pkg/common/httputil/http/client/client_test.go new file mode 100644 index 0000000000000000000000000000000000000000..a31dd309096f9403dca1617231267fc835c462b7 --- /dev/null +++ b/frontend/pkg/common/httputil/http/client/client_test.go @@ -0,0 +1,26 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package client + +import ( + "crypto/tls" + "testing" +) + +func Test_InitTlsConf(t *testing.T) { + InitTlsConf(&tls.Config{}) +} diff --git a/frontend/pkg/common/httputil/http/client/fast/client.go b/frontend/pkg/common/httputil/http/client/fast/client.go new file mode 100644 index 0000000000000000000000000000000000000000..e879718ff2586ef4823832f0c91d9fa215dfceae --- /dev/null +++ b/frontend/pkg/common/httputil/http/client/fast/client.go @@ -0,0 +1,188 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package fast is fasthttp implementation of client +package fast + +import ( + "bytes" + "errors" + "io" + "mime/multipart" + "os" + "path" + "strconv" + "time" + + fhttp "github.com/valyala/fasthttp" + + "frontend/pkg/common/constants" + "frontend/pkg/common/httputil/http" + "frontend/pkg/common/httputil/utils" + "frontend/pkg/common/snerror" + "frontend/pkg/common/uuid" +) + +// FastClient fasthttp implement +type FastClient struct { + Client *fhttp.Client +} + +const ( + DefaultResponseHeadersSize = 16 + DeployTimeout = 90 + Base = 10 +) + +func setRequestHeaders(request *fhttp.Request, headers map[string]string) { + request.Header.Set(constants.HeaderTraceID, uuid.New().String()) + for key, value := range headers { + request.Header.Set(key, value) + } +} + +// ParseFastResponse parse fhttp Response +func ParseFastResponse(response *fhttp.Response) (*http.SuccessResponse, error) { + if response.StatusCode() == fhttp.StatusInternalServerError { + // The call fails and the returned status code is 500, and the body contains the returned error message + return nil, snerror.ConvertBadResponse(response.Body()) + } + if response.StatusCode() == fhttp.StatusOK { + // The call is successful and the returned status code is 200 The body contains the returned information + successResponse := &http.SuccessResponse{ + Body: response.Body(), + Headers: getResponseHeaders(response), + } + return successResponse, nil + } + // Other error codes return error information + return nil, errors.New(fhttp.StatusMessage(response.StatusCode())) +} + +func getResponseHeaders(response *fhttp.Response) map[string]string { + headers := make(map[string]string, DefaultResponseHeadersSize) + response.Header.VisitAll(func(key, value []byte) { + headers[string(key)] = string(value) + }) + return headers +} + +// ProcessMultipartRequestParams process multipart request params into fhttp request +func ProcessMultipartRequestParams(request *fhttp.Request, params map[string]string, + bodyWriter *multipart.Writer, bodyBuffer *bytes.Buffer) (*fhttp.Request, error) { + for key, val := range params { + if err := bodyWriter.WriteField(key, val); err != nil { + return nil, err + } + } + if err := bodyWriter.Close(); err != nil { + return nil, err + } + contentType := bodyWriter.FormDataContentType() + request.Header.SetContentType(contentType) + request.SetBody(bodyBuffer.Bytes()) + return request, nil +} + +func (fast *FastClient) processMultipartRequest(request *fhttp.Request, params map[string]string, + filePath string) (*fhttp.Request, error) { + fileSize := utils.GetFileSize(filePath) + request.Header.Set(http.HeaderContentType, http.Multipart) + request.Header.Set(http.HeaderFileDigest, strconv.FormatInt(fileSize, Base)) + request.SetBodyString(strconv.FormatInt(fileSize, Base)) + + bodyBuffer := &bytes.Buffer{} + bodyWriter := multipart.NewWriter(bodyBuffer) + if err := writeFile(bodyWriter, filePath); err != nil { + return nil, err + } + return ProcessMultipartRequestParams(request, params, bodyWriter, bodyBuffer) +} + +func writeFile(bodyWriter *multipart.Writer, filePath string) error { + var ( + fileWriter io.Writer + err error + ) + + fileWriter, err = bodyWriter.CreateFormFile("file", path.Base(filePath)) + if err != nil { + return err + } + file, err := os.Open(filePath) + if err != nil { + return err + } + defer file.Close() + _, err = io.Copy(fileWriter, file) + if err != nil { + return err + } + return nil +} + +// PostMultipart PostMultipart request +func (fast *FastClient) PostMultipart(url string, params map[string]string, + headers map[string]string, filePath string) (*http.SuccessResponse, error) { + request := fhttp.AcquireRequest() + response := fhttp.AcquireResponse() + setRequestHeaders(request, headers) + request.SetRequestURI(url) + request.Header.SetMethod(fhttp.MethodPost) + request, err := fast.processMultipartRequest(request, params, filePath) + if err != nil { + return nil, err + } + fast.Client.ReadTimeout = DeployTimeout * time.Second + if err := fast.Client.DoTimeout(request, response, DeployTimeout*time.Second); err != nil { + return nil, err + } + return ParseFastResponse(response) +} + +// Get Get request +func (fast *FastClient) Get(url string, headers map[string]string) (*http.SuccessResponse, error) { + request := fhttp.AcquireRequest() + response := fhttp.AcquireResponse() + setRequestHeaders(request, headers) + request.Header.Set(http.HeaderContentType, http.ApplicationJSONUTF8) + request.Header.SetMethod(fhttp.MethodGet) + request.SetRequestURI(url) + + if err := fast.Client.Do(request, response); err != nil { + return nil, err + } + return ParseFastResponse(response) +} + +// PutMultipart PutMultipart request +func (fast *FastClient) PutMultipart(url string, params map[string]string, headers map[string]string, + filePath string) (*http.SuccessResponse, error) { + request := fhttp.AcquireRequest() + response := fhttp.AcquireResponse() + setRequestHeaders(request, headers) + request.SetRequestURI(url) + request.Header.SetMethod(fhttp.MethodPut) + request, err := fast.processMultipartRequest(request, params, filePath) + if err != nil { + return nil, err + } + fast.Client.ReadTimeout = DeployTimeout * time.Second + if err := fast.Client.DoTimeout(request, response, DeployTimeout*time.Second); err != nil { + return nil, err + } + return ParseFastResponse(response) +} diff --git a/frontend/pkg/common/httputil/http/client/fast/client_test.go b/frontend/pkg/common/httputil/http/client/fast/client_test.go new file mode 100644 index 0000000000000000000000000000000000000000..41d8ee338b9a60fc206af8f8941ce37566d335e3 --- /dev/null +++ b/frontend/pkg/common/httputil/http/client/fast/client_test.go @@ -0,0 +1,72 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fast + +import ( + "encoding/json" + "github.com/stretchr/testify/assert" + "github.com/valyala/fasthttp" + "testing" + + "frontend/pkg/common/httputil/http" + "frontend/pkg/common/snerror" +) + +func Test_parseFastResponse(t *testing.T) { + response1 := &fasthttp.Response{} + badResponse := snerror.BadResponse{ + Code: 0, + Message: "500 error", + } + bytes, _ := json.Marshal(badResponse) + response1.SetStatusCode(fasthttp.StatusInternalServerError) + response1.SetBody(bytes) + + response2 := &fasthttp.Response{} + + response3 := &fasthttp.Response{} + response3.SetStatusCode(fasthttp.StatusBadRequest) + + tests := []struct { + name string + response *fasthttp.Response + want *http.SuccessResponse + wantErr bool + }{ + { + name: "test 500", + response: response1, + wantErr: true, + }, + { + name: "test 200", + response: response2, + wantErr: false, + }, + { + name: "test 400", + response: response3, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := ParseFastResponse(tt.response) + assert.Equal(t, err != nil, tt.wantErr) + }) + } +} diff --git a/frontend/pkg/common/httputil/http/const.go b/frontend/pkg/common/httputil/http/const.go new file mode 100644 index 0000000000000000000000000000000000000000..8764a870d0e76e4d4edb8901b4bb8b31e6f6ffd7 --- /dev/null +++ b/frontend/pkg/common/httputil/http/const.go @@ -0,0 +1,32 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package http is http api +package http + +// content +const ( + Multipart = "multipart/form-data" + ApplicationJSONUTF8 = "application/json;charset=UTF-8" +) + +// Header +const ( + HeaderFileDigest = "x-file-digest" + HeaderStorageType = "x-storage-type" + HeaderAuthorization = "authorization" + HeaderContentType = "Content-Type" +) diff --git a/frontend/pkg/common/httputil/http/type.go b/frontend/pkg/common/httputil/http/type.go new file mode 100644 index 0000000000000000000000000000000000000000..e2563a200d1cd2099e886986912ff21107fc2398 --- /dev/null +++ b/frontend/pkg/common/httputil/http/type.go @@ -0,0 +1,35 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package http is http +package http + +// SuccessResponse httpCode 200 成功响应结构体 +type SuccessResponse struct { + Body []byte + Headers map[string]string +} + +// Req 登陆请求 +type Req struct { + UserName string `json:"username"` + Password string `json:"password"` +} + +// Response login响应结构 +type Response struct { + Token string `json:"token"` +} diff --git a/frontend/pkg/common/httputil/utils/file.go b/frontend/pkg/common/httputil/utils/file.go new file mode 100644 index 0000000000000000000000000000000000000000..ef3af124e1f17ef8a3c045a73be64825074dfee3 --- /dev/null +++ b/frontend/pkg/common/httputil/utils/file.go @@ -0,0 +1,52 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "os" + "path/filepath" + + "frontend/pkg/common/reader" +) + +// Exists exists Whether the path exists +func Exists(path string) bool { + if _, err := filepath.Abs(path); err != nil { + return false + } + + if _, err := reader.ReadFileInfoWithTimeout(path); err != nil { + if os.IsExist(err) { + return true + } + return false + } + + return true +} + +// GetFileSize 获取文件大小 +func GetFileSize(path string) int64 { + if !Exists(path) { + return 0 + } + fileInfo, err := reader.ReadFileInfoWithTimeout(path) + if err != nil { + return 0 + } + return fileInfo.Size() +} diff --git a/frontend/pkg/common/httputil/utils/file_test.go b/frontend/pkg/common/httputil/utils/file_test.go new file mode 100644 index 0000000000000000000000000000000000000000..39185c9202f3263780e2f22de1d51844cc26e70d --- /dev/null +++ b/frontend/pkg/common/httputil/utils/file_test.go @@ -0,0 +1,60 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestExists(t *testing.T) { + var cases = []struct { + in string // input + expected bool // expected result + }{ + {"", false}, + {"./file.go", true}, + {"./notexists", false}, + {"/%$&*", false}, + } + for _, c := range cases { + actual := Exists(c.in) + if actual != c.expected { + t.Errorf("Exists(%s) = %v; expected %v", c.in, actual, c.expected) + } + } +} + +func TestGetFileSize(t *testing.T) { + ioutil.WriteFile("./test.txt", []byte("test"), 0666) + var cases = []struct { + in string // input + expectSize int64 // expected result + }{ + {"./test.txt", 4}, + {"./test1.txt", 0}, + } + for _, c := range cases { + + size := GetFileSize(c.in) + assert.Equal(t, size, c.expectSize) + } + os.Remove("./test.txt") +} diff --git a/frontend/pkg/common/httputil/utils/utils.go b/frontend/pkg/common/httputil/utils/utils.go new file mode 100644 index 0000000000000000000000000000000000000000..2dfca80d096a26e2d508fa406e1edffc60367f6e --- /dev/null +++ b/frontend/pkg/common/httputil/utils/utils.go @@ -0,0 +1,33 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import "github.com/gin-gonic/gin" + +// ParseHeader 解析请求头 +func ParseHeader(ctx *gin.Context) map[string]string { + if ctx == nil || ctx.Request == nil || len(ctx.Request.Header) == 0 { + return map[string]string{} + } + headers := make(map[string]string) + for key, values := range ctx.Request.Header { + if len(values) > 0 { + headers[key] = values[0] + } + } + return headers +} diff --git a/frontend/pkg/common/httputil/utils/utils_test.go b/frontend/pkg/common/httputil/utils/utils_test.go new file mode 100644 index 0000000000000000000000000000000000000000..1d2b9887c48f4e3e29e63f7f5d9bb8a5d2951b76 --- /dev/null +++ b/frontend/pkg/common/httputil/utils/utils_test.go @@ -0,0 +1,48 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "net/http" + "testing" + + "github.com/gin-gonic/gin" + "github.com/smartystreets/goconvey/convey" +) + +func TestParseHeader(t *testing.T) { + convey.Convey("test ParseHeader", t, func() { + convey.Convey("when header is empty", func() { + var ctx *gin.Context + result := ParseHeader(ctx) + convey.So(result, convey.ShouldBeEmpty) + }) + + convey.Convey("when header is not empty", func() { + ctx := &gin.Context{ + Request: &http.Request{ + Header: map[string][]string{ + "aa": {"bb", "cc"}, + }, + }, + } + result := ParseHeader(ctx) + convey.So(result, convey.ShouldNotBeEmpty) + convey.So(len(result), convey.ShouldEqual, 1) + }) + }) +} diff --git a/frontend/pkg/common/job/config.go b/frontend/pkg/common/job/config.go new file mode 100644 index 0000000000000000000000000000000000000000..d6904ce3e63da84578ecf2f5581552bd475f5702 --- /dev/null +++ b/frontend/pkg/common/job/config.go @@ -0,0 +1,63 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package job - +package job + +import "frontend/pkg/common/constants" + +// 处理job的对外接口 +const ( + PathParamSubmissionId = "submissionId" + PathGroupJobs = "/api/jobs" + PathGetJobs = constants.DynamicRouterParamPrefix + PathParamSubmissionId + PathDeleteJobs = constants.DynamicRouterParamPrefix + PathParamSubmissionId + PathStopJobs = constants.DynamicRouterParamPrefix + PathParamSubmissionId + "/stop" +) + +const ( + submissionIdPattern = "^[a-z0-9-]{1,64}$" + jobIDPrefix = "app-" + tenantIdKey = "tenantId" +) + +// Response - +type Response struct { + Code int `form:"code" json:"code"` + Message string `form:"message" json:"message"` + Data []byte `form:"data" json:"data"` +} + +// SubmitRequest is SubmitRequest struct +type SubmitRequest struct { + Entrypoint string `form:"entrypoint" json:"entrypoint"` + SubmissionId string `form:"submission_id" json:"submission_id"` + RuntimeEnv *RuntimeEnv `form:"runtime_env" json:"runtime_env" valid:"optional"` + Metadata map[string]string `form:"metadata" json:"metadata" valid:"optional"` + Labels string `form:"labels" json:"labels" valid:"optional"` + CreateOptions map[string]string `form:"createOptions" json:"createOptions" valid:"optional"` + EntrypointResources map[string]float64 `form:"entrypoint_resources" json:"entrypoint_resources" valid:"optional"` + EntrypointNumCpus float64 `form:"entrypoint_num_cpus" json:"entrypoint_num_cpus" valid:"optional"` + EntrypointNumGpus float64 `form:"entrypoint_num_gpus" json:"entrypoint_num_gpus" valid:"optional"` + EntrypointMemory int `form:"entrypoint_memory" json:"entrypoint_memory" valid:"optional"` +} + +// RuntimeEnv args of invoking create_app +type RuntimeEnv struct { + WorkingDir string `form:"working_dir" json:"working_dir" valid:"optional"` + Pip []string `form:"pip" json:"pip" valid:"optional" ` + EnvVars map[string]string `form:"env_vars" json:"env_vars" valid:"optional"` +} diff --git a/frontend/pkg/common/job/handler.go b/frontend/pkg/common/job/handler.go new file mode 100644 index 0000000000000000000000000000000000000000..16ba985e4c8078dd4e7f22e027d8f27be95a091c --- /dev/null +++ b/frontend/pkg/common/job/handler.go @@ -0,0 +1,297 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package job - +package job + +import ( + "encoding/json" + "errors" + "fmt" + "math" + "net/http" + "regexp" + "strings" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" + + "frontend/pkg/common/constants" + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/httputil/utils" + "frontend/pkg/common/uuid" +) + +// SubmitJobHandleReq - +func SubmitJobHandleReq(ctx *gin.Context) *SubmitRequest { + traceID := ctx.Request.Header.Get(constant.HeaderTraceID) + logger := log.GetLogger().With(zap.Any("traceID", traceID)) + var req SubmitRequest + if err := ctx.ShouldBind(&req); err != nil { + logger.Errorf("shouldBind SubmitJob request failed, err: %s", err) + ctx.JSON(http.StatusBadRequest, fmt.Sprintf("shouldBind SubmitJob request failed, err: %v", err)) + return nil + } + err := req.CheckField() + if err != nil { + ctx.JSON(http.StatusBadRequest, err.Error()) + return nil + } + req.EntrypointNumCpus = math.Ceil(req.EntrypointNumCpus * constants.CpuUnitConvert) + req.EntrypointMemory = + int(math.Ceil(float64(req.EntrypointMemory) / constants.MemoryUnitConvert / constants.MemoryUnitConvert)) + reqHeader := utils.ParseHeader(ctx) + if tenantId, ok := reqHeader[constants.HeaderTenantId]; ok { + req.AddCreateOptions(tenantIdKey, tenantId) + } + if labels, ok := reqHeader[constants.HeaderPoolLabel]; ok { + req.Labels = labels + } + logger.Debugf("SubmitJob createApp start, req:%#v", req) + return &req +} + +// SubmitJobHandleRes - +// SubmitJob godoc +// @Summary submit job +// @Description submit a new job +// @Accept json +// @Produce json +// @Router /api/jobs [POST] +// @Param SubmitRequest body SubmitRequest true "提交job时定义的job信息。" +// @Success 200 {object} map[string]string "提交job成功，返回该job的submission_id" +// @Failure 400 {string} string "用户请求错误，包含错误信息" +// @Failure 404 {string} string "该job已经存在" +// @Failure 500 {string} string "服务器处理错误，包含错误信息" +func SubmitJobHandleRes(ctx *gin.Context, resp Response) { + if resp.Code != http.StatusOK || resp.Message != "" { + ctx.JSON(resp.Code, resp.Message) + return + } + var result map[string]string + err := json.Unmarshal(resp.Data, &result) + if err != nil { + ctx.JSON(http.StatusBadRequest, + fmt.Sprintf("unmarshal response data failed, data: %v", resp.Data)) + return + } + ctx.JSON(http.StatusOK, result) + log.GetLogger().Debugf("SubmitJobHandleRes succeed, submission_id: %s", result) +} + +// ListJobsHandleRes - +// ListJobs godoc +// @Summary List Jobs +// @Description list jobs with jobInfo +// @Accept json +// @Produce json +// @Router /api/jobs [GET] +// @Success 200 {array} constant.AppInfo "返回所有jobs的信息" +// @Failure 500 {string} string "服务器处理错误，包含错误信息" +func ListJobsHandleRes(ctx *gin.Context, resp Response) { + traceID := ctx.Request.Header.Get(constant.HeaderTraceID) + logger := log.GetLogger().With(zap.Any("traceID", traceID)) + if resp.Code != http.StatusOK || resp.Message != "" { + ctx.JSON(resp.Code, resp.Message) + return + } + var result []*constant.AppInfo + err := json.Unmarshal(resp.Data, &result) + if err != nil { + ctx.JSON(http.StatusBadRequest, + fmt.Sprintf("unmarshal response data failed, data: %v", resp.Data)) + return + } + ctx.JSON(http.StatusOK, result) + logger.Debugf("ListJobsHandleRes succeed") +} + +// GetJobInfoHandleRes - +// GetJobInfo godoc +// @Summary Get JobInfo +// @Description get jobInfo by submission_id +// @Accept json +// @Produce json +// @Router /api/jobs/{submissionId} [GET] +// @Param submissionId path string true "job的submission_id，以'app-'开头" +// @Success 200 {object} constant.AppInfo "返回submission_id对应的job信息" +// @Failure 404 {string} string "该job不存在" +// @Failure 500 {string} string "服务器处理错误，包含错误信息" +func GetJobInfoHandleRes(ctx *gin.Context, resp Response) { + submissionId := ctx.Param(PathParamSubmissionId) + logger := log.GetLogger().With(zap.Any("SubmissionId", submissionId)) + if resp.Code != http.StatusOK || resp.Message != "" { + ctx.JSON(resp.Code, resp.Message) + return + } + var result *constant.AppInfo + err := json.Unmarshal(resp.Data, &result) + if err != nil { + ctx.JSON(http.StatusBadRequest, + fmt.Sprintf("unmarshal response data failed, data: %v", resp.Data)) + return + } + ctx.JSON(http.StatusOK, result) + logger.Debugf("GetJobInfoHandleRes succeed") +} + +// DeleteJobHandleRes - +// DeleteJob godoc +// @Summary Delete Job +// @Description delete job by submission_id +// @Accept json +// @Produce json +// @Router /api/jobs/{submissionId} [DELETE] +// @Param submissionId path string true "job的submission_id，以'app-'开头" +// @Success 200 {boolean} bool "返回true则说明可以删除对应的job，返回false则说明无法删除job" +// @Failure 403 {string} string "禁止删除job，包含错误信息和job运行状态" +// @Failure 404 {string} string "该job不存在" +// @Failure 500 {string} string "服务器处理错误，包含错误信息" +func DeleteJobHandleRes(ctx *gin.Context, resp Response) { + submissionId := ctx.Param(PathParamSubmissionId) + logger := log.GetLogger().With(zap.Any("SubmissionId", submissionId)) + if resp.Code == http.StatusForbidden { + log.GetLogger().Errorf("forbidden to delete, status: %s", resp.Data) + ctx.JSON(http.StatusOK, false) + return + } + if resp.Code != http.StatusOK || resp.Message != "" { + ctx.JSON(resp.Code, resp.Message) + return + } + ctx.JSON(http.StatusOK, true) + logger.Debugf("DeleteJobHandleRes succeed") +} + +// StopJobHandleRes - +// StopJob godoc +// @Summary Stop Job +// @Description stop job by submission_id +// @Accept json +// @Produce json +// @Router /api/jobs/{submissionId}/stop [POST] +// @Param submissionId path string true "job的submission_id，以'app-'开头" +// @Success 200 {boolean} bool "返回true表示可以停止运行对应的job，返回false表示job当前状态不能被停止" +// @Failure 403 {string} string "禁止删除job，包含错误信息和job运行状态" +// @Failure 404 {string} string "该job不存在" +// @Failure 500 {string} string "服务器处理错误，包含错误信息" +func StopJobHandleRes(ctx *gin.Context, resp Response) { + submissionId := ctx.Param(PathParamSubmissionId) + logger := log.GetLogger().With(zap.Any("SubmissionId", submissionId)) + if resp.Code == http.StatusForbidden { + log.GetLogger().Errorf("forbidden to stop job, status: %s", resp.Data) + ctx.JSON(http.StatusOK, false) + return + } + if resp.Code != http.StatusOK || resp.Message != "" { + ctx.JSON(resp.Code, resp.Message) + return + } + ctx.JSON(http.StatusOK, true) + logger.Debugf("StopJobHandleRes succeed") +} + +// CheckField - +func (req *SubmitRequest) CheckField() error { + if req.Entrypoint == "" { + log.GetLogger().Errorf("entrypoint should not be empty") + return fmt.Errorf("entrypoint should not be empty") + } + if req.RuntimeEnv == nil || req.RuntimeEnv.WorkingDir == "" { + log.GetLogger().Errorf("runtime_env.working_dir should not be empty") + return fmt.Errorf("runtime_env.working_dir should not be empty") + } + if err := req.ValidateResources(); err != nil { + log.GetLogger().Errorf("validateResources error: %s", err.Error()) + return err + } + if err := req.CheckSubmissionId(); err != nil { + log.GetLogger().Errorf("chechk submission_id: %s, error: %s", req.SubmissionId, err.Error()) + return err + } + return nil +} + +// ValidateResources - +func (req *SubmitRequest) ValidateResources() error { + if req.EntrypointNumCpus < 0 { + return errors.New("entrypoint_num_cpus should not be less than 0") + } + if req.EntrypointNumGpus < 0 { + return errors.New("entrypoint_num_gpus should not be less than 0") + } + if req.EntrypointMemory < 0 { + return errors.New("entrypoint_memory should not be less than 0") + } + return nil +} + +// CheckSubmissionId - +func (req *SubmitRequest) CheckSubmissionId() error { + if req.SubmissionId == "" { + return nil + } + if strings.Contains(req.SubmissionId, "driver") { + return errors.New("submission_id should not contain 'driver'") + } + if !strings.HasPrefix(req.SubmissionId, jobIDPrefix) { + req.SubmissionId = jobIDPrefix + req.SubmissionId + } + isMatch, err := regexp.MatchString(submissionIdPattern, req.SubmissionId) + if err != nil || !isMatch { + return fmt.Errorf("regular expression validation error, submissionId: %s, pattern: %s, err: %v", + req.SubmissionId, submissionIdPattern, err) + } + return nil +} + +// NewSubmissionID - +func (req *SubmitRequest) NewSubmissionID() { + if req.SubmissionId == "" { + req.SubmissionId = jobIDPrefix + uuid.New().String() + } +} + +// AddCreateOptions - +func (req *SubmitRequest) AddCreateOptions(key, value string) { + if req.CreateOptions == nil { + req.CreateOptions = map[string]string{} + } + if key != "" { + req.CreateOptions[key] = value + } +} + +// BuildJobResponse - +func BuildJobResponse(data any, code int, err error) Response { + dataBytes, jsonErr := json.Marshal(data) + if jsonErr != nil { + return Response{ + Code: http.StatusInternalServerError, + Message: fmt.Sprintf("marshal job response failed, err: %v", jsonErr), + } + } + var resp Response + resp.Code = code + if data != nil { + resp.Data = dataBytes + } + if err != nil { + resp.Message = err.Error() + } + return resp +} diff --git a/frontend/pkg/common/job/handler_test.go b/frontend/pkg/common/job/handler_test.go new file mode 100644 index 0000000000000000000000000000000000000000..e9554404d6944d18cd65dd9de3b56068ac736b9f --- /dev/null +++ b/frontend/pkg/common/job/handler_test.go @@ -0,0 +1,587 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package job + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/gin-gonic/gin" + "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/common/constants" + "frontend/pkg/common/faas_common/constant" +) + +func TestSubmitJobHandleReq(t *testing.T) { + convey.Convey("test DeleteJobHandleRes", t, func() { + gin.SetMode(gin.TestMode) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + bodyBytes, _ := json.Marshal(SubmitRequest{ + Entrypoint: "", + SubmissionId: "", + RuntimeEnv: &RuntimeEnv{ + WorkingDir: "", + Pip: []string{""}, + EnvVars: map[string]string{}, + }, + Metadata: map[string]string{}, + EntrypointResources: map[string]float64{}, + EntrypointNumCpus: 0, + EntrypointNumGpus: 0, + EntrypointMemory: 0, + }) + reader := bytes.NewBuffer(bodyBytes) + c.Request = &http.Request{ + Method: "POST", + URL: &url.URL{Path: PathGroupJobs}, + Header: http.Header{ + "Content-Type": []string{"application/json"}, + constants.HeaderTenantID: []string{"123456"}, + constants.HeaderPoolLabel: []string{"abc"}, + }, + Body: io.NopCloser(reader), // 使用 io.NopCloser 包装 reader，使其满足 io.ReadCloser 接口 + } + convey.Convey("when process success", func() { + defer gomonkey.ApplyMethodFunc(&SubmitRequest{}, "CheckField", func() error { + return nil + }).Reset() + expectedResult := &SubmitRequest{ + Entrypoint: "", + SubmissionId: "", + RuntimeEnv: &RuntimeEnv{ + WorkingDir: "", + Pip: []string{""}, + EnvVars: map[string]string{}, + }, + Metadata: map[string]string{}, + Labels: "abc", + CreateOptions: map[string]string{ + "tenantId": "123456", + }, + EntrypointResources: map[string]float64{}, + EntrypointNumCpus: 0, + EntrypointNumGpus: 0, + EntrypointMemory: 0, + } + result := SubmitJobHandleReq(c) + convey.So(result, convey.ShouldResemble, expectedResult) + }) + convey.Convey("when CheckField failed", func() { + defer gomonkey.ApplyMethodFunc(&SubmitRequest{}, "CheckField", func() error { + return errors.New("failed CheckField") + }).Reset() + result := SubmitJobHandleReq(c) + convey.So(result, convey.ShouldBeNil) + convey.So(w.Code, convey.ShouldEqual, http.StatusBadRequest) + convey.So(w.Body.String(), convey.ShouldEqual, "\"failed CheckField\"") + }) + }) +} + +func TestSubmitJobHandleRes(t *testing.T) { + convey.Convey("test SubmitJobHandleRes", t, func() { + gin.SetMode(gin.TestMode) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + resp := Response{ + Code: http.StatusOK, + Message: "", + Data: []byte("app-123"), + } + convey.Convey("when statusCode is "+strconv.Itoa(http.StatusNotFound), func() { + resp.Code = http.StatusNotFound + resp.Message = fmt.Sprintf("not found job") + SubmitJobHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusNotFound) + convey.So(w.Body.String(), convey.ShouldEqual, "\"not found job\"") + }) + convey.Convey("when statusCode is "+strconv.Itoa(http.StatusInternalServerError), func() { + resp.Code = http.StatusInternalServerError + resp.Message = fmt.Sprintf("failed get job") + SubmitJobHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusInternalServerError) + convey.So(w.Body.String(), convey.ShouldEqual, "\"failed get job\"") + }) + convey.Convey("when response data is nil", func() { + resp.Data = nil + SubmitJobHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusBadRequest) + convey.So(w.Body.String(), convey.ShouldStartWith, "\"unmarshal response data failed, data:") + }) + convey.Convey("when process success", func() { + marshal, err := json.Marshal(map[string]string{ + "submission_id": "app-123", + }) + resp.Data = marshal + convey.So(err, convey.ShouldBeNil) + SubmitJobHandleRes(c, resp) + expectedResult, err := json.Marshal(map[string]string{ + "submission_id": "app-123", + }) + if err != nil { + t.Errorf("marshal expected result failed, err: %v", err) + } + convey.So(w.Code, convey.ShouldEqual, http.StatusOK) + convey.So(w.Body.String(), convey.ShouldResemble, string(expectedResult)) + }) + }) +} + +func TestListJobsHandleRes(t *testing.T) { + convey.Convey("test ListJobsHandleRes", t, func() { + gin.SetMode(gin.TestMode) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = &http.Request{ + Header: http.Header{ + "Content-Type": []string{"application/json"}, + }, + } + dataBytes, err := json.Marshal([]*constant.AppInfo{ + { + Type: "SUBMISSION", + Entrypoint: "python script.py", + SubmissionID: "app-123", + }, + }) + if err != nil { + t.Errorf("marshal expected result failed, err: %v", err) + } + resp := Response{ + Code: http.StatusOK, + Message: "", + Data: dataBytes, + } + convey.Convey("when statusCode is "+strconv.Itoa(http.StatusInternalServerError), func() { + resp.Code = http.StatusInternalServerError + resp.Message = fmt.Sprintf("failed get job") + ListJobsHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusInternalServerError) + convey.So(w.Body.String(), convey.ShouldEqual, "\"failed get job\"") + }) + convey.Convey("when unmarshal response data failed", func() { + resp.Data = []byte(",aa,") + ListJobsHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusBadRequest) + convey.So(w.Body.String(), convey.ShouldStartWith, "\"unmarshal response data failed") + }) + convey.Convey("when response data is nil", func() { + resp.Data = []byte("[]") + ListJobsHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusOK) + convey.So(w.Body.String(), convey.ShouldEqual, "[]") + }) + convey.Convey("when process success", func() { + ListJobsHandleRes(c, resp) + expectedResult, err := json.Marshal([]*constant.AppInfo{ + { + Type: "SUBMISSION", + Entrypoint: "python script.py", + SubmissionID: "app-123", + }, + }) + if err != nil { + t.Errorf("marshal expected result failed, err: %v", err) + } + convey.So(w.Code, convey.ShouldEqual, http.StatusOK) + convey.So(w.Body.String(), convey.ShouldResemble, string(expectedResult)) + }) + }) +} + +func TestGetJobInfoHandleRes(t *testing.T) { + convey.Convey("test GetJobInfoHandleRes", t, func() { + gin.SetMode(gin.TestMode) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + dataBytes, err := json.Marshal(&constant.AppInfo{ + Type: "SUBMISSION", + Entrypoint: "python script.py", + SubmissionID: "app-123", + }) + if err != nil { + t.Errorf("marshal expected result failed, err: %v", err) + } + resp := Response{ + Code: http.StatusOK, + Message: "", + Data: dataBytes, + } + convey.Convey("when statusCode is "+strconv.Itoa(http.StatusNotFound), func() { + resp.Code = http.StatusNotFound + resp.Message = fmt.Sprintf("not found job") + GetJobInfoHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusNotFound) + convey.So(w.Body.String(), convey.ShouldEqual, "\"not found job\"") + }) + convey.Convey("when statusCode is "+strconv.Itoa(http.StatusInternalServerError), func() { + resp.Code = http.StatusInternalServerError + resp.Message = fmt.Sprintf("failed get job") + GetJobInfoHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusInternalServerError) + convey.So(w.Body.String(), convey.ShouldEqual, "\"failed get job\"") + }) + convey.Convey("when unmarshal response data failed", func() { + resp.Data = []byte(",aa,") + GetJobInfoHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusBadRequest) + convey.So(w.Body.String(), convey.ShouldStartWith, "\"unmarshal response data failed") + }) + convey.Convey("when response data is nil", func() { + resp.Data = nil + GetJobInfoHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusBadRequest) + convey.So(w.Body.String(), convey.ShouldStartWith, "\"unmarshal response data failed") + }) + convey.Convey("when process success", func() { + GetJobInfoHandleRes(c, resp) + expectedResult, err := json.Marshal(&constant.AppInfo{ + Type: "SUBMISSION", + Entrypoint: "python script.py", + SubmissionID: "app-123", + }) + if err != nil { + t.Errorf("marshal expected result failed, err: %v", err) + } + convey.So(w.Code, convey.ShouldEqual, http.StatusOK) + convey.So(w.Body.String(), convey.ShouldResemble, string(expectedResult)) + }) + }) +} + +func TestDeleteJobHandleRes(t *testing.T) { + convey.Convey("test DeleteJobHandleRes", t, func() { + gin.SetMode(gin.TestMode) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + resp := Response{ + Code: http.StatusOK, + Message: "", + Data: []byte("SUCCEEDED"), + } + convey.Convey("when statusCode is "+strconv.Itoa(http.StatusForbidden), func() { + resp.Code = http.StatusForbidden + DeleteJobHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusOK) + convey.So(w.Body.String(), convey.ShouldEqual, "false") + }) + convey.Convey("when statusCode is "+strconv.Itoa(http.StatusBadRequest), func() { + resp.Code = http.StatusBadRequest + resp.Message = fmt.Sprintf("failed delete job") + DeleteJobHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusBadRequest) + convey.So(w.Body.String(), convey.ShouldEqual, "\"failed delete job\"") + }) + convey.Convey("when response data is nil", func() { + resp.Data = nil + DeleteJobHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusOK) + convey.So(w.Body.String(), convey.ShouldEqual, "true") + }) + convey.Convey("when process success", func() { + DeleteJobHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusOK) + convey.So(w.Body.String(), convey.ShouldEqual, "true") + }) + }) +} + +func TestStopJobHandleRes(t *testing.T) { + convey.Convey("test StopJobHandleRes", t, func() { + gin.SetMode(gin.TestMode) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + resp := Response{ + Code: http.StatusOK, + Message: "", + Data: []byte(`SUCCEEDED`), + } + convey.Convey("when statusCode is "+strconv.Itoa(http.StatusForbidden), func() { + resp.Code = http.StatusForbidden + StopJobHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusOK) + convey.So(w.Body.String(), convey.ShouldEqual, "false") + }) + convey.Convey("when statusCode is "+strconv.Itoa(http.StatusBadRequest), func() { + resp.Code = http.StatusBadRequest + resp.Message = fmt.Sprintf("failed stop job") + StopJobHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusBadRequest) + convey.So(w.Body.String(), convey.ShouldEqual, "\"failed stop job\"") + }) + convey.Convey("when response data is nil", func() { + resp.Data = nil + StopJobHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusOK) + convey.So(w.Body.String(), convey.ShouldEqual, "true") + }) + convey.Convey("when process success", func() { + StopJobHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusOK) + convey.So(w.Body.String(), convey.ShouldEqual, "true") + }) + }) +} + +func TestSubmitRequest_CheckField(t *testing.T) { + convey.Convey("test (req *SubmitRequest) CheckField", t, func() { + req := &SubmitRequest{ + Entrypoint: "python script.py", + SubmissionId: "", + RuntimeEnv: &RuntimeEnv{ + WorkingDir: "file:///home/disk/tk/file.zip", + Pip: []string{"numpy==1.24", "scipy==1.11.0"}, + EnvVars: map[string]string{ + "SOURCE_REGION": "suzhou_std", + }, + }, + Metadata: map[string]string{ + "autoscenes_ids": "auto_1-test", + "task_type": "task_1", + "ttl": "1250", + }, + EntrypointResources: map[string]float64{ + "NPU": 0, + }, + EntrypointNumCpus: 0, + EntrypointNumGpus: 0, + EntrypointMemory: 0, + } + convey.Convey("when req.Entrypoint is empty", func() { + req.Entrypoint = "" + err := req.CheckField() + convey.So(err, convey.ShouldBeError, errors.New("entrypoint should not be empty")) + }) + convey.Convey("when req.RuntimeEnv is empty", func() { + req.RuntimeEnv = nil + err := req.CheckField() + convey.So(err, convey.ShouldBeError, errors.New("runtime_env.working_dir should not be empty")) + }) + convey.Convey("when req.RuntimeEnv.WorkingDir is empty", func() { + req.RuntimeEnv.WorkingDir = "" + err := req.CheckField() + convey.So(err, convey.ShouldBeError, errors.New("runtime_env.working_dir should not be empty")) + }) + convey.Convey("when ValidateResources failed", func() { + defer gomonkey.ApplyMethodFunc(&SubmitRequest{}, "ValidateResources", func() error { + return errors.New("failed ValidateResources") + }).Reset() + defer gomonkey.ApplyMethodFunc(&SubmitRequest{}, "CheckSubmissionId", func() error { + return nil + }).Reset() + err := req.CheckField() + convey.So(err, convey.ShouldBeError, errors.New("failed ValidateResources")) + }) + convey.Convey("when CheckSubmissionId failed", func() { + defer gomonkey.ApplyMethodFunc(&SubmitRequest{}, "ValidateResources", func() error { + return nil + }).Reset() + defer gomonkey.ApplyMethodFunc(&SubmitRequest{}, "CheckSubmissionId", func() error { + return errors.New("failed CheckSubmissionId") + }).Reset() + err := req.CheckField() + convey.So(err, convey.ShouldBeError, errors.New("failed CheckSubmissionId")) + }) + convey.Convey("when process success", func() { + defer gomonkey.ApplyMethodFunc(&SubmitRequest{}, "ValidateResources", func() error { + return nil + }).Reset() + defer gomonkey.ApplyMethodFunc(&SubmitRequest{}, "CheckSubmissionId", func() error { + return nil + }).Reset() + err := req.CheckField() + convey.So(err, convey.ShouldBeNil) + }) + }) +} + +func TestSubmitRequest_ValidateResources(t *testing.T) { + convey.Convey("test (req *SubmitRequest) ValidateResources()", t, func() { + req := &SubmitRequest{ + Entrypoint: "python script.py", + SubmissionId: "", + EntrypointResources: map[string]float64{ + "NPU": 0, + }, + EntrypointNumCpus: 0, + EntrypointNumGpus: 0, + EntrypointMemory: 0, + } + convey.Convey("when req.EntrypointNumCpus < 0", func() { + req.EntrypointNumCpus = -0.1 + err := req.ValidateResources() + convey.So(err.Error(), convey.ShouldEqual, "entrypoint_num_cpus should not be less than 0") + }) + convey.Convey("when req.EntrypointNumGpus < 0", func() { + req.EntrypointNumGpus = -0.1 + err := req.ValidateResources() + convey.So(err.Error(), convey.ShouldEqual, "entrypoint_num_gpus should not be less than 0") + }) + convey.Convey("when req.EntrypointMemory < 0", func() { + req.EntrypointMemory = -1 + err := req.ValidateResources() + convey.So(err.Error(), convey.ShouldEqual, "entrypoint_memory should not be less than 0") + }) + convey.Convey("when process success", func() { + err := req.ValidateResources() + convey.So(err, convey.ShouldBeNil) + }) + }) +} + +func TestSubmitRequest_CheckSubmissionId(t *testing.T) { + convey.Convey("test (req *SubmitRequest) CheckSubmissionId()", t, func() { + req := &SubmitRequest{ + Entrypoint: "python script.py", + SubmissionId: "123", + } + convey.Convey("when req.SubmissionId is empty", func() { + req.SubmissionId = "" + err := req.CheckSubmissionId() + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("when req.SubmissionId start with driver", func() { + req.SubmissionId = "driver-123" + err := req.CheckSubmissionId() + convey.So(err.Error(), convey.ShouldEqual, "submission_id should not contain 'driver'") + }) + convey.Convey("when req.SubmissionId doesn't start with 'app-'", func() { + err := req.CheckSubmissionId() + convey.So(err, convey.ShouldBeNil) + convey.So(req.SubmissionId, convey.ShouldStartWith, jobIDPrefix) + }) + convey.Convey("when req.SubmissionId length is 60 without 'app-'", func() { + req.SubmissionId = "023456781234567822345678323456784234567852345678623456787234" + err := req.CheckSubmissionId() + convey.So(err, convey.ShouldBeNil) + convey.So(req.SubmissionId, convey.ShouldStartWith, jobIDPrefix) + }) + convey.Convey("when req.SubmissionId length is more than 60 without 'app-'", func() { + req.SubmissionId = "0234567812345678223456783234567842345678523456786234567872345" + err := req.CheckSubmissionId() + convey.So(err.Error(), convey.ShouldStartWith, "regular expression validation error,") + convey.So(req.SubmissionId, convey.ShouldStartWith, jobIDPrefix) + }) + convey.Convey("when req.SubmissionId length is 64 with 'app-'", func() { + req.SubmissionId = "app-023456781234567822345678323456784234567852345678623456787234" + err := req.CheckSubmissionId() + convey.So(err, convey.ShouldBeNil) + convey.So(req.SubmissionId, convey.ShouldStartWith, jobIDPrefix) + }) + convey.Convey("when req.SubmissionId length is more than 64 with 'app-'", func() { + req.SubmissionId = "app-0234567812345678223456783234567842345678523456786234567872345" + err := req.CheckSubmissionId() + convey.So(err.Error(), convey.ShouldStartWith, "regular expression validation error,") + convey.So(req.SubmissionId, convey.ShouldStartWith, jobIDPrefix) + }) + convey.Convey("when process success", func() { + err := req.CheckSubmissionId() + convey.So(err, convey.ShouldBeNil) + convey.So(req.SubmissionId, convey.ShouldStartWith, jobIDPrefix) + }) + }) +} + +func TestSubmitRequest_NewSubmissionID(t *testing.T) { + convey.Convey("test (req *SubmitRequest) NewSubmissionID()", t, func() { + req := &SubmitRequest{ + Entrypoint: "python script.py", + SubmissionId: "", + } + convey.Convey("when req.SubmissionId is empty", func() { + req.NewSubmissionID() + convey.So(req.SubmissionId, convey.ShouldNotBeEmpty) + convey.So(req.SubmissionId, convey.ShouldStartWith, jobIDPrefix) + }) + }) +} + +func TestSubmitRequest_AddCreateOptions(t *testing.T) { + convey.Convey("test (req *SubmitRequest) AddCreateOptions()", t, func() { + req := &SubmitRequest{ + Entrypoint: "python script.py", + SubmissionId: "123", + } + convey.Convey("when req.CreateOptions is empty", func() { + req.AddCreateOptions("key", "value") + convey.So(len(req.CreateOptions), convey.ShouldEqual, 1) + }) + convey.Convey("when key is empty", func() { + req.AddCreateOptions("", "value") + convey.So(len(req.CreateOptions), convey.ShouldEqual, 0) + }) + convey.Convey("when key is not empty", func() { + req.AddCreateOptions("key", "value") + convey.So(len(req.CreateOptions), convey.ShouldEqual, 1) + }) + }) +} + +func TestBuildJobResponse(t *testing.T) { + convey.Convey("test BuildJobResponse", t, func() { + convey.Convey("when process success", func() { + expectedResult := Response{ + Code: 0, + Message: "", + Data: []byte("test"), + } + result := BuildJobResponse("test", 0, nil) + convey.So(result.Code, convey.ShouldEqual, expectedResult.Code) + convey.So(result.Message, convey.ShouldEqual, expectedResult.Message) + convey.So(string(result.Data), convey.ShouldEqual, "\""+string(expectedResult.Data)+"\"") + }) + convey.Convey("when data is nil", func() { + expectedResult := Response{ + Code: http.StatusOK, + Message: "", + Data: nil, + } + result := BuildJobResponse(nil, http.StatusOK, nil) + convey.So(result, convey.ShouldResemble, expectedResult) + }) + convey.Convey("when response status is "+strconv.Itoa(http.StatusBadRequest), func() { + expectedResult := Response{ + Code: http.StatusBadRequest, + Message: "error request", + Data: nil, + } + result := BuildJobResponse(nil, http.StatusBadRequest, errors.New("error request")) + convey.So(result, convey.ShouldResemble, expectedResult) + }) + convey.Convey("when data marshal failed", func() { + expectedResult := Response{ + Code: http.StatusInternalServerError, + Message: "marshal job response failed, err:", + } + result := BuildJobResponse(func() {}, http.StatusOK, nil) + convey.So(result.Code, convey.ShouldEqual, expectedResult.Code) + convey.So(result.Message, convey.ShouldStartWith, expectedResult.Message) + convey.So(result.Data, convey.ShouldBeNil) + }) + }) +} diff --git a/frontend/pkg/common/reader/reader.go b/frontend/pkg/common/reader/reader.go new file mode 100644 index 0000000000000000000000000000000000000000..c4c74ca274394276757c0811f1d4960f4021a410 --- /dev/null +++ b/frontend/pkg/common/reader/reader.go @@ -0,0 +1,69 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package reader provides ReadFile with timeConsumption +package reader + +import ( + "fmt" + "io/ioutil" + "os" + "time" +) + +// MaxReadFileTime elapsed time allowed to read config file from disk +const MaxReadFileTime = 10 + +// ReadFileWithTimeout is to ReadFile and count timeConsumption at same time +func ReadFileWithTimeout(configFile string) ([]byte, error) { + stopCh := make(chan struct{}) + go printTimeOut(stopCh) + data, err := ioutil.ReadFile(configFile) + close(stopCh) + return data, err +} + +// ReadFileInfoWithTimeout is to Read FileInfo and count timeConsumption at same time +func ReadFileInfoWithTimeout(filePath string) (os.FileInfo, error) { + stopCh := make(chan struct{}) + go printTimeOut(stopCh) + fileInfo, err := os.Stat(filePath) + close(stopCh) + return fileInfo, err +} + +// printTimeOut print error info every 10s after timeout +func printTimeOut(stopCh <-chan struct{}) { + if stopCh == nil { + os.Exit(0) + return + } + timer := time.NewTicker(time.Second * MaxReadFileTime) + count := 0 + for { + <-timer.C + select { + case _, ok := <-stopCh: + if !ok { + timer.Stop() + return + } + default: + count += MaxReadFileTime + fmt.Printf("ReadFile Timeout: elapsed time %ds\n", count) + } + } +} diff --git a/frontend/pkg/common/reader/reader_test.go b/frontend/pkg/common/reader/reader_test.go new file mode 100644 index 0000000000000000000000000000000000000000..7195968c734f619232acc672a0259456341b37d2 --- /dev/null +++ b/frontend/pkg/common/reader/reader_test.go @@ -0,0 +1,64 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package reader provides ReadFile with timeConsumption +package reader + +import ( + "io/ioutil" + "os" + "testing" + "time" + + "github.com/agiledragon/gomonkey" + "github.com/stretchr/testify/assert" +) + +func TestReadFileWithTimeout(t *testing.T) { + patch := gomonkey.ApplyFunc(ioutil.ReadFile, func(string) ([]byte, error) { + return nil, nil + }) + data, _ := ReadFileWithTimeout("/sn/home") + assert.Nil(t, data) + patch.Reset() +} + +func TestReadFileInfoWithTimeout(t *testing.T) { + patch := gomonkey.ApplyFunc(os.Stat, func(string) (os.FileInfo, error) { + return nil, nil + }) + fileInfo, _ := ReadFileInfoWithTimeout("/sn/home") + assert.Nil(t, fileInfo) + patch.Reset() +} + +func TestPrintTimeout(t *testing.T) { + stopCh := make(chan struct{}) + go printTimeOut(stopCh) + time.Sleep(time.Second * 15) + assert.NotNil(t, stopCh) + close(stopCh) +} + +func TestPrintTimeoutErr(t *testing.T) { + test := 0 + patch := gomonkey.ApplyFunc(os.Exit, func(code int) { + test++ + }) + printTimeOut(nil) + assert.EqualValues(t, test, 1) + patch.Reset() +} diff --git a/frontend/pkg/common/utils/helper.go b/frontend/pkg/common/utils/helper.go new file mode 100644 index 0000000000000000000000000000000000000000..dc66b308d4a36155c8e940d96dcfcd57594bbc39 --- /dev/null +++ b/frontend/pkg/common/utils/helper.go @@ -0,0 +1,153 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package utils for common functions +package utils + +import ( + "errors" + "os" + "path/filepath" + "strings" + + "frontend/pkg/common/reader" +) + +const ( + defaultPath = "/home/sn" + defaultBinPath = "/home/sn/bin" + defaultConfigPath = "/home/sn/config/config.json" + defaultLogConfigPath = "/home/sn/config/log.json" + DefaultFunctionPath = "/home/sn/config/function.yaml" +) + +// IsFile returns true if the path is a file +func IsFile(path string) bool { + file, err := reader.ReadFileInfoWithTimeout(path) + if err != nil { + return false + } + return file.Mode().IsRegular() +} + +// IsDir returns true if the path is a dir +func IsDir(path string) bool { + dir, err := reader.ReadFileInfoWithTimeout(path) + if err != nil { + return false + } + + return dir.IsDir() +} + +// FileExists returns true if the path exists +func FileExists(path string) bool { + _, err := reader.ReadFileInfoWithTimeout(path) + if err != nil { + return false + } + return true +} + +// FileSize return path file size +func FileSize(path string) int64 { + fileInfo, err := reader.ReadFileInfoWithTimeout(path) + if err != nil { + return 0 + } + return fileInfo.Size() +} + +// IsHexString judge If Hex String +func IsHexString(str string) bool { + + str = strings.ToLower(str) + + for _, c := range str { + if c < '0' || (c > '9' && c < 'a') || c > 'f' { + return false + } + } + + return true +} + +// ValidateFilePath verify the legitimacy of the file path +func ValidateFilePath(path string) error { + absPath, err := filepath.Abs(path) + if err != nil || !strings.HasPrefix(path, absPath) { + return errors.New("invalid file path, expect to be configured as an absolute path") + } + return nil +} + +// GetBinPath get path of exec bin file +func GetBinPath() (string, error) { + bin, err := os.Executable() + if err != nil { + return "", err + } + binPath := filepath.Dir(bin) + return binPath, nil +} + +// GetConfigPath get config.json file path +func GetConfigPath() (string, error) { + binPath, err := GetBinPath() + if err != nil { + return "", err + } + if binPath == defaultBinPath { + return defaultConfigPath, nil + } + return binPath + "/../config/config.json", nil +} + +// GetFunctionConfigPath get function.yaml file path +func GetFunctionConfigPath() (string, error) { + binPath, err := GetBinPath() + if err != nil { + return "", err + } + if binPath == defaultBinPath { + return DefaultFunctionPath, nil + } + return binPath + "/../config/function.yaml", nil +} + +// GetLogConfigPath get log.json file path +func GetLogConfigPath() (string, error) { + binPath, err := GetBinPath() + if err != nil { + return "", err + } + if binPath == defaultBinPath { + return defaultLogConfigPath, nil + } + return binPath + "/../config/log.json", nil +} + +// GetDefaultPath get default path +func GetDefaultPath() (string, error) { + binPath, err := GetBinPath() + if err != nil { + return "", err + } + if binPath == defaultBinPath { + return defaultPath, nil + } + return binPath + "/..", nil +} diff --git a/frontend/pkg/common/utils/helper_test.go b/frontend/pkg/common/utils/helper_test.go new file mode 100644 index 0000000000000000000000000000000000000000..38865968bab1aa6b2d788168efbb787a899de123 --- /dev/null +++ b/frontend/pkg/common/utils/helper_test.go @@ -0,0 +1,394 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "testing" + + . "github.com/agiledragon/gomonkey" + . "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/suite" +) + +type IsFileTestSuite struct { + suite.Suite + tempDir string +} + +// SetupSuite Setup Suite +func (suite *IsFileTestSuite) SetupSuite() { + var err error + + // Create temp dir for IsFileTestSuite + suite.tempDir, err = ioutil.TempDir("", "isfile-test") + suite.Require().NoError(err) +} + +// TearDownSuite TearDown Suite +func (suite *IsDirTestSuite) TearDownSuite() { + defer os.RemoveAll(suite.tempDir) +} + +// TestPositive Test Positive +func (suite *IsFileTestSuite) TestPositive() { + + // Create temp file + tempFile, err := ioutil.TempFile(suite.tempDir, "temp_file") + suite.Require().NoError(err) + defer os.Remove(tempFile.Name()) + + // Verify that function isFile() returns true when file is created + suite.Require().True(IsFile(tempFile.Name())) + +} + +// TestFileIsNotExist Test File Is Not Exist +func (suite *IsFileTestSuite) TestFileIsNotExist() { + + // Set path to unexisted file + tempFile := filepath.Join(suite.tempDir, "somePath.txt") + + // Verify that function isFile() returns false when file doesn't exist in the system + suite.Require().False(IsFile(tempFile)) +} + +// TestFileIsADirectory Test File Is A Directory +func (suite *IsFileTestSuite) TestFileIsADirectory() { + suite.Require().False(IsFile(suite.tempDir)) +} + +type IsDirTestSuite struct { + suite.Suite + tempDir string +} + +// SetupSuite Setup Suite +func (suite *IsDirTestSuite) SetupSuite() { + var err error + + // Create temp dir for IsDirTestSuite + suite.tempDir, err = ioutil.TempDir("", "isdir-test") + suite.Require().NoError(err) +} + +// TearDownSuite TearDown Suite +func (suite *IsFileTestSuite) TearDownSuite() { + defer os.RemoveAll(suite.tempDir) +} + +// TestPositive Test Positive +func (suite *IsDirTestSuite) TestPositive() { + + // Verify that function IsDir() returns true when directory exists in the system + suite.Require().True(IsDir(suite.tempDir)) +} + +// TestNegative Test Negative +func (suite *IsDirTestSuite) TestNegative() { + + // Create temp file + tempFile, err := ioutil.TempFile(suite.tempDir, "temp_file") + suite.Require().NoError(err) + defer os.Remove(tempFile.Name()) + + // Verify that function IsDir( returns false when file instead of directory is function argument + suite.Require().False(IsDir(tempFile.Name())) +} + +type FileExistTestSuite struct { + suite.Suite + tempDir string +} + +// SetupSuite Setup Suite +func (suite *FileExistTestSuite) SetupSuite() { + var err error + + // Create temp dir for FileExistTestSuite + suite.tempDir, err = ioutil.TempDir("", "file_exists-test") + suite.Require().NoError(err) +} + +// TearDownSuite TearDown Suite +func (suite *FileExistTestSuite) TearDownSuite() { + defer os.RemoveAll(suite.tempDir) +} + +// TestPositive Test Positive +func (suite *FileExistTestSuite) TestPositive() { + + // Create temp file + tempFile, err := ioutil.TempFile(suite.tempDir, "temp_file") + suite.Require().NoError(err) + defer os.Remove(tempFile.Name()) + + // Verify that function FileExists() returns true when file is exist + suite.Require().True(FileExists(tempFile.Name())) +} + +// TestFileNotExist Test File Not Exist +func (suite *FileExistTestSuite) TestFileNotExist() { + + // Set path to unexisted file + tempFile := filepath.Join(suite.tempDir, "somePath.txt") + + // Verify that function FileExists() returns false when file doesn't exist + suite.Require().False(FileExists(tempFile)) +} + +// TestFileIsNotAFile Test File Is Not A File +func (suite *FileExistTestSuite) TestFileIsNotAFile() { + + // Verify that function returns true when folder is exist in the system + suite.Require().True(FileExists(suite.tempDir)) +} + +// TestHelperTestSuite Test Helper Test Suite +func TestHelperTestSuite(t *testing.T) { + suite.Run(t, new(FileExistTestSuite)) + suite.Run(t, new(IsDirTestSuite)) + suite.Run(t, new(IsFileTestSuite)) +} + +// TestHelperTestSuite Test Helper Test Suite +func TestGetDefaultPath(t *testing.T) { + Convey("test get defaultpath", t, func() { + patches := [...]*Patches{ + ApplyFunc(GetBinPath, func() (string, error) { + return defaultBinPath, nil + }), + } + path, res := GetDefaultPath() + So(res, ShouldEqual, nil) + So(path, ShouldEqual, "/home/sn") + for i := range patches { + patches[i].Reset() + } + }) + Convey("test get defaultpath 1 ", t, func() { + patches := [...]*Patches{ + ApplyFunc(GetBinPath, func() (string, error) { + return "/opt", nil + }), + } + path, res := GetDefaultPath() + So(res, ShouldEqual, nil) + So(path, ShouldEqual, "/opt/..") + for i := range patches { + patches[i].Reset() + } + }) + Convey("test get defaultpath 2", t, func() { + patches := [...]*Patches{ + ApplyFunc(GetBinPath, func() (string, error) { + return defaultBinPath, errors.New("failed") + }), + } + path, res := GetDefaultPath() + So(res, ShouldNotEqual, nil) + So(res.Error(), ShouldEqual, "failed") + So(path, ShouldEqual, "") + for i := range patches { + patches[i].Reset() + } + }) +} + +// TestGetFunctionConfigPath - +func TestGetFunctionConfigPath(t *testing.T) { + Convey("test GetFunctionConfigPath", t, func() { + patches := [...]*Patches{ + ApplyFunc(GetBinPath, func() (string, error) { + return "/opt", nil + }), + } + path, res := GetFunctionConfigPath() + So(res, ShouldEqual, nil) + So(path, ShouldEqual, "/opt/../config/function.yaml") + for i := range patches { + patches[i].Reset() + } + }) + Convey("test GetFunctionConfigPath 2", t, func() { + patches := [...]*Patches{ + ApplyFunc(GetBinPath, func() (string, error) { + return defaultBinPath, nil + }), + } + path, res := GetFunctionConfigPath() + So(res, ShouldEqual, nil) + So(path, ShouldEqual, DefaultFunctionPath) + for i := range patches { + patches[i].Reset() + } + }) + Convey("test GetFunctionConfigPath 3", t, func() { + patches := [...]*Patches{ + ApplyFunc(GetBinPath, func() (string, error) { + return defaultBinPath, errors.New("failed") + }), + } + path, res := GetFunctionConfigPath() + So(res, ShouldNotEqual, nil) + So(res.Error(), ShouldEqual, "failed") + So(path, ShouldEqual, "") + for i := range patches { + patches[i].Reset() + } + }) +} + +// TestGetLogConfigPath - +func TestGetLogConfigPath(t *testing.T) { + Convey("test GetLogConfigPath", t, func() { + patches := [...]*Patches{ + ApplyFunc(GetBinPath, func() (string, error) { + return "/opt", nil + }), + } + path, res := GetLogConfigPath() + So(res, ShouldEqual, nil) + So(path, ShouldEqual, "/opt/../config/log.json") + for i := range patches { + patches[i].Reset() + } + }) + Convey("test GetLogConfigPath 2", t, func() { + patches := [...]*Patches{ + ApplyFunc(GetBinPath, func() (string, error) { + return defaultBinPath, nil + }), + } + path, res := GetLogConfigPath() + So(res, ShouldEqual, nil) + So(path, ShouldEqual, defaultLogConfigPath) + for i := range patches { + patches[i].Reset() + } + }) + Convey("test GetLogConfigPath 3", t, func() { + patches := [...]*Patches{ + ApplyFunc(GetBinPath, func() (string, error) { + return defaultBinPath, errors.New("failed") + }), + } + path, res := GetLogConfigPath() + So(res, ShouldNotEqual, nil) + So(res.Error(), ShouldEqual, "failed") + So(path, ShouldEqual, "") + for i := range patches { + patches[i].Reset() + } + }) +} + +// TestGetConfigPath - +func TestGetConfigPath(t *testing.T) { + Convey("test GetConfigPath", t, func() { + patches := [...]*Patches{ + ApplyFunc(GetBinPath, func() (string, error) { + return "/opt", nil + }), + } + path, res := GetConfigPath() + So(res, ShouldEqual, nil) + So(path, ShouldEqual, "/opt/../config/config.json") + for i := range patches { + patches[i].Reset() + } + }) + Convey("test GetConfigPath 2", t, func() { + patches := [...]*Patches{ + ApplyFunc(GetBinPath, func() (string, error) { + return defaultBinPath, nil + }), + } + path, res := GetConfigPath() + So(res, ShouldEqual, nil) + So(path, ShouldEqual, defaultConfigPath) + for i := range patches { + patches[i].Reset() + } + }) + Convey("test GetConfigPath 3", t, func() { + patches := [...]*Patches{ + ApplyFunc(GetBinPath, func() (string, error) { + return defaultBinPath, errors.New("failed") + }), + } + path, res := GetConfigPath() + So(res, ShouldNotEqual, nil) + So(res.Error(), ShouldEqual, "failed") + So(path, ShouldEqual, "") + for i := range patches { + patches[i].Reset() + } + }) +} + +func TestGetBinPath(t *testing.T) { + GetBinPath() +} + +func TestGetResourcePath(t *testing.T) { + GetResourcePath() + + os.Setenv("ResourcePath", "") + GetResourcePath() + + patch := ApplyFunc(exec.LookPath, func(string) (string, error) { + return "", errors.New("test") + }) + GetResourcePath() + patch.Reset() + + patch = ApplyFunc(filepath.Abs, func(string) (string, error) { + return "", errors.New("test") + }) + GetResourcePath() + fmt.Println() + patch.Reset() +} + +func TestIsDir(t *testing.T) { + IsDir("") +} + +func TestGetServicesPath(t *testing.T) { + GetServicesPath() + os.Setenv("ServicesPath", "") + GetServicesPath() + + patch := ApplyFunc(exec.LookPath, func(string) (string, error) { + return "", errors.New("test") + }) + GetServicesPath() + patch.Reset() + + patch = ApplyFunc(filepath.Abs, func(string) (string, error) { + return "", errors.New("test") + }) + GetServicesPath() + fmt.Println() + patch.Reset() +} diff --git a/frontend/pkg/common/utils/net_helper.go b/frontend/pkg/common/utils/net_helper.go new file mode 100644 index 0000000000000000000000000000000000000000..4a6cd4df9f6b3adb654b8b45f8583ce870cb83b0 --- /dev/null +++ b/frontend/pkg/common/utils/net_helper.go @@ -0,0 +1,87 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package utils for common functions +package utils + +import ( + "errors" + "fmt" + "net" + "os" + "runtime" + "strconv" + "strings" + "syscall" + + "frontend/pkg/common/constants" +) + +const ( + normalExitCode = 1 + AddressAlreadyUsedExitCode = 98 + WSAEADDRINUSE = 10048 + addressLen = 2 + ipIndex = 0 + portIndex = 1 +) + +// ProcessBindErrorAndExit will deal with err type +func ProcessBindErrorAndExit(err error) { + fmt.Printf("failed to listen address, err: %s", err.Error()) + if isErrorAddressAlreadyInUse(err) { + os.Exit(AddressAlreadyUsedExitCode) + } + os.Exit(normalExitCode) +} + +func isErrorAddressAlreadyInUse(err error) bool { + var eOsSyscall *os.SyscallError + if !errors.As(err, &eOsSyscall) { + return false + } + var errErrno syscall.Errno + if !errors.As(eOsSyscall, &errErrno) { + return false + } + if errErrno == syscall.EADDRINUSE { + return true + } + if runtime.GOOS == "windows" && errErrno == WSAEADDRINUSE { + return true + } + return false +} + +// CheckAddress check whether the address is valid +func CheckAddress(addr string) bool { + addrArg := strings.Split(addr, ":") + if len(addrArg) != addressLen { + return false + } + ip := net.ParseIP(addrArg[ipIndex]) + if ip == nil { + return false + } + port, err := strconv.Atoi(addrArg[portIndex]) + if err != nil { + return false + } + if port < 0 || port > constants.MaxPort { + return false + } + return true +} diff --git a/frontend/pkg/common/utils/net_helper_test.go b/frontend/pkg/common/utils/net_helper_test.go new file mode 100644 index 0000000000000000000000000000000000000000..01da9661167156a087eef52965265afa297ae5cb --- /dev/null +++ b/frontend/pkg/common/utils/net_helper_test.go @@ -0,0 +1,89 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "errors" + "net" + "os" + "testing" + + "github.com/agiledragon/gomonkey" + "github.com/stretchr/testify/assert" +) + +func TestIsErrorAddressAlreadyInUse(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:55555") + defer func(listener net.Listener) { + _ = listener.Close() + }(listener) + assert.Nil(t, err) + listener2, err := net.Listen("tcp", "127.0.0.1:55555") + assert.True(t, isErrorAddressAlreadyInUse(err)) + if err == nil { + _ = listener2.Close() + } +} + +func TestProcessBindErrorAndExit(t *testing.T) { + flag := false + patches := gomonkey.ApplyFunc(isErrorAddressAlreadyInUse, func(err error) bool { + return flag + }).ApplyFunc(os.Exit, func(code int) { + return + }) + defer patches.Reset() + ProcessBindErrorAndExit(errors.New("mock err")) + flag = true + ProcessBindErrorAndExit(errors.New("mock err2")) +} + +func TestCheckAddress(t *testing.T) { + test := []struct { + addr string + wanted bool + }{ + { + addr: "111", + wanted: false, + }, + { + addr: "asdasd:asdasd", + wanted: false, + }, + { + addr: "127.0.0.1:asd", + wanted: false, + }, + { + addr: "127.0.0.1:994651", + wanted: false, + }, + { + addr: "127.0.0.1:6379", + wanted: true, + }, + } + for _, tt := range test { + res := CheckAddress(tt.addr) + if tt.wanted { + assert.True(t, res) + } else { + assert.False(t, res) + } + } +} diff --git a/frontend/pkg/common/utils/resourcepath.go b/frontend/pkg/common/utils/resourcepath.go new file mode 100644 index 0000000000000000000000000000000000000000..17e3f4656b176d8392ece4b9d91d45f601c8abf9 --- /dev/null +++ b/frontend/pkg/common/utils/resourcepath.go @@ -0,0 +1,59 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package utils for common functions +package utils + +import ( + "fmt" + "os" + "os/exec" + "path" + "path/filepath" + "strings" +) + +// GetResourcePath Get Resource Path +func GetResourcePath() string { + return getPath("ResourcePath", "resource") +} + +// GetServicesPath Get Services Path +func GetServicesPath() string { + return getPath("ServicesPath", "service-config") +} + +func getPath(env, defaultPath string) string { + envPath := os.Getenv(env) + if envPath == "" { + var err error + cliPath, err := exec.LookPath(os.Args[0]) + if err != nil { + return envPath + } + envPath, err = filepath.Abs(filepath.Dir(cliPath)) + // do not return this error + if err != nil { + fmt.Printf("GetResourcePath abs filepath dir error") + } + envPath = strings.Replace(envPath, "\\", "/", -1) + envPath = path.Join(path.Dir(envPath), defaultPath) + } else { + envPath = strings.Replace(envPath, "\\", "/", -1) + } + + return envPath +} diff --git a/frontend/pkg/common/utils/tools.go b/frontend/pkg/common/utils/tools.go new file mode 100644 index 0000000000000000000000000000000000000000..f1ea5a8bf41b461a3cadf39481d55ab2b462b19b --- /dev/null +++ b/frontend/pkg/common/utils/tools.go @@ -0,0 +1,202 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package utils for common functions +package utils + +import ( + "bufio" + "encoding/binary" + "io" + "math" + "net" + "os" + "reflect" + "strconv" + "strings" + "unsafe" + + "github.com/pborman/uuid" + + "frontend/pkg/common/constants" + "frontend/pkg/common/reader" +) + +const ( + // OriginDefaultTimeout is 900 + OriginDefaultTimeout = 900 + // maxTimeout is 100 days + maxTimeout = 100 * 24 * 3600 + bytesToMb = 1024 * 1024 + uint64ArrayLength = 8 +) + +// AzEnv set defaultaz env +func AzEnv() string { + az := os.Getenv(constants.ZoneKey) + if az == "" { + az = constants.DefaultAZ + } + if len(az) > constants.ZoneNameLen { + az = az[0 : constants.ZoneNameLen-1] + } + return az +} + +// Domain2IP convert domain to ip +func Domain2IP(endpoint string) (string, error) { + var host, port string + var err error + host = endpoint + if strings.Contains(endpoint, ":") { + host, port, err = net.SplitHostPort(endpoint) + if err != nil { + return "", err + } + } + if net.ParseIP(host) != nil { + return endpoint, nil + } + ips, err := net.LookupHost(host) + if err != nil { + return "", err + } + if port == "" { + return ips[0], nil + } + return net.JoinHostPort(ips[0], port), nil +} + +// DeepCopy will generate a new copy of original collection type +// currently this function is not recursive so elements will not be deep copied +func DeepCopy(origin interface{}) interface{} { + oriTyp := reflect.TypeOf(origin) + oriVal := reflect.ValueOf(origin) + switch oriTyp.Kind() { + case reflect.Slice: + elemType := oriTyp.Elem() + length := oriVal.Len() + capacity := oriVal.Cap() + newObj := reflect.MakeSlice(reflect.SliceOf(elemType), length, capacity) + reflect.Copy(newObj, oriVal) + return newObj.Interface() + case reflect.Map: + newObj := reflect.MakeMapWithSize(oriTyp, len(oriVal.MapKeys())) + for _, key := range oriVal.MapKeys() { + value := oriVal.MapIndex(key) + newObj.SetMapIndex(key, value) + } + return newObj.Interface() + default: + return nil + } +} + +// ValidateTimeout check timeout +func ValidateTimeout(timeout *int64, defaultTimeout int64) { + if *timeout <= 0 { + *timeout = defaultTimeout + return + } + *timeout = *timeout + defaultTimeout - OriginDefaultTimeout + if *timeout > maxTimeout { + *timeout = maxTimeout + } +} + +// IsDataSystemEnable return the datasystem enable flag +func IsDataSystemEnable() bool { + branch, err := strconv.ParseBool(os.Getenv(constants.DataSystemBranchEnvKey)) + if err != nil { + branch = false + } + return branch +} + +// ClearStringMemory - +func ClearStringMemory(s string) { + bs := *(*[]byte)(unsafe.Pointer(&s)) + ClearByteMemory(bs) +} + +// ClearByteMemory - +func ClearByteMemory(b []byte) { + for i := 0; i < len(b); i++ { + b[i] = 0 + } +} + +// Float64ToByte - +func Float64ToByte(float float64) []byte { + bits := math.Float64bits(float) + bytes := make([]byte, 8) + binary.LittleEndian.PutUint64(bytes, bits) + return bytes +} + +// ByteToFloat64 - +func ByteToFloat64(bytes []byte) float64 { + // bounds check to guarantee safety of function Uint64 + if len(bytes) != uint64ArrayLength { + return 0 + } + bits := binary.LittleEndian.Uint64(bytes) + return math.Float64frombits(bits) +} + +// GetSystemMemoryUsed - +func GetSystemMemoryUsed() float64 { + srcFile, err := os.Open("/sys/fs/cgroup/memory/memory.stat") + if err != nil { + return 0 + } + defer srcFile.Close() + + reader := bufio.NewReader(srcFile) + for { + lineBytes, _, err := reader.ReadLine() + if err == io.EOF { + break + } + + lineStr := string(lineBytes) + if strings.Contains(lineStr, "rss ") { + rssStr := strings.TrimPrefix(lineStr, "rss ") + rssStr = strings.Trim(rssStr, "\n") + + value, err := strconv.ParseInt(rssStr, 10, 64) + if err != nil { + break + } + return float64(value) / bytesToMb + } + } + return 0 +} + +// ExistPath whether path exists +func ExistPath(path string) bool { + _, err := reader.ReadFileInfoWithTimeout(path) + if err != nil && os.IsNotExist(err) { + return false + } + return true +} + +// UniqueID get unique ID +func UniqueID() string { + return uuid.NewRandom().String() +} diff --git a/frontend/pkg/common/utils/tools_test.go b/frontend/pkg/common/utils/tools_test.go new file mode 100644 index 0000000000000000000000000000000000000000..47b0cff2dc2ae2d512d9d1fb64a89c949475d42a --- /dev/null +++ b/frontend/pkg/common/utils/tools_test.go @@ -0,0 +1,169 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "net" + "os" + "testing" + + "github.com/agiledragon/gomonkey" + "github.com/stretchr/testify/assert" + + "frontend/pkg/common/constants" +) + +const ( + DefaultTimeout = 900 +) + +// TestDomain2IP convert domain to ip +// If this test case fails, the problem is caused by inline optimization of the Go compiler. +// go test add "-gcflags="all=-N -l",the case will pass +func TestDomain2IP(t *testing.T) { + patches := [...]*gomonkey.Patches{ + gomonkey.ApplyFunc(net.LookupHost, func(_ string) ([]string, error) { + return []string{"1.1.1.1"}, nil + }), + } + defer func() { + for idx := range patches { + patches[idx].Reset() + } + }() + + type args struct { + endpoint string + } + tests := []struct { + args args + want string + wantErr bool + }{ + { + args{endpoint: "1.1.1.1:9000"}, + "1.1.1.1:9000", + false, + }, + { + args{endpoint: "1.1.1.1"}, + "1.1.1.1", + false, + }, + { + args{endpoint: "test:9000"}, + "1.1.1.1:9000", + false, + }, + { + args{endpoint: "test"}, + "1.1.1.1", + false, + }, + } + for _, tt := range tests { + got, err := Domain2IP(tt.args.endpoint) + if (err != nil) != tt.wantErr { + t.Errorf("Domain2IP() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("Domain2IP() got = %v, want %v", got, tt.want) + } + } +} + +func TestFloat64ToByte(t *testing.T) { + value := 123.45 + bytesValue := Float64ToByte(value) + if ByteToFloat64(bytesValue) != value { + t.Errorf("Float64ToByte and ByteToFloat64 failed") + } + + bytes := []byte{'a'} + ByteToFloat64(bytes) +} + +func TestGetSystemMemoryUsed(t *testing.T) { + if GetSystemMemoryUsed() == 0 { + t.Log("GetSystemMemoryUsed is zero") + } +} + +func TestExistPath(t *testing.T) { + path := os.Args[0] + if !ExistPath(path) { + t.Errorf("test path exist true failed, path: %s", path) + } + if ExistPath(path + "abc") { + t.Errorf("test path exist false failed, path: %s", path+"abc") + } +} + +func TestFileSize(t *testing.T) { + ret := FileSize("test/file") + assert.Equal(t, ret, int64(0)) +} +func TestIsDataSystemEnable(t *testing.T) { + ret := IsDataSystemEnable() + assert.Equal(t, ret, false) + + os.Setenv(constants.DataSystemBranchEnvKey, "t") + ret = IsDataSystemEnable() + assert.Equal(t, ret, true) +} + +func TestUniqueID(t *testing.T) { + uuid1 := UniqueID() + uuid2 := UniqueID() + assert.NotEqual(t, uuid1, uuid2) +} + +func TestDeepCopy(t *testing.T) { + var srcString = "" + copyString := DeepCopy(srcString) + assert.Equal(t, nil, copyString) + + var srcSlice = make([]int, 3) + copySlice := DeepCopy(srcSlice) + assert.Equal(t, 3, len(copySlice.([]int))) + + var srcMap = make(map[int]int) + srcMap[0] = 1 + copyMap := DeepCopy(srcMap) + assert.Equal(t, 1, len(copyMap.(map[int]int))) +} + +func TestValidateTimeout(t *testing.T) { + var timeout int64 + timeout = 0 + ValidateTimeout(&timeout, DefaultTimeout) + assert.Equal(t, int64(DefaultTimeout), timeout) + + timeout = maxTimeout + 1 + ValidateTimeout(&timeout, DefaultTimeout) + assert.Equal(t, int64(maxTimeout), timeout) +} + +func TestAzEnv(t *testing.T) { + var azString = AzEnv() + assert.Equal(t, "defaultaz", azString) +} + +func TestClearByteMemory(t *testing.T) { + ClearByteMemory([]byte{'a'}) +} diff --git a/frontend/pkg/common/uuid/uuid.go b/frontend/pkg/common/uuid/uuid.go new file mode 100644 index 0000000000000000000000000000000000000000..2f04e2f31339fbcbe4f6c409a78870352434ed85 --- /dev/null +++ b/frontend/pkg/common/uuid/uuid.go @@ -0,0 +1,153 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package uuid for common functions +package uuid + +import ( + "crypto/rand" + "crypto/sha1" + "encoding/hex" + "errors" + "hash" + "io" + "strings" +) + +const ( + defaultByteNum = 16 + indexFour = 4 + indexSix = 6 + indexEight = 8 + indexNine = 9 + indexTen = 10 + indexThirteen = 13 + indexFourteen = 14 + indexEighteen = 18 + indexNineteen = 19 + indexTwentyThree = 23 + indexTwentyFour = 24 + indexThirtySix = 36 + defaultSHA1BaseVersion = 5 +) + +// RandomUUID - +type RandomUUID [defaultByteNum]byte + +var ( + rander = rand.Reader // random function + // NameSpaceURL Well known namespace IDs and UUIDs + NameSpaceURL, _ = parseUUID("6ba7b811-9dad-11d1-80b4-00c04fd430c8") +) + +// New - +func New() RandomUUID { + return mustUUID(newRandom()) +} + +func mustUUID(uuid RandomUUID, err error) RandomUUID { + if err != nil { + return RandomUUID{} + } + return uuid +} + +func newRandom() (RandomUUID, error) { + return newRandomFromReader(rander) +} + +func newRandomFromReader(r io.Reader) (RandomUUID, error) { + var randomUUID RandomUUID + _, err := io.ReadFull(r, randomUUID[:]) + if err != nil { + return RandomUUID{}, err + } + randomUUID[indexSix] = (randomUUID[indexSix] & 0x0f) | 0x40 // Version 4 + randomUUID[indexEight] = (randomUUID[indexEight] & 0x3f) | 0x80 // Variant is 10 + return randomUUID, nil +} + +// String- +func (uuid RandomUUID) String() string { + var buf [indexThirtySix]byte + encodeHex(buf[:], uuid) + return string(buf[:]) +} + +func encodeHex(dstBuf []byte, uuid RandomUUID) { + hex.Encode(dstBuf, uuid[:indexFour]) + dstBuf[indexEight] = '-' + hex.Encode(dstBuf[indexNine:indexThirteen], uuid[indexFour:indexSix]) + dstBuf[indexThirteen] = '-' + hex.Encode(dstBuf[indexFourteen:indexEighteen], uuid[indexSix:indexEight]) + dstBuf[indexEighteen] = '-' + hex.Encode(dstBuf[indexNineteen:indexTwentyThree], uuid[indexEight:indexTen]) + dstBuf[indexTwentyThree] = '-' + hex.Encode(dstBuf[indexTwentyFour:], uuid[indexTen:]) +} + +// NewSHA1 - +func NewSHA1(space RandomUUID, data []byte) RandomUUID { + return NewHash(sha1.New(), space, data, defaultSHA1BaseVersion) +} + +// NewHash returns a new RandomUUID derived from the hash of space concatenated with +// data generated by h. The hash should be at least 16 byte in length. The +// first 16 bytes of the hash are used to form the RandomUUID. +func NewHash(sha1Hash hash.Hash, space RandomUUID, data []byte, version int) RandomUUID { + sha1Hash.Reset() + if _, err := sha1Hash.Write(space[:]); err != nil { + return RandomUUID{} + } + if _, err := sha1Hash.Write(data); err != nil { + return RandomUUID{} + } + s := sha1Hash.Sum(nil) + var uuid RandomUUID + copy(uuid[:], s) + // Set the version bits in the RandomUUID. + uuid[6] = (uuid[6] & 0x0f) | uint8((version&0xf)<<4) // The version bits are located at positions 13-15. + // Set the variant bits in the RandomUUID. + uuid[8] = (uuid[8] & 0x3f) | 0x80 // The variant bits are located at positions 8-11 (counting from 0). + return uuid +} + +func parseUUID(uuidStr string) (RandomUUID, error) { + const separator = "-" + + uuidStr = strings.ReplaceAll(uuidStr, separator, "") + + if len(uuidStr) != 32 { // Check if the length of the RandomUUID string is exactly 32 characters (16 bytes). + return RandomUUID{}, errors.New("invalid RandomUUID length") + } + + part1, part2 := uuidStr[:16], uuidStr[16:] // Split the RandomUUID string into two parts, each representing 8 bytes. + + b1, err := hex.DecodeString(part1) + if err != nil { + return RandomUUID{}, err + } + b2, err := hex.DecodeString(part2) + if err != nil { + return RandomUUID{}, err + } + + var uuid RandomUUID + copy(uuid[:8], b1) // Copy the first 8 bytes into the RandomUUID variable. + copy(uuid[8:], b2) // Copy the remaining 8 bytes into the RandomUUID variable. + + return uuid, nil +} diff --git a/frontend/pkg/common/uuid/uuid_test.go b/frontend/pkg/common/uuid/uuid_test.go new file mode 100644 index 0000000000000000000000000000000000000000..1b0951dc3a8540129f84949888461e795d6fb5e3 --- /dev/null +++ b/frontend/pkg/common/uuid/uuid_test.go @@ -0,0 +1,132 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package uuid for common functions +package uuid + +import ( + "testing" +) + +func TestNew(t *testing.T) { + m := make(map[RandomUUID]bool) + for x := 1; x < 32; x++ { + s := New() + if m[s] { + t.Errorf("New returned duplicated RandomUUID %s", s) + } + m[s] = true + } +} + +func TestSHA1(t *testing.T) { + uuid := NewSHA1(NameSpaceURL, []byte("python.org")).String() + want := "7af94e2b-4dd9-50f0-9c9a-8a48519bdef0" + if uuid != want { + t.Errorf("SHA1: got %q expected %q", uuid, want) + } +} + +func Test_parseRandomUUID(t *testing.T) { + type args struct { + uuidStr string + } + validRandomUUID := "6ba7b811-9dad-11d1-80b4-00c04fd430c8" + invalidFormatRandomUUID := "6ba7b8119dad11d180b400c04fd430c8" + illegalCharRandomUUID := "6ba7b811-9dad-11d1-80b4-00c04fd430cG" + shortRandomUUID := "6ba7b811-9dad-11d1-80b4" + longRandomUUID := "6ba7b811-9dad-11d1-80b4-00c04fd430c8-extra" + emptyRandomUUID := "" + tests := []struct { + name string + args args + want bool + wantErr bool + }{ + { + name: "Test_parseRandomUUID_with_validRandomUUID", + args: args{ + uuidStr: validRandomUUID, + }, + want: true, + wantErr: false, + }, + { + name: "Test_parseRandomUUID_with_invalidFormatRandomUUID", + args: args{ + uuidStr: invalidFormatRandomUUID, + }, + want: false, + wantErr: false, + }, + { + name: "Test_parseRandomUUID_with_illegalCharRandomUUID", + args: args{ + uuidStr: illegalCharRandomUUID, + }, + want: false, + wantErr: true, + }, + { + name: "Test_parseRandomUUID_with_shortRandomUUID", + args: args{ + uuidStr: shortRandomUUID, + }, + want: false, + wantErr: true, + }, + { + name: "Test_parseRandomUUID_with_longRandomUUID", + args: args{ + uuidStr: longRandomUUID, + }, + want: false, + wantErr: true, + }, + { + name: "Test_parseRandomUUID_with_emptyRandomUUID", + args: args{ + uuidStr: emptyRandomUUID, + }, + want: false, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseUUID(tt.args.uuidStr) + if (err != nil) != tt.wantErr { + t.Errorf("parseRandomUUID() error = %v, wantErr %v", err, tt.wantErr) + return + } + if (got.String() != tt.args.uuidStr) == tt.want { + t.Errorf("parseRandomUUID() got = %v, want %v", got.String(), tt.args.uuidStr) + } + }) + } +} + +func BenchmarkParseRandomUUID(b *testing.B) { + uuidStr := "6ba7b811-9dad-11d1-80b4-00c04fd430c8" + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := parseUUID(uuidStr) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/frontend/pkg/frontend/README.md b/frontend/pkg/frontend/README.md new file mode 100644 index 0000000000000000000000000000000000000000..99aca70f96b27ad8bec8c8b08903bc9ce9b6782c --- /dev/null +++ b/frontend/pkg/frontend/README.md @@ -0,0 +1,21 @@ +# Faas Frontend + +See the Frontend structure as below, + +```text ++------+ +------------+ +----------------+ +--------------+ +----------------+ +| | --> | Invocation | --> | | --> | InstancePool | --> | FaaS Scheduler | +| | +------------+ | FunctionInvoke | +--------------+ +----------------+ +| | +------------+ | | +--------------+ +| | --> | Trigger | --> | | --> | Go Runtime | +| HTTP | +------------+ +----------------+ +--------------+ +| | +------------+ +----------------+ +| | --> | Alias | --> | Alias Route | +| | +------------+ +----------------+ +| | +------------+ +----------------+ +| | --> | Worker | --> | Worker Route | ++------+ +------------+ +----------------+ +``` + + +HTTP -> PROCESS -> GO RUNTIME \ No newline at end of file diff --git a/frontend/pkg/frontend/api/api.go b/frontend/pkg/frontend/api/api.go new file mode 100644 index 0000000000000000000000000000000000000000..8554d4e81743829364a718b94aaae9d247d04b94 --- /dev/null +++ b/frontend/pkg/frontend/api/api.go @@ -0,0 +1,126 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package api wraps different api versions, and can be easily switched between different versions +// API provides http handlers used by fast-http, the handlers should only do http context checking and should dispatch +// the actual logic to +package api + +import ( + "github.com/gin-gonic/gin" + + "frontend/pkg/common/constants" + "frontend/pkg/common/faas_common/tracer" + commonJob "frontend/pkg/common/job" + "frontend/pkg/frontend/api/app" + "frontend/pkg/frontend/api/datasystem" + "frontend/pkg/frontend/api/functionsystem" + "frontend/pkg/frontend/api/job" + "frontend/pkg/frontend/api/lease" + "frontend/pkg/frontend/api/v1" + "frontend/pkg/frontend/common" + "frontend/pkg/frontend/frontendsdkadapter/handler" +) + +const ( + // naming convention: url + method + description + urlPostInvoke = "/serverless/v1/functions/" + common.GinUrnParamMark + + common.FunctionUrnParam + "/invocations" + urlStreamSubscribe = "/serverless/v1/stream/subscribe" + urlGetHealthCheck = "/healthz" + urlClusterHealthy = "/serverless/v1/componentshealth" + // url to faasmanager + urlLease = "/client/v1/lease" + urlLeaseKeepAlive = "/client/v1/lease/keepalive" + // url to frontend + urlPreCreate = "/serverless/v1/posix/instance/create" + urlPreInvoke = "/serverless/v1/posix/instance/invoke" + urlPreKill = "/serverless/v1/posix/instance/kill" + urlCreate = "/frontend/v1/instance/create" + urlInvoke = "/frontend/v1/instance/invoke" + urlKill = "/frontend/v1/instance/kill" + // url to datasystem + urlPut = "/datasystem/v1/obj/put" + urlGet = "/datasystem/v1/obj/get" + urlIncreaseRef = "/datasystem/v1/obj/increaseref" + urlDecreaseRef = "/datasystem/v1/obj/decreaseref" + urlKvSet = "/datasystem/v1/kv/set" + urlKvGet = "/datasystem/v1/kv/get" + urlKvDel = "/datasystem/v1/kv/del" + urlKvMSetTx = "/datasystem/v1/kv/msettx" + urlUpload = "/serverless/v2/data/kv/multiset" + urlDownload = "/serverless/v2/data/kv/multiget" + urlDelete = "/serverless/v2/data/kv/multidel" + urlExecute = "/serverless/v2/aggregation/execute" + // url to app + urlGroupApp = "/app/v1" + urlCreateApp = "/posix/instance/create" + urlListApp = "/list" + urlGetAppInfo = "/getappinfo" + + constants.DynamicRouterParamPrefix + commonJob.PathParamSubmissionId + urlStopApp = "/posix/kill" + + constants.DynamicRouterParamPrefix + commonJob.PathParamSubmissionId + urlDeleteApp = "/delete" + + constants.DynamicRouterParamPrefix + commonJob.PathParamSubmissionId +) + +// InitRoute - +func InitRoute(r *gin.Engine) { + r.GET(urlGetHealthCheck, v1.HealthzHandler) + r.GET(urlClusterHealthy, v1.ClusterHealthHandler) // Health check + r.POST(urlPostInvoke, tracer.WrapGinHandler(v1.InvokeHandler)) // Invocation + r.GET(urlStreamSubscribe, v1.SubscribeHandler) // Subscribe Stream + r.PUT(urlLease, lease.NewLeaseHandler) + r.DELETE(urlLease, lease.DelLeaseHandler) + r.POST(urlLeaseKeepAlive, lease.KeepAliveHandler) + r.POST(urlPreCreate, frontend.CreateHandler) + r.POST(urlPreInvoke, frontend.InvokeHandler) + r.POST(urlPreKill, frontend.KillHandler) + r.POST(urlCreate, frontend.CreateHandler) + r.POST(urlInvoke, frontend.InvokeHandler) + r.POST(urlKill, frontend.KillHandler) + r.POST(urlPut, datasystem.PutHandler) + r.POST(urlGet, datasystem.GetHandler) + r.POST(urlIncreaseRef, datasystem.IncreaseRefHandler) + r.POST(urlDecreaseRef, datasystem.DecreaseRefHandler) + r.POST(urlKvSet, datasystem.KvSetHandler) + r.POST(urlKvMSetTx, datasystem.KvMSetTxHandler) + r.POST(urlKvGet, datasystem.KvGetHandler) + r.POST(urlKvDel, datasystem.KvDelHandler) + r.NoRoute(v1.ProxyHandler) + r.POST(urlUpload, handler.MultiSetHandler) + r.POST(urlDownload, handler.MultiGetHandler) + r.POST(urlDelete, handler.MultiDelHandler) + r.POST(urlExecute, handler.ExecuteHandler) + // app 外部请求经过dashboard，再请求到frontend，处理job + appGroup := r.Group(urlGroupApp) + { + appGroup.POST(urlCreateApp, app.CreateHandler) + appGroup.GET(urlListApp, app.ListHandler) + appGroup.GET(urlGetAppInfo, app.GetInfoHandler) + appGroup.DELETE(urlDeleteApp, app.DeleteHandler) + appGroup.POST(urlStopApp, app.StopHandler) + } + // job 外部请求直接访问frontend，处理job + jobGroup := r.Group(commonJob.PathGroupJobs) + { + jobGroup.POST("", job.SubmitJobHandler) + jobGroup.GET("", job.ListJobsHandler) + jobGroup.GET(commonJob.PathGetJobs, job.GetJobInfoHandler) + jobGroup.DELETE(commonJob.PathDeleteJobs, job.DeleteJobHandler) + jobGroup.POST(commonJob.PathStopJobs, job.StopJobHandler) + } +} diff --git a/frontend/pkg/frontend/api/api_test.go b/frontend/pkg/frontend/api/api_test.go new file mode 100644 index 0000000000000000000000000000000000000000..41216eb17dd4c20e32062a800ed5bf4ff8d00956 --- /dev/null +++ b/frontend/pkg/frontend/api/api_test.go @@ -0,0 +1,66 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package api wraps different api versions, and can be easily switched between different versions +// API provides http handlers used by fast-http, the handlers should only do http context checking and should dispatch +// the actual logic to +package api + +import ( + "testing" + + "github.com/gin-gonic/gin" + + "frontend/pkg/frontend/config" +) + +var cfg = `{ + "slaQuota":1000, + "functionCapability":1, + "authenticationEnable":false, + "trafficLimitDisable":true, + "http":{ + "resptimeout":5, + "workerInstanceReadTimeOut":5, + "maxRequestBodySize":6 + }, + "dataSystemConfig":{ + "uploadWriteMode":"NoneL2Cache", + "executeWriteMode":"NoneL2Cache", + "uploadTTLSec":86400, + "executeTTLSec":1800, + "timeoutMs":60000 + }, + "businessType":1 + }` + +func TestInitRoute(t *testing.T) { + config.InitFunctionConfig([]byte(cfg)) + type args struct { + r *gin.Engine + } + tests := []struct { + name string + args args + }{ + {"case1 init route caas", args{r: gin.New()}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + InitRoute(tt.args.r) + }) + } +} diff --git a/frontend/pkg/frontend/api/app/handler.go b/frontend/pkg/frontend/api/app/handler.go new file mode 100644 index 0000000000000000000000000000000000000000..99a0c89f393e9dc8134b7f1a89d1df72430366f9 --- /dev/null +++ b/frontend/pkg/frontend/api/app/handler.go @@ -0,0 +1,281 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package app - used for car BU +package app + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/types" + "frontend/pkg/common/job" + "frontend/pkg/frontend/common/util" + "frontend/pkg/frontend/instancemanager" +) + +// CreateHandler - +func CreateHandler(ctx *gin.Context) { + body, err := io.ReadAll(ctx.Request.Body) + if err != nil { + log.GetLogger().Errorf("failed to read request body error %v", err) + SetCtxResponse(ctx, nil, http.StatusInternalServerError, + fmt.Errorf("failed to read request body error %v", err)) + return + } + reqBody := &job.SubmitRequest{} + err = json.Unmarshal(body, reqBody) + if err != nil { + log.GetLogger().Errorf("create app unmarshal request failed, err: %v", err) + SetCtxResponse(ctx, nil, http.StatusBadRequest, + fmt.Errorf("create app unmarshal request failed, err: %s", err)) + return + } + respBody, repCode, err := SubmitApp(reqBody) + SetCtxResponse(ctx, respBody, repCode, err) +} + +// SubmitApp - +func SubmitApp(reqBody *job.SubmitRequest) (map[string]string, int, error) { + logger := log.GetLogger().With(zap.Any("SubmissionId", reqBody.SubmissionId)) + logger.Debugf("start to submit app") + funcMeta := api.FunctionMeta{ + FuncName: constant.FunctionNameApp, + FuncID: constant.AppFuncId, + Api: api.ActorApi, + Language: api.Python, + Name: &reqBody.SubmissionId, + } + invokeOpts := createInvokeOpts(reqBody) + logger.Debugf("begin to invoke libruntime api: CreateInstanceByLibRt") + instanceId, err := util.NewClient().CreateInstanceByLibRt(funcMeta, []api.Arg{}, invokeOpts) + if err != nil { + logger.Errorf("create app failed, err: %v", err) + return nil, http.StatusInternalServerError, + fmt.Errorf("create app failed, submissionId:[%s], err: %v", reqBody.SubmissionId, err) + } + logger.Debugf("submit app success") + return map[string]string{ + "submission_id": instanceId, + }, http.StatusOK, nil +} + +// ListHandler - +func ListHandler(ctx *gin.Context) { + respBody, repCode, err := ListApps(ctx) + SetCtxResponse(ctx, respBody, repCode, err) +} + +// ListApps - +func ListApps(ctx *gin.Context) ([]*constant.AppInfo, int, error) { + traceID := ctx.Request.Header.Get(constant.HeaderTraceID) + log.GetLogger().Debugf("start to list apps, traceID:%s", traceID) + logger := log.GetLogger().With(zap.Any("traceID", traceID)) + apps, err := instancemanager.ListAppsInfo() + if err != nil { + logger.Errorf("list apps failed, err: %v", err) + return nil, http.StatusInternalServerError, fmt.Errorf("list apps failed, err: %w", err) + } + logger.Debugf("list apps success") + return apps, http.StatusOK, nil +} + +// GetInfoHandler - +func GetInfoHandler(ctx *gin.Context) { + submissionId := ctx.Param(job.PathParamSubmissionId) + respBody, repCode, err := GetAppInfo(submissionId) + SetCtxResponse(ctx, respBody, repCode, err) +} + +// GetAppInfo - +func GetAppInfo(submissionId string) (*constant.AppInfo, int, error) { + logger := log.GetLogger().With(zap.Any("SubmissionId", submissionId)) + logger.Debugf("start to get app") + appInfo, err := instancemanager.GetAppDetailsByID(submissionId) + if err != nil { + logger.Errorf("not found app, err: %v", err) + return nil, http.StatusNotFound, + fmt.Errorf("not found app, submissionId: %s, err: %w", submissionId, err) + } + logger.Debugf("get app success") + return appInfo, http.StatusOK, nil +} + +// DeleteHandler - +func DeleteHandler(ctx *gin.Context) { + respBody, repCode, err := DeleteApp(ctx) + SetCtxResponse(ctx, respBody, repCode, err) +} + +// DeleteApp - +func DeleteApp(ctx *gin.Context) (string, int, error) { + submissionId := ctx.Param(job.PathParamSubmissionId) + logger := log.GetLogger().With(zap.Any("SubmissionId", submissionId)) + logger.Debugf("start to delete app") + status := instancemanager.GetAppStatusByID(submissionId) + if status == "" { + logger.Errorf("the app does not exist") + return "", http.StatusNotFound, fmt.Errorf("the app does not exist, submissionId:[%s]", submissionId) + } + if status != constant.AppStatusSucceeded && status != constant.AppStatusFailed && status != constant.AppStatusStopped { + logger.Errorf("the app isn't allow to delete, status: %s", status) + return status, http.StatusForbidden, + fmt.Errorf("the app isn't allow to delete, submissionId:[%s], status: %s", submissionId, status) + } + logger.Debugf("send signal to kernel to delete app, submissionId: %s", submissionId) + err := util.NewClient().KillByLibRt(submissionId, constant.KillSignalVal, []byte("the job was manually deleted")) + if err != nil { + logger.Errorf("delete app failed, status:[%s] err: %v", status, err) + return status, http.StatusInternalServerError, + fmt.Errorf("delete app failed, submissionId:[%s], status:[%s] err: %v", submissionId, status, err) + } + logger.Debugf("delete app success") + return status, http.StatusOK, nil +} + +// StopHandler - +func StopHandler(ctx *gin.Context) { + respBody, repCode, err := StopApp(ctx) + SetCtxResponse(ctx, respBody, repCode, err) +} + +// StopApp - +func StopApp(ctx *gin.Context) (string, int, error) { + submissionId := ctx.Param(job.PathParamSubmissionId) + logger := log.GetLogger().With(zap.Any("SubmissionId", submissionId)) + logger.Debugf("start to stop app") + status := instancemanager.GetAppStatusByID(submissionId) + if status == "" { + logger.Errorf("the app does not exist") + return "", http.StatusNotFound, fmt.Errorf("the app does not exist, submissionId:[%s]", submissionId) + } + if status != constant.AppStatusRunning { + logger.Errorf("the app isn't allow to stop, status: %s", status) + return status, http.StatusForbidden, + fmt.Errorf("the app isn't allow to stop, submissionId:[%s], status: %s", submissionId, status) + } + err := util.NewClient().KillByLibRt(submissionId, constant.StopAppSignalVal, []byte("the job was manually stopped")) + if err != nil { + logger.Debugf("stop app failed, status:[%s] err: %v", status, err) + } + logger.Debugf("stop app success") + return status, http.StatusOK, nil +} + +// SetCtxResponse set ctx response +func SetCtxResponse(ctx *gin.Context, data interface{}, code int, err error) { + if data == nil { + log.GetLogger().Warnf("the body of ctx response is empty") + } + ctx.JSON(code, job.BuildJobResponse(data, code, err)) +} + +func createInvokeOpts(reqBody *job.SubmitRequest) api.InvokeOptions { + logger := log.GetLogger().With(zap.Any("SubmissionId", reqBody.SubmissionId)) + invokeOpts := api.InvokeOptions{ + Cpu: int(reqBody.EntrypointNumCpus), + Memory: reqBody.EntrypointMemory, + CustomResources: reqBody.EntrypointResources, + CreateOpt: buildCreateOpt(reqBody), + Timeout: constant.AppInvokeTimeout, + } + if reqBody.Labels != "" { + invokeOpts.ScheduleAffinities = generateScheduleAffinity(invokeOpts.ScheduleAffinities, reqBody.Labels) + } + logger.Debugf("create app invokeOpts is: %v", invokeOpts) + return invokeOpts +} + +func buildCreateOpt(reqBody *job.SubmitRequest) map[string]string { + logger := log.GetLogger().With(zap.Any("SubmissionId", reqBody.SubmissionId)) + createOpt := make(map[string]string) + if reqBody.CreateOptions != nil { + createOpt = reqBody.CreateOptions + } + createOpt[constant.EntryPointKey] = reqBody.Entrypoint + if reqBody.RuntimeEnv != nil { + if len(reqBody.RuntimeEnv.Pip) != 0 { + pipCmd := fmt.Sprintf("%s %s && %s", constant.PipInstallPrefix, + strings.Join(reqBody.RuntimeEnv.Pip, " "), constant.PipCheckSuffix) + createOpt[constant.PostStartExec] = pipCmd + } + if reqBody.RuntimeEnv.WorkingDir != "" { + codePath := reqBody.RuntimeEnv.WorkingDir + delegateDownLoadValue := types.LocalMetaData{ + StorageType: constant.WorkingDirType, + CodePath: codePath, + } + workingDir, err := json.Marshal(delegateDownLoadValue) + if err != nil { + logger.Warnf("workingDir JSON marshaling failed: %s", err.Error()) + } + createOpt[constant.DelegateDownloadKey] = string(workingDir) + } + if len(reqBody.RuntimeEnv.EnvVars) != 0 { + envVars := reqBody.RuntimeEnv.EnvVars + envVarsJsonByte, err := json.Marshal(envVars) + if err != nil { + logger.Warnf("env Vars JSON marshaling failed: %s", err.Error()) + } + createOpt[constant.DelegateEnvVar] = string(envVarsJsonByte) + } + if len(reqBody.Metadata) != 0 { + userMetadataJsonByte, err := json.Marshal(reqBody.Metadata) + if err != nil { + logger.Warnf("userMetadata JSON marshaling failed: %s", err.Error()) + } + createOpt[constant.UserMetadataKey] = string(userMetadataJsonByte) + } + } + return createOpt +} + +func generateScheduleAffinity(scheduleAffinity []api.Affinity, label string) []api.Affinity { + if label == "" { + return scheduleAffinity + } + labels := strings.Split(label, ",") + for _, poolLabel := range labels { + if strings.TrimSpace(poolLabel) == constant.UnUseAntiOtherLabelsKey { + continue + } + affinity := api.Affinity{ + Kind: api.AffinityKindResource, + Affinity: api.PreferredAffinity, + PreferredPriority: true, + PreferredAntiOtherLabels: !strings.Contains(label, constant.UnUseAntiOtherLabelsKey), + LabelOps: []api.LabelOperator{ + { + Type: api.LabelOpExists, + LabelKey: strings.TrimSpace(poolLabel), + LabelValues: nil, + }, + }, + } + scheduleAffinity = append(scheduleAffinity, affinity) + } + return scheduleAffinity +} diff --git a/frontend/pkg/frontend/api/app/handler_test.go b/frontend/pkg/frontend/api/app/handler_test.go new file mode 100644 index 0000000000000000000000000000000000000000..cd16cb6ae74e46834a37b8679069afd2aea4b550 --- /dev/null +++ b/frontend/pkg/frontend/api/app/handler_test.go @@ -0,0 +1,444 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/gin-gonic/gin" + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/types" + mockUtils "frontend/pkg/common/faas_common/utils" + "frontend/pkg/common/job" + "frontend/pkg/frontend/common/util" + "frontend/pkg/frontend/instancemanager" +) + +func TestCreateHandler(t *testing.T) { + mock := &mockUtils.FakeLibruntimeSdkClient{} + util.SetAPIClientLibruntime(mock) + req1 := &job.SubmitRequest{ + Entrypoint: "sleep 200", + SubmissionId: "app-scrpit-1", + RuntimeEnv: &job.RuntimeEnv{ + WorkingDir: "file:///home/ray/codeNew/god_gt_factory/file/lidar_seg_042801.zip", + Pip: []string{"numpy==1.24", "scipy==1.25"}, + EnvVars: map[string]string{ + "SOURCE_REGION": "suzhou_std", + "DEPLOY_REGION": "suzhou_std", + }, + }, + EntrypointNumCpus: 300, + EntrypointNumGpus: 0, + EntrypointMemory: 0, + EntrypointResources: map[string]float64{ + "NPU": 0, + }, + } + + convey.Convey("create successfully", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + body, _ := json.Marshal(req1) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + util.SetAPIClientLibruntime(mock) + CreateHandler(ctx) + assert.Equal(t, http.StatusOK, rw.Code) + }) + + convey.Convey("io read failed", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + body, _ := json.Marshal(req1) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + p := gomonkey.ApplyFunc(io.ReadAll, func(r io.Reader) ([]byte, error) { + return nil, fmt.Errorf("error") + }) + defer p.Reset() + CreateHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusInternalServerError) + }) + + convey.Convey("unmarshal failed", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer([]byte("aaa"))) + util.SetAPIClientLibruntime(mock) + CreateHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + }) + + convey.Convey("create failed", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + body, _ := json.Marshal(req1) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + util.SetAPIClientLibruntime(mock) + p := gomonkey.ApplyFunc((*mockUtils.FakeLibruntimeSdkClient).CreateInstance, func(_ *mockUtils.FakeLibruntimeSdkClient, funcMeta api.FunctionMeta, args []api.Arg, invokeOpt api.InvokeOptions) (instanceID string, err error) { + return "", fmt.Errorf("error") + }) + defer p.Reset() + CreateHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusInternalServerError) + }) + +} + +func TestGetInfoHandler(t *testing.T) { + + instancemanager.StoreAppInfo("app-123456", &types.InstanceSpecification{ + InstanceID: "app-123456", + StartTime: "", + InstanceStatus: types.InstanceStatus{ + Code: 3, // RUNNING + ExitCode: 0, + }, + CreateOptions: map[string]string{ + "POST_START_EXEC": "pip3.9 install numpy==1.24 scipy==1.25", + "DELEGATE_DOWNLOAD": "{\"code_path\":\"file:///home/ray/codeNew/god_gt_factory/file/lidar_seg_042801.zip\",\"storage_type\":\"working_dir\"}", + "ENTRYPOINT": "sleep 200", + "DELEGATE_ENV_VAR": "{\"DEPLOY_REGION\":\"suzhou_std\",\"SOURCE_REGION\":\"suzhou_std\"}", + "USER_PROVIDED_METADATA": "{\"autoscenes_ids\":\"auto_1-test\",\"task_type\":\"task_1\",\"ttl\":\"1250\"}", + }, + }) + convey.Convey("get info successfully", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + ctx.Request, _ = http.NewRequest(http.MethodGet, "/app/v1/getappinfo/app-123456", nil) + ctx.AddParam("submissionId", "app-123456") + GetInfoHandler(ctx) + assert.Equal(t, http.StatusOK, rw.Code) + }) + + convey.Convey("get info failed", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + ctx.Request, _ = http.NewRequest(http.MethodGet, "/app/v1/getappinfo/app-123", nil) + GetInfoHandler(ctx) + assert.Equal(t, http.StatusNotFound, rw.Code) + }) +} + +func TestListHandler(t *testing.T) { + instancemanager.StoreAppInfo("app-script-1", &types.InstanceSpecification{ + InstanceID: "app-script-1", + StartTime: "", + InstanceStatus: types.InstanceStatus{ + Code: 3, // RUNNING + ExitCode: 0, + }, + CreateOptions: map[string]string{ + "POST_START_EXEC": "pip3.9 install numpy==1.24 scipy==1.25", + "DELEGATE_DOWNLOAD": "{\"code_path\":\"file:///home/ray/codeNew/god_gt_factory/file/lidar_seg_042801.zip\",\"storage_type\":\"working_dir\"}", + "ENTRYPOINT": "sleep 200", + "DELEGATE_ENV_VAR": "{\"DEPLOY_REGION\":\"suzhou_std\",\"SOURCE_REGION\":\"suzhou_std\"}", + "USER_PROVIDED_METADATA": "{\"autoscenes_ids\":\"auto_1-test\",\"task_type\":\"task_1\",\"ttl\":\"1250\"}", + }, + }) + instancemanager.StoreAppInfo("app-script-2", &types.InstanceSpecification{ + InstanceID: "app-script-2", + StartTime: "", + InstanceStatus: types.InstanceStatus{ + Code: 3, // RUNNING + ExitCode: 0, + }, + CreateOptions: map[string]string{ + "POST_START_EXEC": "pip3.9 install numpy==1.24 scipy==1.25", + "DELEGATE_DOWNLOAD": "{\"code_path\":\"file:///home/ray/codeNew/god_gt_factory/file/lidar_seg_042801.zip\",\"storage_type\":\"working_dir\"}", + "ENTRYPOINT": "sleep 200", + "DELEGATE_ENV_VAR": "{\"DEPLOY_REGION\":\"suzhou_std\",\"SOURCE_REGION\":\"suzhou_std\"}", + "USER_PROVIDED_METADATA": "{\"autoscenes_ids\":\"auto_1-test\",\"task_type\":\"task_1\",\"ttl\":\"1250\"}", + }, + }) + convey.Convey("get info successfully", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + ctx.Request, _ = http.NewRequest(http.MethodGet, "/app/v1/list", nil) + ListHandler(ctx) + assert.Equal(t, http.StatusOK, rw.Code) + }) +} + +func TestKillHandler(t *testing.T) { + mock := &mockUtils.FakeLibruntimeSdkClient{} + util.SetAPIClientLibruntime(mock) + instancemanager.StoreAppInfo("app-script-1", &types.InstanceSpecification{ + InstanceID: "app-script-1", + StartTime: "", + InstanceStatus: types.InstanceStatus{ + Code: 3, // RUNNING + ExitCode: 0, + }, + CreateOptions: map[string]string{ + "POST_START_EXEC": "pip3.9 install numpy==1.24 scipy==1.25", + "DELEGATE_DOWNLOAD": "{\"code_path\":\"file:///home/ray/codeNew/god_gt_factory/file/lidar_seg_042801.zip\",\"storage_type\":\"working_dir\"}", + "ENTRYPOINT": "python script.py", + "DELEGATE_ENV_VAR": "{\"DEPLOY_REGION\":\"suzhou_std\",\"SOURCE_REGION\":\"suzhou_std\"}", + "USER_PROVIDED_METADATA": "{\"autoscenes_ids\":\"auto_1-test\",\"task_type\":\"task_1\",\"ttl\":\"1250\"}", + }, + }) + instancemanager.StoreAppInfo("app-script-2", &types.InstanceSpecification{ + InstanceID: "app-script-2", + StartTime: "", + InstanceStatus: types.InstanceStatus{ + Code: 1, + ExitCode: 0, + }, + CreateOptions: map[string]string{ + "POST_START_EXEC": "pip3.9 install numpy==1.24 scipy==1.25", + "DELEGATE_DOWNLOAD": "{\"code_path\":\"file:///home/ray/codeNew/god_gt_factory/file/lidar_seg_042801.zip\",\"storage_type\":\"working_dir\"}", + "ENTRYPOINT": "python script.py", + "DELEGATE_ENV_VAR": "{\"DEPLOY_REGION\":\"suzhou_std\",\"SOURCE_REGION\":\"suzhou_std\"}", + "USER_PROVIDED_METADATA": "{\"autoscenes_ids\":\"auto_1-test\",\"task_type\":\"task_1\",\"ttl\":\"1250\"}", + }, + }) + convey.Convey("kill successfully", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + ctx.Request, _ = http.NewRequest("", "", nil) + ctx.AddParam("submissionId", "app-script-2") + StopHandler(ctx) + assert.Equal(t, http.StatusForbidden, rw.Code) + }) + + convey.Convey("kill failed, status not allowed", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + ctx.AddParam("submissionId", "app-script-1") + ctx.Request, _ = http.NewRequest("", "", nil) + StopHandler(ctx) + assert.Equal(t, http.StatusOK, rw.Code) + }) + + convey.Convey("kill failed, job not existed", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + ctx.AddParam("submissionId", "app-script-3") + ctx.Request, _ = http.NewRequest("", "", nil) + StopHandler(ctx) + assert.Equal(t, http.StatusNotFound, rw.Code) + }) +} + +func TestDeleteHandler(t *testing.T) { + mock := &mockUtils.FakeLibruntimeSdkClient{} + util.SetAPIClientLibruntime(mock) + instancemanager.StoreAppInfo("app-script-1", &types.InstanceSpecification{ + InstanceID: "app-script-1", + StartTime: "", + InstanceStatus: types.InstanceStatus{ + Code: 3, // RUNNING + ExitCode: 0, + }, + CreateOptions: map[string]string{ + "POST_START_EXEC": "pip3.9 install numpy==1.24 scipy==1.25", + "DELEGATE_DOWNLOAD": "{\"code_path\":\"file:///home/ray/codeNew/god_gt_factory/file/lidar_seg_042801.zip\",\"storage_type\":\"working_dir\"}", + "ENTRYPOINT": "python script.py", + "DELEGATE_ENV_VAR": "{\"DEPLOY_REGION\":\"suzhou_std\",\"SOURCE_REGION\":\"suzhou_std\"}", + "USER_PROVIDED_METADATA": "{\"autoscenes_ids\":\"auto_1-test\",\"task_type\":\"task_1\",\"ttl\":\"1250\"}", + }, + }) + instancemanager.StoreAppInfo("app-script-2", &types.InstanceSpecification{ + InstanceID: "app-script-2", + StartTime: "", + InstanceStatus: types.InstanceStatus{ + Code: 7, + ExitCode: 0, + }, + CreateOptions: map[string]string{ + "POST_START_EXEC": "pip3.9 install numpy==1.24 scipy==1.25", + "DELEGATE_DOWNLOAD": "{\"code_path\":\"file:///home/ray/codeNew/god_gt_factory/file/lidar_seg_042801.zip\",\"storage_type\":\"working_dir\"}", + "ENTRYPOINT": "python script.py", + "DELEGATE_ENV_VAR": "{\"DEPLOY_REGION\":\"suzhou_std\",\"SOURCE_REGION\":\"suzhou_std\"}", + "USER_PROVIDED_METADATA": "{\"autoscenes_ids\":\"auto_1-test\",\"task_type\":\"task_1\",\"ttl\":\"1250\"}", + }, + }) + convey.Convey("delete successfully", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + ctx.Request, _ = http.NewRequest(http.MethodGet, "", nil) + ctx.AddParam("submissionId", "app-script-2") + DeleteHandler(ctx) + assert.Equal(t, http.StatusOK, rw.Code) + }) + + convey.Convey("delete failed, status not allowed", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + ctx.Request, _ = http.NewRequest(http.MethodGet, "", nil) + ctx.AddParam("submissionId", "app-script-1") + DeleteHandler(ctx) + assert.Equal(t, http.StatusForbidden, rw.Code) + }) + + convey.Convey("kill failed, job not existed", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + ctx.Request, _ = http.NewRequest(http.MethodGet, "/app/v1/getappinfo/app-script-3", nil) + ctx.AddParam("submissionId", "app-script-3") + DeleteHandler(ctx) + assert.Equal(t, http.StatusNotFound, rw.Code) + }) +} + +func TestSetCtxResponse(t *testing.T) { + convey.Convey("test SetCtxResponse", t, func() { + gin.SetMode(gin.TestMode) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + convey.Convey("when process success", func() { + jsonBytes, err := json.Marshal(job.Response{ + Code: http.StatusOK, + Data: nil, + }) + if err != nil { + t.Errorf("json.Marshal failed: %v", err) + } + SetCtxResponse(c, nil, http.StatusOK, nil) + convey.So(w.Code, convey.ShouldEqual, http.StatusOK) + convey.So(w.Body.String(), convey.ShouldEqual, string(jsonBytes)) + }) + }) +} + +func TestCreateInvokeOpts(t *testing.T) { + convey.Convey("test createInvokeOpts", t, func() { + convey.Convey("when process success", func() { + defer gomonkey.ApplyFunc(buildCreateOpt, func(reqBody *job.SubmitRequest) map[string]string { + return map[string]string{"abc": "123"} + }).Reset() + defer gomonkey.ApplyFunc(generateScheduleAffinity, func(scheduleAffinity []api.Affinity, label string) []api.Affinity { + return []api.Affinity{ + { + Kind: api.AffinityKindResource, + Affinity: api.PreferredAffinity, + PreferredPriority: true, + PreferredAntiOtherLabels: !strings.Contains(label, constant.UnUseAntiOtherLabelsKey), + LabelOps: []api.LabelOperator{ + { + Type: api.LabelOpExists, + LabelKey: strings.TrimSpace(label), + LabelValues: nil, + }, + }, + }, + } + }).Reset() + reqBody := &job.SubmitRequest{ + EntrypointNumCpus: 0, + EntrypointMemory: 0, + EntrypointResources: map[string]float64{"abc": 0}, + Labels: "poolLabel", + } + expectedResult := api.InvokeOptions{ + Cpu: 0, + Memory: 0, + CustomResources: map[string]float64{"abc": 0}, + CreateOpt: map[string]string{"abc": "123"}, + Timeout: constant.AppInvokeTimeout, + ScheduleAffinities: []api.Affinity{ + { + Kind: api.AffinityKindResource, + Affinity: api.PreferredAffinity, + PreferredPriority: true, + PreferredAntiOtherLabels: !strings.Contains("poolLabel", constant.UnUseAntiOtherLabelsKey), + LabelOps: []api.LabelOperator{ + { + Type: api.LabelOpExists, + LabelKey: strings.TrimSpace("poolLabel"), + LabelValues: nil, + }, + }, + }, + }, + } + result := createInvokeOpts(reqBody) + convey.So(result, convey.ShouldResemble, expectedResult) + }) + }) +} + +func TestBuildCreateOpt(t *testing.T) { + convey.Convey("test buildCreateOpt", t, func() { + convey.Convey("when reqBody is empty", func() { + reqBody := &job.SubmitRequest{} + result := buildCreateOpt(reqBody) + convey.So(result[constant.EntryPointKey], convey.ShouldEqual, reqBody.Entrypoint) + }) + convey.Convey("when reqBody.CreateOptions is not nil", func() { + reqBody := &job.SubmitRequest{ + CreateOptions: map[string]string{"abc": "123"}, + } + result := buildCreateOpt(reqBody) + convey.So(result, convey.ShouldResemble, map[string]string{ + "abc": "123", + constant.EntryPointKey: "", + }) + }) + convey.Convey("when reqBody.RuntimeEnv is not nil", func() { + reqBody := &job.SubmitRequest{ + RuntimeEnv: &job.RuntimeEnv{ + WorkingDir: "file:///home/disk/tk/file.zip", + Pip: []string{"numpy==1.24", "scipy==1.11.0"}, + EnvVars: map[string]string{ + "SOURCE_REGION": "suzhou_std", + }, + }, + Metadata: map[string]string{ + "autoscenes_ids": "auto_1-test", + }, + } + result := buildCreateOpt(reqBody) + convey.So(result, convey.ShouldResemble, map[string]string{ + constant.DelegateDownloadKey: "{\"storage_type\":\"working_dir\",\"code_path\":\"file:///home/disk/tk/file.zip\"}", + constant.DelegateEnvVar: "{\"SOURCE_REGION\":\"suzhou_std\"}", + constant.EntryPointKey: "", + constant.PostStartExec: "pip3.9 install numpy==1.24 scipy==1.11.0 && pip3.9 check", + constant.UserMetadataKey: "{\"autoscenes_ids\":\"auto_1-test\"}", + }) + }) + }) +} + +func TestGenerateScheduleAffinity(t *testing.T) { + convey.Convey("test generateScheduleAffinity", t, func() { + convey.Convey("when scheduleAffinity is nil, and labels is empty", func() { + result := generateScheduleAffinity(nil, "") + convey.So(result, convey.ShouldBeNil) + }) + convey.Convey("when scheduleAffinity is nil, label equal "+constant.UnUseAntiOtherLabelsKey, func() { + result := generateScheduleAffinity(nil, constant.UnUseAntiOtherLabelsKey) + convey.So(result, convey.ShouldBeNil) + }) + convey.Convey("when scheduleAffinity is nil, label equal 'aaa,bbb'", func() { + result := generateScheduleAffinity(nil, "aaa,bbb") + convey.So(len(result), convey.ShouldEqual, 2) + }) + }) +} diff --git a/frontend/pkg/frontend/api/datasystem/handler.go b/frontend/pkg/frontend/api/datasystem/handler.go new file mode 100644 index 0000000000000000000000000000000000000000..5a75236f5fd4a8912d3b75abe04a83133c244ed1 --- /dev/null +++ b/frontend/pkg/frontend/api/datasystem/handler.go @@ -0,0 +1,396 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package datasystem the api of datasystem scene +package datasystem + +import ( + "io" + "net/http" + + "github.com/gin-gonic/gin" + "google.golang.org/protobuf/proto" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/datasystemclient" + "frontend/pkg/common/faas_common/grpc/pb/data" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/frontend/api/functionsystem" + "frontend/pkg/frontend/common/httputil" +) + +const ( + errPramInvalid = 2 // error code from datasystem: K_INVALID + errNone = 0 +) + +// PutHandler the handler of put +func PutHandler(ctx *gin.Context) { + tenantID, traceID := getHeaderPrams(ctx) + body, err := io.ReadAll(ctx.Request.Body) + if err != nil { + log.GetLogger().Errorf("failed to read request body error %s", err.Error()) + frontend.SetCtxResponse(ctx, nil, http.StatusInternalServerError) + return + } + msg := &data.PutRequest{} + err = proto.Unmarshal(body, msg) + if err != nil { + log.GetLogger().Errorf("failed to parse object put request message, err: %s", err) + frontend.SetCtxResponse(ctx, nil, http.StatusBadRequest) + return + } + if msg.ObjectId == "" || msg.ObjectData == nil { + log.GetLogger().Errorf("failed to parse object put request message, empty id or data") + frontend.SetCtxResponse(ctx, nil, http.StatusBadRequest) + return + } + log.GetLogger().Infof("receive object put request, traceID: %s", traceID) + state := datasystemclient.ObjPut(msg, &datasystemclient.Config{TenantID: tenantID}, traceID) + response := &data.PutResponse{Code: int32(state.Code)} + if state.Err != nil { + response.Message = state.Err.Error() + } + if int32(state.Code) == errNone { + response.Message = "object put success" + } + log.GetLogger().Debugf("receive object put response, traceID: %s, code: %d", traceID, state.Code) + respBody, parseErr := proto.Marshal(response) + if parseErr != nil { + log.GetLogger().Errorf("proto Marshal failed, err: %s", parseErr) + frontend.SetCtxResponse(ctx, respBody, http.StatusBadRequest) + return + } + frontend.SetCtxResponse(ctx, respBody, http.StatusOK) +} + +// GetHandler the handler of get +func GetHandler(ctx *gin.Context) { + tenantID, traceID := getHeaderPrams(ctx) + body, err := io.ReadAll(ctx.Request.Body) + if err != nil { + log.GetLogger().Errorf("failed to read request body error %s", err.Error()) + frontend.SetCtxResponse(ctx, nil, http.StatusInternalServerError) + return + } + msg := &data.GetRequest{} + err = proto.Unmarshal(body, msg) + if err != nil { + log.GetLogger().Errorf("failed to parse object get request message, err: %s", err) + frontend.SetCtxResponse(ctx, nil, http.StatusBadRequest) + return + } + if msg.ObjectIds == nil { + log.GetLogger().Errorf("failed to parse object get request message, empty object ids") + frontend.SetCtxResponse(ctx, nil, http.StatusBadRequest) + return + } + log.GetLogger().Infof("receive object get request, traceID: %s", traceID) + values, state := datasystemclient.ObjGet(msg, &datasystemclient.Config{TenantID: tenantID}, traceID) + response := &data.GetResponse{ + Code: int32(state.Code), + Buffers: values, + } + if state.Err != nil { + response.Message = state.Err.Error() + } + if int32(state.Code) == errNone { + response.Message = "object get success" + } + log.GetLogger().Debugf("receive object get response, traceID: %s, code: %d", traceID, state.Code) + respBody, parseErr := proto.Marshal(response) + if parseErr != nil { + log.GetLogger().Errorf("proto Marshal failed, err: %s", parseErr) + frontend.SetCtxResponse(ctx, respBody, http.StatusBadRequest) + return + } + frontend.SetCtxResponse(ctx, respBody, http.StatusOK) +} + +// IncreaseRefHandler the handler of increaseref +func IncreaseRefHandler(ctx *gin.Context) { + tenantID, traceID := getHeaderPrams(ctx) + body, err := io.ReadAll(ctx.Request.Body) + if err != nil { + log.GetLogger().Errorf("failed to read request body error %s", err.Error()) + frontend.SetCtxResponse(ctx, nil, http.StatusInternalServerError) + return + } + msg := &data.IncreaseRefRequest{} + err = proto.Unmarshal(body, msg) + if err != nil { + log.GetLogger().Errorf("failed to parse increase ref request message, err: %s", err) + frontend.SetCtxResponse(ctx, nil, http.StatusBadRequest) + return + } + if msg.ObjectIds == nil || msg.RemoteClientId == "" { + log.GetLogger().Errorf("failed to parse increase ref request message, empty ids") + frontend.SetCtxResponse(ctx, nil, http.StatusBadRequest) + return + } + log.GetLogger().Infof("receive increase ref request, traceID: %s", traceID) + values, state := datasystemclient.GIncreaseRef(msg, &datasystemclient.Config{TenantID: tenantID}, traceID) + response := &data.IncreaseRefResponse{ + Code: int32(state.Code), + FailedObjectIds: values, + } + if state.Err != nil { + response.Message = state.Err.Error() + } + if int32(state.Code) == errNone { + response.Message = "increase ref success" + } + log.GetLogger().Debugf("receive increase ref response, traceID: %s, code: %d", traceID, state.Code) + respBody, parseErr := proto.Marshal(response) + if parseErr != nil { + log.GetLogger().Errorf("proto Marshal failed, err: %s", parseErr) + frontend.SetCtxResponse(ctx, respBody, http.StatusBadRequest) + return + } + frontend.SetCtxResponse(ctx, respBody, http.StatusOK) +} + +// DecreaseRefHandler the handler of decreaseref +func DecreaseRefHandler(ctx *gin.Context) { + tenantID, traceID := getHeaderPrams(ctx) + body, err := io.ReadAll(ctx.Request.Body) + if err != nil { + log.GetLogger().Errorf("failed to read request body error %s", err.Error()) + frontend.SetCtxResponse(ctx, nil, http.StatusInternalServerError) + return + } + msg := &data.DecreaseRefRequest{} + err = proto.Unmarshal(body, msg) + if err != nil { + log.GetLogger().Errorf("failed to parse decrease ref request message, err: %s", err) + frontend.SetCtxResponse(ctx, nil, http.StatusBadRequest) + return + } + if msg.ObjectIds == nil || msg.RemoteClientId == "" { + log.GetLogger().Errorf("failed to parse decrease ref request message, empty ids") + frontend.SetCtxResponse(ctx, nil, http.StatusBadRequest) + return + } + log.GetLogger().Infof("receive decrease ref request, traceID: %s", traceID) + values, state := datasystemclient.GDecreaseRef(msg, &datasystemclient.Config{TenantID: tenantID}, traceID) + response := &data.DecreaseRefResponse{ + Code: int32(state.Code), + FailedObjectIds: values, + } + if state.Err != nil { + response.Message = state.Err.Error() + } + + if int32(state.Code) == errNone { + response.Message = "decrease ref success" + } + log.GetLogger().Debugf("receive decrease ref response, traceID: %s, code: %d", traceID, state.Code) + respBody, parseErr := proto.Marshal(response) + if parseErr != nil { + log.GetLogger().Errorf("proto Marshal failed, err: %s", parseErr) + frontend.SetCtxResponse(ctx, respBody, http.StatusBadRequest) + return + } + frontend.SetCtxResponse(ctx, respBody, http.StatusOK) +} + +// KvSetHandler the handler of kv-set +func KvSetHandler(ctx *gin.Context) { + tenantID, traceID := getHeaderPrams(ctx) + body, err := io.ReadAll(ctx.Request.Body) + if err != nil { + log.GetLogger().Errorf("failed to read request body error %s", err.Error()) + frontend.SetCtxResponse(ctx, nil, http.StatusInternalServerError) + return + } + msg := &data.KvSetRequest{} + err = proto.Unmarshal(body, msg) + if err != nil { + log.GetLogger().Errorf("failed to parse kv set request message, err: %s", err) + frontend.SetCtxResponse(ctx, nil, http.StatusBadRequest) + return + } + if msg.Key == "" || msg.Value == nil { + log.GetLogger().Errorf("failed to parse kv set request message, empty key or value") + frontend.SetCtxResponse(ctx, nil, http.StatusBadRequest) + return + } + log.GetLogger().Infof("receives kv set request, traceID: %s", traceID) + state := datasystemclient.Set(msg, &datasystemclient.Config{TenantID: tenantID}, traceID) + response := &data.KvSetResponse{ + Code: int32(state.Code), + } + if state.Err != nil { + response.Message = state.Err.Error() + } + + if int32(state.Code) == errNone { + response.Message = "kv set success" + } + log.GetLogger().Debugf("receive kv set response, traceID: %s, code: %d", traceID, state.Code) + respBody, parseErr := proto.Marshal(response) + if parseErr != nil { + log.GetLogger().Errorf("proto Marshal failed, err: %s", parseErr) + frontend.SetCtxResponse(ctx, respBody, http.StatusBadRequest) + return + } + frontend.SetCtxResponse(ctx, respBody, http.StatusOK) +} + +// KvMSetTxHandler the handler of kv-mSetTx +func KvMSetTxHandler(ctx *gin.Context) { + tenantID, traceID := getHeaderPrams(ctx) + body, err := io.ReadAll(ctx.Request.Body) + if err != nil { + log.GetLogger().Errorf("failed to read request body error %s", err.Error()) + frontend.SetCtxResponse(ctx, nil, http.StatusInternalServerError) + return + } + msg := &data.KvMSetTxRequest{} + err = proto.Unmarshal(body, msg) + if err != nil { + log.GetLogger().Errorf("failed to parse kv multi set tx request message, err: %s", err) + frontend.SetCtxResponse(ctx, nil, http.StatusBadRequest) + return + } + if msg.Keys == nil || msg.Values == nil { + log.GetLogger().Errorf("failed to parse kv multi set tx request message, empty keys or values") + frontend.SetCtxResponse(ctx, nil, http.StatusBadRequest) + return + } + if len(msg.Keys) != len(msg.Values) { + log.GetLogger().Errorf("failed to parse kv multi set tx request message, "+ + "keys size: %d isn't equal to values size: %d", len(msg.Keys), len(msg.Values)) + frontend.SetCtxResponse(ctx, nil, http.StatusBadRequest) + return + } + log.GetLogger().Infof("receives kv multi set tx request, traceID: %s", traceID) + state := datasystemclient.MSetTx(msg, &datasystemclient.Config{TenantID: tenantID}, traceID) + response := &data.KvMSetTxResponse{ + Code: int32(state.Code), + } + if state.Err != nil { + response.Message = state.Err.Error() + } + + if int32(state.Code) == errNone { + response.Message = "kv multi set tx success" + } + log.GetLogger().Debugf("receive kv multi set tx response, traceID: %s, code: %d", traceID, state.Code) + respBody, parseErr := proto.Marshal(response) + if parseErr != nil { + log.GetLogger().Errorf("proto Marshal failed, err: %s", parseErr) + frontend.SetCtxResponse(ctx, respBody, http.StatusBadRequest) + return + } + frontend.SetCtxResponse(ctx, respBody, http.StatusOK) +} + +// KvGetHandler the handler of kv-get +func KvGetHandler(ctx *gin.Context) { + tenantID, traceID := getHeaderPrams(ctx) + body, err := io.ReadAll(ctx.Request.Body) + if err != nil { + log.GetLogger().Errorf("failed to read request body error %s", err.Error()) + frontend.SetCtxResponse(ctx, nil, http.StatusInternalServerError) + return + } + msg := &data.KvGetRequest{} + err = proto.Unmarshal(body, msg) + if err != nil { + log.GetLogger().Errorf("failed to parse kv get request message, err: %s", err) + frontend.SetCtxResponse(ctx, nil, http.StatusBadRequest) + return + } + if msg.Keys == nil { + log.GetLogger().Errorf("failed to parse kv get request message, empty keys") + frontend.SetCtxResponse(ctx, nil, http.StatusBadRequest) + return + } + log.GetLogger().Infof("receives kv get request, traceID: %s", traceID) + values, state := datasystemclient.Get(msg, &datasystemclient.Config{TenantID: tenantID}, traceID) + response := &data.KvGetResponse{ + Code: int32(state.Code), + Values: values, + } + if state.Err != nil { + response.Message = state.Err.Error() + } + + if int32(state.Code) == errNone { + response.Message = "kv get success" + } + log.GetLogger().Debugf("receive kv get response, traceID: %s, code: %d", traceID, state.Code) + respBody, parseErr := proto.Marshal(response) + if parseErr != nil { + log.GetLogger().Errorf("proto Marshal failed, err: %s", parseErr) + frontend.SetCtxResponse(ctx, respBody, http.StatusBadRequest) + return + } + frontend.SetCtxResponse(ctx, respBody, http.StatusOK) +} + +// KvDelHandler the handler of kv-del +func KvDelHandler(ctx *gin.Context) { + tenantID, traceID := getHeaderPrams(ctx) + body, err := io.ReadAll(ctx.Request.Body) + if err != nil { + log.GetLogger().Errorf("failed to read request body error %s", err.Error()) + frontend.SetCtxResponse(ctx, nil, http.StatusInternalServerError) + return + } + msg := &data.KvDelRequest{} + err = proto.Unmarshal(body, msg) + if err != nil { + log.GetLogger().Errorf("failed to parse kv del request message, err: %s", err) + frontend.SetCtxResponse(ctx, nil, http.StatusBadRequest) + return + } + if msg.Keys == nil { + log.GetLogger().Errorf("failed to parse kv del request message, empty keys") + frontend.SetCtxResponse(ctx, nil, http.StatusBadRequest) + return + } + log.GetLogger().Infof("receive kv del request, traceID: %s", traceID) + values, state := datasystemclient.Del(msg, &datasystemclient.Config{TenantID: tenantID}, traceID) + response := &data.KvDelResponse{ + Code: int32(state.Code), + FailedKeys: values, + } + if state.Err != nil { + response.Message = state.Err.Error() + } + + log.GetLogger().Debugf("receive kv del response, traceID: %s, code: %d", traceID, state.Code) + if int32(state.Code) == errNone { + response.Message = "kv del success" + } + log.GetLogger().Debugf("receive kv del response, traceID: %s, code: %d", traceID, state.Code) + respBody, parseErr := proto.Marshal(response) + if parseErr != nil { + log.GetLogger().Errorf("proto Marshal failed after receive kv del response, err: %s", parseErr) + frontend.SetCtxResponse(ctx, respBody, http.StatusBadRequest) + return + } + frontend.SetCtxResponse(ctx, respBody, http.StatusOK) +} + +func getHeaderPrams(ctx *gin.Context) (string, string) { + tenantID := httputil.GetCompatibleGinHeader(ctx.Request, constant.HeaderTenantID, "tenantId") + traceID := httputil.GetCompatibleGinHeader(ctx.Request, constant.HeaderTraceID, "traceId") + log.GetLogger().Debugf("check tenantID and traceID in request header: %s, %s", tenantID, traceID) + return tenantID, traceID +} diff --git a/frontend/pkg/frontend/api/datasystem/handler_test.go b/frontend/pkg/frontend/api/datasystem/handler_test.go new file mode 100644 index 0000000000000000000000000000000000000000..8d3e89916a93b12dbb4de64501249b84a4871538 --- /dev/null +++ b/frontend/pkg/frontend/api/datasystem/handler_test.go @@ -0,0 +1,640 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package datasystem + +import ( + "bytes" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/gin-gonic/gin" + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + "google.golang.org/protobuf/proto" + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/datasystemclient" + "frontend/pkg/common/faas_common/grpc/pb/commonargs" + "frontend/pkg/common/faas_common/grpc/pb/data" +) + +func TestPutHandler(t *testing.T) { + errMsg := &data.PutRequest{ + WriteMode: 0, + ConsistencyType: 0, + NestedObjectIds: nil, + } + body, _ := proto.Marshal(errMsg) + convey.Convey("parse failed", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + PutHandler(ctx) + assert.Equal(t, http.StatusBadRequest, rw.Code) + }) + + msg := &data.PutRequest{ + ObjectData: []byte("123"), + ObjectId: "objectId", + WriteMode: 0, + ConsistencyType: 0, + NestedObjectIds: nil, + } + body, _ = proto.Marshal(msg) + convey.Convey("put failed", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + objPut := gomonkey.ApplyFunc(datasystemclient.ObjPut, + func(req *data.PutRequest, config *datasystemclient.Config, traceID string) api.ErrorInfo { + return api.ErrorInfo{Err: errors.New("put failed"), + Code: int(commonargs.ErrorCode_ERR_INNER_SYSTEM_ERROR)} + }) + defer objPut.Reset() + PutHandler(ctx) + assert.Equal(t, http.StatusOK, rw.Code) + response := &data.PutResponse{} + err := proto.Unmarshal(rw.Body.Bytes(), response) + assert.Equal(t, nil, err) + assert.Equal(t, int32(commonargs.ErrorCode_ERR_INNER_SYSTEM_ERROR), response.Code) + assert.Equal(t, "put failed", response.Message) + + rw = httptest.NewRecorder() + ctx, _ = gin.CreateTestContext(rw) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + defer gomonkey.ApplyFunc(proto.Unmarshal, func(b []byte, m proto.Message) error { + return errors.New("proto unmarshal error") + }).Reset() + PutHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + }) + convey.Convey("put succeed", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + objPut := gomonkey.ApplyFunc(datasystemclient.ObjPut, + func(req *data.PutRequest, config *datasystemclient.Config, traceID string) api.ErrorInfo { + return api.ErrorInfo{Code: 0, Err: errors.New("")} + }) + defer objPut.Reset() + PutHandler(ctx) + assert.Equal(t, http.StatusOK, rw.Code) + response := &data.PutResponse{} + err := proto.Unmarshal(rw.Body.Bytes(), response) + assert.Equal(t, nil, err) + assert.Equal(t, int32(commonargs.ErrorCode_ERR_NONE), response.Code) + }) + convey.Convey("internal error", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + patch := gomonkey.ApplyFunc(io.ReadAll, + func(r io.Reader) ([]byte, error) { + return nil, errors.New("some error") + }) + defer patch.Reset() + PutHandler(ctx) + assert.Equal(t, http.StatusInternalServerError, rw.Code) + }) + convey.Convey("internal error", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + ctx.Request.Header.Add(constant.HeaderTenantID, "tenantId") + ctx.Request.Header.Add(constant.HeaderTraceID, "traceId") + patch := gomonkey.ApplyFunc(proto.Marshal, + func(m proto.Message) ([]byte, error) { + return nil, errors.New("some error") + }) + defer patch.Reset() + PutHandler(ctx) + assert.Equal(t, http.StatusBadRequest, rw.Code) + }) +} + +func TestGetHandler(t *testing.T) { + msg := &data.GetRequest{ + ObjectIds: []string{"objectId"}, + TimeoutMs: 0, + } + body, _ := proto.Marshal(msg) + convey.Convey("get failed", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + objGet := gomonkey.ApplyFunc(datasystemclient.ObjGet, + func(req *data.GetRequest, config *datasystemclient.Config, traceID string) ([][]byte, api.ErrorInfo) { + return nil, api.ErrorInfo{Err: errors.New("get failed"), + Code: int(commonargs.ErrorCode_ERR_INNER_SYSTEM_ERROR)} + }) + defer objGet.Reset() + GetHandler(ctx) + assert.Equal(t, http.StatusOK, rw.Code) + response := &data.GetResponse{} + err := proto.Unmarshal(rw.Body.Bytes(), response) + assert.Equal(t, nil, err) + assert.Equal(t, int32(commonargs.ErrorCode_ERR_INNER_SYSTEM_ERROR), response.Code) + assert.Equal(t, "get failed", response.Message) + + rw = httptest.NewRecorder() + ctx, _ = gin.CreateTestContext(rw) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + defer gomonkey.ApplyFunc(proto.Unmarshal, func(b []byte, m proto.Message) error { + return errors.New("proto unmarshal error") + }).Reset() + GetHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + }) + convey.Convey("get succeed", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + objGet := gomonkey.ApplyFunc(datasystemclient.ObjGet, + func(req *data.GetRequest, config *datasystemclient.Config, traceID string) ([][]byte, api.ErrorInfo) { + return nil, api.ErrorInfo{Err: errors.New(""), Code: int(commonargs.ErrorCode_ERR_NONE)} + }) + defer objGet.Reset() + GetHandler(ctx) + assert.Equal(t, http.StatusOK, rw.Code) + response := &data.GetResponse{} + err := proto.Unmarshal(rw.Body.Bytes(), response) + assert.Equal(t, nil, err) + assert.Equal(t, int32(commonargs.ErrorCode_ERR_NONE), response.Code) + }) + convey.Convey("internal error", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + patch := gomonkey.ApplyFunc(io.ReadAll, + func(r io.Reader) ([]byte, error) { + return nil, errors.New("some error") + }) + defer patch.Reset() + GetHandler(ctx) + assert.Equal(t, http.StatusInternalServerError, rw.Code) + }) + convey.Convey("internal error", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + patch := gomonkey.ApplyFunc(proto.Marshal, + func(m proto.Message) ([]byte, error) { + return nil, errors.New("some error") + }) + defer patch.Reset() + GetHandler(ctx) + assert.Equal(t, http.StatusBadRequest, rw.Code) + }) +} + +func TestIncreaseRefHandler(t *testing.T) { + convey.Convey("IncreaseRefHandler", t, func() { + convey.Convey("failed to parse increase ref request message, empty ids", func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + errMsg := &data.IncreaseRefRequest{} + body, _ := proto.Marshal(errMsg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + IncreaseRefHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + + defer gomonkey.ApplyFunc(proto.Unmarshal, func(b []byte, m proto.Message) error { + return errors.New("proto unmarshal error") + }).Reset() + IncreaseRefHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + }) + + convey.Convey("OK", func() { + defer gomonkey.ApplyFunc(datasystemclient.GIncreaseRef, func(req *data.IncreaseRefRequest, config *datasystemclient.Config, traceID string) ([]string, api.ErrorInfo) { + return []string{}, api.ErrorInfo{ + Code: 0, + Err: fmt.Errorf("nil err"), + } + }).Reset() + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + errMsg := &data.IncreaseRefRequest{ + RemoteClientId: "123456", + ObjectIds: []string{"123", "345"}, + } + body, _ := proto.Marshal(errMsg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + IncreaseRefHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + }) + }) + convey.Convey("internal error", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + body, _ := proto.Marshal(&data.IncreaseRefRequest{ + RemoteClientId: "123456", + ObjectIds: []string{"123", "345"}, + }) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + patch := gomonkey.ApplyFunc(io.ReadAll, + func(r io.Reader) ([]byte, error) { + return nil, errors.New("some error") + }) + defer patch.Reset() + IncreaseRefHandler(ctx) + assert.Equal(t, http.StatusInternalServerError, rw.Code) + }) + convey.Convey("internal error", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + body, _ := proto.Marshal(&data.IncreaseRefRequest{ + RemoteClientId: "123456", + ObjectIds: []string{"123", "345"}, + }) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + patch := gomonkey.ApplyFunc(proto.Marshal, + func(m proto.Message) ([]byte, error) { + return nil, errors.New("some error") + }) + defer patch.Reset() + IncreaseRefHandler(ctx) + assert.Equal(t, http.StatusBadRequest, rw.Code) + }) +} + +func TestDecreaseRefHandler(t *testing.T) { + convey.Convey("DecreaseRefHandler", t, func() { + convey.Convey("failed to parse decrease ref request message", func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + errMsg := &data.DecreaseRefRequest{} + body, _ := proto.Marshal(errMsg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + DecreaseRefHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + + defer gomonkey.ApplyFunc(proto.Unmarshal, func(b []byte, m proto.Message) error { + return errors.New("proto unmarshal error") + }).Reset() + DecreaseRefHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + }) + + convey.Convey("OK", func() { + defer gomonkey.ApplyFunc(datasystemclient.GDecreaseRef, func(req *data.DecreaseRefRequest, config *datasystemclient.Config, traceID string) ([]string, api.ErrorInfo) { + return []string{}, api.ErrorInfo{ + Code: 0, + Err: fmt.Errorf("nil err"), + } + }).Reset() + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + errMsg := &data.DecreaseRefRequest{ + RemoteClientId: "123456", + ObjectIds: []string{"123", "345"}, + } + body, _ := proto.Marshal(errMsg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + DecreaseRefHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + }) + }) + convey.Convey("internal error", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + body, _ := proto.Marshal(&data.DecreaseRefRequest{ + RemoteClientId: "123456", + ObjectIds: []string{"123", "345"}, + }) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + patch := gomonkey.ApplyFunc(io.ReadAll, + func(r io.Reader) ([]byte, error) { + return nil, errors.New("some error") + }) + defer patch.Reset() + DecreaseRefHandler(ctx) + assert.Equal(t, http.StatusInternalServerError, rw.Code) + }) + convey.Convey("internal error", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + body, _ := proto.Marshal(&data.DecreaseRefRequest{ + RemoteClientId: "123456", + ObjectIds: []string{"123", "345"}, + }) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + patch := gomonkey.ApplyFunc(proto.Marshal, + func(m proto.Message) ([]byte, error) { + return nil, errors.New("some error") + }) + defer patch.Reset() + DecreaseRefHandler(ctx) + assert.Equal(t, http.StatusBadRequest, rw.Code) + }) +} + +func TestKvSetHandler(t *testing.T) { + convey.Convey("KvSetHandler", t, func() { + convey.Convey("failed to parse kv set request message", func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + errMsg := &data.KvSetRequest{} + body, _ := proto.Marshal(errMsg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + KvSetHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + + defer gomonkey.ApplyFunc(proto.Unmarshal, func(b []byte, m proto.Message) error { + return errors.New("proto unmarshal error") + }).Reset() + KvSetHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + }) + + convey.Convey("OK", func() { + defer gomonkey.ApplyFunc(datasystemclient.Set, func(req *data.KvSetRequest, config *datasystemclient.Config, traceID string) api.ErrorInfo { + return api.ErrorInfo{ + Code: 0, + Err: fmt.Errorf("nil err"), + } + }).Reset() + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + errMsg := &data.KvSetRequest{ + Key: "test", + Value: []byte("VALUE"), + } + body, _ := proto.Marshal(errMsg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + KvSetHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + }) + }) + convey.Convey("internal error", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + body, _ := proto.Marshal(&data.KvSetRequest{ + Key: "test", + Value: []byte("VALUE"), + }) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + patch := gomonkey.ApplyFunc(io.ReadAll, + func(r io.Reader) ([]byte, error) { + return nil, errors.New("some error") + }) + defer patch.Reset() + KvSetHandler(ctx) + assert.Equal(t, http.StatusInternalServerError, rw.Code) + }) + convey.Convey("internal error", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + body, _ := proto.Marshal(&data.KvSetRequest{ + Key: "test", + Value: []byte("VALUE"), + }) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + patch := gomonkey.ApplyFunc(proto.Marshal, + func(m proto.Message) ([]byte, error) { + return nil, errors.New("some error") + }) + defer patch.Reset() + KvSetHandler(ctx) + assert.Equal(t, http.StatusBadRequest, rw.Code) + }) +} + +func TestKvMSetTxHandler(t *testing.T) { + convey.Convey("KvMSetTxHandler", t, func() { + convey.Convey("failed to parse kv mset tx request message", func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + errMsg := &data.KvMSetTxRequest{} + body, _ := proto.Marshal(errMsg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + KvMSetTxHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + + defer gomonkey.ApplyFunc(proto.Unmarshal, func(b []byte, m proto.Message) error { + return errors.New("proto unmarshal error") + }).Reset() + KvMSetTxHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + }) + + convey.Convey("OK", func() { + defer gomonkey.ApplyFunc(datasystemclient.MSetTx, func(req *data.KvMSetTxRequest, + config *datasystemclient.Config, traceID string) api.ErrorInfo { + return api.ErrorInfo{ + Code: 0, + Err: fmt.Errorf("nil err"), + } + }).Reset() + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + errMsg := &data.KvMSetTxRequest{ + Keys: []string{"test"}, + Values: [][]byte{[]byte("VALUE")}, + } + body, _ := proto.Marshal(errMsg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + KvMSetTxHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + }) + }) + convey.Convey("internal error", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + body, _ := proto.Marshal(&data.KvMSetTxRequest{ + Keys: []string{"test"}, + Values: [][]byte{[]byte("VALUE")}, + }) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + patch := gomonkey.ApplyFunc(io.ReadAll, + func(r io.Reader) ([]byte, error) { + return nil, errors.New("some error") + }) + defer patch.Reset() + KvMSetTxHandler(ctx) + assert.Equal(t, http.StatusInternalServerError, rw.Code) + }) + convey.Convey("internal error", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + body, _ := proto.Marshal(&data.KvMSetTxRequest{ + Keys: []string{"test"}, + Values: [][]byte{[]byte("VALUE")}, + }) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + patch := gomonkey.ApplyFunc(proto.Marshal, + func(m proto.Message) ([]byte, error) { + return nil, errors.New("some error") + }) + defer patch.Reset() + KvMSetTxHandler(ctx) + assert.Equal(t, http.StatusBadRequest, rw.Code) + }) + convey.Convey("internal error", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + body, _ := proto.Marshal(&data.KvMSetTxRequest{ + Keys: []string{"test", "test2"}, + Values: [][]byte{[]byte("VALUE")}, + }) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + KvMSetTxHandler(ctx) + assert.Equal(t, http.StatusBadRequest, rw.Code) + }) +} + +func TestKvGetHandler(t *testing.T) { + convey.Convey("KvGetHandler", t, func() { + convey.Convey("failed to parse kv get request message", func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + errMsg := &data.KvGetRequest{} + body, _ := proto.Marshal(errMsg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + KvGetHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + + defer gomonkey.ApplyFunc(proto.Unmarshal, func(b []byte, m proto.Message) error { + return errors.New("proto unmarshal error") + }).Reset() + KvGetHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + }) + + convey.Convey("OK", func() { + defer gomonkey.ApplyFunc(datasystemclient.Get, func(req *data.KvGetRequest, config *datasystemclient.Config, traceID string) ([][]byte, api.ErrorInfo) { + return [][]byte{}, api.ErrorInfo{ + Code: 0, + Err: fmt.Errorf("nil err"), + } + }).Reset() + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + errMsg := &data.KvGetRequest{ + Keys: []string{"test"}, + } + body, _ := proto.Marshal(errMsg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + KvGetHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + }) + }) + convey.Convey("internal error", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + body, _ := proto.Marshal(&data.KvGetRequest{ + Keys: []string{"test"}, + }) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + patch := gomonkey.ApplyFunc(io.ReadAll, + func(r io.Reader) ([]byte, error) { + return nil, errors.New("some error") + }) + defer patch.Reset() + KvGetHandler(ctx) + assert.Equal(t, http.StatusInternalServerError, rw.Code) + }) + convey.Convey("internal error", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + body, _ := proto.Marshal(&data.KvGetRequest{ + Keys: []string{"test"}, + }) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + patch := gomonkey.ApplyFunc(proto.Marshal, + func(m proto.Message) ([]byte, error) { + return nil, errors.New("some error") + }) + defer patch.Reset() + KvGetHandler(ctx) + assert.Equal(t, http.StatusBadRequest, rw.Code) + }) +} + +func TestKvDelHandler(t *testing.T) { + convey.Convey("KvDelHandler", t, func() { + convey.Convey("failed to parse kv set request message", func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + errMsg := &data.KvDelRequest{} + body, _ := proto.Marshal(errMsg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + KvDelHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + + defer gomonkey.ApplyFunc(proto.Unmarshal, func(b []byte, m proto.Message) error { + return errors.New("proto unmarshal error") + }).Reset() + KvDelHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + }) + + convey.Convey("OK", func() { + defer gomonkey.ApplyFunc(datasystemclient.Del, func(req *data.KvDelRequest, config *datasystemclient.Config, traceID string) ([][]byte, api.ErrorInfo) { + return [][]byte{}, api.ErrorInfo{ + Code: 0, + Err: fmt.Errorf("nil err"), + } + }).Reset() + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + errMsg := &data.KvDelRequest{ + Keys: []string{"test"}, + } + body, _ := proto.Marshal(errMsg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + KvDelHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + }) + }) + convey.Convey("internal error", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + body, _ := proto.Marshal(&data.KvDelRequest{ + Keys: []string{"test"}, + }) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + patch := gomonkey.ApplyFunc(io.ReadAll, + func(r io.Reader) ([]byte, error) { + return nil, errors.New("some error") + }) + defer patch.Reset() + KvDelHandler(ctx) + assert.Equal(t, http.StatusInternalServerError, rw.Code) + }) + convey.Convey("internal error", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + body, _ := proto.Marshal(&data.KvDelRequest{ + Keys: []string{"test"}, + }) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + patch := gomonkey.ApplyFunc(proto.Marshal, + func(m proto.Message) ([]byte, error) { + return nil, errors.New("some error") + }) + defer patch.Reset() + KvDelHandler(ctx) + assert.Equal(t, http.StatusBadRequest, rw.Code) + }) +} diff --git a/frontend/pkg/frontend/api/functionsystem/handler.go b/frontend/pkg/frontend/api/functionsystem/handler.go new file mode 100644 index 0000000000000000000000000000000000000000..73553e74033d403ff164127f95b5b445520e3206 --- /dev/null +++ b/frontend/pkg/frontend/api/functionsystem/handler.go @@ -0,0 +1,105 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package frontend the api of frontend +package frontend + +import ( + "io" + "net/http" + + "github.com/gin-gonic/gin" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/frontend/common/httputil" + "frontend/pkg/frontend/common/util" + "frontend/pkg/frontend/serverstatus" +) + +// CreateHandler the handler of create +func CreateHandler(ctx *gin.Context) { + remoteClientID, traceID := getHeaderPrams(ctx) + log.GetLogger().Infof("%s|receive instance create request, remoteClientID: %s", traceID, remoteClientID) + body, err := io.ReadAll(ctx.Request.Body) + if err != nil { + log.GetLogger().Errorf("failed to read request body error %s", err.Error()) + SetCtxResponse(ctx, nil, http.StatusInternalServerError) + return + } + resp, err := util.NewClient().CreateInstanceRaw(body) + log.GetLogger().Debugf("receive instance create response, msg: %s", resp) + if err != nil { + SetCtxResponse(ctx, []byte(err.Error()), http.StatusBadRequest) + } + SetCtxResponse(ctx, resp, http.StatusOK) +} + +// InvokeHandler the handler of invoke +func InvokeHandler(ctx *gin.Context) { + remoteClientID, traceID := getHeaderPrams(ctx) + log.GetLogger().Infof("%s|receive instance invoke request, remoteClientID: %s", traceID, remoteClientID) + body, err := io.ReadAll(ctx.Request.Body) + if err != nil { + log.GetLogger().Errorf("failed to read request body error %s", err.Error()) + SetCtxResponse(ctx, nil, http.StatusInternalServerError) + return + } + notify, err := util.NewClient().InvokeInstanceRaw(body) + log.GetLogger().Debugf("receive instance invoke response, msg: %s", notify) + if err != nil { + SetCtxResponse(ctx, []byte(err.Error()), http.StatusBadRequest) + } + SetCtxResponse(ctx, notify, http.StatusOK) +} + +// KillHandler the handler of kill +func KillHandler(ctx *gin.Context) { + remoteClientID, traceID := getHeaderPrams(ctx) + log.GetLogger().Infof("%s|receives instance kill request, remoteClientID: %s", traceID, remoteClientID) + body, err := io.ReadAll(ctx.Request.Body) + if err != nil { + log.GetLogger().Errorf("failed to read request body error %s", err.Error()) + SetCtxResponse(ctx, nil, http.StatusInternalServerError) + return + } + resp, err := util.NewClient().KillRaw(body) + log.GetLogger().Debugf("receive instance kill response, msg: %s", resp) + if err != nil { + SetCtxResponse(ctx, []byte(err.Error()), http.StatusBadRequest) + } + SetCtxResponse(ctx, resp, http.StatusOK) +} + +func getHeaderPrams(ctx *gin.Context) (string, string) { + remoteClientID := httputil.GetCompatibleGinHeader(ctx.Request, constant.HeaderRemoteClientId, "remoteClientId") + traceID := httputil.GetCompatibleGinHeader(ctx.Request, constant.HeaderTraceID, "traceId") + return remoteClientID, traceID +} + +// SetCtxResponse set ctx response +func SetCtxResponse(ctx *gin.Context, body []byte, statusCode int) { + if len(body) == 0 { + log.GetLogger().Warnf("the body of ctx response is empty") + } + ctx.Writer.WriteHeader(statusCode) + if serverstatus.IsShutdown() { + ctx.Writer.Header().Set("Connection", "close") + } + if _, err := ctx.Writer.Write(body); err != nil { + log.GetLogger().Errorf("failed to set response body in context error %s", err.Error()) + } +} diff --git a/frontend/pkg/frontend/api/functionsystem/handler_test.go b/frontend/pkg/frontend/api/functionsystem/handler_test.go new file mode 100644 index 0000000000000000000000000000000000000000..7521f1c2d57ed2522151c1b16c6af8bfe6a37890 --- /dev/null +++ b/frontend/pkg/frontend/api/functionsystem/handler_test.go @@ -0,0 +1,171 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package frontend + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/gin-gonic/gin" + "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/common/faas_common/constant" + mockUtils "frontend/pkg/common/faas_common/utils" + "frontend/pkg/frontend/common/util" +) + +func Test_CreateHandler(t *testing.T) { + convey.Convey("test CreateHandler", t, func() { + mock := &mockUtils.FakeLibruntimeSdkClient{} + util.SetAPIClientLibruntime(mock) + convey.Convey("read body error", func() { + defer gomonkey.ApplyFunc(io.ReadAll, func(r io.Reader) ([]byte, error) { + return []byte{}, errors.New("read body error") + }).Reset() + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + reqBody := "test body" + bodyMarshal, _ := json.Marshal(reqBody) + ctx.Request, _ = http.NewRequest("POST", "/test", bytes.NewBuffer(bodyMarshal)) + ctx.Request.Header.Set("remoteClientId", "test-client-id") + CreateHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusInternalServerError) + }) + convey.Convey("CreateHandler success", func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + reqBody := "test body" + bodyMarshal, _ := json.Marshal(reqBody) + ctx.Request, _ = http.NewRequest("POST", "/test", bytes.NewBuffer(bodyMarshal)) + ctx.Request.Header.Set("remoteClientId", "test-client-id") + CreateHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + }) + convey.Convey("CreateHandler failed", func() { + defer gomonkey.ApplyMethod(reflect.TypeOf(&mockUtils.FakeLibruntimeSdkClient{}), + "CreateInstanceRaw", + func(_ *mockUtils.FakeLibruntimeSdkClient, createReqRaw []byte) (createRespRaw []byte, err error) { + return []byte{}, errors.New("CreateInstanceRaw error") + }).Reset() + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + reqBody := "test body" + bodyMarshal, _ := json.Marshal(reqBody) + ctx.Request, _ = http.NewRequest("POST", "/test", bytes.NewBuffer(bodyMarshal)) + ctx.Request.Header.Set("remoteClientId", "test-client-id") + CreateHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + }) + }) +} + +func Test_InvokeHandler(t *testing.T) { + convey.Convey("test InvokeHandler", t, func() { + mock := &mockUtils.FakeLibruntimeSdkClient{} + util.SetAPIClientLibruntime(mock) + convey.Convey("read body error", func() { + defer gomonkey.ApplyFunc(io.ReadAll, func(r io.Reader) ([]byte, error) { + return []byte{}, errors.New("read body error") + }).Reset() + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + reqBody := "test body" + bodyMarshal, _ := json.Marshal(reqBody) + ctx.Request, _ = http.NewRequest("POST", "/test", bytes.NewBuffer(bodyMarshal)) + ctx.Request.Header.Set(constant.HeaderRemoteClientId, "test-client-id") + InvokeHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusInternalServerError) + }) + convey.Convey("InvokeHandler success", func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + reqBody := "test body" + bodyMarshal, _ := json.Marshal(reqBody) + ctx.Request, _ = http.NewRequest("POST", "/test", bytes.NewBuffer(bodyMarshal)) + ctx.Request.Header.Set(constant.HeaderRemoteClientId, "test-client-id") + InvokeHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + }) + convey.Convey("InvokeHandler failed", func() { + defer gomonkey.ApplyMethod(reflect.TypeOf(&mockUtils.FakeLibruntimeSdkClient{}), + "InvokeByInstanceIdRaw", + func(_ *mockUtils.FakeLibruntimeSdkClient, invokeReqRaw []byte) (resultRaw []byte, err error) { + return []byte{}, errors.New("InvokeByInstanceIdRaw error") + }).Reset() + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + reqBody := "test body" + bodyMarshal, _ := json.Marshal(reqBody) + ctx.Request, _ = http.NewRequest("POST", "/test", bytes.NewBuffer(bodyMarshal)) + ctx.Request.Header.Set(constant.HeaderRemoteClientId, "test-client-id") + InvokeHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + }) + }) +} + +func Test_KillHandler(t *testing.T) { + convey.Convey("test KillHandler", t, func() { + mock := &mockUtils.FakeLibruntimeSdkClient{} + util.SetAPIClientLibruntime(mock) + convey.Convey("read body error", func() { + defer gomonkey.ApplyFunc(io.ReadAll, func(r io.Reader) ([]byte, error) { + return []byte{}, errors.New("read body error") + }).Reset() + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + reqBody := "test body" + bodyMarshal, _ := json.Marshal(reqBody) + ctx.Request, _ = http.NewRequest("POST", "/test", bytes.NewBuffer(bodyMarshal)) + ctx.Request.Header.Set("remoteClientId", "test-client-id") + KillHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusInternalServerError) + }) + convey.Convey("KillHandler success", func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + reqBody := "test body" + bodyMarshal, _ := json.Marshal(reqBody) + ctx.Request, _ = http.NewRequest("POST", "/test", bytes.NewBuffer(bodyMarshal)) + ctx.Request.Header.Set("remoteClientId", "test-client-id") + KillHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + }) + convey.Convey("KillHandler failed", func() { + defer gomonkey.ApplyMethod(reflect.TypeOf(&mockUtils.FakeLibruntimeSdkClient{}), + "KillRaw", + func(_ *mockUtils.FakeLibruntimeSdkClient, killReqRaw []byte) (killRespRaw []byte, err error) { + return []byte{}, errors.New("KillRaw error") + }).Reset() + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + reqBody := "test body" + bodyMarshal, _ := json.Marshal(reqBody) + ctx.Request, _ = http.NewRequest("POST", "/test", bytes.NewBuffer(bodyMarshal)) + ctx.Request.Header.Set("remoteClientId", "test-client-id") + KillHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + }) + }) +} diff --git a/frontend/pkg/frontend/api/job/handler.go b/frontend/pkg/frontend/api/job/handler.go new file mode 100644 index 0000000000000000000000000000000000000000..c332ea85851e12d741e99e891ba2fa2cdc285518 --- /dev/null +++ b/frontend/pkg/frontend/api/job/handler.go @@ -0,0 +1,83 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package job for handle request +package job + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/job" + "frontend/pkg/frontend/api/app" +) + +// SubmitJobHandler - +func SubmitJobHandler(ctx *gin.Context) { + traceID := ctx.Request.Header.Get(constant.HeaderTraceID) + logger := log.GetLogger().With(zap.Any("traceID", traceID)) + req := job.SubmitJobHandleReq(ctx) + if req == nil { + return + } + if req.SubmissionId == "" { + req.NewSubmissionID() + } else { + _, statusCode, err := app.GetAppInfo(req.SubmissionId) + if err != nil { + if statusCode != http.StatusNotFound { + logger.Errorf("failed GetAppInfo, submissionId: %s, err: %v", req.SubmissionId, err) + ctx.JSON(http.StatusInternalServerError, + fmt.Sprintf("failed GetAppInfo, submissionId: %s, err: %v", req.SubmissionId, err)) + return + } + } + if statusCode == http.StatusOK { + logger.Errorf("submit job has already exist, submissionId: %s", req.SubmissionId) + ctx.JSON(http.StatusBadRequest, fmt.Sprintf("submit job has already exist, submissionId: %s", + req.SubmissionId)) + return + } + } + logger.Debugf("start to SubmitApp, req:%#v", req) + job.SubmitJobHandleRes(ctx, job.BuildJobResponse(app.SubmitApp(req))) +} + +// ListJobsHandler - +func ListJobsHandler(ctx *gin.Context) { + job.ListJobsHandleRes(ctx, job.BuildJobResponse(app.ListApps(ctx))) +} + +// GetJobInfoHandler - +func GetJobInfoHandler(ctx *gin.Context) { + submissionId := ctx.Param(job.PathParamSubmissionId) + job.GetJobInfoHandleRes(ctx, job.BuildJobResponse(app.GetAppInfo(submissionId))) +} + +// DeleteJobHandler - +func DeleteJobHandler(ctx *gin.Context) { + job.DeleteJobHandleRes(ctx, job.BuildJobResponse(app.DeleteApp(ctx))) +} + +// StopJobHandler - +func StopJobHandler(ctx *gin.Context) { + job.StopJobHandleRes(ctx, job.BuildJobResponse(app.StopApp(ctx))) +} diff --git a/frontend/pkg/frontend/api/job/handler_test.go b/frontend/pkg/frontend/api/job/handler_test.go new file mode 100644 index 0000000000000000000000000000000000000000..ad2aebaeeb8e5f64a09f45b87136003bb8c815b0 --- /dev/null +++ b/frontend/pkg/frontend/api/job/handler_test.go @@ -0,0 +1,326 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package job + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/gin-gonic/gin" + "github.com/smartystreets/goconvey/convey" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/constants" + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/types" + mockUtils "frontend/pkg/common/faas_common/utils" + "frontend/pkg/common/job" + "frontend/pkg/frontend/common/util" + "frontend/pkg/frontend/instancemanager" +) + +func addApps(submissionId string) []*constant.AppInfo { + var result []*constant.AppInfo + return append(result, buildAppInfo(submissionId)) +} + +func buildAppInfo(submissionId string) *constant.AppInfo { + return &constant.AppInfo{ + Key: submissionId, + Type: "SUBMISSION", + SubmissionID: submissionId, + RuntimeEnv: map[string]interface{}{ + "working_dir": "", + "pip": "", + "envVars": "", + }, + DriverInfo: constant.DriverInfo{ + ID: submissionId, + }, + Status: "RUNNING", + } +} + +func storeDefaultApp(key string, code int32, statusType int32) { + instancemanager.StoreAppInfo(key, &types.InstanceSpecification{ + InstanceID: key, + StartTime: "", + InstanceStatus: types.InstanceStatus{ + Code: code, + Type: statusType, + ExitCode: 0, + }, + }) +} + +func TestListJobsHandler(t *testing.T) { + // list处理优先执行，因为存储job的appInfo是一个全局sync.map，所以其它用例添加了元素后可能会对查询结果造成一定影响 + convey.Convey("test ListJobsHandler", t, func() { + gin.SetMode(gin.TestMode) + rw := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rw) + c.Request = &http.Request{ + Method: http.MethodGet, + URL: &url.URL{Path: job.PathGroupJobs}, + Header: http.Header{ + "Content-Type": []string{"application/json"}, + constants.HeaderTenantID: []string{"123456"}, + constants.HeaderPoolLabel: []string{"abc"}, + }, + Body: nil, // 使用 io.NopCloser 包装 reader，使其满足 io.ReadCloser 接口 + } + convey.Convey("when process success", func() { + ListJobsHandler(c) + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + convey.So(rw.Body.String(), convey.ShouldStartWith, "[") + }) + convey.Convey("when list job is not empty", func() { + storeDefaultApp("app-frontend-list1", 3, 0) + ListJobsHandler(c) + expectedResult, err := json.Marshal(addApps("app-frontend-list1")) + if err != nil { + t.Errorf("marshal expected result failed, err: %v", err) + } + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + convey.So(rw.Body.String(), convey.ShouldEqual, string(expectedResult)) + }) + }) +} + +func TestSubmitJobHandler(t *testing.T) { + convey.Convey("test SubmitJobHandler", t, func() { + mock := &mockUtils.FakeLibruntimeSdkClient{} + submissionId := "app-frontend-job-submit1" + util.SetAPIClientLibruntime(mock) + gin.SetMode(gin.TestMode) + rw := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rw) + bodyBytes, _ := json.Marshal(job.SubmitRequest{ + Entrypoint: "sleep 200", + SubmissionId: submissionId, + RuntimeEnv: &job.RuntimeEnv{ + WorkingDir: "file:///home/ray/codeNew/god_gt_factory/file/lidar_seg_042801.zip", + Pip: []string{"numpy==1.24", "scipy==1.25"}, + EnvVars: map[string]string{ + "SOURCE_REGION": "suzhou_std", + "DEPLOY_REGION": "suzhou_std", + }, + }, + Metadata: map[string]string{ + "autoscenes_ids": "auto_1-test", + "task_type": "task_1", + "ttl": "1250", + }, + EntrypointResources: map[string]float64{ + "NPU": 0, + }, + EntrypointNumCpus: 300, + EntrypointNumGpus: 0, + EntrypointMemory: 0, + }) + reader := bytes.NewBuffer(bodyBytes) + c.Request = &http.Request{ + Method: http.MethodPost, + URL: &url.URL{Path: job.PathGroupJobs}, + Header: http.Header{ + "Content-Type": []string{"application/json"}, + constants.HeaderTenantID: []string{"123456"}, + constants.HeaderPoolLabel: []string{"abc"}, + }, + Body: io.NopCloser(reader), // 使用 io.NopCloser 包装 reader，使其满足 io.ReadCloser 接口 + } + convey.Convey("when process success", func() { + defer gomonkey.ApplyMethodFunc(mock, "CreateInstance", + func(funcMeta api.FunctionMeta, args []api.Arg, invokeOpt api.InvokeOptions) (instanceID string, err error) { + return submissionId, nil + }).Reset() + SubmitJobHandler(c) + expectedResult, err := json.Marshal(map[string]string{ + "submission_id": submissionId, + }) + if err != nil { + t.Errorf("marshal expected result failed, err: %v", err) + } + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + convey.So(rw.Body.String(), convey.ShouldResemble, string(expectedResult)) + }) + convey.Convey("when app is exist", func() { + storeDefaultApp(submissionId, 3, 0) + defer gomonkey.ApplyMethodFunc(mock, "CreateInstance", + func(funcMeta api.FunctionMeta, args []api.Arg, invokeOpt api.InvokeOptions) (instanceID string, err error) { + return submissionId, nil + }).Reset() + SubmitJobHandler(c) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + convey.So(rw.Body.String(), convey.ShouldStartWith, "\"submit job has already exist, submissionId") + }) + }) +} + +func TestGetJobInfoHandler(t *testing.T) { + convey.Convey("test GetJobInfoHandler", t, func() { + submissionId := "app-frontend-get1" + gin.SetMode(gin.TestMode) + rw := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rw) + c.Request = &http.Request{ + Method: http.MethodGet, + URL: &url.URL{Path: job.PathGroupJobs + "/" + submissionId}, + Header: http.Header{ + "Content-Type": []string{"application/json"}, + constants.HeaderTenantID: []string{"123456"}, + constants.HeaderPoolLabel: []string{"abc"}, + }, + Body: nil, // 使用 io.NopCloser 包装 reader，使其满足 io.ReadCloser 接口 + } + c.AddParam(job.PathParamSubmissionId, submissionId) + convey.Convey("when get job is not found", func() { + GetJobInfoHandler(c) + convey.So(rw.Code, convey.ShouldEqual, http.StatusNotFound) + convey.So(rw.Body.String(), convey.ShouldStartWith, "\"not found app, submissionId") + }) + convey.Convey("when process success", func() { + storeDefaultApp(submissionId, 3, 0) + GetJobInfoHandler(c) + expectedResult, err := json.Marshal(buildAppInfo(submissionId)) + if err != nil { + t.Errorf("marshal expected result failed, err: %v", err) + } + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + convey.So(rw.Body.String(), convey.ShouldEqual, string(expectedResult)) + }) + }) +} + +func TestDeleteJobHandler(t *testing.T) { + convey.Convey("test DeleteJobHandler", t, func() { + mock := &mockUtils.FakeLibruntimeSdkClient{} + util.SetAPIClientLibruntime(mock) + submissionId := "app-frontend-delete1" + gin.SetMode(gin.TestMode) + rw := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rw) + c.Request = &http.Request{ + Method: http.MethodGet, + URL: &url.URL{Path: job.PathGroupJobs + "/" + submissionId}, + Header: http.Header{ + "Content-Type": []string{"application/json"}, + constants.HeaderTenantID: []string{"123456"}, + constants.HeaderPoolLabel: []string{"abc"}, + }, + Body: nil, // 使用 io.NopCloser 包装 reader，使其满足 io.ReadCloser 接口 + } + c.AddParam(job.PathParamSubmissionId, submissionId) + convey.Convey("when job is not found", func() { + DeleteJobHandler(c) + convey.So(rw.Code, convey.ShouldEqual, http.StatusNotFound) + convey.So(rw.Body.String(), convey.ShouldStartWith, "\"the app does not exist, submissionId") + }) + convey.Convey("when process success", func() { + storeDefaultApp(submissionId, 6, 1) + DeleteJobHandler(c) + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + convey.So(rw.Body.String(), convey.ShouldEqual, "true") + }) + convey.Convey("when job is forbidden to delete, status: SUCCEEDED", func() { + storeDefaultApp(submissionId, 3, 1) + DeleteJobHandler(c) + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + convey.So(rw.Body.String(), convey.ShouldEqual, "false") + }) + convey.Convey("when job is forbidden to delete, status: STOPPED", func() { + storeDefaultApp(submissionId, 3, 6) + DeleteJobHandler(c) + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + convey.So(rw.Body.String(), convey.ShouldEqual, "false") + }) + convey.Convey("when job is forbidden to delete, status: FAILED", func() { + storeDefaultApp(submissionId, 3, 0) + DeleteJobHandler(c) + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + convey.So(rw.Body.String(), convey.ShouldEqual, "false") + }) + convey.Convey("when statusCode is "+strconv.Itoa(http.StatusInternalServerError), func() { + defer gomonkey.ApplyMethodFunc(mock, "Kill", + func(instanceID string, signal int, payload []byte) error { + return errors.New("failed delete app") + }).Reset() + storeDefaultApp(submissionId, 6, 1) + DeleteJobHandler(c) + convey.So(rw.Code, convey.ShouldEqual, http.StatusInternalServerError) + convey.So(rw.Body.String(), convey.ShouldStartWith, "\"delete app failed") + }) + }) +} + +func TestStopJobHandler(t *testing.T) { + convey.Convey("test StopJobHandler", t, func() { + mock := &mockUtils.FakeLibruntimeSdkClient{} + util.SetAPIClientLibruntime(mock) + submissionId := "app-frontend-stop1" + gin.SetMode(gin.TestMode) + rw := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rw) + c.Request = &http.Request{ + Method: http.MethodGet, + URL: &url.URL{Path: job.PathGroupJobs + "/" + submissionId + "/stop"}, + Header: http.Header{ + "Content-Type": []string{"application/json"}, + constants.HeaderTenantID: []string{"123456"}, + constants.HeaderPoolLabel: []string{"abc"}, + }, + Body: nil, // 使用 io.NopCloser 包装 reader，使其满足 io.ReadCloser 接口 + } + c.AddParam(job.PathParamSubmissionId, submissionId) + convey.Convey("when job is not found", func() { + StopJobHandler(c) + convey.So(rw.Code, convey.ShouldEqual, http.StatusNotFound) + convey.So(rw.Body.String(), convey.ShouldStartWith, "\"the app does not exist, submissionId") + }) + convey.Convey("when process success", func() { + storeDefaultApp(submissionId, 3, 1) + StopJobHandler(c) + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + convey.So(rw.Body.String(), convey.ShouldEqual, "true") + }) + convey.Convey("when job is forbidden to stop, status: RUNNING", func() { + storeDefaultApp(submissionId, 6, 1) + StopJobHandler(c) + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + convey.So(rw.Body.String(), convey.ShouldEqual, "false") + }) + convey.Convey("when statusCode is "+strconv.Itoa(http.StatusInternalServerError), func() { + defer gomonkey.ApplyMethodFunc(mock, "Kill", + func(instanceID string, signal int, payload []byte) error { + return errors.New("failed stop app") + }).Reset() + storeDefaultApp(submissionId, 3, 1) + StopJobHandler(c) + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + convey.So(rw.Body.String(), convey.ShouldEqual, "true") + }) + }) +} diff --git a/frontend/pkg/frontend/api/lease/handler.go b/frontend/pkg/frontend/api/lease/handler.go new file mode 100644 index 0000000000000000000000000000000000000000..4b26ca756c042ed1f9b8ab653723aed19dd40c9b --- /dev/null +++ b/frontend/pkg/frontend/api/lease/handler.go @@ -0,0 +1,117 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package lease the api of lease scene +package lease + +import ( + "io" + "net/http" + + "github.com/gin-gonic/gin" + "google.golang.org/protobuf/proto" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/grpc/pb/lease" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/frontend/api/functionsystem" + "frontend/pkg/frontend/common/httputil" + "frontend/pkg/frontend/remoteclientlease" +) + +// NewLeaseHandler the handler of put +func NewLeaseHandler(ctx *gin.Context) { + traceID := getHeaderPrams(ctx) + msg, isOk := getLeaseRequest(ctx) + if !isOk { + return + } + log.GetLogger().Infof("receive new lease request, traceID: %s", traceID) + response := remoteclientlease.NewLease(msg.RemoteClientId, traceID) + respBody, parseErr := proto.Marshal(response) + if parseErr != nil { + log.GetLogger().Errorf("proto Marshal failed, err: %s", parseErr) + frontend.SetCtxResponse(ctx, respBody, http.StatusBadRequest) + return + } + log.GetLogger().Debugf("new lease response:%v, traceID: %s", response, traceID) + frontend.SetCtxResponse(ctx, respBody, http.StatusOK) +} + +// DelLeaseHandler the handler of del +func DelLeaseHandler(ctx *gin.Context) { + traceID := getHeaderPrams(ctx) + msg, isOk := getLeaseRequest(ctx) + if !isOk { + return + } + log.GetLogger().Infof("receive delete lease request, traceId: %s", traceID) + response := remoteclientlease.DelLease(msg.RemoteClientId, traceID) + respBody, parseErr := proto.Marshal(response) + if parseErr != nil { + log.GetLogger().Errorf("proto Marshal failed, err: %s", parseErr) + frontend.SetCtxResponse(ctx, respBody, http.StatusBadRequest) + return + } + log.GetLogger().Debugf("delete lease response:%v, traceID: %s", response, traceID) + frontend.SetCtxResponse(ctx, respBody, http.StatusOK) +} + +// KeepAliveHandler the handler of keep-alive +func KeepAliveHandler(ctx *gin.Context) { + traceID := getHeaderPrams(ctx) + msg, isOk := getLeaseRequest(ctx) + if !isOk { + return + } + log.GetLogger().Infof("receive keep-alive lease request, traceID: %s", traceID) + response := remoteclientlease.KeepAlive(msg.RemoteClientId, traceID) + respBody, parseErr := proto.Marshal(response) + if parseErr != nil { + log.GetLogger().Errorf("proto Marshal failed, err: %s", parseErr) + frontend.SetCtxResponse(ctx, respBody, http.StatusBadRequest) + return + } + log.GetLogger().Debugf("keep alive lease response:%v, traceID: %s", response, traceID) + frontend.SetCtxResponse(ctx, respBody, http.StatusOK) +} + +func getLeaseRequest(ctx *gin.Context) (*lease.LeaseRequest, bool) { + body, err := io.ReadAll(ctx.Request.Body) + if err != nil { + log.GetLogger().Errorf("failed to read request body error %s", err.Error()) + return nil, false + } + msg := &lease.LeaseRequest{} + err = proto.Unmarshal(body, msg) + if err != nil { + log.GetLogger().Errorf("failed to parse lease request, err: %s", err) + frontend.SetCtxResponse(ctx, nil, http.StatusBadRequest) + return nil, false + } + if msg.RemoteClientId == "" { + log.GetLogger().Errorf("failed to parse lease request, empty remote client id") + frontend.SetCtxResponse(ctx, nil, http.StatusBadRequest) + return nil, false + } + return msg, true +} + +func getHeaderPrams(ctx *gin.Context) string { + traceID := httputil.GetCompatibleGinHeader(ctx.Request, constant.HeaderTraceID, "traceId") + log.GetLogger().Debugf("check traceID in request header: %s", traceID) + return traceID +} diff --git a/frontend/pkg/frontend/api/lease/handler_test.go b/frontend/pkg/frontend/api/lease/handler_test.go new file mode 100644 index 0000000000000000000000000000000000000000..3b48454a7ef1effb6c9559e94c650ff6cc9f35c4 --- /dev/null +++ b/frontend/pkg/frontend/api/lease/handler_test.go @@ -0,0 +1,387 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package lease + +import ( + "bytes" + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/gin-gonic/gin" + "github.com/smartystreets/goconvey/convey" + clientv3 "go.etcd.io/etcd/client/v3" + "google.golang.org/protobuf/proto" + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/grpc/pb/lease" + "frontend/pkg/common/faas_common/types" + mockUtils "frontend/pkg/common/faas_common/utils" + "frontend/pkg/frontend/common/util" + "frontend/pkg/frontend/remoteclientlease" +) + +var in = &types.InstanceInfo{ + TenantID: "test-TenantID", + FunctionName: "test-faasmanager", + Version: "", + InstanceName: "test-faasnamager-instance", +} + +type KvMock struct { +} + +func (k *KvMock) Put(ctx context.Context, key, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error) { + return nil, nil +} + +func (k *KvMock) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + response := &clientv3.GetResponse{} + response.Count = 10 + return response, nil +} + +func (k *KvMock) Delete(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.DeleteResponse, error) { + //TODO implement me + panic("implement me") +} + +func (k *KvMock) Compact(ctx context.Context, rev int64, opts ...clientv3.CompactOption) (*clientv3.CompactResponse, error) { + //TODO implement me + panic("implement me") +} + +func (k *KvMock) Do(ctx context.Context, op clientv3.Op) (clientv3.OpResponse, error) { + //TODO implement me + panic("implement me") +} + +func (k *KvMock) Txn(ctx context.Context) clientv3.Txn { + //TODO implement me + panic("implement me") +} + +var event = etcd3.Event{ + Type: etcd3.PUT, + Key: "/sn/instance/business/yrk/tenant/1/function/test-faasmanager/version/latest/defaultaz/requestID/test-faasnamager-instance", + Value: []byte(`{ + "instanceID": "test-faasnamager-instance", + "instanceStatus": { + "code": 3, + "msg": "running" + }}`)} + +func TestNewLeaseHandler(t *testing.T) { + util.SetAPIClientLibruntime(&mockUtils.FakeLibruntimeSdkClient{}) + defer gomonkey.ApplyMethod(reflect.TypeOf(&mockUtils.FakeLibruntimeSdkClient{}), "GetAsync", + func(_ *mockUtils.FakeLibruntimeSdkClient, objectID string, cb api.GetAsyncCallback) { + cb([]byte{}, nil) + }).Reset() + convey.Convey("NewLeaseHandler", t, func() { + convey.Convey("failed to parse lease request, empty remote client id", func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + msg := &lease.LeaseRequest{} + body, _ := proto.Marshal(msg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + ctx.Request.Header.Add(constant.HeaderTraceID, "test-traceID") + remoteclientlease.UpdateFaasManager(&event, in) + NewLeaseHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + }) + convey.Convey("failed to unmarshal msg", func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + msg := &lease.LeaseRequest{ + RemoteClientId: "test-clientID", + } + body, _ := proto.Marshal(msg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + ctx.Request.Header.Add(constant.HeaderTraceID, "test-traceID") + remoteclientlease.UpdateFaasManager(&event, in) + defer gomonkey.ApplyFunc(proto.Unmarshal, func(b []byte, m proto.Message) error { + return errors.New("proto unmarshal error") + }).Reset() + NewLeaseHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + }) + convey.Convey("invoke failed", func() { + defer gomonkey.ApplyFunc(etcd3.GetRouterEtcdClient, func() *etcd3.EtcdClient { + return &etcd3.EtcdClient{Client: &clientv3.Client{KV: &KvMock{}}} + }).Reset() + defer gomonkey.ApplyMethod(reflect.TypeOf(&mockUtils.FakeLibruntimeSdkClient{}), "GetAsync", + func(_ *mockUtils.FakeLibruntimeSdkClient, objectID string, cb api.GetAsyncCallback) { + cb([]byte{}, errors.New("invoke failed")) + }).Reset() + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + msg := &lease.LeaseRequest{ + RemoteClientId: "test-clientID", + } + body, _ := proto.Marshal(msg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + ctx.Request.Header.Add(constant.HeaderTraceID, "test-traceID") + remoteclientlease.UpdateFaasManager(&event, in) + NewLeaseHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + }) + convey.Convey("OK", func() { + p := gomonkey.ApplyFunc(etcd3.GetRouterEtcdClient, func() *etcd3.EtcdClient { + return &etcd3.EtcdClient{Client: &clientv3.Client{KV: &KvMock{}}} + }) + defer p.Reset() + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + msg := &lease.LeaseRequest{ + RemoteClientId: "test-clientID", + } + body, _ := proto.Marshal(msg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + ctx.Request.Header.Add(constant.HeaderTraceID, "test-traceID") + remoteclientlease.UpdateFaasManager(&event, in) + NewLeaseHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + }) + convey.Convey("internal error", func() { + p := gomonkey.ApplyFunc(etcd3.GetRouterEtcdClient, func() *etcd3.EtcdClient { + return &etcd3.EtcdClient{Client: &clientv3.Client{KV: &KvMock{}}} + }) + defer p.Reset() + p2 := gomonkey.ApplyFunc((*KvMock).Put, func(_ *KvMock, ctx context.Context, key, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error) { + return nil, fmt.Errorf("error") + }) + defer p2.Reset() + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + msg := &lease.LeaseRequest{ + RemoteClientId: "test-clientID", + } + body, _ := proto.Marshal(msg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + patch := gomonkey.ApplyFunc(proto.Marshal, + func(m proto.Message) ([]byte, error) { + return nil, errors.New("some error") + }) + defer patch.Reset() + NewLeaseHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + }) + }) +} + +func TestDelLeaseHandler(t *testing.T) { + util.SetAPIClientLibruntime(&mockUtils.FakeLibruntimeSdkClient{}) + defer gomonkey.ApplyMethod(reflect.TypeOf(&mockUtils.FakeLibruntimeSdkClient{}), "GetAsync", + func(_ *mockUtils.FakeLibruntimeSdkClient, objectID string, cb api.GetAsyncCallback) { + cb([]byte{}, nil) + }).Reset() + convey.Convey("DelLeaseHandler", t, func() { + convey.Convey("failed to parse lease request, empty remote client id", func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + msg := &lease.LeaseRequest{} + body, _ := proto.Marshal(msg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + ctx.Request.Header.Add("traceId", "test-traceID") + remoteclientlease.UpdateFaasManager(&event, in) + DelLeaseHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + }) + convey.Convey("failed to unmarshal msg", func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + msg := &lease.LeaseRequest{ + RemoteClientId: "test-clientID", + } + body, _ := proto.Marshal(msg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + ctx.Request.Header.Add("traceId", "test-traceID") + remoteclientlease.UpdateFaasManager(&event, in) + defer gomonkey.ApplyFunc(proto.Unmarshal, func(b []byte, m proto.Message) error { + return errors.New("proto unmarshal error") + }).Reset() + DelLeaseHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + }) + convey.Convey("invoke failed", func() { + defer gomonkey.ApplyFunc(etcd3.GetRouterEtcdClient, func() *etcd3.EtcdClient { + return &etcd3.EtcdClient{Client: &clientv3.Client{KV: &KvMock{}}} + }).Reset() + defer gomonkey.ApplyMethod(reflect.TypeOf(&mockUtils.FakeLibruntimeSdkClient{}), "GetAsync", + func(_ *mockUtils.FakeLibruntimeSdkClient, objectID string, cb api.GetAsyncCallback) { + cb([]byte{}, errors.New("invoke failed")) + }).Reset() + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + msg := &lease.LeaseRequest{ + RemoteClientId: "test-clientID", + } + body, _ := proto.Marshal(msg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + ctx.Request.Header.Add("traceId", "test-traceID") + remoteclientlease.UpdateFaasManager(&event, in) + DelLeaseHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + }) + convey.Convey("OK", func() { + p := gomonkey.ApplyFunc(etcd3.GetRouterEtcdClient, func() *etcd3.EtcdClient { + return &etcd3.EtcdClient{Client: &clientv3.Client{KV: &KvMock{}}} + }) + defer p.Reset() + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + msg := &lease.LeaseRequest{ + RemoteClientId: "test-clientID", + } + body, _ := proto.Marshal(msg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + ctx.Request.Header.Add("traceId", "test-traceID") + remoteclientlease.UpdateFaasManager(&event, in) + DelLeaseHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + }) + convey.Convey("internal error", func() { + p := gomonkey.ApplyFunc(etcd3.GetRouterEtcdClient, func() *etcd3.EtcdClient { + return &etcd3.EtcdClient{Client: &clientv3.Client{KV: &KvMock{}}} + }) + defer p.Reset() + p2 := gomonkey.ApplyFunc((*KvMock).Put, func(_ *KvMock, ctx context.Context, key, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error) { + return nil, fmt.Errorf("error") + }) + defer p2.Reset() + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + msg := &lease.LeaseRequest{ + RemoteClientId: "test-clientID", + } + body, _ := proto.Marshal(msg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + patch := gomonkey.ApplyFunc(proto.Marshal, + func(m proto.Message) ([]byte, error) { + return nil, errors.New("some error") + }) + defer patch.Reset() + DelLeaseHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + }) + }) +} + +func TestKeepAliveHandler(t *testing.T) { + util.SetAPIClientLibruntime(&mockUtils.FakeLibruntimeSdkClient{}) + defer gomonkey.ApplyMethod(reflect.TypeOf(&mockUtils.FakeLibruntimeSdkClient{}), "GetAsync", + func(_ *mockUtils.FakeLibruntimeSdkClient, objectID string, cb api.GetAsyncCallback) { + cb([]byte{}, nil) + }).Reset() + convey.Convey("KeepAliveHandler", t, func() { + convey.Convey("failed to parse lease request, empty remote client id", func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + msg := &lease.LeaseRequest{} + body, _ := proto.Marshal(msg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + ctx.Request.Header.Add("traceId", "test-traceID") + remoteclientlease.UpdateFaasManager(&event, in) + KeepAliveHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + }) + convey.Convey("failed to unmarshal msg", func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + msg := &lease.LeaseRequest{ + RemoteClientId: "test-clientID", + } + body, _ := proto.Marshal(msg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + ctx.Request.Header.Add("traceId", "test-traceID") + remoteclientlease.UpdateFaasManager(&event, in) + defer gomonkey.ApplyFunc(proto.Unmarshal, func(b []byte, m proto.Message) error { + return errors.New("proto unmarshal error") + }).Reset() + KeepAliveHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + }) + convey.Convey("invoke failed", func() { + defer gomonkey.ApplyFunc(etcd3.GetRouterEtcdClient, func() *etcd3.EtcdClient { + return &etcd3.EtcdClient{Client: &clientv3.Client{KV: &KvMock{}}} + }).Reset() + defer gomonkey.ApplyMethod(reflect.TypeOf(&mockUtils.FakeLibruntimeSdkClient{}), "GetAsync", + func(_ *mockUtils.FakeLibruntimeSdkClient, objectID string, cb api.GetAsyncCallback) { + cb([]byte{}, errors.New("invoke failed")) + }).Reset() + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + msg := &lease.LeaseRequest{ + RemoteClientId: "test-clientID", + } + body, _ := proto.Marshal(msg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + ctx.Request.Header.Add("traceId", "test-traceID") + remoteclientlease.UpdateFaasManager(&event, in) + KeepAliveHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + }) + convey.Convey("OK", func() { + p := gomonkey.ApplyFunc(etcd3.GetRouterEtcdClient, func() *etcd3.EtcdClient { + return &etcd3.EtcdClient{Client: &clientv3.Client{KV: &KvMock{}}} + }) + defer p.Reset() + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + msg := &lease.LeaseRequest{ + RemoteClientId: "test-clientID", + } + body, _ := proto.Marshal(msg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + ctx.Request.Header.Add("traceId", "test-traceID") + remoteclientlease.UpdateFaasManager(&event, in) + KeepAliveHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + }) + convey.Convey("internal error", func() { + p := gomonkey.ApplyFunc(etcd3.GetRouterEtcdClient, func() *etcd3.EtcdClient { + return &etcd3.EtcdClient{Client: &clientv3.Client{KV: &KvMock{}}} + }) + defer p.Reset() + p2 := gomonkey.ApplyFunc((*KvMock).Put, func(_ *KvMock, ctx context.Context, key, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error) { + return nil, fmt.Errorf("error") + }) + defer p2.Reset() + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + msg := &lease.LeaseRequest{ + RemoteClientId: "test-clientID", + } + body, _ := proto.Marshal(msg) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(body)) + patch := gomonkey.ApplyFunc(proto.Marshal, + func(m proto.Message) ([]byte, error) { + return nil, errors.New("some error") + }) + defer patch.Reset() + KeepAliveHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + }) + }) +} diff --git a/frontend/pkg/frontend/api/v1/componentshealth.go b/frontend/pkg/frontend/api/v1/componentshealth.go new file mode 100644 index 0000000000000000000000000000000000000000..9d7ae99fe5d55f5252e9c81f5186c60cbf17ecd7 --- /dev/null +++ b/frontend/pkg/frontend/api/v1/componentshealth.go @@ -0,0 +1,56 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package v1 - +package v1 + +import ( + "encoding/json" + + "github.com/valyala/fasthttp" + + "frontend/pkg/common/faas_common/logger/log" +) + +const ( + healthy = "healthy" + + task = "functiontask" + instanceManager = "instancemanager" + functionAccessor = "functionaccessor" +) + +var components = [...]string{task, instanceManager, functionAccessor} + +type healthyStatus string + +// ComponentsHealthHandler - handler components health check request +func ComponentsHealthHandler(ctx *fasthttp.RequestCtx) { + // Temporary realization. return all healthy response same with fg + resultMap := make(map[string]healthyStatus, len(components)) + resultMap[functionAccessor] = healthy + resultMap[task] = healthy + resultMap[instanceManager] = healthy + bytes, err := json.Marshal(resultMap) + if err != nil { + log.GetLogger().Errorf("failed to marshal resultMap, error: %s", err.Error()) + ctx.Response.SetBody([]byte(err.Error())) + } else { + ctx.Response.SetBody(bytes) + } + ctx.Response.SetStatusCode(fasthttp.StatusOK) + return +} diff --git a/frontend/pkg/frontend/api/v1/componentshealth_test.go b/frontend/pkg/frontend/api/v1/componentshealth_test.go new file mode 100644 index 0000000000000000000000000000000000000000..95360932ed8f0b4cbaf0c76ffd719e2ab5333e23 --- /dev/null +++ b/frontend/pkg/frontend/api/v1/componentshealth_test.go @@ -0,0 +1,38 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package v1 + +import ( + "encoding/json" + "github.com/smartystreets/goconvey/convey" + "github.com/valyala/fasthttp" + "testing" +) + +func TestComponentsHealthHandler(t *testing.T) { + convey.Convey("ComponentsHealthHandler", t, func() { + ctx := &fasthttp.RequestCtx{} + ComponentsHealthHandler(ctx) + convey.So(ctx.Response.StatusCode(), convey.ShouldEqual, fasthttp.StatusOK) + resultMap := make(map[string]string) + err := json.Unmarshal(ctx.Response.Body(), &resultMap) + convey.So(err, convey.ShouldBeNil) + convey.So(resultMap["functiontask"], convey.ShouldEqual, "healthy") + convey.So(resultMap["instancemanager"], convey.ShouldEqual, "healthy") + convey.So(resultMap["functionaccessor"], convey.ShouldEqual, "healthy") + }) +} diff --git a/frontend/pkg/frontend/api/v1/healthz.go b/frontend/pkg/frontend/api/v1/healthz.go new file mode 100644 index 0000000000000000000000000000000000000000..04f474a722decbdfe725e1e0a160840df69f7b3b --- /dev/null +++ b/frontend/pkg/frontend/api/v1/healthz.go @@ -0,0 +1,37 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package v1 - +package v1 + +import ( + "net/http" + + "github.com/gin-gonic/gin" + + "frontend/pkg/frontend/clusterhealth" +) + +// HealthzHandler - +func HealthzHandler(ctx *gin.Context) { + ctx.Writer.WriteHeader(http.StatusOK) +} + +// ClusterHealthHandler - +func ClusterHealthHandler(ctx *gin.Context) { + clusterhealth.CheckClusterHealth(ctx.Writer, ctx.Request) + return +} diff --git a/frontend/pkg/frontend/api/v1/healthz_test.go b/frontend/pkg/frontend/api/v1/healthz_test.go new file mode 100644 index 0000000000000000000000000000000000000000..9edb284f38b4ec1faa1eda7c215dddc1959641ca --- /dev/null +++ b/frontend/pkg/frontend/api/v1/healthz_test.go @@ -0,0 +1,50 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package v1 + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/gin-gonic/gin" + "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/frontend/clusterhealth" +) + +func TestHealthzHandler(t *testing.T) { + convey.Convey("HealthzHandler", t, func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + HealthzHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + }) +} + +func TestClusterHealthHandler(t *testing.T) { + convey.Convey("ClusterHealthHandler", t, func() { + gomonkey.ApplyFunc(clusterhealth.CheckClusterHealth, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + ClusterHealthHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + }) +} diff --git a/frontend/pkg/frontend/api/v1/invoke.go b/frontend/pkg/frontend/api/v1/invoke.go new file mode 100644 index 0000000000000000000000000000000000000000..be1a5f6e23984ccd7771bac0412c565dc65c3e24 --- /dev/null +++ b/frontend/pkg/frontend/api/v1/invoke.go @@ -0,0 +1,259 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package v1 - +package v1 + +import ( + "fmt" + "io/ioutil" + "net/http" + "strings" + "time" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" + + "frontend/pkg/common/faas_common/aliasroute" + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/snerror" + "frontend/pkg/common/faas_common/statuscode" + "frontend/pkg/common/faas_common/urnutils" + "frontend/pkg/frontend/common" + "frontend/pkg/frontend/common/httpconstant" + "frontend/pkg/frontend/common/httputil" + "frontend/pkg/frontend/config" + frontendlog "frontend/pkg/frontend/log" + "frontend/pkg/frontend/middleware" + "frontend/pkg/frontend/responsehandler" + "frontend/pkg/frontend/stream" + "frontend/pkg/frontend/types" +) + +// InvokeHandler - +// Invocation godoc +// @Summary Invoke FaaS +// @Description 通过HTTP请求调用FaaS函数 +// @Accept json +// @Produce json +// @Router /serverless/v1/functions/{function-urn}/invocations [POST] +// @Param X-Instance-Cpu header string false "指定函数实例使用的CPU大小" +// @Param X-Instance-Memory header string false "指定函数实例使用的内存大小" +// @Param X-Instance-Custom-Resource header string false "指定函数实例使用的自定义资源大小" +// @Param X-Invoke-Alias header string false "指定函数的别名进行调用" +// @Param X-Stream-Apig-Event header string false "流式调用时通过此 Header 指定调用事件" +// @Param X-Log-Type header string false "指定函数调用是否需要日志回显，Tail标识需要回显" +// @Param X-Pool-Label header string false "指定函数实例池化启动时使用的资源池" +// @Param function-urn path string true "用户函数的URN" +// @Param invoke-event body string true "用户函数处理事件" +// @Success 200 {string} string "调用成功返回，格式由用户函数决定" +// @Failure 500 {object} types.InvokeErrorResponse "调用报错返回，包含错误码和错误信息" +// @Header 200,500 {string} X-Inner-Code "调用结果内部返回码" +// @Header 200 {string} X-Billing-Duration "本次调用计费信息" +// @Header 200 {string} X-Invoke-Summary "本次调用摘要信息" +// @Header 200 {string} X-Log-Result "调用过程中产生日志" +func InvokeHandler(ctx *gin.Context) { + traceID := httputil.InitTraceID(ctx) + logger := log.GetLogger().With(zap.Any("traceId", traceID)) + logger.Infof("invoking handler receives one request") + + processCtx, err := buildProcessContext(ctx, traceID) + if err != nil { + logger.Errorf("failed to set processCtx req, error: %s", err.Error()) + writeHTTPResponse(ctx, processCtx) + return + } + defer writeInterfaceLog(processCtx) + logger = logger.With(zap.Any("funcKey", processCtx.FuncKey)) + if err := middleware.Invoker.Handle(processCtx); err != nil { + logger.Errorf("invoke failed,error: %s", err.Error()) + } + writeHTTPResponse(ctx, processCtx) + sessionId := processCtx.ReqHeader[httpconstant.HeaderInstanceSession] + instanceLabel := processCtx.ReqHeader[httpconstant.HeaderInstanceLabel] + logger.Infof("invoke function success, totalTime %f, sessionId %s, instanceLabel %s", + time.Since(processCtx.StartTime).Seconds(), sessionId, instanceLabel) +} + +func writeInterfaceLog(invokeCtx *types.InvokeProcessContext) { + if invokeCtx.RequestTraceInfo == nil { + log.GetLogger().Errorf("write invoke interface log failed, traceIno is nil") + return + } + + totalTime := time.Since(invokeCtx.StartTime) + tenantId := invokeCtx.RequestTraceInfo.TenantID + funcName := invokeCtx.RequestTraceInfo.FuncName + version := invokeCtx.RequestTraceInfo.Version + + splits := strings.Split(invokeCtx.FuncKey, urnutils.FunctionKeySep) // {tenantid}@{funtionName}@{version} + + if len(splits) == 3 && config.GetConfig().BusinessType != constant.BusinessTypeFG { // magicnumber + tenantId = splits[0] // tenantIdIndex + funcName = splits[1] // funcNameIndex + version = splits[2] // versionIndex + } + + message := "OK" + if invokeCtx.StatusCode != http.StatusOK { + message = string(invokeCtx.RespBody) + } + if len(message) > 100 { // 仅保留前100个字符 + message = message[:100] // 仅保留前100个字符 + } + // tenantId | funcName | version | sessionId | instanceLabel | statusCode | code | totalCost | + logContent := fmt.Sprintf("invocation |%s | %s | %s | %s | %s | %d | %s | %.2f | %s", + tenantId, funcName, version, + invokeCtx.ReqHeader[httpconstant.HeaderInstanceSession], + invokeCtx.ReqHeader[httpconstant.HeaderInstanceLabel], + invokeCtx.StatusCode, + invokeCtx.RespHeader[httpconstant.HeaderInnerCode], + totalTime.Seconds()*1000, // 秒转换成毫秒 + message) + + frontendlog.Write(logContent) +} + +func buildProcessContext(ctx *gin.Context, traceID string) (processCtx *types.InvokeProcessContext, err error) { + processCtx = types.CreateInvokeProcessContext() + processCtx.TraceID = traceID + processCtx.RequestID = traceID + + var ( + funcUrn urnutils.FunctionURN + plainURN string + ) + defer func() { + if err != nil { + processCtx.StatusCode = http.StatusBadRequest + responsehandler.SetErrorInContextWithDefault(processCtx, err, statuscode.FrontendStatusBadRequest, + err.Error()) + } + }() + err = handleRequestBodyAndStream(ctx, processCtx, traceID) + if err != nil { + return + } + processCtx.ReqHeader = readHeaders(ctx.Request.Header) + processCtx.ReqPath = ctx.Request.URL.Path + processCtx.ReqMethod = ctx.Request.Method + processCtx.ReqQuery = ctx.Request.URL.RawQuery + funcUrn, plainURN, err = extractFunctionURN(ctx, processCtx.ReqHeader) + if err != nil { + return + } + processCtx.FuncKey = urnutils.CombineFunctionKey(funcUrn.TenantID, funcUrn.FuncName, funcUrn.FuncVersion) + if config.GetConfig().BusinessType == constant.BusinessTypeFG { + if err = processContextForFG(ctx, processCtx, plainURN, funcUrn); err != nil { + return + } + } + return +} + +func handleRequestBodyAndStream(ctx *gin.Context, processCtx *types.InvokeProcessContext, traceID string) error { + stream.BuildStreamContext(ctx, processCtx) + if stream.IsHTTPUploadStream(ctx.Request) { + if !config.GetConfig().StreamEnable { + log.GetLogger().With(zap.String("traceID", traceID)).Warnf("not enable to support http stream") + return snerror.New(statuscode.HTTPStreamNOTEnableError, statuscode.InternalErrorMessage) + } + processCtx.IsHTTPUploadStream = true + } else { + var err error + processCtx.ReqBody, err = ioutil.ReadAll(ctx.Request.Body) + if err != nil { + return err + } + } + return nil +} + +func writeHTTPResponse(ctx *gin.Context, processCtx *types.InvokeProcessContext) { + // It has to be in this order. 1. set header 2.writeHeader 3.write + writeHeadersToResponse(processCtx.RespHeader, ctx.Writer.Header()) + ctx.Writer.WriteHeader(processCtx.StatusCode) + _, err := ctx.Writer.Write(processCtx.RespBody) + if err != nil { + log.GetLogger().Errorf("failed to write response body error %s", err.Error()) + } +} + +func readHeaders(header http.Header) map[string]string { + headerMap := make(map[string]string) + for key := range header { + headerMap[key] = header.Get(key) + } + return headerMap +} + +func writeHeadersToResponse(headerMap map[string]string, header http.Header) { + for key, value := range headerMap { + header.Set(key, value) + } +} + +func extractFunctionURN(c *gin.Context, reqHeaders map[string]string) (urnutils.FunctionURN, string, error) { + plainURN := c.Param(common.FunctionUrnParam) + params := make(map[string]string) + for k, v := range reqHeaders { + params[strings.ToLower(k)] = v + } + functionURN := aliasroute.GetAliases().GetFuncVersionURNWithParams(plainURN, params) + functionInfo, err := urnutils.GetFunctionInfo(functionURN) + if err != nil { + return urnutils.FunctionURN{}, "", err + } + return functionInfo, plainURN, nil +} + +func processContextForFG(c *gin.Context, processCtx *types.InvokeProcessContext, + plainURN string, functionInfo urnutils.FunctionURN) error { + anonymizeURN := urnutils.AnonymizeTenantURN(plainURN) + + log.GetLogger().Debugf("request URN is coming: %s, alias: %s traceID: %s", + anonymizeURN, c.Request.Header.Get(constant.HeaderInvokeAlias), processCtx.TraceID) + + if err := functionInfo.Valid(); err != nil { + return fmt.Errorf("invalid function name,err is %s", err) + } + if functionInfo.BusinessID == "" || functionInfo.TenantID == "" || functionInfo.FuncName == "" { + return fmt.Errorf("wrong function name %s", plainURN) + } + + urn, version := getURNWithVersion(functionInfo.FuncVersion, plainURN) + processCtx.RequestTraceInfo = &types.RequestTraceInfo{ + URN: urn, + AnonymizeURN: anonymizeURN, + BusinessID: functionInfo.BusinessID, + TenantID: functionInfo.TenantID, + FuncName: functionInfo.FuncName, + Version: version, + } + return nil +} + +func getURNWithVersion(version string, plainURN string) (string, string) { + var newURN string + if version == "" { + version = "latest" + newURN = plainURN + urnutils.URNSep + version + } else { + newURN = plainURN + } + return newURN, version +} diff --git a/frontend/pkg/frontend/api/v1/invoke_test.go b/frontend/pkg/frontend/api/v1/invoke_test.go new file mode 100644 index 0000000000000000000000000000000000000000..ff1d7ec1490922dc1a382d6a9217a4bcdfad58af --- /dev/null +++ b/frontend/pkg/frontend/api/v1/invoke_test.go @@ -0,0 +1,497 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package v1 - +package v1 + +import ( + "bytes" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "reflect" + "strconv" + "strings" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/gin-gonic/gin" + "github.com/smartystreets/goconvey/convey" + "github.com/valyala/fasthttp" + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/constant" + frontend "frontend/pkg/common/faas_common/grpc/pb/function" + "frontend/pkg/common/faas_common/localauth" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/monitor" + "frontend/pkg/common/faas_common/statuscode" + "frontend/pkg/common/faas_common/tls" + commontype "frontend/pkg/common/faas_common/types" + "frontend/pkg/common/faas_common/urnutils" + "frontend/pkg/frontend/common/httpconstant" + "frontend/pkg/frontend/common/util" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/functionmeta" + "frontend/pkg/frontend/functiontask" + "frontend/pkg/frontend/instancemanager" + "frontend/pkg/frontend/invocation" + "frontend/pkg/frontend/middleware" + "frontend/pkg/frontend/responsehandler" + "frontend/pkg/frontend/schedulerproxy" + "frontend/pkg/frontend/stream" + "frontend/pkg/frontend/tenanttrafficlimit" + "frontend/pkg/frontend/types" +) + +func constructFakeInvokeRequest(funcName, reqBody string, rw http.ResponseWriter) *gin.Context { + ctx, _ := gin.CreateTestContext(rw) + bodyMarshal, _ := json.Marshal(reqBody) + ctx.Request, _ = http.NewRequest("POST", "/test", bytes.NewBuffer(bodyMarshal)) + ctx.AddParam("function-urn", funcName) + return ctx +} + +type fakeClient struct { +} + +func (f *fakeClient) AcquireInstance(functionKey string, req util.AcquireOption) (*commontype.InstanceAllocationInfo, error) { + //TODO implement me + panic("implement me") +} + +func (f *fakeClient) ReleaseInstance(allocation *commontype.InstanceAllocationInfo, abnormal bool) { + //TODO implement me + panic("implement me") +} + +func (f *fakeClient) Invoke(req util.InvokeRequest) ([]byte, error) { + //TODO implement me + panic("implement me") +} + +func (f *fakeClient) CreateInstanceRaw(createReq []byte) ([]byte, error) { + return nil, nil +} +func (f *fakeClient) InvokeInstanceRaw(invokeReq []byte) ([]byte, error) { + return nil, nil +} +func (f *fakeClient) KillRaw(killReq []byte) ([]byte, error) { + return nil, nil +} +func (c *fakeClient) CreateInstanceByLibRt(funcMeta api.FunctionMeta, args []api.Arg, invokeOpt api.InvokeOptions) (instanceID string, err error) { + InstanceID := "" + return InstanceID, nil +} +func (c *fakeClient) KillByLibRt(instanceID string, signal int, payload []byte) error { + return nil +} + +// InvokeByName copy from faasinvoker_test.go +func (f *fakeClient) InvokeByName(request util.InvokeRequest) ([]byte, error) { + req := &types.CallReq{ + Header: map[string]string{}, + } + json.Unmarshal(request.Args[1].Data, req) + + resp := &types.CallResp{ + InnerCode: strconv.Itoa(statuscode.InnerResponseSuccessCode), + Body: req.Body, + } + return json.Marshal(resp) +} + +func (f *fakeClient) IsHealth() bool { + return true +} + +func (f *fakeClient) IsDsHealth() bool { + return true +} + +type fakeFailedClient struct { +} + +func (c *fakeFailedClient) AcquireInstance(functionKey string, req util.AcquireOption) (*commontype.InstanceAllocationInfo, error) { + //TODO implement me + panic("implement me") +} + +func (c *fakeFailedClient) ReleaseInstance(allocation *commontype.InstanceAllocationInfo, abnormal bool) { + //TODO implement me + panic("implement me") +} + +func (c *fakeFailedClient) Invoke(req util.InvokeRequest) ([]byte, error) { + //TODO implement me + panic("implement me") +} + +func (c *fakeFailedClient) IsLibruntime() bool { + return false +} +func (c *fakeFailedClient) CreateInstanceRaw(createReq []byte) ([]byte, error) { + return nil, nil +} +func (c *fakeFailedClient) InvokeInstanceRaw(invokeReq []byte) ([]byte, error) { + return nil, nil +} +func (c *fakeFailedClient) KillRaw(killReq []byte) ([]byte, error) { + return nil, nil +} + +func (f *fakeFailedClient) IsHealth() bool { + return false +} + +func (f *fakeFailedClient) IsDsHealth() bool { + return true +} + +// Invoke - +func (c *fakeFailedClient) InvokeByName(request util.InvokeRequest) ([]byte, error) { + req := &types.CallReq{ + Header: map[string]string{}, + } + json.Unmarshal(request.Args[1].Data, req) + + resp := &types.CallResp{ + InnerCode: strconv.Itoa(statuscode.InternalErrorCode), + Body: json.RawMessage("\"runtime initialization timed out after 3s\""), + } + res, _ := json.Marshal(resp) + return res, errors.New("runtime initialization timed out after 3s") +} + +func (c *fakeFailedClient) CreateInstance(req *frontend.CreateRequest) (*frontend.CreateResponse, error) { + //TODO implement me + panic("implement me") +} +func (c *fakeFailedClient) InvokeInstance(req *frontend.InvokeRequest) (*frontend.NotifyRequest, error) { + //TODO implement me + panic("implement me") +} +func (c *fakeFailedClient) Kill(req *frontend.KillRequest) (*frontend.KillResponse, error) { + //TODO implement me + panic("implement me") +} + +func (c *fakeFailedClient) CreateInstanceByLibRt(funcMeta api.FunctionMeta, args []api.Arg, invokeOpt api.InvokeOptions) (instanceID string, err error) { + InstanceID := "" + return InstanceID, nil +} +func (c *fakeFailedClient) KillByLibRt(instanceID string, signal int, payload []byte) error { + return nil +} + +func fakeCaaSInvokeHandler(ctx *types.InvokeProcessContext) error { + return nil +} + +func Test_InvokeHandler(t *testing.T) { + patches := []*gomonkey.Patches{ + gomonkey.ApplyFunc(util.NewClient, func() util.Client { + return &fakeClient{} + }), + gomonkey.ApplyFunc(functionmeta.LoadFuncSpec, func(funcKey string) (*commontype.FuncSpec, bool) { + return &commontype.FuncSpec{FunctionKey: funcKey, FuncMetaData: commontype.FuncMetaData{Timeout: 10}}, true + }), + // new mock + gomonkey.ApplyFunc(config.GetConfig, func() *types.Config { + return &types.Config{ + HTTPConfig: &types.FrontendHTTP{MaxRequestBodySize: 1}, + MemoryEvaluatorConfig: &types.MemoryEvaluatorConfig{ + RequestMemoryEvaluator: 2, + }, + DefaultTenantLimitQuota: 1800, + } + }), + gomonkey.ApplyMethod(reflect.TypeOf(instancemanager.GetFaaSSchedulerInstanceManager()), "IsExist", func(_ *instancemanager.FaaSSchedulerInstanceManager) bool { + return true + }), + } + defer func() { + for _, patch := range patches { + time.Sleep(100 * time.Millisecond) + patch.Reset() + } + }() + fgAdapter := &invocation.FGAdapter{} + responsehandler.Handler = fgAdapter.MakeResponseHandler() + middleware.Invoker = fgAdapter.MakeInvoker() + urnutils.SetSeparator(urnutils.TenantProductSplitStr) + stopCh := make(chan struct{}) + _ = monitor.InitMemMonitor(stopCh) + funcNameDemo := "functions/sn:cn:yrk:xxxxxxxxxxx:function:0@base@testpythonbase001:latest" + reqBody := "test body" + schedulerproxy.Proxy.Add(&commontype.InstanceInfo{InstanceName: "instance1", InstanceID: "instance1", Address: "127.0.0.1"}, log.GetLogger()) + + convey.Convey("stream not enable", t, func() { + rw := httptest.NewRecorder() + ctx := constructFakeInvokeRequest("", reqBody, rw) + defer gomonkey.ApplyFunc(stream.IsHTTPUploadStream, func(r interface{}) bool { + return true + }).Reset() + InvokeHandler(ctx) + t.Logf("test stream not enable, rsp: %s", rw.Body.String()) + convey.So(rw.Body.String(), convey.ShouldContainSubstring, "internal system error") + convey.So(rw.Code, convey.ShouldEqual, http.StatusInternalServerError) + }) + + testFgStreamException(t, funcNameDemo, reqBody) + + convey.Convey("big body", t, func() { + rw := httptest.NewRecorder() + reqBigBody := strings.Repeat("a", 6*1024*1024) + ctx := constructFakeInvokeRequest(funcNameDemo, reqBigBody, rw) + InvokeHandler(ctx) + t.Logf("req body len: %d\n", rw.Body.Len()) + convey.So(rw.Body.String(), convey.ShouldContainSubstring, "the size of request body is beyond") + convey.So(rw.Code, convey.ShouldEqual, http.StatusInternalServerError) + }) + + convey.Convey("failed to set processCtx req", t, func() { + rw := httptest.NewRecorder() + ctx := constructFakeInvokeRequest("", reqBody, rw) + ctx.Params = make(gin.Params, 0, 0) + InvokeHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + }) + + convey.Convey("tenant traffic limit", t, func() { + rw := httptest.NewRecorder() + ctx := constructFakeInvokeRequest(funcNameDemo, reqBody, rw) + defer gomonkey.ApplyFunc(tenanttrafficlimit.Limit, func(tenantID string) error { + return errors.New("traffic limit") + }).Reset() + InvokeHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, http.StatusInternalServerError) + }) + convey.Convey("invoke success", t, func() { + defer gomonkey.ApplyFunc(util.NewClient, func() util.Client { + return &fakeClient{} + }).Reset() + rw := httptest.NewRecorder() + ctx := constructFakeInvokeRequest(funcNameDemo, reqBody, rw) + InvokeHandler(ctx) + t.Logf("body %s\n", rw.Body.String()) + convey.So(rw.Body.String(), convey.ShouldEqual, "\"test body\"") + convey.So(rw.Code, convey.ShouldEqual, 200) + }) + convey.Convey("invoke failed", t, func() { + defer gomonkey.ApplyFunc(util.NewClient, func() util.Client { + return &fakeFailedClient{} + }).Reset() + rw := httptest.NewRecorder() + ctx := constructFakeInvokeRequest(funcNameDemo, reqBody, rw) + InvokeHandler(ctx) + t.Logf("body %s\n", rw.Body.String()) + convey.So(rw.Body.String(), convey.ShouldContainSubstring, "runtime initialization timed out after 3s") + convey.So(rw.Code, convey.ShouldEqual, 500) + }) + convey.Convey("invoke for fg success", t, func() { + resp := &commontype.InstanceResponse{ + InstanceAllocationInfo: commontype.InstanceAllocationInfo{ + FuncKey: "xxxxxxxxxxx/0@base@testpythonbase001/latest", + ThreadID: "lease1-1", + InstanceID: "lease1", LeaseInterval: 100000, + }, + ErrorCode: constant.InsReqSuccessCode, + ErrorMessage: "", + SchedulerTime: 0, + } + body, _ := json.Marshal(resp) + c := &fasthttp.Client{} + defer gomonkey.ApplyMethod(reflect.TypeOf(c), + "DoTimeout", func(c *fasthttp.Client, req *fasthttp.Request, + resp *fasthttp.Response, timeout time.Duration) error { + resp.Header.Set(constant.HeaderInnerCode, "0") + resp.Header.Set(constant.HeaderWorkerCost, "20") + resp.Header.Set(constant.HeaderCallNode, "node1") + resp.Header.Set(constant.HeaderCallInstance, "instance1") + resp.SetBody(body) + resp.SetStatusCode(200) + return nil + }).Reset() + defer gomonkey.ApplyMethod(reflect.TypeOf(functiontask.GetBusProxies()), "IsBusProxyHealthy", + func(_ *functiontask.BusProxies, _ string, _ string) bool { + return true + }).Reset() + defer gomonkey.ApplyFunc(config.GetConfig, func() *types.Config { + return &types.Config{ + FunctionInvokeBackend: constant.BackendTypeFG, + MemoryEvaluatorConfig: &types.MemoryEvaluatorConfig{ + RequestMemoryEvaluator: 2, + }, + DefaultTenantLimitQuota: 1800, + HTTPConfig: &types.FrontendHTTP{ + WorkerInstanceReadTimeOut: 60, + MaxRequestBodySize: 1, + }, + HTTPSConfig: &tls.InternalHTTPSConfig{}, + E2EMaxDelayTime: 60, + LocalAuth: &localauth.AuthConfig{ + AKey: "ak", + SKey: "sk", + Duration: 5, + }, + InvokeMaxRetryTimes: 3, + RetryConfig: &types.RetryConfig{}, + } + + }).Reset() + rw := httptest.NewRecorder() + ctx := constructFakeInvokeRequest(funcNameDemo, reqBody, rw) + InvokeHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, 200) + convey.So(rw.Header().Get(constant.HeaderCallNode), convey.ShouldEqual, "node1") + convey.So(rw.Header().Get(constant.HeaderCallInstance), convey.ShouldEqual, "instance1") + time.Sleep(150 * time.Millisecond) + }) + convey.Convey("invoke for fg failed", t, func() { + resp := &commontype.InstanceResponse{ + InstanceAllocationInfo: commontype.InstanceAllocationInfo{ + FuncKey: "xxxxxxxxxxx/0@base@testpythonbase001/latest", + ThreadID: "lease1-1", + InstanceID: "lease1", LeaseInterval: 100000, + }, + ErrorCode: constant.InsReqSuccessCode, + ErrorMessage: "", + SchedulerTime: 0, + } + body, _ := json.Marshal(resp) + defer gomonkey.ApplyMethod(reflect.TypeOf(functiontask.GetBusProxies()), "IsBusProxyHealthy", + func(_ *functiontask.BusProxies, _ string, _ string) bool { + return true + }).Reset() + c := &fasthttp.Client{} + defer gomonkey.ApplyMethod(reflect.TypeOf(c), + "DoTimeout", func(c *fasthttp.Client, req *fasthttp.Request, + resp *fasthttp.Response, timeout time.Duration) error { + resp.Header.Set(constant.HeaderInnerCode, "200500") + resp.Header.Set(constant.HeaderWorkerCost, "20") + resp.Header.Set(constant.HeaderCallNode, "node1") + resp.Header.Set(constant.HeaderCallInstance, "instance1") + resp.SetBody(body) + resp.SetStatusCode(200) + return nil + }).Reset() + defer gomonkey.ApplyFunc(config.GetConfig, func() *types.Config { + return &types.Config{ + FunctionInvokeBackend: constant.BackendTypeFG, + MemoryEvaluatorConfig: &types.MemoryEvaluatorConfig{ + RequestMemoryEvaluator: 2, + }, + DefaultTenantLimitQuota: 1800, + HTTPConfig: &types.FrontendHTTP{ + WorkerInstanceReadTimeOut: 60, + MaxRequestBodySize: 1, + }, + HTTPSConfig: &tls.InternalHTTPSConfig{}, + E2EMaxDelayTime: 60, + LocalAuth: &localauth.AuthConfig{ + AKey: "ak", + SKey: "sk", + Duration: 5, + }, + InvokeMaxRetryTimes: 2, + RetryConfig: &types.RetryConfig{ + InstanceExceptionRetry: true, + }, + } + }).Reset() + rw := httptest.NewRecorder() + ctx := constructFakeInvokeRequest(funcNameDemo, reqBody, rw) + InvokeHandler(ctx) + convey.So(rw.Code, convey.ShouldEqual, 200) + convey.So(rw.Header().Get(constant.HeaderInnerCode), convey.ShouldEqual, "200500") + time.Sleep(150 * time.Millisecond) + }) + convey.Convey("grace exit", t, func() { + middleware.GraceExit() + rw := httptest.NewRecorder() + ctx := constructFakeInvokeRequest(funcNameDemo, reqBody, rw) + InvokeHandler(ctx) + t.Logf("body: %s\n", rw.Body.String()) + convey.So(rw.Body.String(), convey.ShouldEqual, "frontend exiting") + convey.So(rw.Code, convey.ShouldEqual, http.StatusBadRequest) + }) +} + +func TestExtractFunctionKey(t *testing.T) { + functionURN := "sn:cn:yrk:12345678901234561234567890123456:function:0@yrservice@test-faas-python-runtime-001" + ctx := &gin.Context{} + ctx.AddParam("function-urn", functionURN) + type args struct { + ctx *gin.Context + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "TestExtractFunctionKey", + args: args{ + ctx: ctx, + }, + want: "12345678901234561234567890123456/0@yrservice@test-faas-python-runtime-001/", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + funcUrn, _, err := extractFunctionURN(ctx, make(map[string]string)) + funcKey := urnutils.CombineFunctionKey(funcUrn.TenantID, funcUrn.FuncName, funcUrn.FuncVersion) + if (err != nil) != tt.wantErr { + t.Errorf("ExtractFunctionKey() error = %v, wantErr %v", err, tt.wantErr) + return + } + if funcKey != tt.want { + t.Errorf("ExtractFunctionKey() got = %v, want %v", funcKey, tt.want) + } + }) + } +} + +func mockFgStreamReqConfig() func() *types.Config { + return func() *types.Config { + return &types.Config{ + FunctionInvokeBackend: constant.BackendTypeFG, + MemoryEvaluatorConfig: &types.MemoryEvaluatorConfig{ + RequestMemoryEvaluator: 2, + }, + DefaultTenantLimitQuota: 1800, + HTTPConfig: &types.FrontendHTTP{ + WorkerInstanceReadTimeOut: 60, + MaxRequestBodySize: 1, + MaxStreamRequestBodySize: 1, + }, + HTTPSConfig: &tls.InternalHTTPSConfig{}, + E2EMaxDelayTime: 60, + LocalAuth: &localauth.AuthConfig{ + AKey: "ak", + SKey: "sk", + Duration: 5}, + InvokeMaxRetryTimes: 3, + RetryConfig: &types.RetryConfig{}, + StreamEnable: true, + } + } +} diff --git a/frontend/pkg/frontend/api/v1/proxyhandler.go b/frontend/pkg/frontend/api/v1/proxyhandler.go new file mode 100644 index 0000000000000000000000000000000000000000..49b0893c7654289f5bcf7242d923d0a29b50c256 --- /dev/null +++ b/frontend/pkg/frontend/api/v1/proxyhandler.go @@ -0,0 +1,48 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package v1 + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/frontend/common" + "frontend/pkg/frontend/common/httputil" + "frontend/pkg/frontend/functionmeta" +) + +// ProxyHandler - +func ProxyHandler(ctx *gin.Context) { + traceID := httputil.InitTraceID(ctx) + logger := log.GetLogger().With(zap.Any("traceId", traceID)) + logger.Infof("route proxy handler receives one request") + path := ctx.Request.URL.Path + funcSpc, ok := functionmeta.LoadFuncSpecWithPath(path, traceID) + if !ok { + logger.Infof("load funcSpec with path failed, path: %s,traceID %s", path, traceID) + ctx.Writer.WriteHeader(http.StatusNotFound) + ctx.String(http.StatusNotFound, "404 page not found") + return + } + ctx.AddParam(common.FunctionUrnParam, funcSpc.FuncMetaData.FunctionVersionURN) + ctx.Request.Header.Set(constant.HeaderRequestID, traceID) + InvokeHandler(ctx) +} diff --git a/frontend/pkg/frontend/api/v1/proxyhandler_test.go b/frontend/pkg/frontend/api/v1/proxyhandler_test.go new file mode 100644 index 0000000000000000000000000000000000000000..e3533459eaa4fbe9bd81bc77fdd8f83695cec7fc --- /dev/null +++ b/frontend/pkg/frontend/api/v1/proxyhandler_test.go @@ -0,0 +1,70 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package v1 - +package v1 + +import ( + "bytes" + "net/http" + "net/http/httptest" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/gin-gonic/gin" + "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/common/faas_common/types" + "frontend/pkg/frontend/common" + "frontend/pkg/frontend/functionmeta" +) + +func TestProxyHandler(t *testing.T) { + convey.Convey("invoke path not found", t, func() { + defer gomonkey.ApplyFunc(functionmeta.LoadFuncSpecWithPath, + func(path string, traceID string) (*types.FuncSpec, bool) { + return &types.FuncSpec{}, false + }).Reset() + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + ctx.Request, _ = http.NewRequest("POST", "/hello", bytes.NewBuffer([]byte("hello"))) + ProxyHandler(ctx) + convey.So(rw.Body.String(), convey.ShouldEqual, "404 page not found") + convey.So(rw.Code, convey.ShouldEqual, http.StatusNotFound) + }) + convey.Convey("proxy success", t, func() { + defer gomonkey.ApplyFunc(functionmeta.LoadFuncSpecWithPath, + func(path string, traceID string) (*types.FuncSpec, bool) { + return &types.FuncSpec{ + FuncMetaData: types.FuncMetaData{ + Name: "testFunc", + FunctionVersionURN: "123/testFunc/latest", + }, + }, true + }).Reset() + defer gomonkey.ApplyFunc(InvokeHandler, func(ctx *gin.Context) { + ctx.Writer.WriteHeader(http.StatusOK) + ctx.String(http.StatusOK, "ok") + }).Reset() + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + ctx.Request, _ = http.NewRequest("POST", "/hello", bytes.NewBuffer([]byte("hello"))) + ProxyHandler(ctx) + convey.So(ctx.Param(common.FunctionUrnParam), convey.ShouldEqual, "123/testFunc/latest") + convey.So(rw.Body.String(), convey.ShouldEqual, "ok") + convey.So(rw.Code, convey.ShouldEqual, http.StatusOK) + }) +} diff --git a/frontend/pkg/frontend/api/v1/subscribe.go b/frontend/pkg/frontend/api/v1/subscribe.go new file mode 100644 index 0000000000000000000000000000000000000000..6f6d6e9274d4cb0dc950ec871096bc5344167542 --- /dev/null +++ b/frontend/pkg/frontend/api/v1/subscribe.go @@ -0,0 +1,82 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package v1 - +package v1 + +import ( + "io" + "net/http" + + "github.com/gin-gonic/gin" + + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/frontend/common/httpconstant" + "frontend/pkg/frontend/stream" + "frontend/pkg/frontend/types" +) + +// SubscribeHandler - +func SubscribeHandler(ctx *gin.Context) { + processCtx := types.CreateInvokeProcessContext() + setStreamProcessCtxReq(ctx, processCtx) + streamName := processCtx.ReqHeader[httpconstant.HeaderStreamName] + log.GetLogger().Infof("subscribe handler receives one request with stream name: %s", streamName) + err := stream.SubscribeHandler(processCtx) + if err != nil { + setStreamProcessCtxResp(ctx, processCtx) + log.GetLogger().Errorf("failed to handle request stream %s, error: %s", streamName, err.Error()) + } else { + errorSubscribe := stream.StartSubscribeStream(processCtx, ctx) + if errorSubscribe != nil { + log.GetLogger().Errorf("failed to subscribe stream %s, error: %s", + streamName, errorSubscribe.Error()) + setStreamProcessCtxResp(ctx, processCtx) + } + } +} + +func setStreamProcessCtxReq(ctx *gin.Context, processCtx *types.InvokeProcessContext) { + body, err := io.ReadAll(ctx.Request.Body) + if err != nil { + log.GetLogger().Errorf("get request body failed, err: %v", err) + ctx.Writer.WriteHeader(http.StatusUnauthorized) + ctx.Writer.WriteString("get request body") + return + } + processCtx.ReqBody = body + processCtx.ReqHeader = readStreamReqHeaders(ctx) +} + +func setStreamProcessCtxResp(ctx *gin.Context, processCtx *types.InvokeProcessContext) { + ctx.Writer.WriteHeader(processCtx.StatusCode) + ctx.Writer.Write(processCtx.RespBody) + writeHeadersToStreamResponse(processCtx.RespHeader, ctx) +} + +func readStreamReqHeaders(ctx *gin.Context) map[string]string { + headerMap := make(map[string]string) + for key := range ctx.Request.Header { + headerMap[key] = ctx.Request.Header.Get(key) + } + return headerMap +} + +func writeHeadersToStreamResponse(headers map[string]string, ctx *gin.Context) { + for key, value := range headers { + ctx.Writer.Header().Set(key, value) + } +} diff --git a/frontend/pkg/frontend/api/v1/subscribe_test.go b/frontend/pkg/frontend/api/v1/subscribe_test.go new file mode 100644 index 0000000000000000000000000000000000000000..0fc4bc9c47bf40ec186cf0f6a40da07818630601 --- /dev/null +++ b/frontend/pkg/frontend/api/v1/subscribe_test.go @@ -0,0 +1,97 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package v1 - +package v1 + +import ( + "bytes" + "net/http" + "net/http/httptest" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/gin-gonic/gin" + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + + "frontend/pkg/common/faas_common/datasystemclient" + "frontend/pkg/frontend/common/httpconstant" + "frontend/pkg/frontend/common/util" + "frontend/pkg/frontend/invocation" + "frontend/pkg/frontend/middleware" + "frontend/pkg/frontend/responsehandler" +) + +func constructFakeSubscribeRequest(streamName string, timeoutMs string, expectReceiveNum string) (*gin.Context, *httptest.ResponseRecorder) { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer([]byte(""))) + ctx.Request.Header.Set(httpconstant.HeaderStreamName, streamName) + ctx.Request.Header.Set(httpconstant.HeaderTimeoutMs, timeoutMs) + ctx.Request.Header.Set(httpconstant.HeaderExpectNum, expectReceiveNum) + return ctx, rw +} + +func Test_Subscribe(t *testing.T) { + patches := []*gomonkey.Patches{ + gomonkey.ApplyFunc(util.NewClient, func() util.Client { + return &fakeClient{} + }), + gomonkey.ApplyFunc(datasystemclient.SubscribeStream, func(param datasystemclient.SubscribeParam, + ctx datasystemclient.StreamCtx) error { + t.Logf("MockSubscribeStream") + return nil + }), + } + defer func() { + for _, patch := range patches { + patch.Reset() + } + }() + fgAdapter := &invocation.FGAdapter{} + responsehandler.Handler = fgAdapter.MakeResponseHandler() + middleware.Invoker = fgAdapter.MakeInvoker() + convey.Convey("subscribe success", t, func() { + ctx, rw := constructFakeSubscribeRequest("test_stream_name", "100", "1") + SubscribeHandler(ctx) + assert.Equal(t, http.StatusOK, rw.Code) + }) + + convey.Convey("subscribe failed by invalid stream name", t, func() { + ctx, rw := constructFakeSubscribeRequest("", "100", "1") + SubscribeHandler(ctx) + assert.Equal(t, http.StatusBadRequest, rw.Code) + }) + + convey.Convey("subscribe failed by invalid timoutMs", t, func() { + ctx, rw := constructFakeSubscribeRequest("test_stream_name", "", "1") + SubscribeHandler(ctx) + assert.Equal(t, http.StatusBadRequest, rw.Code) + }) + + convey.Convey("subscribe failed by invalid expectReceiveNum", t, func() { + ctx, rw := constructFakeSubscribeRequest("test_stream_name", "100", "-1") + SubscribeHandler(ctx) + assert.Equal(t, http.StatusBadRequest, rw.Code) + }) + + convey.Convey("subscribe failed by invalid timeout", t, func() { + ctx, rw := constructFakeSubscribeRequest("test_stream_name", "-1", "1") + SubscribeHandler(ctx) + assert.Equal(t, http.StatusBadRequest, rw.Code) + }) +} diff --git a/frontend/pkg/frontend/clusterhealth/clusterhealth.go b/frontend/pkg/frontend/clusterhealth/clusterhealth.go new file mode 100644 index 0000000000000000000000000000000000000000..02f6e8825371bef4977e223edd2940682cc40577 --- /dev/null +++ b/frontend/pkg/frontend/clusterhealth/clusterhealth.go @@ -0,0 +1,197 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package clusterhealth - +package clusterhealth + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "time" + + "github.com/valyala/fasthttp" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/localauth" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/tls" + "frontend/pkg/frontend/common/httputil" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/functiontask" +) + +const ( + healthy = "healthy" + unhealthy = "unhealthy" + subhealthy = "subhealthy" + unknown = "unknown" + + task = "functiontask" + instanceManager = "instancemanager" + functionAccessor = "functionaccessor" + + defaultHealthTimeOut = 5 * time.Second + + headerRouterEtcdState = "X-Router-Etcd-State" +) + +var components = [...]string{task, instanceManager, functionAccessor} + +type healthyStatus string + +// CheckClusterHealth - +func CheckClusterHealth(w http.ResponseWriter, r *http.Request) { + resultMap := make(map[string]healthyStatus, len(components)) + + initComponentsStatus(resultMap) + if !etcd3.GetRouterEtcdClient().GetEtcdStatusLostContact() { + resultMap[functionAccessor] = subhealthy + resultMap[task] = unknown + resultMap[instanceManager] = unknown + } + if authCheckReq(r) != nil { + w.WriteHeader(http.StatusUnauthorized) + log.GetLogger().Errorf("failed to return auth error") + return + } + + // check task and instance manager + err := checkCoreComponentsHealth(resultMap, r) + if err != nil { + log.GetLogger().Errorf("failed to check component health %s", err.Error()) + w.WriteHeader(http.StatusInternalServerError) + bytes, err := json.Marshal(resultMap) + if err != nil { + _, err := w.Write([]byte(err.Error())) + if err != nil { + log.GetLogger().Errorf("failed to write rsp err %s", err.Error()) + } + } + _, err = w.Write(bytes) + if err != nil { + log.GetLogger().Errorf("failed to write rsp err %s", err.Error()) + } + return + } + w.WriteHeader(http.StatusOK) + return +} + +func initComponentsStatus(resultMap map[string]healthyStatus) { + if resultMap != nil { + for _, component := range components { + resultMap[component] = healthy + } + } +} + +func checkCoreComponentsHealth(resultMap map[string]healthyStatus, r *http.Request) error { + if resultMap == nil { + return fmt.Errorf("failed to check, result map is nil") + } + + if resultMap[functionAccessor] == subhealthy { + return errors.New("frontend health status is subhealthy") + } + + if functiontask.GetBusProxies().GetNum() == 0 { + resultMap[task] = unhealthy + resultMap[instanceManager] = unknown + return errors.New("no available proxy to request") + } + return sendClusterHealthCheckToTask(resultMap, r) +} + +func sendClusterHealthCheckToTask(resultMap map[string]healthyStatus, r *http.Request) error { + if resultMap == nil { + log.GetLogger().Errorf("healthyStatus resultMap is nil") + return fmt.Errorf("failed to check instance manager healthy") + } + // Traverse all nodes. If one node is available, the node is available. + req := fasthttp.AcquireRequest() + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseRequest(req) + defer fasthttp.ReleaseResponse(resp) + var err error + needReturn := false + functiontask.GetBusProxies().DoRange(func(nodeID string, nodeIP string) bool { + setComponentHealthReq(req, nodeIP, r.Header.Get(constant.HeaderAuthTimestamp), + r.Header.Get(constant.HeaderAuthorization)) + proxyClient := httputil.GetGlobalClient() + err = proxyClient.DoTimeout(req, resp, defaultHealthTimeOut) + if err != nil { + log.GetLogger().Errorf("failed to request proxy %s, err %s", req.URI().String(), err.Error()) + return true + } + if resp == nil { + return true + } + if resp.StatusCode() == http.StatusInternalServerError { + log.GetLogger().Errorf("failed to request proxy, err %s ", string(resp.Body())) + if string(resp.Header.Peek(headerRouterEtcdState)) == "false" { + resultMap[instanceManager] = subhealthy + } else { + resultMap[instanceManager] = unhealthy + } + err = fmt.Errorf("failed to check instance manager healthy") + needReturn = true + return false + } + if resp.StatusCode() == http.StatusOK { + needReturn = true + return false + } + err = fmt.Errorf("failed to request proxy") + log.GetLogger().Errorf("failed to request proxy err code %d uri %s", resp.StatusCode(), req.URI().String()) + return true + }) + if needReturn { + return err + } + // all tasks are unhealthy + resultMap[task] = unhealthy + resultMap[instanceManager] = unknown + return err +} + +func setComponentHealthReq(req *fasthttp.Request, nodeIP string, timeStamp string, authorization string) { + req.Header.SetMethod(http.MethodGet) + req.Header.Set(constant.HeaderAuthorization, authorization) + req.Header.Set(constant.HeaderAuthTimestamp, timeStamp) + req.Header.ResetConnectionClose() + req.SetRequestURI("/componenthealth") + req.Header.SetHost(fmt.Sprintf("%s:%s", nodeIP, constant.BusProxyHTTPPort)) + req.URI().SetScheme(tls.GetURLScheme(config.GetConfig().HTTPSConfig.HTTPSEnable)) +} + +func authCheckReq(r *http.Request) error { + functionConfig := config.GetConfig() + if !functionConfig.AuthenticationEnable { + return nil + } + requestSign := r.Header.Get(constant.HeaderAuthorization) + timestamp := r.Header.Get(constant.HeaderAuthTimestamp) + err := localauth.AuthCheckLocally(functionConfig.LocalAuth.AKey, config.GetConfig().LocalAuth.SKey, + requestSign, timestamp, functionConfig.LocalAuth.Duration) + if err != nil { + log.GetLogger().Errorf("failed to check authorization of URL locally, error: %s", err.Error()) + return err + } + return nil +} diff --git a/frontend/pkg/frontend/clusterhealth/clusterhealth_test.go b/frontend/pkg/frontend/clusterhealth/clusterhealth_test.go new file mode 100644 index 0000000000000000000000000000000000000000..2c2fc3d9ba391457d0a77b0c4ead0b39812a6f03 --- /dev/null +++ b/frontend/pkg/frontend/clusterhealth/clusterhealth_test.go @@ -0,0 +1,216 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package clusterhealth + +import ( + "encoding/json" + "errors" + "io/ioutil" + "net/http" + "net/http/httptest" + "reflect" + "strconv" + "strings" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/valyala/fasthttp" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/localauth" + "frontend/pkg/common/faas_common/tls" + "frontend/pkg/frontend/common/httputil" + config2 "frontend/pkg/frontend/config" + "frontend/pkg/frontend/functiontask" + "frontend/pkg/frontend/types" +) + +func clearGetProxies() { + clearList := []string{} + functiontask.GetBusProxies().DoRange(func(nodeID, nodeIP string) bool { + clearList = append(clearList, nodeID) + return true + }) + for _, nodeID := range clearList { + functiontask.GetBusProxies().Delete(nodeID) + } +} + +func TestClusterHealth(t *testing.T) { + clearGetProxies() + c := &types.Config{ + AuthenticationEnable: true, + MemoryEvaluatorConfig: &types.MemoryEvaluatorConfig{ + RequestMemoryEvaluator: 2, + }, + DefaultTenantLimitQuota: 1800, + HTTPConfig: &types.FrontendHTTP{ + WorkerInstanceReadTimeOut: 60, + MaxRequestBodySize: 100, + }, + HTTPSConfig: &tls.InternalHTTPSConfig{}, + E2EMaxDelayTime: 60, + LocalAuth: &localauth.AuthConfig{ + AKey: "ak", + SKey: "sk", + Duration: 5, + }, + InvokeMaxRetryTimes: 3, + RetryConfig: &types.RetryConfig{}, + HeartbeatConfig: &types.HeartbeatConfig{ + HeartbeatTimeout: 1, + HeartbeatInterval: 2, + HeartbeatTimeoutThreshold: 3, + }, + } + + patched := []*gomonkey.Patches{ + gomonkey.ApplyFunc(localauth.AuthCheckLocally, func(ak string, sk string, requestSign string, timestamp string, duration int) error { + if timestamp == "" { + return errors.New("no auth check info") + } + return nil + }), + gomonkey.ApplyPrivateMethod(reflect.TypeOf(&functiontask.BusProxy{}), "startMonitor", func(_ *functiontask.BusProxy, ch chan struct{}, status *int32) { + return + }), + gomonkey.ApplyFunc(config2.GetConfig, func() *types.Config { + return c + }), + } + defer func() { + for i := range patched { + patched[i].Reset() + } + }() + + router := gin.New() + router.GET("/serverless/v1/componentshealth", func(context *gin.Context) { + CheckClusterHealth(context.Writer, context.Request) + }) + + httpClient := httputil.GetGlobalClient() + + tests := []struct { + name string + proxyStatusMap map[string]int + headerMap map[string]string + timestamp string + exceptedCode int + expectResultMap map[string]string + }{ + { + name: "no proxy", + proxyStatusMap: map[string]int{}, + timestamp: strconv.Itoa(int(time.Now().Unix())), + exceptedCode: http.StatusInternalServerError, + expectResultMap: map[string]string{task: unhealthy, instanceManager: unknown, functionAccessor: healthy}, + }, + { + name: "frontend lost router etcd contact", + proxyStatusMap: map[string]int{}, + timestamp: strconv.Itoa(int(time.Now().Unix())), + headerMap: map[string]string{functionAccessor: "false"}, + exceptedCode: http.StatusInternalServerError, + expectResultMap: map[string]string{task: unknown, instanceManager: unknown, functionAccessor: subhealthy}, + }, + { + name: "proxy all ok", + timestamp: strconv.Itoa(int(time.Now().Unix())), + proxyStatusMap: map[string]int{"127.0.0.1": http.StatusOK, "127.0.0.2": http.StatusOK}, + exceptedCode: http.StatusOK, + }, + { + name: "instance manager is not ok", + timestamp: strconv.Itoa(int(time.Now().Unix())), + proxyStatusMap: map[string]int{"127.0.0.1": http.StatusInternalServerError, "127.0.0.2": http.StatusInternalServerError}, + exceptedCode: http.StatusInternalServerError, + expectResultMap: map[string]string{task: healthy, instanceManager: unhealthy, functionAccessor: healthy}, + }, + { + name: "instance manager lost router etcd contact", + timestamp: strconv.Itoa(int(time.Now().Unix())), + proxyStatusMap: map[string]int{"127.0.0.1": http.StatusInternalServerError, "127.0.0.2": http.StatusInternalServerError}, + headerMap: map[string]string{functionAccessor: "true", instanceManager: "false"}, + exceptedCode: http.StatusInternalServerError, + expectResultMap: map[string]string{task: healthy, instanceManager: subhealthy, functionAccessor: healthy}, + }, + { + name: "proxy all not ok", + timestamp: strconv.Itoa(int(time.Now().Unix())), + proxyStatusMap: map[string]int{"127.0.0.1": http.StatusBadRequest, "127.0.0.2": http.StatusBadRequest}, + exceptedCode: http.StatusInternalServerError, + expectResultMap: map[string]string{task: unhealthy, instanceManager: unknown, functionAccessor: healthy}, + }, + { + name: "auth check failed", + proxyStatusMap: map[string]int{"127.0.0.1": http.StatusOK, "127.0.0.2": http.StatusOK}, + exceptedCode: http.StatusUnauthorized, + }, + } + for _, test := range tests { + for nodeIP, _ := range test.proxyStatusMap { + nodeKey := "/sn/workers/business/yrk/tenant/0/function/function-task/version/$latest/defaultaz/" + nodeIP + functiontask.GetBusProxies().Add(nodeKey, nodeIP) + } + patches := []*gomonkey.Patches{ + gomonkey.ApplyMethod(reflect.TypeOf(httpClient), "DoTimeout", + func(_ *fasthttp.Client, req *fasthttp.Request, resp *fasthttp.Response, timeout time.Duration) error { + nodeIP := strings.Split(string(req.Host()), ":")[0] + if test.headerMap != nil && test.headerMap[instanceManager] == "false" { + resp.Header.Set(headerRouterEtcdState, "false") + } + resp.SetStatusCode(test.proxyStatusMap[nodeIP]) + return nil + }), + gomonkey.ApplyFunc(etcd3.GetRouterEtcdClient, func() *etcd3.EtcdClient { + return &etcd3.EtcdClient{} + }), + gomonkey.ApplyFunc((*etcd3.EtcdClient).GetEtcdStatusLostContact, func(_ *etcd3.EtcdClient) bool { + if test.headerMap != nil && test.headerMap[functionAccessor] == "false" { + return false + } + return true + }), + } + req, _ := http.NewRequest(http.MethodGet, "/serverless/v1/componentshealth", nil) + + req.Header.Set(constant.HeaderAuthTimestamp, test.timestamp) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + // check response status code + assert.Equal(t, test.exceptedCode, w.Result().StatusCode, test.name) + t.Logf("after check expect code") + // cluster is healthy only response http.StatusOK without responseBody + if test.exceptedCode == http.StatusInternalServerError { + expectResponseBody, _ := json.Marshal(test.expectResultMap) + actualResponseBody, _ := ioutil.ReadAll(w.Result().Body) + assert.Equal(t, string(expectResponseBody), string(actualResponseBody), test.name) + } + // 清理函数 + func() { + clearGetProxies() + for _, p := range patches { + p.Reset() + } + }() + } +} diff --git a/frontend/pkg/frontend/common/constant.go b/frontend/pkg/frontend/common/constant.go new file mode 100644 index 0000000000000000000000000000000000000000..c004aa39b3842a77e972192f3883d222cb68229f --- /dev/null +++ b/frontend/pkg/frontend/common/constant.go @@ -0,0 +1,28 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package common + +import "time" + +const ( + // FunctionUrnParam - + FunctionUrnParam = "function-urn" + // GinUrnParamMark - + GinUrnParamMark = ":" + // GracefulShutdownTimeOut - + GracefulShutdownTimeOut = 1300 * time.Second +) diff --git a/frontend/pkg/frontend/common/httpconstant/constant.go b/frontend/pkg/frontend/common/httpconstant/constant.go new file mode 100644 index 0000000000000000000000000000000000000000..22c793d1bcdd3051c7f7ba7107f8c635341d26ca --- /dev/null +++ b/frontend/pkg/frontend/common/httpconstant/constant.go @@ -0,0 +1,133 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package httpconstant - +package httpconstant + +const ( + // SemicolonReplacer replaces ";" to other character to solve golang.org/issue/25192 + SemicolonReplacer = "#" +) + +// Http request constant for http trigger +const ( + // HeaderInvokeURN - + HeaderInvokeURN = "X-Tag-VersionUrn" + // HeaderCPUSize is cpu size specified by invoke + HeaderCPUSize = "X-Instance-Cpu" + // HeaderMemorySize is cpu memory specified by invoke + HeaderMemorySize = "X-Instance-Memory" + // HeaderPoolLabel is pool label + HeaderPoolLabel = "X-Pool-Label" + // HeaderInvokeTag - + HeaderInvokeTag = "X-Invoke-Tag" + // HeaderInstanceLabel - + HeaderInstanceLabel = "X-Instance-Label" + // HeaderContentType - + HeaderContentType = "Content-Type" + // HeaderBillingDuration - + HeaderBillingDuration = "X-Billing-Duration" + // HeaderInnerCode - + HeaderInnerCode = "X-Inner-Code" + // HeaderInvokeSummary - + HeaderInvokeSummary = "X-Invoke-Summary" + // HeaderLogResult - + HeaderLogResult = "X-Log-Result" + // HeaderLogType - + HeaderLogType = "X-Log-Type" + // DefaultLogFlag is the default flag for log + DefaultLogFlag = "None" + // HeaderAuthTimestamp is the timestamp for authorization + HeaderAuthTimestamp = "X-Timestamp-Auth" + // HeaderAuthorization is authorization + HeaderAuthorization = "authorization" + // AppID name of module + AppID = "frontendinvoke" + // HeaderInvokeAlias indicates alias of current invocation + HeaderInvokeAlias = "x-invoke-alias" + // HeaderRetryFlag - + HeaderRetryFlag = "X-Retry-Flag" + // HeaderWorkerCost - + HeaderWorkerCost = "X-Worker-Cost" + // HeaderCallInstance - + HeaderCallInstance = "X-Call-Instance" + // HeaderCallNode - + HeaderCallNode = "X-Call-Node" + // AuthType - + AuthType = "X-Authorization-Type" + // AuthHeader - + AuthHeader = "Authorization" + // SAAuthHeader - + SAAuthHeader = "service-account" + // HeaderInstanceSession - + HeaderInstanceSession = "X-Instance-Session" +) + +const ( + // ContentTypeHeaderKey - + ContentTypeHeaderKey = "Content-Type" + // ApplicationJSON - + ApplicationJSON = "application/json" + // StreamContentType - + StreamContentType = "application/octet-stream" + // FormContentType - + FormContentType = "application/x-www-form-urlencoded" + // MultipartFormContentType - + MultipartFormContentType = "multipart/form-data" + + // ContentType - + ContentType = "Content-Type" +) + +const ( + // DefaultGraphReadBufferSize In FunctionGraph mode, the size of this message needs to be set to 32 KB. + DefaultGraphReadBufferSize = 32 * 1024 +) + +const ( + // HeaderLuBanNTraceID - + HeaderLuBanNTraceID = "lubanops-ntrace-id" + // HeaderLuBanGTraceID - + HeaderLuBanGTraceID = "lubanops-gtrace-id" + // HeaderLuBanSpanID - + HeaderLuBanSpanID = "lubanops-nspan-id" + // HeaderLuBanEvnID - + HeaderLuBanEvnID = "lubanops-nenv-id" + // HeaderLuBanEventID - + HeaderLuBanEventID = "lubanops-sevent-id" + // HeaderLuBanDomainID - + HeaderLuBanDomainID = "lubanops-ndomain-id" +) + +const ( + // CrossHeaderKeyCrossCluster - + CrossHeaderKeyCrossCluster = "X-System-Cross-Cluster" + // CrossHeaderKeyClusterID - + CrossHeaderKeyClusterID = "X-System-Cluster-Id" + // CrossHeaderKeyTimestamp - + CrossHeaderKeyTimestamp = "X-System-Timestamp" + // CrossHeaderKeySignature - + CrossHeaderKeySignature = "X-System-Signature" +) + +const ( + // HeaderStreamName - + HeaderStreamName = "X-Stream-Name" + // HeaderExpectNum - + HeaderExpectNum = "X-Expect-Num" + // HeaderTimeoutMs - + HeaderTimeoutMs = "X-Timeout-Ms" +) diff --git a/frontend/pkg/frontend/common/httputil/util.go b/frontend/pkg/frontend/common/httputil/util.go new file mode 100644 index 0000000000000000000000000000000000000000..ca1b0d0f1e6ce3700def697bad8ea39a4d2f5f26 --- /dev/null +++ b/frontend/pkg/frontend/common/httputil/util.go @@ -0,0 +1,431 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package httputil - +package httputil + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "os" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "github.com/gin-gonic/gin" + "github.com/valyala/fasthttp" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/localauth" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/snerror" + "frontend/pkg/common/faas_common/statuscode" + commontls "frontend/pkg/common/faas_common/tls" + "frontend/pkg/common/faas_common/utils" + "frontend/pkg/common/uuid" + "frontend/pkg/frontend/common/httpconstant" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/responsehandler" + "frontend/pkg/frontend/types" +) + +const ( + createInstanceErrorMessageIndex = 2 + + invokeErrorMsgFormat = `failed to invoke, code: (\w+),msg: ([\s\S]*)$` + invokeErrorMessageFormat = `failed to invoke, code: (\w+), message: ([\s\S]*)$` + invokeErrorEmptyResp = "failed to get invoke response: invokeRsp is nil" + invokeErrorAcquireTimeout = "acquire instance timeout" + invokeErrorCodeEmptyResp = "3008" + invokeErrorCodeAcquireTimeout = "3009" + invokeUnknownCode = "3010" + invokeUnknownMsg = "unknownMsg" + invokeErrorCodeIndex = 1 + invokeErrorMessageIndex = 2 +) +const ( + // MaxClientConcurrency - + MaxClientConcurrency = 10000 + // WorkerInstanceMaxIdleConnDuration define max connction time + WorkerInstanceMaxIdleConnDuration = 10 * time.Second + // MaxConnsPerWorker limit the max connection + MaxConnsPerWorker = 10240 + // HeartbeatDialTimeOut heartbeat dial timeout + HeartbeatDialTimeOut = 2 * time.Second + defaultWriteTimeout = time.Duration(30) * time.Second + // defaultGraphReadBufferSize is default readBuffer size is 6k (4k for log) + defaultGraphReadBufferSize = 6 * 1024 + // DialTimeOut set timeout limit + DialTimeOut = 10 * time.Second +) + +const ( + // scheduler fast http client config + readTimeout = 600 * time.Second + writeTimeout = 600 * time.Second + readBufSize = 1 * 1024 + maxIdleConnDuration = 5 * time.Second + dialTimeout = 10 * time.Second +) + +const defaultSyncRequestTimeout = 900 + +var ( + globalClient = struct { + // common fast http client + c *fasthttp.Client + sync.Once + }{} + + heartbeatClient = struct { + // heartbeat fast http client + hc *fasthttp.Client + sync.Once + }{} + schedulerClient = struct { + // heartbeat fast http client + sc *fasthttp.Client + sync.Once + }{} +) +var ( + tcpDialer = fasthttp.TCPDialer{Concurrency: MaxClientConcurrency} +) + +// GetHeartbeatClient return heart beat client +func GetHeartbeatClient() *fasthttp.Client { + heartbeatClient.Do(func() { + heartbeatClient.hc = newFastHTTPClient(HeartbeatDialTimeOut) + }) + return heartbeatClient.hc +} + +// GetGlobalClient returns global client of fastHttp +func GetGlobalClient() *fasthttp.Client { + globalClient.Do(func() { + globalClient.c = newFastHTTPClient(DialTimeOut) + }) + return globalClient.c +} + +// GetSchedulerClient returns fastHttp client of scheduler +func GetSchedulerClient() *fasthttp.Client { + schedulerClient.Do(func() { + schedulerClient.sc = newSchedulerClient() + }) + return schedulerClient.sc +} + +func newSchedulerClient() *fasthttp.Client { + var tlsConfig *tls.Config + if config.GetConfig().HTTPSConfig.HTTPSEnable { + tlsConfig = commontls.GetClientTLSConfig() + if tlsConfig != nil { + tlsConfig.NextProtos = []string{"http/1.1"} + } + } + return &fasthttp.Client{ + MaxIdleConnDuration: maxIdleConnDuration, + MaxConnsPerHost: MaxClientConcurrency, + ReadTimeout: readTimeout, + WriteTimeout: writeTimeout, + ReadBufferSize: readBufSize, + TLSConfig: tlsConfig, + Dial: func(addr string) (net.Conn, error) { + return tcpDialer.DialTimeout(addr, dialTimeout) + }, + } +} + +// NewFastHTTPClient create fasthttp client +func newFastHTTPClient(dialTimeout time.Duration) *fasthttp.Client { + var tlsConfig *tls.Config + if config.GetConfig().HTTPSConfig.HTTPSEnable { + newCfg := *commontls.GetClientTLSConfig() + newCfg.NextProtos = []string{"http/1.1"} + tlsConfig = &newCfg + } + return &fasthttp.Client{ + MaxIdemponentCallAttempts: 1, + MaxIdleConnDuration: WorkerInstanceMaxIdleConnDuration, + MaxConnsPerHost: MaxConnsPerWorker, + ReadTimeout: time.Duration(config.GetConfig().HTTPConfig.WorkerInstanceReadTimeOut) * time.Second, + WriteTimeout: defaultWriteTimeout, + ReadBufferSize: defaultGraphReadBufferSize, + TLSConfig: tlsConfig, + Dial: func(addr string) (net.Conn, error) { + return tcpDialer.DialTimeout(addr, dialTimeout) + }, + } +} + +// InitTraceID init trace ID +func InitTraceID(ctx *gin.Context) string { + var traceID string + if config.GetConfig().BusinessType == constant.BusinessTypeWiseCloud { + traceID = ctx.Request.Header.Get(constant.CaaSHeaderTraceID) + } else { + traceID = ctx.Request.Header.Get(constant.HeaderRequestID) + } + switch { + case traceID == "": + traceID = uuid.New().String() + log.GetLogger().Infof("x-request-id is empty, generates a traceID: %s", traceID) + case len(traceID) > constant.MaxTraceIDLength: + traceID = traceID[:constant.MaxTraceIDLength] + default: + } + ctx.Request.Header.Set(constant.HeaderTraceID, traceID) + return traceID +} + +// GetTimeFromResp - +func GetTimeFromResp(timeTaken float64) time.Duration { + return time.Duration(timeTaken) * time.Millisecond +} + +// TranslateInvokeMsgToCallReq is used to extract the http header +// and insert it into the body for executor use. +func TranslateInvokeMsgToCallReq(ctx *types.InvokeProcessContext) ([]byte, error) { + var err error + req := &types.CallReq{ + Header: make(map[string]string, len(ctx.ReqHeader)), + } + if len(ctx.ReqBody) == 0 { + req.Body = json.RawMessage(`{}`) + } else { + req.Body = ctx.ReqBody + } + for key, value := range ctx.ReqHeader { + req.Header[key] = value + } + if config.GetConfig().BusinessType == constant.BusinessTypeWiseCloud { + req.Header[constant.CaaSHeaderRequestID] = ctx.TraceID + } + if config.GetConfig().BusinessType == constant.BusinessTypeFG { + req.Header = decryptSensitiveHeader(req.Header) + req.Header[constant.FGHeaderRequestID] = ctx.TraceID + } + req.Header["X-Trace-Id"] = ctx.TraceID + req.Path = ctx.ReqPath + req.Method = ctx.ReqMethod + req.Query = ctx.ReqQuery + reqMsg, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("failed to marshal request data: %s", err) + } + return reqMsg, nil +} + +func decryptSensitiveHeader(header map[string]string) map[string]string { + decryptHeaderToExtra(header, constant.FGHeaderAccessKey) + decryptHeaderToExtra(header, constant.FGHeaderSecretKey) + decryptHeaderToExtra(header, constant.FGHeaderAuthToken) + decryptHeaderToExtra(header, constant.FGHeaderSecurityAccessKey) + decryptHeaderToExtra(header, constant.FGHeaderSecuritySecretKey) + decryptHeaderToExtra(header, constant.FGHeaderSecurityToken) + return header +} + +func decryptHeaderToExtra(header map[string]string, headerName string) { + if header == nil { + return + } + if header[headerName] != "" { + decryptHeaderValue, err := localauth.Decrypt(header[headerName]) + if err != nil { + log.GetLogger().Warnf("failed to decrypt %s", headerName) + header[headerName] = "" + } else { + header[headerName] = string(decryptHeaderValue) + } + utils.ClearByteMemory(decryptHeaderValue) + } +} + +// unmarshalInitResp - +func unmarshalInitResp(message []byte) (*types.InitResp, error) { + // There is a possibility that the kernel returns slice[Message] or Message, which will be rectified later. + // To avoid this problem, the first and last slice identifiers are removed. + if bytes.HasPrefix(message, []byte("[")) && bytes.HasSuffix(message, []byte("]")) { + // trim [ from the beginning and ] from the end + message = bytes.TrimPrefix(message, []byte("[")) + message = bytes.TrimSuffix(message, []byte("]")) + } + + respMsg := &types.InitResp{} + err := json.Unmarshal(message, respMsg) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal init response data: %s", err) + } + return respMsg, nil +} + +// HandleCreateInstanceError - +func HandleCreateInstanceError(ctx *types.InvokeProcessContext, err error) bool { + snErr, ok := err.(snerror.SNError) + // if error code of sn error equals to InternalErrCode then it's a create error from kernel and should be processed + // further in logic below + if ok && snErr.Code() != statuscode.InternalErrorCode { + responsehandler.SetErrorInContext(ctx, snErr.Code(), fmt.Sprintf(`"%s"`, strings.ReplaceAll(snErr.Error(), + `"`, ""))) + return true + } + errMessage := err.Error() + respMsg, err := unmarshalInitResp([]byte(errMessage)) + if err != nil { + log.GetLogger().Errorf("failed to translate response data, traceID: %s, err: %s", + ctx.TraceID, err.Error()) + responsehandler.SetErrorInContext(ctx, statuscode.InternalErrorCode, errMessage) + return true + } + errorCode, err := strconv.Atoi(respMsg.ErrorCode) + if err != nil { + log.GetLogger().Errorf("failed to get error code, traceID: %s, err: %s", + ctx.TraceID, err.Error()) + responsehandler.SetErrorInContext(ctx, statuscode.InternalErrorCode, errMessage) + return true + } + responsehandler.SetErrorInContext(ctx, errorCode, respMsg.Message) + return true +} + +// HandleInvokeError - +func HandleInvokeError(ctx *types.InvokeProcessContext, err error) { + switch { + // failed to create an instance + case HandleCreateInstanceError(ctx, err): + return + default: + responsehandler.SetErrorInContext(ctx, statuscode.InternalErrorCode, err.Error()) + return + } +} + +// JudgeRetry - +func JudgeRetry(err error, ctx *types.InvokeProcessContext) { + var errCode int + if errInfo, ok := err.(api.ErrorInfo); ok { + errCode = errInfo.Code + } else { + submatches := getMatches(err) + if submatches == nil || len(submatches) <= invokeErrorMessageIndex || + submatches[createInstanceErrorMessageIndex] == "" { + return + } + errCode, err = strconv.Atoi(submatches[invokeErrorCodeIndex]) + if err != nil { + log.GetLogger().Warnf("failed to get error code from invoke ErrMsg, err: %s", err.Error()) + return + } + } + switch errCode { + case statuscode.ErrInstanceExitedCode, statuscode.ErrRequestBetweenRuntimeBusCode, statuscode.ErrInstanceNotFound, + statuscode.ErrRequestBetweenRuntimeFrontendCode, statuscode.ErrAcquireTimeoutCode, + statuscode.ErrInstanceCircuitCode, statuscode.ErrInstanceEvicted: + ctx.ShouldRetry = true + default: + log.GetLogger().Warnf("unable to support handle errCode: %d", errCode) + } +} + +func getMatches(err error) []string { + re := regexp.MustCompile(invokeErrorMessageFormat) + submatches := re.FindStringSubmatch(err.Error()) + if submatches != nil && len(submatches) > invokeErrorMessageIndex && + submatches[createInstanceErrorMessageIndex] != "" { + return submatches + } + re = regexp.MustCompile(invokeErrorMsgFormat) + submatches = re.FindStringSubmatch(err.Error()) + if submatches != nil && len(submatches) > invokeErrorMessageIndex && + submatches[createInstanceErrorMessageIndex] != "" { + return submatches + } + if strings.Contains(err.Error(), invokeErrorEmptyResp) { + return []string{"", invokeErrorCodeEmptyResp, invokeErrorEmptyResp} + } + if strings.Contains(err.Error(), invokeErrorAcquireTimeout) { + return []string{"", invokeErrorCodeAcquireTimeout, invokeErrorAcquireTimeout} + } + return []string{"", invokeUnknownCode, invokeUnknownMsg} +} + +// GetSyncRequestTimeout returns sync request timeout +func GetSyncRequestTimeout() int64 { + return syncRequestTimeout +} + +var syncRequestTimeout = parseDefaultSyncRequestTimeout() + +func parseDefaultSyncRequestTimeout() int64 { + env := os.Getenv("SYNC_REQUEST_TIMEOUT") + if env == "" { + return defaultSyncRequestTimeout + } + val, err := strconv.Atoi(env) + if err != nil || val <= 0 { + return defaultSyncRequestTimeout + } + return int64(val) +} + +// AddAuthorizationHeaderForFG - +func AddAuthorizationHeaderForFG(proxyReq *fasthttp.Request) { + authorization, timestamp := localauth.SignLocally(config.GetConfig().LocalAuth.AKey, + config.GetConfig().LocalAuth.SKey, httpconstant.AppID, config.GetConfig().LocalAuth.Duration) + proxyReq.Header.Set(constant.HeaderAuthTimestamp, timestamp) + proxyReq.Header.Set(constant.HeaderAuthorization, authorization) +} + +// ReadLimitedBody - +func ReadLimitedBody(inputStream io.Reader, maxReadSize int64) ([]byte, error) { + reader := io.LimitReader(inputStream, maxReadSize+1) + body, err := io.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("read body from gin request error") + } + if int64(len(body)) > maxReadSize { + return nil, fmt.Errorf("body is beyond maximum: %d", maxReadSize) + } + return body, nil +} + +// GetCompatibleHeader - +func GetCompatibleHeader(headers map[string]string, primaryHeader, secondaryHeader string) string { + if value, ok := headers[primaryHeader]; ok && value != "" { + return value + } + return headers[secondaryHeader] +} + +// GetCompatibleGinHeader get invoke label from res key +func GetCompatibleGinHeader(req *http.Request, primaryHeader string, secondaryHeader string) string { + headerValue := req.Header.Get(primaryHeader) + if headerValue == "" { + headerValue = req.Header.Get(secondaryHeader) + } + return headerValue +} diff --git a/frontend/pkg/frontend/common/httputil/util_test.go b/frontend/pkg/frontend/common/httputil/util_test.go new file mode 100644 index 0000000000000000000000000000000000000000..de1b163e4203ced9bdeee10e87807b105f59b922 --- /dev/null +++ b/frontend/pkg/frontend/common/httputil/util_test.go @@ -0,0 +1,271 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package httputil + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "frontend/pkg/common/faas_common/localauth" + "io" + "net/http" + "net/http/httptest" + "os" + "strconv" + "strings" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/gin-gonic/gin" + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + "github.com/valyala/fasthttp" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/snerror" + "frontend/pkg/common/faas_common/statuscode" + "frontend/pkg/common/faas_common/tls" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/responsehandler" + "frontend/pkg/frontend/types" +) + +func TestGetClient(t *testing.T) { + defer gomonkey.ApplyFunc(config.GetConfig, func() *types.Config { + return &types.Config{ + HTTPConfig: &types.FrontendHTTP{}, + HTTPSConfig: &tls.InternalHTTPSConfig{}, + } + }).Reset() + convey.Convey("TestGetClient", t, func() { + client := GetHeartbeatClient() + convey.So(client, convey.ShouldNotBeNil) + client = GetGlobalClient() + convey.So(client, convey.ShouldNotBeNil) + }) +} + +func TestTranslateInvokeMsgToCallReq(t *testing.T) { + convey.Convey("TranslateInvokeMsgToCallReq", t, func() { + defer gomonkey.ApplyFunc(json.Marshal, func(v interface{}) ([]byte, error) { + return nil, errors.New("json marshal error") + }).Reset() + _, err := TranslateInvokeMsgToCallReq(&types.InvokeProcessContext{ + ReqHeader: map[string]string{}, + RespHeader: map[string]string{}, + }) + convey.So(err, convey.ShouldBeError) + }) + + convey.Convey("TranslateInvokeMsgToCallReq success", t, func() { + defer gomonkey.ApplyFunc(config.GetConfig, func() *types.Config { + return &types.Config{ + BusinessType: constant.BusinessTypeWiseCloud, + } + }).Reset() + _, err := TranslateInvokeMsgToCallReq(&types.InvokeProcessContext{ + ReqHeader: map[string]string{}, + RespHeader: map[string]string{}, + }) + convey.So(err, convey.ShouldBeNil) + }) +} + +func TestJudgeRetry(t *testing.T) { + convey.Convey("test JudgeRetry", t, func() { + convey.Convey("test failed", func() { + processCtx := types.CreateInvokeProcessContext() + err := errors.New("failed to test") + JudgeRetry(err, processCtx) + convey.So(processCtx.ShouldRetry, convey.ShouldEqual, false) + }) + convey.Convey("test success 1", func() { + processCtx := types.CreateInvokeProcessContext() + err := errors.New("failed to invoke, code: 1007, message: exit") + JudgeRetry(err, processCtx) + convey.So(processCtx.ShouldRetry, convey.ShouldEqual, true) + }) + convey.Convey("test success 2", func() { + processCtx := types.CreateInvokeProcessContext() + err := errors.New("failed to invoke, code: 3001, message: exit") + JudgeRetry(err, processCtx) + convey.So(processCtx.ShouldRetry, convey.ShouldEqual, true) + }) + convey.Convey("test success 3", func() { + processCtx := types.CreateInvokeProcessContext() + err := errors.New("failed to get invoke response: invokeRsp is nil, XXX") + JudgeRetry(err, processCtx) + convey.So(processCtx.ShouldRetry, convey.ShouldEqual, true) + }) + }) +} + +func TestInitTraceID(t *testing.T) { + convey.Convey("InitTraceID", t, func() { + convey.Convey("not empty", func() { + defer gomonkey.ApplyFunc(config.GetConfig, func() *types.Config { + return &types.Config{ + BusinessType: constant.BusinessTypeWiseCloud, + } + }).Reset() + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(nil)) + ctx.Request.Header.Add("constant.CaaSHeaderTraceID", "test-traceID") + id := InitTraceID(ctx) + convey.So(id, convey.ShouldNotEqual, "test-traceID") + }) + + convey.Convey("empty trace id", func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(nil)) + ctx.Request.Header.Add(constant.HeaderRequestID, "") + id := InitTraceID(ctx) + convey.So(id, convey.ShouldNotEqual, "") + }) + + convey.Convey("long trace id", func() { + rw := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(rw) + ctx.Request, _ = http.NewRequest("", "", bytes.NewBuffer(nil)) + traceID := "" + for i := 0; i < constant.MaxTraceIDLength; i++ { + traceID += strconv.Itoa(i) + } + fmt.Println("123:", len(traceID)) + ctx.Request.Header.Add(constant.HeaderRequestID, traceID) + id := InitTraceID(ctx) + convey.So(len(id), convey.ShouldEqual, constant.MaxTraceIDLength) + }) + }) +} + +func TestGetSyncRequestTimeout(t *testing.T) { + convey.Convey("GetSyncRequestTimeout", t, func() { + requestTimeout := GetSyncRequestTimeout() + convey.So(requestTimeout, convey.ShouldEqual, defaultSyncRequestTimeout) + }) +} + +func TestParseDefaultSyncRequestTimeout(t *testing.T) { + convey.Convey("TestParseDefaultSyncRequestTimeout", t, func() { + os.Setenv("SYNC_REQUEST_TIMEOUT", "0") + requestTimeout := parseDefaultSyncRequestTimeout() + convey.So(requestTimeout, convey.ShouldEqual, defaultSyncRequestTimeout) + os.Setenv("SYNC_REQUEST_TIMEOUT", "60") + requestTimeout = parseDefaultSyncRequestTimeout() + convey.So(requestTimeout, convey.ShouldEqual, 60) + os.Unsetenv("SYNC_REQUEST_TIMEOUT") + }) +} + +func TestGetSchedulerClient(t *testing.T) { + convey.Convey("TestGetSchedulerClient", t, func() { + gomonkey.ApplyFunc(config.GetConfig, func() *types.Config { + return &types.Config{ + HTTPSConfig: &tls.InternalHTTPSConfig{ + HTTPSEnable: true, + }, + } + }) + client := GetSchedulerClient() + convey.So(client.ReadTimeout, convey.ShouldEqual, readTimeout) + }) + +} + +func TestReadLimitedBody_Error(t *testing.T) { + input := strings.NewReader("test input") + maxReadSize := int64(5) + + patches := gomonkey.ApplyFunc(io.ReadAll, func(r io.Reader) ([]byte, error) { + return nil, fmt.Errorf("mocked read error") + }) + defer patches.Reset() + + _, err := ReadLimitedBody(input, maxReadSize) + + assert.NotNil(t, err) + assert.Equal(t, "read body from gin request error", err.Error()) +} + +func TestHandleCreateInstanceError(t *testing.T) { + setCode := 0 + defer gomonkey.ApplyFunc(responsehandler.SetErrorInContext, func(ctx *types.InvokeProcessContext, innerCode int, + message interface{}) { + setCode = innerCode + }).Reset() + convey.Convey("TestHandleCreateInstanceError", t, func() { + ctx := &types.InvokeProcessContext{} + snErr := snerror.New(1234, "some error") + flag := HandleCreateInstanceError(ctx, snErr) + convey.So(flag, convey.ShouldBeTrue) + convey.So(setCode, convey.ShouldEqual, 1234) + + err := errors.New("some error") + flag = HandleCreateInstanceError(ctx, err) + convey.So(flag, convey.ShouldBeTrue) + convey.So(setCode, convey.ShouldEqual, statuscode.InternalErrorCode) + + err = errors.New(`{"errorCode": "invalid"}`) + flag = HandleCreateInstanceError(ctx, err) + convey.So(flag, convey.ShouldBeTrue) + convey.So(setCode, convey.ShouldEqual, statuscode.InternalErrorCode) + }) +} + +func TestAddAuthorizationHeaderForFG(t *testing.T) { + defer gomonkey.ApplyFunc(config.GetConfig, func() *types.Config { + return &types.Config{ + LocalAuth: &localauth.AuthConfig{}, + } + }).Reset() + convey.Convey("TestAddAuthorizationHeaderForFG", t, func() { + request := &fasthttp.Request{} + AddAuthorizationHeaderForFG(request) + value := request.Header.Peek(constant.HeaderAuthTimestamp) + convey.So(value, convey.ShouldNotBeEmpty) + value = request.Header.Peek(constant.HeaderAuthorization) + convey.So(value, convey.ShouldNotBeEmpty) + }) +} + +func TestGetCompatibleHeader(t *testing.T) { + convey.Convey("TestGetCompatibleHeader", t, func() { + header := map[string]string{"primary": "aaa", "secondary": "bbb"} + value := GetCompatibleHeader(header, "primary", "secondary") + convey.So(value, convey.ShouldEqual, "aaa") + + header["primary"] = "" + value = GetCompatibleHeader(header, "primary", "secondary") + convey.So(value, convey.ShouldEqual, "bbb") + }) +} + +func TestGetCompatibleGinHeader(t *testing.T) { + convey.Convey("TestGetCompatibleGinHeader", t, func() { + request := &http.Request{Header: map[string][]string{"Primary": {"aaa"}, "Secondary": {"bbb"}}} + value := GetCompatibleGinHeader(request, "primary", "secondary") + convey.So(value, convey.ShouldEqual, "aaa") + + request.Header["Primary"] = []string{} + value = GetCompatibleGinHeader(request, "primary", "secondary") + convey.So(value, convey.ShouldEqual, "bbb") + }) +} diff --git a/frontend/pkg/frontend/common/util/client.go b/frontend/pkg/frontend/common/util/client.go new file mode 100644 index 0000000000000000000000000000000000000000..3a2b68b504c8451b89b6c9300e40f5207f5e877e --- /dev/null +++ b/frontend/pkg/frontend/common/util/client.go @@ -0,0 +1,335 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package util - +package util + +import ( + "fmt" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/types" + "frontend/pkg/common/faas_common/utils" + "frontend/pkg/frontend/common/httpconstant" +) + +const ( + maxInvokeRetries = 5 +) + +type invokerLibruntime interface { + CreateInstance(funcMeta api.FunctionMeta, args []api.Arg, + invokeOpt api.InvokeOptions) (instanceID string, err error) + InvokeByInstanceId(funcMeta api.FunctionMeta, instanceID string, args []api.Arg, + invokeOpt api.InvokeOptions) (returnObjectID string, err error) + InvokeByFunctionName(funcMeta api.FunctionMeta, args []api.Arg, + invokeOpt api.InvokeOptions) (returnObjectID string, err error) + AcquireInstance(state string, funcMeta api.FunctionMeta, + acquireOpt api.InvokeOptions) (api.InstanceAllocation, error) + + ReleaseInstance(allocation api.InstanceAllocation, stateID string, abnormal bool, option api.InvokeOptions) + Kill(instanceID string, signal int, payload []byte) (err error) + + CreateInstanceRaw(createReqRaw []byte) (createRespRaw []byte, err error) + InvokeByInstanceIdRaw(invokeReqRaw []byte) (resultRaw []byte, err error) + KillRaw(killReqRaw []byte) (killRespRaw []byte, err error) + + SaveState(state []byte) (stateID string, err error) + LoadState(checkpointID string) (state []byte, err error) + + Exit(code int, message string) + + KVSet(key string, value []byte, param api.SetParam) (err error) + KVSetWithoutKey(value []byte, param api.SetParam) (key string, err error) + KVGet(key string, timeoutms uint) (value []byte, err error) + KVGetMulti(keys []string, timeoutms uint) (values [][]byte, err error) + KVDel(key string) (err error) + KVDelMulti(keys []string) (failedKeys []string, err error) + + SetTraceID(traceID string) + + Put(objectID string, value []byte, param api.PutParam, nestedObjectIDs ...string) (err error) + Get(objectIDs []string, timeoutMs int) (data [][]byte, err error) + GIncreaseRef(objectIDs []string, remoteClientID ...string) (failedIDs []string, err error) + GDecreaseRef(objectIDs []string, remoteClientID ...string) (failedIDs []string, err error) + GetAsync(objectID string, cb api.GetAsyncCallback) + + GetFormatLogger() api.FormatLogger + GetCredential() api.Credential + SetTenantID(tenantID string) error + IsHealth() bool + IsDsHealth() bool +} + +var clientLibruntime invokerLibruntime + +// SetAPIClientLibruntime set the client provided by the runtime +func SetAPIClientLibruntime(rt invokerLibruntime) { + clientLibruntime = rt +} + +// AcquireOption holds the options for acquireInstance +type AcquireOption struct { + DesignateInstanceID string + SchedulerFuncKey string + SchedulerID string + RequestID string + TraceID string + FuncSig string + ResourceSpecs map[string]int64 + Timeout int64 + TrafficLimited bool +} + +// InvokeRequest - +type InvokeRequest struct { + Function string + InstanceID string + TraceID string + Args []*api.Arg + SchedulerID string + SchedulerFuncKey string + RequestID string + FuncSig string + PoolLabel string + InstanceSession *types.InstanceSessionConfig + InvokeTag map[string]string + InstanceLabel string + ReturnObjectIDs []string + ResourceSpecs map[string]int64 + AcquireTimeout int64 + InvokeTimeout int64 + TrafficLimited bool + RetryTimes int + BusinessType string + TenantID string +} + +// Client is used to invoke an instance and wait for its response +type Client interface { + AcquireInstance(functionKey string, req AcquireOption) (*types.InstanceAllocationInfo, error) + ReleaseInstance(allocation *types.InstanceAllocationInfo, abnormal bool) + Invoke(req InvokeRequest) ([]byte, error) + InvokeByName(req InvokeRequest) ([]byte, error) + CreateInstanceRaw(createReq []byte) ([]byte, error) + InvokeInstanceRaw(invokeReq []byte) ([]byte, error) + KillRaw(killReq []byte) ([]byte, error) + CreateInstanceByLibRt(funcMeta api.FunctionMeta, args []api.Arg, + invokeOpt api.InvokeOptions) (instanceID string, err error) + KillByLibRt(instanceID string, signal int, payload []byte) (err error) + IsHealth() bool + IsDsHealth() bool +} + +// NewClient return a client used to invoke other functions +func NewClient() Client { + return newDefaultClientLibruntime(clientLibruntime) +} + +func newDefaultClientLibruntime(librtcli invokerLibruntime) *defaultClient { + return &defaultClient{clientLibruntime: librtcli} +} + +type defaultClient struct { + clientLibruntime invokerLibruntime +} + +func (c *defaultClient) AcquireInstance(functionKey string, req AcquireOption) (*types.InstanceAllocationInfo, error) { + var err error + var instanceAllocation api.InstanceAllocation + functionMeta := api.FunctionMeta{ + FuncID: functionKey, + Sig: req.FuncSig, + Name: &req.DesignateInstanceID, + Api: api.FaaSApi, + } + option := convertAcquireOption(req) + if instanceAllocation, err = c.clientLibruntime.AcquireInstance("", functionMeta, option); err != nil { + return nil, err + } + return &types.InstanceAllocationInfo{ + FuncKey: instanceAllocation.FuncKey, + FuncSig: instanceAllocation.FuncSig, + InstanceID: instanceAllocation.InstanceID, + ThreadID: instanceAllocation.LeaseID, + LeaseInterval: instanceAllocation.LeaseInterval, + }, nil +} + +func (c *defaultClient) ReleaseInstance(allocation *types.InstanceAllocationInfo, abnormal bool) { + instanceAllocation := api.InstanceAllocation{ + FuncKey: allocation.FuncKey, + FuncSig: allocation.FuncSig, + InstanceID: allocation.InstanceID, + LeaseID: allocation.ThreadID, + LeaseInterval: allocation.LeaseInterval, + } + c.clientLibruntime.ReleaseInstance(instanceAllocation, "", abnormal, api.InvokeOptions{}) +} + +func deepCopyArgs(args []*api.Arg, tenantID string) []api.Arg { + rtArgs := make([]api.Arg, len(args)) + for idx, val := range args { + rtArgs[idx] = api.Arg{ + Type: val.Type, + Data: val.Data, + TenantID: tenantID, + } + } + return rtArgs +} + +func (c *defaultClient) Invoke(req InvokeRequest) ([]byte, error) { + wait := make(chan struct{}, 1) + var ( + res []byte + resErr, err error + ) + funcMeta := api.FunctionMeta{FuncID: req.Function, Api: api.FaaSApi} + funcArgs := deepCopyArgs(req.Args, "") + invokeOpts := api.InvokeOptions{TraceID: req.TraceID, CustomExtensions: req.InvokeTag, RetryTimes: req.RetryTimes, + Timeout: int(req.InvokeTimeout)} + var objID string + objID, err = c.clientLibruntime.InvokeByInstanceId(funcMeta, req.InstanceID, funcArgs, invokeOpts) + + c.clientLibruntime.GetAsync(objID, func(result []byte, err error) { + res = result + resErr = err + wait <- struct{}{} + if _, err := c.clientLibruntime.GDecreaseRef([]string{objID}); err != nil { + fmt.Printf("failed to decrease object ref,err: %s", err.Error()) + } + }) + if err != nil { + return res, fmt.Errorf("failed to invoke by instance id request, req: %#v, err: %s", req, err.Error()) + } + <-wait + return res, resErr +} + +func convertInvokeOption(req InvokeRequest) api.InvokeOptions { + cpu, mem, customRes := LibruntimeCustomResources(req.ResourceSpecs) + invokeLabels := map[string]string{} + if req.InstanceLabel != "" { + invokeLabels[httpconstant.HeaderInstanceLabel] = req.InstanceLabel + } + invokeOpt := api.InvokeOptions{ + Cpu: cpu, + Memory: mem, + InvokeLabels: invokeLabels, + CustomResources: customRes, + CustomExtensions: req.InvokeTag, + SchedulerFunctionID: req.SchedulerFuncKey, + SchedulerInstanceIDs: []string{req.SchedulerID}, + TraceID: req.TraceID, + Timeout: int(req.InvokeTimeout), + AcquireTimeout: int(req.AcquireTimeout), + } + if req.InstanceSession != nil { + invokeOpt.InstanceSession = &api.InstanceSessionConfig{ + SessionID: req.InstanceSession.SessionID, + SessionTTL: req.InstanceSession.SessionTTL, + Concurrency: req.InstanceSession.Concurrency, + } + } + return invokeOpt +} + +func convertAcquireOption(req AcquireOption) api.InvokeOptions { + cpu, mem, customRes := LibruntimeCustomResources(req.ResourceSpecs) + invokeOpt := api.InvokeOptions{ + Cpu: cpu, + Memory: mem, + CustomResources: customRes, + SchedulerFunctionID: req.SchedulerFuncKey, + SchedulerInstanceIDs: []string{req.SchedulerID}, + TraceID: req.TraceID, + RetryTimes: maxInvokeRetries, + Timeout: int(req.Timeout), + AcquireTimeout: int(req.Timeout), + TrafficLimited: req.TrafficLimited, + } + return invokeOpt +} + +// InvokeByName - +func (c *defaultClient) InvokeByName(req InvokeRequest) ([]byte, error) { + wait := make(chan struct{}, 1) + var ( + res []byte + resErr, err error + ) + funcMeta := api.FunctionMeta{ + FuncID: req.Function, + Name: &req.InstanceID, + Sig: req.FuncSig, + Api: utils.GetAPIType(req.BusinessType), + PoolLabel: req.PoolLabel, + } + funcArgs := deepCopyArgs(req.Args, req.TenantID) + invokeOpt := convertInvokeOption(req) + var objID string + objID, err = c.clientLibruntime.InvokeByFunctionName(funcMeta, funcArgs, invokeOpt) + if err != nil { + return nil, err + } + c.clientLibruntime.GetAsync(objID, func(result []byte, err error) { + res = result + resErr = err + wait <- struct{}{} + if _, err := c.clientLibruntime.GDecreaseRef([]string{objID}); err != nil { + fmt.Printf("failed to decrease object ref,err: %s", err.Error()) + } + }) + <-wait + return res, resErr +} + +func (c *defaultClient) CreateInstanceRaw(createReq []byte) ([]byte, error) { + resp, err := c.clientLibruntime.CreateInstanceRaw(createReq) + return resp, err +} + +func (c *defaultClient) InvokeInstanceRaw(invokeReq []byte) ([]byte, error) { + notify, err := c.clientLibruntime.InvokeByInstanceIdRaw(invokeReq) + return notify, err +} + +func (c *defaultClient) KillByLibRt(instanceID string, signal int, payload []byte) error { + return c.clientLibruntime.Kill(instanceID, signal, payload) +} + +func (c *defaultClient) CreateInstanceByLibRt( + funcMeta api.FunctionMeta, + args []api.Arg, + invokeOpt api.InvokeOptions, +) (string, error) { + return c.clientLibruntime.CreateInstance(funcMeta, args, invokeOpt) +} + +func (c *defaultClient) KillRaw(killReq []byte) ([]byte, error) { + resp, err := c.clientLibruntime.KillRaw(killReq) + return resp, err +} + +func (c *defaultClient) IsHealth() bool { + return c.clientLibruntime.IsHealth() +} + +func (c *defaultClient) IsDsHealth() bool { + return c.clientLibruntime.IsHealth() +} diff --git a/frontend/pkg/frontend/common/util/client_test.go b/frontend/pkg/frontend/common/util/client_test.go new file mode 100644 index 0000000000000000000000000000000000000000..b957d30b32abb1419012bfb1bc6c4822a4255ad3 --- /dev/null +++ b/frontend/pkg/frontend/common/util/client_test.go @@ -0,0 +1,99 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package util + +import ( + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + . "github.com/smartystreets/goconvey/convey" + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/constant" + mockUtils "frontend/pkg/common/faas_common/utils" + "frontend/pkg/common/uuid" +) + +func TestNewClientLibruntime(t *testing.T) { + mock := &mockUtils.FakeLibruntimeSdkClient{} + Convey("TestNewClientLibruntime", t, func() { + testInstID := uuid.New().String() + returnObjID := uuid.New().String() + result := []byte(uuid.New().String()) + req := InvokeRequest{ + Function: "test", + Args: nil, + InstanceID: testInstID, + } + + patches := []*gomonkey.Patches{ + gomonkey.ApplyMethod(reflect.TypeOf(mock), "GetAsync", + func(_ *mockUtils.FakeLibruntimeSdkClient, objectID string, cb api.GetAsyncCallback) { + cb(result, nil) + return + }), + gomonkey.ApplyMethod(reflect.TypeOf(mock), "InvokeByFunctionName", + func(_ *mockUtils.FakeLibruntimeSdkClient, funcMeta api.FunctionMeta, args []api.Arg, + invokeOpt api.InvokeOptions) (string, error) { + return testInstID, nil + }), + gomonkey.ApplyMethod(reflect.TypeOf(mock), "InvokeByInstanceId", + func(_ *mockUtils.FakeLibruntimeSdkClient, funcMeta api.FunctionMeta, instanceID string, args []api.Arg, + invokeOpt api.InvokeOptions) (string, error) { + So(instanceID, ShouldEqual, testInstID) + return returnObjID, nil + }), + } + defer func() { + for _, patch := range patches { + patch.Reset() + } + }() + + client := newDefaultClientLibruntime(mock) + So(client, ShouldNotBeNil) + res, err := client.InvokeByName(req) + So(err, ShouldBeNil) + So(res, ShouldResemble, result) + + res, err = client.Invoke(req) + So(err, ShouldBeNil) + So(res, ShouldResemble, result) + }) +} + +func Test_defaultClient_AcquireInstance(t *testing.T) { + Convey("test AcquireInstance", t, func() { + Convey("baseline", func() { + mock := &mockUtils.FakeLibruntimeSdkClient{} + client := newDefaultClientLibruntime(mock) + instance, err := client.AcquireInstance("func", AcquireOption{ + DesignateInstanceID: "id", + FuncSig: "aaa", + ResourceSpecs: map[string]int64{ + constant.ResourceCPUName: 1000, + constant.ResourceMemoryName: 1000, + }, + Timeout: 100, + TrafficLimited: false, + }) + So(err, ShouldBeNil) + So(instance, ShouldNotBeNil) + }) + }) +} diff --git a/frontend/pkg/frontend/common/util/common.go b/frontend/pkg/frontend/common/util/common.go new file mode 100644 index 0000000000000000000000000000000000000000..53a4ccfbe3730087da860d66c9136427b2b8b216 --- /dev/null +++ b/frontend/pkg/frontend/common/util/common.go @@ -0,0 +1,155 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package util - +package util + +import ( + "bytes" + "encoding/json" + "fmt" + "strconv" + "time" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/log" + commonType "frontend/pkg/common/faas_common/types" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/types" +) + +// Retry - +func Retry(execute func() error, shouldRetry func() bool, maxRetries int, sleep time.Duration) error { + var err error + for i := 0; i < maxRetries; i++ { + err = execute() + if err == nil { + return nil + } + if shouldRetry() { + time.Sleep(sleep) + } else { + return err + } + } + return err +} + +// LibruntimeCustomResources - +func LibruntimeCustomResources(res map[string]int64) (int, int, map[string]float64) { + var cpu, mem int + rtRes := make(map[string]float64, len(res)) + for k, v := range res { + rtRes[k] = float64(v) + if k == constant.ResourceCPUName { + cpu = int(v) + } else if k == constant.ResourceMemoryName { + mem = int(v) + } + } + return cpu, mem, rtRes +} + +// ConvertResourceSpecs - +func ConvertResourceSpecs(ctx *types.InvokeProcessContext, funcSpec *commonType.FuncSpec) (map[string]int64, error) { + if config.GetConfig().BusinessType == constant.BusinessTypeWiseCloud { + return nil, nil + } + resourceSpecs := make(map[string]int64) + setCPUMemory(ctx, funcSpec, resourceSpecs) + if funcSpec.ResourceMetaData.CustomResources != "" { + var customResources map[string]int64 + if err := json.Unmarshal([]byte(funcSpec.ResourceMetaData.CustomResources), &customResources); err != nil { + log.GetLogger().Errorf("failed to unmarshal custom resources %s", err.Error()) + return nil, err + } + for resourceType, resource := range customResources { + if resource > constant.MinCustomResourcesSize { + resourceSpecs[resourceType] = resource + } else { + log.GetLogger().Warnf("ignore invalid value %f of custom resource %s", resource, resourceType) + } + } + } + return resourceSpecs, nil +} + +func setCPUMemory(ctx *types.InvokeProcessContext, funcSpec *commonType.FuncSpec, resourceSpecs map[string]int64) { + if resourceSpecs == nil { + return + } + resourceSpecs[constant.ResourceCPUName] = funcSpec.ResourceMetaData.CPU + resourceSpecs[constant.ResourceMemoryName] = funcSpec.ResourceMetaData.Memory + if ctx == nil || ctx.ReqHeader == nil { + return + } + if cpuString := PeekIgnoreCase(ctx.ReqHeader, constant.HeaderCPUSize); cpuString != "" { + cpu, err := strconv.Atoi(cpuString) + if err != nil { + log.GetLogger().Warnf("invalid value %s from request header", constant.HeaderCPUSize) + resourceSpecs[constant.ResourceCPUName] = funcSpec.ResourceMetaData.CPU + } else { + resourceSpecs[constant.ResourceCPUName] = int64(cpu) + } + } + + if memoryString := PeekIgnoreCase(ctx.ReqHeader, constant.HeaderMemorySize); memoryString != "" { + memory, err := strconv.Atoi(memoryString) + if err != nil { + log.GetLogger().Warnf("invalid value %s from request header", constant.ResourceMemoryName) + resourceSpecs[constant.ResourceMemoryName] = funcSpec.ResourceMetaData.Memory + } else { + resourceSpecs[constant.ResourceMemoryName] = int64(memory) + } + } +} + +// UnmarshalCallResp - +func UnmarshalCallResp(message []byte) (*types.CallResp, error) { + respMsg := &types.CallResp{} + err := json.Unmarshal(message, respMsg) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal call response data: %s", err) + } + return respMsg, nil +} + +// GetAcquireTimeout - +func GetAcquireTimeout(funcSpec *commonType.FuncSpec) int64 { + acquireTimeout := funcSpec.ExtendedMetaData.Initializer.Timeout + if funcSpec.FuncMetaData.Runtime == constant.CustomContainerRuntimeType { + acquireTimeout += constant.CustomImageExtraTimeout + } + // if acquireTimeout is 0,use default 120s set by libruntime, add CommonExtraTimeout two times, one for scheduler, + // one for kernel + if acquireTimeout > 0 { + acquireTimeout += 2*constant.CommonExtraTimeout + constant.KernelScheduleTimeout + } + return acquireTimeout +} + +// PeekIgnoreCase Compatible with uppercase and lowercase letters +func PeekIgnoreCase(reqHeader map[string]string, name string) string { + if value, ok := reqHeader[name]; ok { + return value + } + for key, value := range reqHeader { + if bytes.EqualFold([]byte(key), []byte(name)) { + return value + } + } + return "" +} diff --git a/frontend/pkg/frontend/common/util/common_test.go b/frontend/pkg/frontend/common/util/common_test.go new file mode 100644 index 0000000000000000000000000000000000000000..0a5eba693dc8a2a58b0cfcd538d19470f816532d --- /dev/null +++ b/frontend/pkg/frontend/common/util/common_test.go @@ -0,0 +1,151 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package util - +package util + +import ( + "errors" + "reflect" + "testing" + "time" + + "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/common/faas_common/constant" + commontype "frontend/pkg/common/faas_common/types" + "frontend/pkg/frontend/types" +) + +func TestConvertResourceSpecs(t *testing.T) { + type args struct { + ctx *types.InvokeProcessContext + funcSpec *commontype.FuncSpec + } + ctx := &types.InvokeProcessContext{ + ReqHeader: map[string]string{}, + } + tests := []struct { + name string + args args + want map[string]int64 + wantErr bool + }{ + {"case1 succeed to convert custom resource", + args{ + ctx: ctx, + funcSpec: &commontype.FuncSpec{ + ResourceMetaData: commontype.ResourceMetaData{CPU: 300, Memory: 128, CustomResources: "{\"nvidia.com/gpu\":8}"}, + }}, + map[string]int64{"CPU": 300, "Memory": 128, "nvidia.com/gpu": 8}, false}, + {"case2 failed to unmarshal", + args{ + ctx: ctx, + funcSpec: &commontype.FuncSpec{ + ResourceMetaData: commontype.ResourceMetaData{CPU: 300, Memory: 128, CustomResources: "b"}, + }}, + nil, true}, + {"case3 failed to un", + args{ + ctx: ctx, + funcSpec: &commontype.FuncSpec{ + ResourceMetaData: commontype.ResourceMetaData{CPU: 300, Memory: 128, CustomResources: "{\"nvidia.com/gpu\":0}"}}, + }, + map[string]int64{"CPU": 300, "Memory": 128}, false}, + {"case4 succeed to convert custom resource from header", + args{ + ctx: &types.InvokeProcessContext{ + ReqHeader: map[string]string{constant.HeaderCPUSize: "400", constant.HeaderMemorySize: "256"}, + }, + funcSpec: &commontype.FuncSpec{ + ResourceMetaData: commontype.ResourceMetaData{CPU: 300, Memory: 128, CustomResources: "{\"nvidia.com/gpu\":8}"}, + }}, + map[string]int64{"CPU": 400, "Memory": 256, "nvidia.com/gpu": 8}, false}, + {"case5 failed to convert custom resource from header", + args{ + ctx: &types.InvokeProcessContext{ + ReqHeader: map[string]string{constant.HeaderCPUSize: "aaa", constant.HeaderMemorySize: "bbb"}, + }, + funcSpec: &commontype.FuncSpec{ + ResourceMetaData: commontype.ResourceMetaData{CPU: 300, Memory: 128, CustomResources: "{\"nvidia.com/gpu\":8}"}, + }}, + map[string]int64{"CPU": 300, "Memory": 128, "nvidia.com/gpu": 8}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ConvertResourceSpecs(tt.args.ctx, tt.args.funcSpec) + if (err != nil) != tt.wantErr { + t.Errorf("ConvertResourceSpecs() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ConvertResourceSpecs() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestRetry(t *testing.T) { + convey.Convey("Retry", t, func() { + retryCount := 0 + do := func() error { + if retryCount > 2 { + return nil + } + return errors.New("should retry") + } + ifRetry := func() bool { + retryCount++ + if retryCount > 2 { + return false + } + return true + } + err := Retry(do, ifRetry, 3, 500*time.Millisecond) + convey.So(err.Error(), convey.ShouldEqual, "should retry") + + retryCount = 0 + do = func() error { + if retryCount > 1 { + return nil + } + return errors.New("should retry") + } + err = Retry(do, ifRetry, 3, 500*time.Millisecond) + convey.So(err, convey.ShouldBeNil) + }) +} + +func TestPeekIgnoreCase(t *testing.T) { + tests := []struct { + reqHeader map[string]string + name string + want string + }{ + {map[string]string{"Content-Type": "application/json"}, "Content-Type", "application/json"}, + {map[string]string{"content-type": "application/json"}, "Content-Type", "application/json"}, + {map[string]string{"CONTENT-TYPE": "application/json"}, "content-type", "application/json"}, + {map[string]string{}, "Content-Type", ""}, + {map[string]string{"X-Custom-Header": "value"}, "Content-Type", ""}, + } + + for _, tt := range tests { + got := PeekIgnoreCase(tt.reqHeader, tt.name) + if got != tt.want { + t.Errorf("PeekIgnoreCase(%v, %v) = %v, want %v", tt.reqHeader, tt.name, got, tt.want) + } + } +} diff --git a/frontend/pkg/frontend/config/config.go b/frontend/pkg/frontend/config/config.go new file mode 100644 index 0000000000000000000000000000000000000000..23ac9b12a3e2df9d664daf1b9a5f734542c8003e --- /dev/null +++ b/frontend/pkg/frontend/config/config.go @@ -0,0 +1,352 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package config is used to keep the config used by the faas frontend function +package config + +import ( + "encoding/json" + "fmt" + "os" + "sync" + + "github.com/asaskevich/govalidator/v11" + + "frontend/pkg/common/faas_common/alarm" + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/crypto" + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/redisclient" + "frontend/pkg/common/faas_common/sts" + commonType "frontend/pkg/common/faas_common/types" + "frontend/pkg/common/faas_common/utils" + "frontend/pkg/frontend/types" +) + +const ( + // DefaultTimeout defines the timeout of invoke + DefaultTimeout = 1200 + defaultE2EMaxDelay = 50 + defaultRPCClientConcurrentNum = 1 + defaultStreamLengthLimitMb = 1024 + defaultDataSystemPayloadLimitByte = 32 +) + +const ( + defaultLowerMemoryPercent = 0.6 + defaultHighMemoryPercent = 0.8 + defaultStatefulHighMemoryPercent = 0.85 + defaultMemoryRefreshInterval = 20 + defaultBodyThreshold = 40000 + defaultRequestMemoryEvaluator = 2 + defaultTenantLimitQuota = 1800 + // heartbeat default config + defaultHeartbeatTimeout = 2 + defaultHeartbeatInterval = 3 + defaultHeartbeatTimeoutThreshold = 3 +) + +const ( + localAuthConfigEnvKey = "PAAS_CRYPTO_PATH" + defaultLocalAuthConfigPath = "/home/sn/resource/cipher" +) + +var ( + fConfig = &types.Config{} + nativeAz = "" + loadAzOnce sync.Once +) + +// MetricServerConfig define monitoring server config +type MetricServerConfig struct { + ServerAddr string `json:"serverAddr,omitempty" valid:",optional"` + ServerMode string `json:"serverMode,omitempty" valid:",optional"` + Password string `json:"password,omitempty" valid:",optional"` + EnableTLS bool `json:"enableTLS,omitempty" valid:",optional"` + TimeoutConf redisclient.TimeoutConf `json:"timeoutConf,omitempty" valid:",optional"` +} + +// GetConfig return the current fConfig +func GetConfig() *types.Config { + return fConfig +} + +// SetConfig set the current fConfig +func SetConfig(conf types.Config) { + fConfig = &conf +} + +// InitFunctionConfig is used to initialize the config +func InitFunctionConfig(data []byte) error { + err := json.Unmarshal(data, &fConfig) + if err != nil { + return fmt.Errorf("failed to parse the config data: %s", err) + } + err = loadFunctionConfig(fConfig) + if err != nil { + return err + } + initDefaultTenantLimitQuota() + initDefaultMemoryControlConfig() + initDefaultMemoryEvaluatorConfig() + initDefaultLocalAuthConfig() + initDefaultHeartbeatConfig() + initDefaultHTTPConfig() + return nil +} + +// loadFunctionConfig is used to initialize the config +func loadFunctionConfig(config *types.Config) error { + if config.BusinessType == constant.BusinessTypeWiseCloud && config.DataSystemConfig == nil { + return fmt.Errorf("invalid config: empty data system config in caas type") + } + if config.RouterEtcd.UseSecret { + etcd3.SetETCDTLSConfig(&config.RouterEtcd) + } + if config.MetaEtcd.UseSecret { + etcd3.SetETCDTLSConfig(&config.MetaEtcd) + } + _, err := govalidator.ValidateStruct(config) + if err != nil { + return fmt.Errorf("invalid config: %s", err) + } + err = setAlarmEnv(config) + if err != nil { + return err + } + if config.RawStsConfig.StsEnable { + if err = sts.InitStsSDK(config.RawStsConfig.ServerConfig); err != nil { + log.GetLogger().Errorf("failed to init sts sdk, err: %s", err.Error()) + return err + } + if err = os.Setenv(sts.EnvSTSEnable, "true"); err != nil { + log.GetLogger().Errorf("failed to set env of %s, err: %s", sts.EnvSTSEnable, err.Error()) + return err + } + } + + if config.SccConfig.Enable && crypto.InitializeSCC(config.SccConfig) != nil { + return fmt.Errorf("failed to initialize scc") + } + + return nil +} + +// InitModuleConfig initializes config for module +func InitModuleConfig() error { + configFromEnv, err := loadConfigFromEnv() + if err != nil { + log.GetLogger().Errorf("loadConfigFromEnv failed err %s", err) + return err + } + + fConfig = configFromEnv + fConfig.EtcdLeaseConfig = &types.EtcdLeaseConfig{} + + log.GetLogger().Infof("init config success, authenticationEnable: %v", fConfig.AuthenticationEnable) + + utils.ValidateTimeout(&fConfig.HTTPConfig.RespTimeOut, DefaultTimeout) + utils.ValidateTimeout(&fConfig.HTTPConfig.WorkerInstanceReadTimeOut, DefaultTimeout) + if _, err = govalidator.ValidateStruct(fConfig); err != nil { + log.GetLogger().Errorf("initConfigData error: %s", err.Error()) + return err + } + if fConfig.E2EMaxDelayTime <= 0 { + fConfig.E2EMaxDelayTime = defaultE2EMaxDelay + } + if fConfig.RPCClientConcurrentNum < 1 { + fConfig.RPCClientConcurrentNum = defaultRPCClientConcurrentNum + } + loadAzOnce.Do(func() { + var exist bool + nativeAz, exist = os.LookupEnv(fConfig.Runtime.AvailableZoneKey) + if !exist || nativeAz == "" { + nativeAz = constant.DefaultAZ + } + if len(nativeAz) > constant.ZoneNameLen { + nativeAz = nativeAz[0 : constant.ZoneNameLen-1] + } + }) + initDefaultTenantLimitQuota() + initDefaultMemoryControlConfig() + initDefaultMemoryEvaluatorConfig() + initDefaultHeartbeatConfig() + initDefaultHTTPConfig() + return nil +} + +func loadConfigFromEnv() (*types.Config, error) { + configJSON := os.Getenv(ConfigEnvKey) + config := &types.Config{} + err := json.Unmarshal([]byte(configJSON), config) + if err != nil { + return nil, err + } + return config, nil +} + +// RecoverConfig will recover config +func RecoverConfig(stateConfig types.Config) error { + fConfig = &types.Config{} + err := utils.DeepCopyObj(stateConfig, &fConfig) + if err != nil { + return err + } + err = setAlarmEnv(fConfig) + if err != nil { + return err + } + log.GetLogger().Infof("configuration recovered ") + return nil +} + +// InitEtcd - init router etcd and meta etcd +func InitEtcd(stopCh <-chan struct{}) error { + if &fConfig == nil { + return fmt.Errorf("config is not initialized") + } + if err := etcd3.InitRouterEtcdClient(fConfig.RouterEtcd, fConfig.AlarmConfig, stopCh); err != nil { + return fmt.Errorf("faaSFrontend failed to init route etcd: %s", err.Error()) + } + + if err := etcd3.InitMetaEtcdClient(fConfig.MetaEtcd, fConfig.AlarmConfig, stopCh); err != nil { + return fmt.Errorf("faaSFrontend failed to init metadata etcd: %s", err.Error()) + } + + if len(fConfig.CAEMetaEtcd.Servers) != 0 { + if err := etcd3.InitCAEMetaEtcdClient(fConfig.CAEMetaEtcd, fConfig.AlarmConfig, stopCh); err != nil { + return fmt.Errorf("faaSFrontend failed to init cae metadata etcd: %s", err.Error()) + } + log.GetLogger().Infof("init CAEMetaEtcd success") + } + + if len(fConfig.DataSystemEtcd.Servers) != 0 { + if err := etcd3.InitDataSystemEtcdClient(fConfig.DataSystemEtcd, fConfig.AlarmConfig, stopCh); err != nil { + return fmt.Errorf("faaSFrontend failed to init dataSystemEtcd etcd: %s", err.Error()) + } + log.GetLogger().Infof("init dataSystemEtcd success") + } + + return nil +} + +// ClearSensitiveInfo - +func ClearSensitiveInfo() { + if &fConfig == nil { + return + } + utils.ClearStringMemory(fConfig.RouterEtcd.Password) + utils.ClearStringMemory(fConfig.MetaEtcd.Password) +} + +func setAlarmEnv(fConfig *types.Config) error { + if !fConfig.AlarmConfig.EnableAlarm { + log.GetLogger().Infof("enable alarm is false") + return nil + } + utils.SetClusterNameEnv(fConfig.ClusterName) + alarm.SetAlarmEnv(fConfig.AlarmConfig.AlarmLogConfig) + alarm.SetXiangYunFourConfigEnv(fConfig.AlarmConfig.XiangYunFourConfig) + err := alarm.SetPodIP() + if err != nil { + return err + } + return nil +} + +func initDefaultLocalAuthConfig() { + LocalAuthCryptoPath := defaultLocalAuthConfigPath + if fConfig.AuthConfig.LocalAuthConfig.LocalAuthCryptoPath != "" { + LocalAuthCryptoPath = fConfig.AuthConfig.LocalAuthConfig.LocalAuthCryptoPath + } + err := os.Setenv(localAuthConfigEnvKey, LocalAuthCryptoPath) + if err != nil { + log.GetLogger().Warnf("initDefaultLocalAuthConfig error, error is %s", err.Error()) + return + } + return +} + +func initDefaultMemoryEvaluatorConfig() { + if fConfig.MemoryEvaluatorConfig == nil { + fConfig.MemoryEvaluatorConfig = &types.MemoryEvaluatorConfig{} + } + if fConfig.MemoryEvaluatorConfig.RequestMemoryEvaluator <= 0 { + fConfig.MemoryEvaluatorConfig.RequestMemoryEvaluator = defaultRequestMemoryEvaluator + } + log.GetLogger().Infof("RequestMemoryEvaluator %f", fConfig.MemoryEvaluatorConfig.RequestMemoryEvaluator) +} + +func initDefaultHTTPConfig() { + if fConfig.HTTPConfig == nil { + fConfig.HTTPConfig = &types.FrontendHTTP{} + } + if fConfig.HTTPConfig.MaxStreamRequestBodySize == 0 { + fConfig.HTTPConfig.MaxStreamRequestBodySize = defaultStreamLengthLimitMb + } + + if fConfig.HTTPConfig.MaxDataSystemMultiDataBodySize == 0 { + fConfig.HTTPConfig.MaxDataSystemMultiDataBodySize = defaultDataSystemPayloadLimitByte + } + if fConfig.HTTPConfig.ServerListenPort == 0 { + fConfig.HTTPConfig.ServerListenPort = HTTPServerListenPort + } +} + +func initDefaultMemoryControlConfig() { + if fConfig.MemoryControlConfig == nil { + fConfig.MemoryControlConfig = &commonType.MemoryControlConfig{} + } + + if fConfig.MemoryControlConfig.LowerMemoryPercent <= 0 { + fConfig.MemoryControlConfig.LowerMemoryPercent = defaultLowerMemoryPercent + } + if fConfig.MemoryControlConfig.HighMemoryPercent <= 0 { + fConfig.MemoryControlConfig.HighMemoryPercent = defaultHighMemoryPercent + } + if fConfig.MemoryControlConfig.StatefulHighMemPercent <= 0 { + fConfig.MemoryControlConfig.StatefulHighMemPercent = defaultStatefulHighMemoryPercent + } + if fConfig.MemoryControlConfig.MemDetectIntervalMs <= 0 { + fConfig.MemoryControlConfig.MemDetectIntervalMs = defaultMemoryRefreshInterval + } + if fConfig.MemoryControlConfig.BodyThreshold <= 0 { + fConfig.MemoryControlConfig.BodyThreshold = defaultBodyThreshold + } +} + +func initDefaultTenantLimitQuota() { + if fConfig.DefaultTenantLimitQuota <= 0 { + fConfig.DefaultTenantLimitQuota = defaultTenantLimitQuota + } + log.GetLogger().Infof("defaultTenantLimitQuota %d", fConfig.DefaultTenantLimitQuota) +} + +func initDefaultHeartbeatConfig() { + if fConfig.HeartbeatConfig == nil { + fConfig.HeartbeatConfig = &types.HeartbeatConfig{} + } + if fConfig.HeartbeatConfig.HeartbeatTimeout <= 0 { + fConfig.HeartbeatConfig.HeartbeatTimeout = defaultHeartbeatTimeout + } + if fConfig.HeartbeatConfig.HeartbeatInterval <= 0 { + fConfig.HeartbeatConfig.HeartbeatInterval = defaultHeartbeatInterval + } + if fConfig.HeartbeatConfig.HeartbeatTimeoutThreshold <= 0 { + fConfig.HeartbeatConfig.HeartbeatTimeoutThreshold = defaultHeartbeatTimeoutThreshold + } +} diff --git a/frontend/pkg/frontend/config/config_test.go b/frontend/pkg/frontend/config/config_test.go new file mode 100644 index 0000000000000000000000000000000000000000..250667ecd5676853b35a683a849a48e96326b569 --- /dev/null +++ b/frontend/pkg/frontend/config/config_test.go @@ -0,0 +1,334 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package config is used to keep the config used by the faas frontend function +package config + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/asaskevich/govalidator/v11" + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + + "frontend/pkg/common/faas_common/alarm" + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/sts" + "frontend/pkg/common/faas_common/sts/raw" + "frontend/pkg/common/faas_common/utils" + mockUtils "frontend/pkg/common/faas_common/utils" + "frontend/pkg/frontend/types" +) + +func Test_InitFunctionConfig(t *testing.T) { + convey.Convey("init config error 1", t, func() { + cfg := &types.Config{} + inputCfg, _ := json.Marshal(cfg) + defer gomonkey.ApplyFunc(json.Unmarshal, func(data []byte, v any) error { + return fmt.Errorf("unmarshal error") + }).Reset() + err := InitFunctionConfig(inputCfg) + convey.So(err, convey.ShouldNotBeNil) + }) + + convey.Convey("init config error 2", t, func() { + defer gomonkey.ApplyFunc(govalidator.ValidateStruct, func(s interface{}) (bool, error) { + return true, nil + }).Reset() + cfg := &types.Config{} + cfg.BusinessType = constant.BusinessTypeWiseCloud + cfg.DataSystemConfig = nil + inputCfg, _ := json.Marshal(cfg) + err := InitFunctionConfig(inputCfg) + convey.So(err, convey.ShouldNotBeNil) + }) + + convey.Convey("init config error 3", t, func() { + defer gomonkey.ApplyFunc(govalidator.ValidateStruct, func(s interface{}) (bool, error) { + return true, nil + }).Reset() + defer gomonkey.ApplyFunc(setAlarmEnv, func(FConfig *types.Config) error { + return fmt.Errorf("setAlarmEnv error ") + }).Reset() + cfg := &types.Config{} + inputCfg, _ := json.Marshal(cfg) + err := InitFunctionConfig(inputCfg) + convey.So(err, convey.ShouldNotBeNil) + }) + + convey.Convey("init config success", t, func() { + cfg := &types.Config{} + inputCfg, _ := json.Marshal(cfg) + err := InitFunctionConfig(inputCfg) + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("init default maxStreamRequestBodySize", t, func() { + defer gomonkey.ApplyFunc(govalidator.ValidateStruct, func(s interface{}) (bool, error) { + return true, nil + }).Reset() + cfg := &types.Config{} + inputCfg, _ := json.Marshal(cfg) + err := InitFunctionConfig(inputCfg) + convey.So(err, convey.ShouldBeNil) + convey.ShouldEqual(GetConfig().HTTPConfig.MaxStreamRequestBodySize, 1024) + }) + convey.Convey("init config success", t, func() { + defer gomonkey.ApplyFunc(govalidator.ValidateStruct, func(s interface{}) (bool, error) { + return true, nil + }).Reset() + cfg := &types.Config{} + inputCfg, _ := json.Marshal(cfg) + err := InitFunctionConfig(inputCfg) + convey.So(err, convey.ShouldBeNil) + }) +} + +func Test_InitModuleConfig(t *testing.T) { + convey.Convey("init from env failed", t, func() { + err := InitModuleConfig() + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("init from env json error", t, func() { + defer gomonkey.ApplyFunc(os.Getenv, func(key string) string { + return `{"http":{"maxRequestBodySize":5}, "cpu":500, "memory":500}` + }).Reset() + err := InitModuleConfig() + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("init from env success", t, func() { + defer gomonkey.ApplyFunc(os.Getenv, func(key string) string { + return `{"http":{"maxRequestBodySize":5}, "cpu":500, "memory":500, "metaEtcd":{"servers":[]}, "routerEtcd":{"servers":[]}, "slaQuota":1000, "runtime":{}}` + }).Reset() + err := InitModuleConfig() + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("init default maxStreamRequestBodySize", t, func() { + defer gomonkey.ApplyFunc(os.Getenv, func(key string) string { + return `{"http":{"maxRequestBodySize":5}, "cpu":500, "memory":500, "metaEtcd":{"servers":[]}, "routerEtcd":{"servers":[]}, "slaQuota":1000, "runtime":{}}` + }).Reset() + err := InitModuleConfig() + convey.So(err, convey.ShouldBeNil) + convey.ShouldEqual(GetConfig().HTTPConfig.MaxStreamRequestBodySize, 1024) + }) +} + +func TestRecoverConfig(t *testing.T) { + cfgByte := []byte(`{"Config":{ + "slaQuota": 1000, + "functionCapability": 1, + "authenticationEnable": false, + "trafficLimitDisable": true, + "http": { + "resptimeout": 5, + "workerInstanceReadTimeOut": 5, + "maxRequestBodySize": 6 + }, + "routerEtcd": { + "servers": ["1.2.3.4:1234"], + "user": "tom", + "password": "**" + }, + "metaEtcd": { + "servers": ["1.2.3.4:5678"], + "user": "tom", + "password": "**" + } + }}`) + tests := []struct { + name string + wantErr bool + }{ + { + name: "TestRecoverConfig", + wantErr: false, + }, + } + cfg := types.Config{} + json.Unmarshal(cfgByte, &cfg) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := RecoverConfig(cfg); (err != nil) != tt.wantErr { + t.Errorf("RecoverConfig() error = %v, wantErr %v", err, tt.wantErr) + cfgTest := GetConfig() + assert.Equal(t, cfgTest.SLAQuota, 1000) + assert.Equal(t, cfgTest.RouterEtcd.Servers, []string{"1.2.3.4:1234"}) + } + }) + } + + convey.Convey("RecoverConfig error", t, func() { + defer gomonkey.ApplyFunc(utils.DeepCopyObj, func(src interface{}, dst interface{}) error { + return fmt.Errorf("DeepCopyObj error") + }).Reset() + err := RecoverConfig(cfg) + convey.So(err, convey.ShouldNotBeNil) + }) +} + +func TestInitConfig(t *testing.T) { + cfg := []byte(`{ + "slaQuota": 1000, + "functionCapability": 1, + "authenticationEnable": false, + "trafficLimitDisable": true, + "rawStsConfig": {"stsEnable": true}, + "http": { + "resptimeout": 5, + "workerInstanceReadTimeOut": 5, + "maxRequestBodySize": 6 + }, + "routerEtcd": { + "servers": ["1.2.3.4:1234"], + "user": "tom", + "password": "**" + }, + "metaEtcd": { + "servers": ["1.2.3.4:5678"], + "user": "tom", + "password": "**" + } + }`) + type args struct { + data []byte + } + tests := []struct { + name string + args args + wantErr assert.ErrorAssertionFunc + patchesFunc mockUtils.PatchesFunc + }{ + {"case1 succeed to init config when caas", args{data: cfg}, assert.NoError, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(govalidator.ValidateStruct, func(s interface{}) (bool, error) { return true, nil }), + gomonkey.ApplyFunc(sts.InitStsSDK, func(serverCfg raw.ServerConfig) error { return nil }), + }) + return patches + }}, + {"case2 failed to init config when caas", args{data: cfg}, assert.Error, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(govalidator.ValidateStruct, func(s interface{}) (bool, error) { return true, nil }), + gomonkey.ApplyFunc(sts.InitStsSDK, func(serverCfg raw.ServerConfig) error { return errors.New("e") }), + }) + return patches + }}, + {"case3 failed to init config when caas 2", args{data: cfg}, assert.Error, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyFunc(govalidator.ValidateStruct, func(s interface{}) (bool, error) { return true, nil }), + gomonkey.ApplyFunc(sts.InitStsSDK, func(serverCfg raw.ServerConfig) error { return nil }), + gomonkey.ApplyFunc(os.Setenv, func(key, value string) error { return errors.New("e") }), + }) + return patches + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + patches := tt.patchesFunc() + tt.wantErr(t, InitFunctionConfig(tt.args.data), fmt.Sprintf("InitFunctionConfig(%v)", tt.args.data)) + patches.ResetAll() + }) + } +} + +func TestInitEtcd(t *testing.T) { + type args struct { + stopCh <-chan struct{} + } + tests := []struct { + name string + args args + wantErr assert.ErrorAssertionFunc + patchesFunc mockUtils.PatchesFunc + }{ + {"case1 succeed to init etcd", args{stopCh: make(<-chan struct{})}, assert.NoError, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyMethod(reflect.TypeOf(&etcd3.EtcdInitParam{}), "InitClient", func(_ *etcd3.EtcdInitParam) error { return nil }), + }) + return patches + }}, + {"case2 failed to init etcd", args{stopCh: make(<-chan struct{})}, assert.Error, func() mockUtils.PatchSlice { + patches := mockUtils.InitPatchSlice() + patches.Append(mockUtils.PatchSlice{ + gomonkey.ApplyMethod(reflect.TypeOf(&etcd3.EtcdInitParam{}), "InitClient", func(_ *etcd3.EtcdInitParam) error { return errors.New("e") }), + }) + return patches + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + patches := tt.patchesFunc() + tt.wantErr(t, InitEtcd(tt.args.stopCh), fmt.Sprintf("InitEtcd(%v)", tt.args.stopCh)) + patches.ResetAll() + }) + } +} + +func TestClearSensitiveInfo(t *testing.T) { + cfg := []byte(`{ + "slaQuota": 1000, + "functionCapability": 1, + "authenticationEnable": false, + "trafficLimitDisable": true, + "rawStsConfig": {"stsEnable": true}, + "smsConfig": {"accessKey": "ak"}, + "http": { + "resptimeout": 5, + "workerInstanceReadTimeOut": 5, + "maxRequestBodySize": 6 + }, + "routerEtcd": { + "servers": ["1.2.3.4:1234"], + "user": "tom", + "password": "**" + }, + "metaEtcd": { + "servers": ["1.2.3.4:5678"], + "user": "tom", + "password": "**" + } + }`) + defer gomonkey.ApplyFunc(govalidator.ValidateStruct, func(s interface{}) (bool, error) { + return true, nil + }).Reset() + _ = InitFunctionConfig(cfg) + tests := []struct { + name string + }{ + {"case1"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ClearSensitiveInfo() + }) + } +} + +func TestSetAlarmEnv(t *testing.T) { + err := setAlarmEnv(&types.Config{AlarmConfig: alarm.Config{ + EnableAlarm: true, + }}) + assert.Nil(t, err) +} diff --git a/frontend/pkg/frontend/config/constant.go b/frontend/pkg/frontend/config/constant.go new file mode 100644 index 0000000000000000000000000000000000000000..f8d76e81f46f4b08f2df300ae3ae28fd782c3c13 --- /dev/null +++ b/frontend/pkg/frontend/config/constant.go @@ -0,0 +1,27 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package config is used to keep the config used by the faas frontend function +package config + +const ( + // HTTPServerListenPort is the listen port of frontend http server + HTTPServerListenPort = 8888 + // ConfigFilePath defines config file path of frontend + ConfigFilePath = "/home/sn/config/config.json" + // ConfigEnvKey defines config env key of frontend + ConfigEnvKey = "FRONTEND_CONFIG" +) diff --git a/frontend/pkg/frontend/config/hotload_config.go b/frontend/pkg/frontend/config/hotload_config.go new file mode 100644 index 0000000000000000000000000000000000000000..f57c64c0ffd06bf8270fcf6e4f02448bb8f73fc4 --- /dev/null +++ b/frontend/pkg/frontend/config/hotload_config.go @@ -0,0 +1,195 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package config is used to keep the config used by the faas frontend function +package config + +import ( + "encoding/json" + "io/ioutil" + + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/monitor" + types2 "frontend/pkg/common/faas_common/types" + "frontend/pkg/frontend/types" +) + +var ( + configWatcher monitor.FileWatcher + configChangedCallback ChangedCallback +) + +// ChangedCallback config change callback func +type ChangedCallback func() + +// WatchConfig describe config watch api +func WatchConfig(configPath string, stopCh <-chan struct{}, callback ChangedCallback) error { + + watcher, err := monitor.CreateFileWatcher(stopCh) + if err != nil { + return err + } + configWatcher = watcher + configChangedCallback = callback + configWatcher.RegisterCallback(configPath, hotLoadConfig) + return nil +} + +func hotLoadConfig(filename string, opType monitor.OpType) { + log.GetLogger().Infof("file %s hot load start", filename) + config, err := loadConfig(filename) + if err != nil { + log.GetLogger().Errorf("hotLoadConfig failed file: %s, opType: %d, err: %s", + filename, opType, err.Error()) + return + } + hotLoadMemoryControlConfig(config) + hotLoadMemoryEvaluatorConfig(config) + hotLoadEtcdLeaseConfig(config) + hotLoadCAEEtcdConfig(config) + hotLoadMetaEtcdConfig(config) + hotLoadRouterEtcdConfig(config) + + if configChangedCallback != nil { + configChangedCallback() + } +} + +func loadConfig(configPath string) (*types.Config, error) { + data, err := ioutil.ReadFile(configPath) + if err != nil { + log.GetLogger().Errorf("read file error, file path is %s", configPath) + return nil, err + } + config := &types.Config{} + err = json.Unmarshal(data, config) + if err != nil { + log.GetLogger().Errorf("failed to parse the config data: %s", err) + return nil, err + } + err = loadFunctionConfig(config) + if err != nil { + return nil, err + } + return config, err +} + +func hotLoadMemoryControlConfig(newAllConfig *types.Config) { + if newAllConfig.MemoryControlConfig == nil { + return + } + updateMemoryControlConfig(newAllConfig.MemoryControlConfig, GetConfig().MemoryControlConfig) +} + +func hotLoadMemoryEvaluatorConfig(newAllConfig *types.Config) { + if newAllConfig.MemoryEvaluatorConfig == nil { + return + } + oldConfig := GetConfig().MemoryEvaluatorConfig + newConfig := newAllConfig.MemoryEvaluatorConfig + if newConfig.RequestMemoryEvaluator > 0 { + log.GetLogger().Infof("RequestMemoryEvaluator update old: %f, new: %f", + oldConfig.RequestMemoryEvaluator, newConfig.RequestMemoryEvaluator) + oldConfig.RequestMemoryEvaluator = newConfig.RequestMemoryEvaluator + } +} + +func hotLoadEtcdLeaseConfig(newAllConfig *types.Config) { + if newAllConfig.EtcdLeaseConfig == nil { + return + } + if GetConfig().EtcdLeaseConfig == nil { + GetConfig().EtcdLeaseConfig = &types.EtcdLeaseConfig{} + } + newConfig := newAllConfig.EtcdLeaseConfig + oldConfig := GetConfig().EtcdLeaseConfig + if newConfig.LeaseTTL > 0 { + oldConfig.LeaseTTL = newConfig.LeaseTTL + log.GetLogger().Infof("LeaseTTL update, new: %d", newConfig.LeaseTTL) + + } + if newConfig.RenewTTL > 0 { + oldConfig.RenewTTL = newConfig.RenewTTL + log.GetLogger().Infof("RenewTTL update, new: %d", newConfig.RenewTTL) + } +} + +func hotLoadCAEEtcdConfig(newAllConfig *types.Config) { + if newAllConfig.CAEMetaEtcd.Servers != nil && len(newAllConfig.CAEMetaEtcd.Servers) > 0 { + newConfig := newAllConfig.CAEMetaEtcd + oldConfig := GetConfig().CAEMetaEtcd + oldConfig.Servers = newConfig.Servers + log.GetLogger().Infof("etcd serverList update, new: %v", newConfig.Servers) + } + return +} + +func hotLoadMetaEtcdConfig(newAllConfig *types.Config) { + if newAllConfig.MetaEtcd.Servers != nil && len(newAllConfig.MetaEtcd.Servers) > 0 { + newConfig := newAllConfig.MetaEtcd + oldConfig := GetConfig().MetaEtcd + oldConfig.Servers = newConfig.Servers + log.GetLogger().Infof("etcd serverList update, new: %v", newConfig.Servers) + } + return +} + +func hotLoadRouterEtcdConfig(newAllConfig *types.Config) { + if newAllConfig.RouterEtcd.Servers != nil && len(newAllConfig.RouterEtcd.Servers) > 0 { + newConfig := newAllConfig.RouterEtcd + oldConfig := GetConfig().RouterEtcd + oldConfig.Servers = newConfig.Servers + log.GetLogger().Infof("etcd serverList update, new: %v", newConfig.Servers) + } + return +} + +// UpdateMemoryControlConfig update memory control config +func updateMemoryControlConfig(newConfig *types2.MemoryControlConfig, oldConfig *types2.MemoryControlConfig) { + if newConfig == nil || oldConfig == nil { + log.GetLogger().Infof("MemoryControlConfig is nil") + return + } + if newConfig.LowerMemoryPercent > 0 { + log.GetLogger().Infof("LowerMemoryPercent update old: %f, new: %f", + oldConfig.LowerMemoryPercent, newConfig.LowerMemoryPercent) + oldConfig.LowerMemoryPercent = newConfig.LowerMemoryPercent + } + + if newConfig.HighMemoryPercent > 0 { + log.GetLogger().Infof("HighMemoryPercent update old: %f, new: %f", + oldConfig.HighMemoryPercent, newConfig.HighMemoryPercent) + oldConfig.HighMemoryPercent = newConfig.HighMemoryPercent + } + + if newConfig.StatefulHighMemPercent > 0 { + log.GetLogger().Infof("StatefulHighMemPercent update old: %f, new: %f", + oldConfig.StatefulHighMemPercent, newConfig.StatefulHighMemPercent) + oldConfig.StatefulHighMemPercent = newConfig.StatefulHighMemPercent + } + + if newConfig.BodyThreshold > 0 { + log.GetLogger().Infof("BodyThreshold update old: %d, new: %d", + oldConfig.BodyThreshold, newConfig.BodyThreshold) + oldConfig.BodyThreshold = newConfig.BodyThreshold + } + + if newConfig.MemDetectIntervalMs > 0 { + log.GetLogger().Infof("MemDetectIntervalMs update old: %d, new: %d", + oldConfig.MemDetectIntervalMs, newConfig.MemDetectIntervalMs) + oldConfig.MemDetectIntervalMs = newConfig.MemDetectIntervalMs + } +} diff --git a/frontend/pkg/frontend/config/hotload_config_test.go b/frontend/pkg/frontend/config/hotload_config_test.go new file mode 100644 index 0000000000000000000000000000000000000000..8680baf946b5b5965bec0e08e66ed899c4aae41a --- /dev/null +++ b/frontend/pkg/frontend/config/hotload_config_test.go @@ -0,0 +1,306 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package config + +import ( + "encoding/json" + "fmt" + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/monitor" + commonType "frontend/pkg/common/faas_common/types" + "frontend/pkg/frontend/types" + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + "io/ioutil" + "testing" +) + +const ( + serverPort = "8888" + defaultMetaEtcdCafile = "/home/sn/resource/ca/ca.pem" + defaultMetaEtcdCertfile = "/home/sn/resource/ca/cert.pem" + defaultMetaEtcdKeyfile = "/home/sn/resource/ca/key.pem" + + defaultRouterEtcdCafile = "/home/sn/resource/routerEtcd/ca.pem" + defaultRouterEtcdCertfile = "/home/sn/resource/routerEtcd/cert.pem" + defaultRouterEtcdKeyfile = "/home/sn/resource/routerEtcd/key.pem" +) + +var ( + watcher *monitor.MockFileWatcher + maxTimeout = 100*24*3600 + 1 + testConfig = &types.Config{ + CPU: 5, + Memory: 100, + SLAQuota: 10, + HTTPConfig: &types.FrontendHTTP{ + RespTimeOut: int64(maxTimeout), + WorkerInstanceReadTimeOut: int64(maxTimeout), + MaxRequestBodySize: 6, + }, + MetaEtcd: etcd3.EtcdConfig{ + Servers: []string{"127.0.0.1:2379"}, + User: "root", + Password: "0000", + CaFile: defaultMetaEtcdCafile, + CertFile: defaultMetaEtcdCertfile, + KeyFile: defaultMetaEtcdKeyfile, + SslEnable: true, + }, + CAEMetaEtcd: etcd3.EtcdConfig{ + Servers: []string{"127.0.0.1:2379"}, + User: "root", + Password: "00001", + CaFile: defaultMetaEtcdCafile, + CertFile: defaultMetaEtcdCertfile, + KeyFile: defaultMetaEtcdKeyfile, + SslEnable: true, + }, + RouterEtcd: etcd3.EtcdConfig{ + Servers: []string{"127.0.0.2:2379"}, + User: "root", + Password: "1111", + CaFile: defaultRouterEtcdCafile, + CertFile: defaultRouterEtcdCertfile, + KeyFile: defaultRouterEtcdKeyfile, + SslEnable: true, + }, + MemoryControlConfig: &commonType.MemoryControlConfig{ + LowerMemoryPercent: 0.5, + BodyThreshold: 10, + HighMemoryPercent: 0.7, + MemDetectIntervalMs: 100, + }, + MemoryEvaluatorConfig: &types.MemoryEvaluatorConfig{ + RequestMemoryEvaluator: 2, + }, + EtcdLeaseConfig: &types.EtcdLeaseConfig{ + LeaseTTL: 10, + RenewTTL: 10, + }, + } + + testConfig2 = &types.Config{ + CPU: 5, + Memory: 100, + SLAQuota: 10, + HTTPConfig: &types.FrontendHTTP{ + RespTimeOut: int64(maxTimeout), + WorkerInstanceReadTimeOut: int64(maxTimeout), + MaxRequestBodySize: 6, + }, + MetaEtcd: etcd3.EtcdConfig{ + SslEnable: true, + }, + CAEMetaEtcd: etcd3.EtcdConfig{ + SslEnable: true, + }, + RouterEtcd: etcd3.EtcdConfig{ + SslEnable: true, + }, + } +) + +func createMockFileWatcher(stopCh <-chan struct{}) (monitor.FileWatcher, error) { + watcher = &monitor.MockFileWatcher{ + Callbacks: map[string]monitor.FileChangedCallback{}, + StopCh: stopCh, + EventChan: make(chan string, 1), + } + return watcher, nil +} + +func TestWatchConfig(t *testing.T) { + convey.Convey("TestWatchConfig error", t, func() { + defer gomonkey.ApplyFunc(monitor.CreateFileWatcher, func(stopCh <-chan struct{}) (monitor.FileWatcher, error) { + return nil, fmt.Errorf("ioutil.ReadFile error") + }).Reset() + stopCh := make(chan struct{}, 1) + err := WatchConfig(ConfigFilePath, stopCh, nil) + if err != nil { + return + } + convey.So(err, convey.ShouldNotBeNil) + }) +} + +func TestHotLoadConfig(t *testing.T) { + convey.Convey("TestHotLoadConfig OK", t, func() { + patches := gomonkey.NewPatches() + data, _ := json.Marshal(testConfig) + patches.ApplyFunc(ioutil.ReadFile, func() ([]byte, error) { + fmt.Println(string(data)) + return data, nil + }) + defer func() { + patches.Reset() + }() + + SetConfig(*testConfig) + initDefaultMemoryControlConfig() + initDefaultMemoryEvaluatorConfig() + + monitor.SetCreator(createMockFileWatcher) + + stopCh := make(chan struct{}, 1) + callbackChan := make(chan int, 1) + WatchConfig(ConfigFilePath, stopCh, func() { + callbackChan <- 1 + fmt.Println("do call back") + }) + + watcher.EventChan <- ConfigFilePath + + <-callbackChan + convey.So(GetConfig().MemoryControlConfig.LowerMemoryPercent, convey.ShouldEqual, + testConfig.MemoryControlConfig.LowerMemoryPercent) + convey.So(GetConfig().MemoryControlConfig.BodyThreshold, convey.ShouldEqual, + testConfig.MemoryControlConfig.BodyThreshold) + convey.So(GetConfig().MemoryControlConfig.HighMemoryPercent, convey.ShouldEqual, + testConfig.MemoryControlConfig.HighMemoryPercent) + convey.So(GetConfig().MemoryControlConfig.MemDetectIntervalMs, convey.ShouldEqual, + testConfig.MemoryControlConfig.MemDetectIntervalMs) + convey.So(GetConfig().MemoryEvaluatorConfig.RequestMemoryEvaluator, convey.ShouldEqual, + testConfig.MemoryEvaluatorConfig.RequestMemoryEvaluator) + + convey.So(GetConfig().MetaEtcd.SslEnable, convey.ShouldEqual, + testConfig.MetaEtcd.SslEnable) + convey.So(GetConfig().CAEMetaEtcd.SslEnable, convey.ShouldEqual, + testConfig.CAEMetaEtcd.SslEnable) + convey.So(GetConfig().RouterEtcd.SslEnable, convey.ShouldEqual, + testConfig.RouterEtcd.SslEnable) + close(stopCh) + }) + + convey.Convey("TestHotLoadConfig OK 2", t, func() { + patches := gomonkey.NewPatches() + data, _ := json.Marshal(testConfig2) + patches.ApplyFunc(ioutil.ReadFile, func() ([]byte, error) { + fmt.Println(string(data)) + return data, nil + }) + defer func() { + patches.Reset() + }() + + SetConfig(*testConfig) + initDefaultMemoryControlConfig() + initDefaultMemoryEvaluatorConfig() + + monitor.SetCreator(createMockFileWatcher) + + stopCh := make(chan struct{}, 1) + callbackChan := make(chan int, 1) + WatchConfig(ConfigFilePath, stopCh, func() { + callbackChan <- 1 + fmt.Println("do call back") + }) + + watcher.EventChan <- ConfigFilePath + + <-callbackChan + convey.So(GetConfig().MemoryControlConfig.LowerMemoryPercent, convey.ShouldEqual, + testConfig.MemoryControlConfig.LowerMemoryPercent) + convey.So(GetConfig().MemoryControlConfig.BodyThreshold, convey.ShouldEqual, + testConfig.MemoryControlConfig.BodyThreshold) + convey.So(GetConfig().MemoryControlConfig.HighMemoryPercent, convey.ShouldEqual, + testConfig.MemoryControlConfig.HighMemoryPercent) + convey.So(GetConfig().MemoryControlConfig.MemDetectIntervalMs, convey.ShouldEqual, + testConfig.MemoryControlConfig.MemDetectIntervalMs) + convey.So(GetConfig().MemoryEvaluatorConfig.RequestMemoryEvaluator, convey.ShouldEqual, + testConfig.MemoryEvaluatorConfig.RequestMemoryEvaluator) + + convey.So(GetConfig().MetaEtcd.SslEnable, convey.ShouldEqual, + testConfig.MetaEtcd.SslEnable) + convey.So(GetConfig().CAEMetaEtcd.SslEnable, convey.ShouldEqual, + testConfig.CAEMetaEtcd.SslEnable) + convey.So(GetConfig().RouterEtcd.SslEnable, convey.ShouldEqual, + testConfig.RouterEtcd.SslEnable) + close(stopCh) + }) +} + +func TestLoadConfig(t *testing.T) { + convey.Convey("TestLoadConfig error 0", t, func() { + defer gomonkey.ApplyFunc(ioutil.ReadFile, func() ([]byte, error) { + return nil, fmt.Errorf("ioutil.ReadFile error") + }).Reset() + _, err := loadConfig(ConfigFilePath) + convey.So(err, convey.ShouldNotBeNil) + }) + + convey.Convey("TestLoadConfig error 1", t, func() { + patches := gomonkey.NewPatches() + data, _ := json.Marshal(testConfig) + patches.ApplyFunc(ioutil.ReadFile, func() ([]byte, error) { + fmt.Println(string(data)) + return data, nil + }) + patches.ApplyFunc(json.Unmarshal, func(data []byte, v any) error { + return fmt.Errorf("json.Unmarshal error") + }) + defer func() { + patches.Reset() + }() + _, err := loadConfig(ConfigFilePath) + convey.So(err, convey.ShouldNotBeNil) + }) + + convey.Convey("TestLoadConfig error 2", t, func() { + patches := gomonkey.NewPatches() + data, _ := json.Marshal(testConfig) + patches.ApplyFunc(ioutil.ReadFile, func() ([]byte, error) { + fmt.Println(string(data)) + return data, nil + }) + patches.ApplyFunc(loadFunctionConfig, func(config *types.Config) error { + return fmt.Errorf("loadFunctionConfig error") + }) + defer func() { + patches.Reset() + }() + _, err := loadConfig(ConfigFilePath) + convey.So(err, convey.ShouldNotBeNil) + }) +} + +func TestUpdateMemoryControlConfig(t *testing.T) { + convey.Convey("TestUpdateMemoryControlConfig", t, func() { + updateMemoryControlConfig(nil, nil) + oldConfig := &commonType.MemoryControlConfig{} + newConfig := &commonType.MemoryControlConfig{ + LowerMemoryPercent: 0.6, + StatefulHighMemPercent: 0.85, + } + updateMemoryControlConfig(newConfig, oldConfig) + convey.So(oldConfig.LowerMemoryPercent, convey.ShouldEqual, newConfig.LowerMemoryPercent) + convey.So(oldConfig.StatefulHighMemPercent, convey.ShouldEqual, newConfig.StatefulHighMemPercent) + }) + convey.Convey("TestUpdateMemoryControlConfig 2", t, func() { + oldConfig := &commonType.MemoryControlConfig{} + newConfig := &commonType.MemoryControlConfig{ + LowerMemoryPercent: 0.6, + HighMemoryPercent: 0.8, + StatefulHighMemPercent: 0.85, + MemDetectIntervalMs: 1, + BodyThreshold: 2, + } + updateMemoryControlConfig(newConfig, oldConfig) + convey.So(oldConfig.LowerMemoryPercent, convey.ShouldEqual, newConfig.LowerMemoryPercent) + convey.So(oldConfig.StatefulHighMemPercent, convey.ShouldEqual, newConfig.StatefulHighMemPercent) + }) +} diff --git a/frontend/pkg/frontend/frontendsdk/api.go b/frontend/pkg/frontend/frontendsdk/api.go new file mode 100644 index 0000000000000000000000000000000000000000..a5ac314018df5bf5e33c8e5b6d81cc72fca10105 --- /dev/null +++ b/frontend/pkg/frontend/frontendsdk/api.go @@ -0,0 +1,34 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package frontendsdk is sdk +package frontendsdk + +import "github.com/valyala/fasthttp" + +// FrontendAPI - frontend SDK api +type FrontendAPI interface { + Init(configFilePath string) error + InvokeHandler(ctx *InvokeProcessContext) error + UploadWithKeyRetry(value []byte, config *Config, param SetParam, traceID string) (string, error) + DownloadArrayRetry(keys []string, config *Config, traceID string) ([][]byte, error) + DeleteArrayRetry(keys []string, config *Config, traceID string) ([]string, error) + SubscribeStream(param SubscribeParam, ctx StreamCtx) error + ExecShutdownHandler(signum int) + CheckLocalDataSystemStatusReady() bool + CheckFrontendIsHealth() bool + Auth(ctx *fasthttp.RequestCtx, ak string, sk []byte) bool +} diff --git a/frontend/pkg/frontend/frontendsdk/frontendimpl.go b/frontend/pkg/frontend/frontendsdk/frontendimpl.go new file mode 100644 index 0000000000000000000000000000000000000000..8df3daa8f1c16935ffe59b31605ddc7b70033f62 --- /dev/null +++ b/frontend/pkg/frontend/frontendsdk/frontendimpl.go @@ -0,0 +1,294 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package frontendsdk + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "os" + + "github.com/magiconair/properties" + "github.com/valyala/fasthttp" + + "yuanrong.org/kernel/runtime/libruntime/api" + "yuanrong.org/kernel/runtime/libruntime/common" + "yuanrong.org/kernel/runtime/posixsdk" + + "frontend/pkg/common/faas_common/datasystemclient" + "frontend/pkg/common/faas_common/instanceconfig" + "frontend/pkg/common/faas_common/logger/log" + commontypes "frontend/pkg/common/faas_common/types" + "frontend/pkg/common/faas_common/utils" + "frontend/pkg/common/faas_common/wisecloudtool/serviceaccount" + "frontend/pkg/frontend/common/util" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/functionmeta" + "frontend/pkg/frontend/instanceconfigmanager" + "frontend/pkg/frontend/instancemanager" + "frontend/pkg/frontend/invocation" + "frontend/pkg/frontend/responsehandler" + "frontend/pkg/frontend/schedulerproxy" + "frontend/pkg/frontend/subscriber" + "frontend/pkg/frontend/types" + "frontend/pkg/frontend/watcher" + "frontend/pkg/frontend/wisecloud" +) + +var ( + stopCh = make(chan struct{}) +) + +const ( + initArgsFilePathEnvKey = "INIT_ARGS_FILE_PATH" + dataSystemAddr = "DATASYSTEM_ADDR" + dsWorkerPort = "31501" + functionSystemPort = "32568" + functionName = "0/0-system-faasfrontend/$latest" + runtimeID = "faas_frontend_libruntime" + instanceIDPrefix = "driver-faas-frontend" + logFileName = "frontendsdk" + logConfigKey = "LOG_CONFIG" + localHostAddr = "127.0.0.1" + defaultJobID = "12345678" +) + +const ( + podName = "POD_NAME" + podIP = "POD_IP" + nodeIP = "NODE_IP" +) + +// Frontend - FrontendAPI implement +type Frontend struct{} + +// Init - init frontend SDK +func (f *Frontend) Init(configFilePath string) error { + if configFilePath == "" { + return fmt.Errorf("config file path is empty") + } + runtimeConfig, err := parseRuntimeCfgAndSetEnv(configFilePath) + if err != nil { + return fmt.Errorf("parse runtime config error: %s", err.Error()) + } + err = log.InitRunLog(logFileName, true) + if err != nil { + return fmt.Errorf("init logger error, err %s", err.Error()) + } + funcExecution := posixsdk.NewSDKPosixFuncExecutionWithHandler(posixsdk.RegisterHandler{ + InitHandler: initSDKHandler, + }) + err = posixsdk.InitRuntime(runtimeConfig, funcExecution) + if err != nil { + utils.SafeCloseChannel(stopCh) + log.GetLogger().Errorf("init runtime failed: %s", err.Error()) + return err + } + go posixsdk.Run() + wisecloud.NewColdStartProvider(&config.GetConfig().WiseCloudConfig.ServiceAccountJwt) + log.GetLogger().Infof("init frontend sdk successfully") + return nil +} + +func setEnv(configFilePath string, cfg *types.Config) error { + logConfig, err := json.Marshal(cfg.Runtime.LogConfig) + if err != nil { + return err + } + err = os.Setenv(logConfigKey, string(logConfig)) + if err != nil { + return err + } + err = os.Setenv(initArgsFilePathEnvKey, configFilePath) + if err != nil { + return err + } + err = os.Setenv(dataSystemAddr, fmt.Sprintf("%s:%s", os.Getenv(nodeIP), dsWorkerPort)) + if err != nil { + return err + } + return nil +} + +func parseRuntimeCfgAndSetEnv(configFilePath string) (*common.Configuration, error) { + data, err := os.ReadFile(configFilePath) + if err != nil { + return nil, fmt.Errorf("read config failed, err %s", err.Error()) + } + cfg := &types.Config{} + err = json.Unmarshal(data, cfg) + if err != nil { + return nil, fmt.Errorf("unmarshal config failed, err %s", err.Error()) + } + jobID := os.Getenv(podName) + if jobID == "" { + jobID = defaultJobID + } + fsAddr := os.Getenv(podIP) + if fsAddr == "" { + fsAddr = localHostAddr + } + runtimeCfg := &common.Configuration{ + RuntimeID: runtimeID, + InstanceID: fmt.Sprintf("%s-%s", instanceIDPrefix, jobID), + FunctionName: functionName, + LogLevel: cfg.Runtime.LogConfig.Level, + FSAddress: fmt.Sprintf("%s:%s", fsAddr, functionSystemPort), + LogPath: cfg.Runtime.LogConfig.FilePath, + JobID: jobID, + DriverMode: true, + MaxConcurrencyCreateNum: 5000, + EnableSigaction: cfg.Runtime.EnableSigaction, + } + err = setEnv(configFilePath, cfg) + if err != nil { + return nil, err + } + return runtimeCfg, nil +} + + +// InvokeHandler - +func (f *Frontend) InvokeHandler(ctx *InvokeProcessContext) error { + return invocation.InvokeHandler(ctx) +} + +// UploadWithKeyRetry - +func (f *Frontend) UploadWithKeyRetry(value []byte, config *Config, param SetParam, traceID string) (string, error) { + return datasystemclient.UploadWithKeyRetry(value, config, param, traceID) +} + +// DownloadArrayRetry - +func (f *Frontend) DownloadArrayRetry(keys []string, config *Config, traceID string) ([][]byte, error) { + return datasystemclient.DownloadArrayRetry(keys, config, traceID) +} + +// DeleteArrayRetry - +func (f *Frontend) DeleteArrayRetry(keys []string, config *Config, traceID string) ([]string, error) { + return datasystemclient.DeleteArrayRetry(keys, config, traceID) +} + +// SubscribeStream - +func (f *Frontend) SubscribeStream(param SubscribeParam, ctx StreamCtx) error { + return datasystemclient.SubscribeStream(param, ctx) +} + +func initSDKHandler(args []api.Arg, rt api.LibruntimeAPI) ([]byte, error) { + var err error + if err = config.InitFunctionConfig(args[0].Data); err != nil { + log.GetLogger().Errorf("init frontend config fail, err: %s", err) + return []byte{}, err + } + if err = config.InitEtcd(stopCh); err != nil { + log.GetLogger().Errorf("failed to init etcd ,err:%s", err.Error()) + return []byte{}, err + } + + if err = watcher.StartWatch(stopCh); err != nil { + log.GetLogger().Errorf("failed to watch etcd ,err:%s", err.Error()) + return []byte{}, err + } + initSubscribe() + util.SetAPIClientLibruntime(rt) + schedulerproxy.Proxy.RTAPI = rt + datasystemclient.SetStreamEnable(config.GetConfig().StreamEnable) + datasystemclient.InitDataSystemLibruntime(config.GetConfig().DataSystemConfig, rt, stopCh) + responsehandler.Handler = (&invocation.FGAdapter{}).MakeResponseHandler() + return []byte{}, nil +} + +func initSubscribe() { + functionmeta.GetFunctionMetaDataSubject().Subscribe(&subscriber.Observer{ + Update: func(data interface{}) {}, + Delete: func(data interface{}) { + functionMeta, ok := data.(*commontypes.FuncSpec) + if !ok { + return + } + wisecloud.GetMetricsManager().ProcessFunctionDelete(functionMeta) + wisecloud.GetQueueManager().ProcessFunctionDelete(functionMeta) + }, + }) + functionmeta.GetFunctionMetaDataSubject().StartLoop(stopCh) + + instanceconfigmanager.GetInstanceConfigSubject().Subscribe(&subscriber.Observer{ + Update: func(data interface{}) {}, + Delete: func(data interface{}) { + insConfig, ok := data.(*instanceconfig.Configuration) + if !ok { + return + } + wisecloud.GetMetricsManager().ProcessInsConfigDelete(insConfig) + wisecloud.GetQueueManager().ProcessInsConfigDelete(insConfig) + }, + }) + instanceconfigmanager.GetInstanceConfigSubject().StartLoop(stopCh) + + instancemanager.GetInstanceSubject().Subscribe(&subscriber.Observer{ + Update: func(data interface{}) { + instance, ok := data.(*commontypes.InstanceSpecification) + if !ok { + return + } + wisecloud.GetQueueManager().ProcessInstanceUpdate(instance) + }, + Delete: func(data interface{}) { + instance, ok := data.(*commontypes.InstanceSpecification) + if !ok { + return + } + wisecloud.GetMetricsManager().ProcessInstanceDelete(instance) + }, + }) + instancemanager.GetInstanceSubject().StartLoop(stopCh) +} + +// ExecShutdownHandler - +func (f *Frontend) ExecShutdownHandler(signum int) { + log.GetLogger().Infof("recv signal %d", signum) + log.GetLogger().Sync() + + defer func() { + log.GetLogger().Infof("recv signal %d over", signum) + log.GetLogger().Sync() + }() + posixsdk.ExecShutdownHandler(signum) +} + +// CheckFrontendIsHealth - +func (f *Frontend) CheckFrontendIsHealth() bool { + return util.NewClient().IsHealth() +} + +// CheckLocalDataSystemStatusReady 检查本节点数据系统状态是否为ready +func (f *Frontend) CheckLocalDataSystemStatusReady() bool { + return !config.GetConfig().StreamEnable || datasystemclient.IsLocalDataSystemStatusReady() +} + +// Auth - +func (f *Frontend) Auth(ctx *fasthttp.RequestCtx, ak string, hexSk []byte) bool { + sk, err := hex.DecodeString(string(hexSk)) + if err != nil { + sk = hexSk + } + return wisecloud.Auth(ctx, ak, sk) +} + +// NewFrontend - +func NewFrontend() FrontendAPI { + return &Frontend{} +} diff --git a/frontend/pkg/frontend/frontendsdk/frontendimpl_test.go b/frontend/pkg/frontend/frontendsdk/frontendimpl_test.go new file mode 100644 index 0000000000000000000000000000000000000000..bc6696c52c520f4d1158303e6304a3bf5b04ecd4 --- /dev/null +++ b/frontend/pkg/frontend/frontendsdk/frontendimpl_test.go @@ -0,0 +1,239 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package frontendsdk + +import ( + "encoding/json" + "errors" + "os" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/magiconair/properties" + "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/common/faas_common/datasystemclient" + "frontend/pkg/common/faas_common/logger/config" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/sts/raw" + commonType "frontend/pkg/common/faas_common/types" + "frontend/pkg/common/faas_common/wisecloudtool" + wisecloudtypes "frontend/pkg/common/faas_common/wisecloudtool/types" + frontendConfig "frontend/pkg/frontend/config" + "frontend/pkg/frontend/frontendsdk/posixsdk" + "frontend/pkg/frontend/types" + "frontend/pkg/frontend/watcher" + "frontend/pkg/frontend/wisecloud" + "yuanrong.org/kernel/runtime/libruntime/api" + "yuanrong.org/kernel/runtime/libruntime/common" + "yuanrong.org/kernel/runtime/libruntime/execution" +) + +func TestInit(t *testing.T) { + convey.Convey("Test frontend init", t, func() { + frontend := NewFrontend() + cfg := &types.Config{ + Runtime: types.RuntimeConfig{ + LogConfig: config.CoreInfo{ + Level: "DEBUG", + }, + SystemAuthConfig: types.SystemAuthConfig{ + Enable: true, + AccessKey: "ak", + SecretKey: "sk", + }, + }, + RawStsConfig: raw.StsConfig{ + StsEnable: true, + }, + } + data, _ := json.Marshal(cfg) + convey.Convey("config file is empty", func() { + err := frontend.Init("") + convey.So(err.Error(), convey.ShouldContainSubstring, "config file path is empty") + }) + convey.Convey("read config file error", func() { + err := frontend.Init("/home/sn/config.json") + convey.So(err.Error(), convey.ShouldContainSubstring, "read config failed") + }) + convey.Convey("Unmarshal config error", func() { + defer gomonkey.ApplyFunc(os.ReadFile, func(name string) ([]byte, error) { + return []byte{}, nil + }).Reset() + err := frontend.Init("/home/sn/config.json") + convey.So(err.Error(), convey.ShouldContainSubstring, "unmarshal config failed") + }) + convey.Convey("sts init failed", func() { + defer gomonkey.ApplyFunc(os.ReadFile, func(name string) ([]byte, error) { + return data, nil + }).Reset() + err := frontend.Init("/home/sn/config.json") + convey.So(err.Error(), convey.ShouldContainSubstring, "failed to init sts sdk") + }) + }) +} + +func TestInitSDKHandler(t *testing.T) { + convey.Convey("Test initSDKHandler", t, func() { + args := []api.Arg{{ + Data: []byte("hello"), + }} + convey.Convey("init frontend config fail", func() { + defer gomonkey.ApplyFunc(frontendConfig.InitFunctionConfig, func(data []byte) error { + return errors.New("failed to parse the config data") + }).Reset() + _, err := initSDKHandler(args, nil) + convey.So(err.Error(), convey.ShouldContainSubstring, "failed to parse the config data") + }) + + convey.Convey("failed to init etcd", func() { + defer gomonkey.ApplyFunc(frontendConfig.InitFunctionConfig, func(data []byte) error { + return nil + }).Reset() + defer gomonkey.ApplyFunc(frontendConfig.InitEtcd, func(stopCh <-chan struct{}) error { + return errors.New("failed to init etcd") + }).Reset() + _, err := initSDKHandler(args, nil) + convey.So(err.Error(), convey.ShouldContainSubstring, "failed to init etcd") + }) + + convey.Convey("failed to watch etcd", func() { + defer gomonkey.ApplyFunc(frontendConfig.InitFunctionConfig, func(data []byte) error { + return nil + }).Reset() + defer gomonkey.ApplyFunc(frontendConfig.InitEtcd, func(stopCh <-chan struct{}) error { + return nil + }).Reset() + defer gomonkey.ApplyFunc(watcher.StartWatch, func(stopCh <-chan struct{}) error { + return errors.New("failed to watch etcd") + }).Reset() + _, err := initSDKHandler(args, nil) + convey.So(err.Error(), convey.ShouldContainSubstring, "failed to watch etcd") + }) + + convey.Convey("success", func() { + defer gomonkey.ApplyFunc(frontendConfig.InitFunctionConfig, func(data []byte) error { + return nil + }).Reset() + defer gomonkey.ApplyFunc(frontendConfig.InitEtcd, func(stopCh <-chan struct{}) error { + return nil + }).Reset() + defer gomonkey.ApplyFunc(watcher.StartWatch, func(stopCh <-chan struct{}) error { + return nil + }).Reset() + defer gomonkey.ApplyFunc(datasystemclient.InitDataSystemLibruntime, func(cfg *commonType.DataSystemConfig, + rt api.LibruntimeAPI, stopCh <-chan struct{}) { + }).Reset() + defer gomonkey.ApplyFunc(wisecloud.NewColdStartProvider, func(serviceAccountJwt *wisecloudtypes.ServiceAccountJwt) *wisecloudtool.PodOperator { + return nil + }).Reset() + _, err := initSDKHandler(args, nil) + convey.So(err, convey.ShouldBeNil) + }) + }) +} + +func TestSetEnvError(t *testing.T) { + convey.Convey("Test SetEnv Error", t, func() { + cfg := &types.Config{ + Runtime: types.RuntimeConfig{ + LogConfig: config.CoreInfo{ + Level: "INFO", + }, + }, + } + configFilePath := "/home/sn/config" + convey.Convey("json Marshal error ", func() { + defer gomonkey.ApplyFunc(json.Marshal, func(v interface{}) ([]byte, error) { + return nil, errors.New("json marshal error") + }).Reset() + err := setEnv(configFilePath, cfg) + convey.So(err.Error(), convey.ShouldContainSubstring, "json marshal error") + }) + convey.Convey("set logConfigKey error ", func() { + defer gomonkey.ApplyFunc(os.Setenv, func(key, value string) error { + if key == logConfigKey { + return errors.New("set env logConfigKey error") + } + return nil + }).Reset() + err := setEnv(configFilePath, cfg) + convey.So(err.Error(), convey.ShouldContainSubstring, "set env logConfigKey error") + }) + convey.Convey("set initArgsFilePathEnvKey error ", func() { + defer gomonkey.ApplyFunc(os.Setenv, func(key, value string) error { + if key == initArgsFilePathEnvKey { + return errors.New("set env initArgsFilePathEnvKey error") + } + return nil + }).Reset() + err := setEnv(configFilePath, cfg) + convey.So(err.Error(), convey.ShouldContainSubstring, "set env initArgsFilePathEnvKey error") + }) + convey.Convey("set dataSystemAddr error ", func() { + defer gomonkey.ApplyFunc(os.Setenv, func(key, value string) error { + if key == dataSystemAddr { + return errors.New("set env dataSystemAddr error") + } + return nil + }).Reset() + err := setEnv(configFilePath, cfg) + convey.So(err.Error(), convey.ShouldContainSubstring, "set env dataSystemAddr error") + }) + }) +} + +func TestCheckLocalDataSystemStatusReady(t *testing.T) { + convey.Convey("test check local dataSystem status ready", t, func() { + frontend := NewFrontend() + convey.Convey("test check local dataSystem status, when streamEnable is false", func() { + defer gomonkey.ApplyFunc(frontendConfig.GetConfig, func() *types.Config { + return &types.Config{ + StreamEnable: false, + } + }).Reset() + result := frontend.CheckLocalDataSystemStatusReady() + convey.So(result, convey.ShouldBeTrue) + }) + + convey.Convey("test check local dataSystem status, when it's not local dataSystem or not ready", func() { + defer gomonkey.ApplyFunc(frontendConfig.GetConfig, func() *types.Config { + return &types.Config{ + StreamEnable: true, + } + }).Reset() + defer gomonkey.ApplyFunc(datasystemclient.IsLocalDataSystemStatusReady, func() bool { + return false + }).Reset() + result := frontend.CheckLocalDataSystemStatusReady() + convey.So(result, convey.ShouldBeFalse) + }) + + convey.Convey("test check local dataSystem status, when it's local dataSystem and ready", func() { + defer gomonkey.ApplyFunc(frontendConfig.GetConfig, func() *types.Config { + return &types.Config{ + StreamEnable: true, + } + }).Reset() + defer gomonkey.ApplyFunc(datasystemclient.IsLocalDataSystemStatusReady, func() bool { + return true + }).Reset() + result := frontend.CheckLocalDataSystemStatusReady() + convey.So(result, convey.ShouldBeTrue) + }) + }) +} diff --git a/frontend/pkg/frontend/frontendsdk/posixsdk/mockposixsdk.go b/frontend/pkg/frontend/frontendsdk/posixsdk/mockposixsdk.go new file mode 100644 index 0000000000000000000000000000000000000000..126819bfe513dbbf160d9c57835eff1d87f2ebb4 --- /dev/null +++ b/frontend/pkg/frontend/frontendsdk/posixsdk/mockposixsdk.go @@ -0,0 +1,50 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package posixsdk +package posixsdk + +import ( + "yuanrong.org/kernel/runtime/libruntime/api" + "yuanrong.org/kernel/runtime/libruntime/common" + "yuanrong.org/kernel/runtime/libruntime/execution" +) + +type initHandlerType = func([]api.Arg, api.LibruntimeAPI) ([]byte, error) + +// RegisterHandler - +type RegisterHandler struct { + InitHandler initHandlerType +} + +// Run - +func Run() { +} + +// InitRuntime init runtime +func InitRuntime(conf *common.Configuration, intfs execution.FunctionExecutionIntfs) error { + return nil +} + +// NewSDKPosixFuncExecutionWithHandler - +func NewSDKPosixFuncExecutionWithHandler(registerHandler RegisterHandler) execution.FunctionExecutionIntfs { + return nil +} + +// ExecShutdownHandler - +func ExecShutdownHandler(signum int) { + return +} diff --git a/frontend/pkg/frontend/frontendsdk/type.go b/frontend/pkg/frontend/frontendsdk/type.go new file mode 100644 index 0000000000000000000000000000000000000000..e241cfd61671c0b0cd30941c4f4031eb506fa96d --- /dev/null +++ b/frontend/pkg/frontend/frontendsdk/type.go @@ -0,0 +1,48 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package frontendsdk + +import ( + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/datasystemclient" + "frontend/pkg/frontend/types" +) + +// InvokeProcessContext - +type InvokeProcessContext = types.InvokeProcessContext + +// Config - +type Config = datasystemclient.Config + +// SetParam - +type SetParam = api.SetParam + +// WriteModeEnum - +type WriteModeEnum = api.WriteModeEnum + +// RequestTraceInfo - +type RequestTraceInfo = types.RequestTraceInfo + +// StreamCtx - +type StreamCtx = datasystemclient.StreamCtx + +// FastHttpCtxAdapter - +type FastHttpCtxAdapter = datasystemclient.FastHttpCtxAdapter + +// SubscribeParam - +type SubscribeParam = datasystemclient.SubscribeParam diff --git a/frontend/pkg/frontend/frontendsdkadapter/assembler/response_assembler.go b/frontend/pkg/frontend/frontendsdkadapter/assembler/response_assembler.go new file mode 100644 index 0000000000000000000000000000000000000000..2aed1eccc57497342cb8d1d0a5f9cd768362650d --- /dev/null +++ b/frontend/pkg/frontend/frontendsdkadapter/assembler/response_assembler.go @@ -0,0 +1,151 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package assembler +package assembler + +import ( + "encoding/json" + + "github.com/gin-gonic/gin" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/snerror" + "frontend/pkg/common/faas_common/statuscode" + "frontend/pkg/frontend/common/httpconstant" + "frontend/pkg/frontend/frontendsdkadapter/models" +) + +// NewMultiSetSuccessResponse - +func NewMultiSetSuccessResponse(dataKeyList []string, traceID string) *models.MultiSetSuccessResponse { + return &models.MultiSetSuccessResponse{ + CommonRspHeader: models.CommonRspHeader{ + InnerCode: 0, + TraceID: traceID, + }, + DataKeyList: models.DataKeyList{DataKeys: dataKeyList}, + } +} + +// NewMultiGetSuccessResponse - +func NewMultiGetSuccessResponse(payloadInfo *models.DataSystemPayloadInfo, data [][]byte, + traceID string) *models.MultiGetSuccessResponse { + return &models.MultiGetSuccessResponse{ + CommonRspHeader: models.CommonRspHeader{ + InnerCode: 0, + TraceID: traceID, + }, + DataSystemPayloadInfo: payloadInfo, + RawData: data, + } +} + +// NewMultiDelSuccessResponse - +func NewMultiDelSuccessResponse( + traceID string) *models.MultiDelSuccessResponse { + return &models.MultiDelSuccessResponse{ + CommonRspHeader: models.CommonRspHeader{ + InnerCode: 0, + TraceID: traceID, + }, + SuccessResponse: models.SuccessResponse{ + Code: 0, + Message: "success", + }, + } +} + +// NewCommonErrorResponseByBody - +func NewCommonErrorResponseByBody(body []byte, errMsg string, traceID string) *models.CommonErrorResponse { + response := &models.CommonErrorResponse{} + response.TraceID = traceID + response.ContentType = httpconstant.ApplicationJSON + if err := json.Unmarshal(body, &response.ErrorRsp); err != nil { + response.ErrorRsp.Code = statuscode.FrontendStatusInternalError + response.ErrorRsp.Message = errMsg + } + response.InnerCode = int32(response.ErrorRsp.Code) + return response +} + +// NewCommonErrorResponse - +func NewCommonErrorResponse(code int, err string, traceID string) *models.CommonErrorResponse { + if code == 0 { + code = statuscode.FrontendStatusInternalError + } + return NewCommonErrorResponseByError(snerror.New(code, err), traceID) +} + +// NewCommonErrorResponseByError - +func NewCommonErrorResponseByError(err error, traceID string) *models.CommonErrorResponse { + snError := convertError(err) + response := &models.CommonErrorResponse{} + response.ContentType = httpconstant.ApplicationJSON + + response.TraceID = traceID + + response.InnerCode = int32(snError.Code()) + response.ErrorRsp.Code = snError.Code() + response.ErrorRsp.Message = snError.Error() + return response +} + +// CommonErrorWriter - +type CommonErrorWriter struct{} + +// WriteErrorToResponse - 用于不带trace在filter层的错误返回 +func (d *CommonErrorWriter) WriteErrorToResponse(ctx *gin.Context, err error) { + traceID := ctx.Request.Header.Get(constant.HeaderTraceID) + errResponse := NewCommonErrorResponseByError(err, traceID) + _ = errResponse.WriteResponse(ctx) +} + +// WriteResponse - +func WriteResponse(ctx *gin.Context, response models.ResponseWriter, errResponse *models.CommonErrorResponse, + traceID string) { + if errResponse != nil { + _ = errResponse.WriteResponse(ctx) + return + } + + err := response.WriteResponse(ctx) + if err != nil { + errResponse := NewCommonErrorResponseByError(err, traceID) + _ = errResponse.WriteResponse(ctx) + return + } +} + +// NewExecuteSuccessResponse - +func NewExecuteSuccessResponse(payloadInfo *models.DataSystemPayloadInfo, data [][]byte, + traceID string) *models.ExecuteSuccessResponse { + return &models.ExecuteSuccessResponse{ + CommonRspHeader: models.CommonRspHeader{ + InnerCode: 0, + TraceID: traceID, + }, + DataSystemPayloadInfo: payloadInfo, + RawData: data, + } +} + +func convertError(err error) snerror.SNError { + snErr, ok := err.(snerror.SNError) + if ok { + return snErr + } + return snerror.New(statuscode.FrontendStatusInternalError, err.Error()) +} diff --git a/frontend/pkg/frontend/frontendsdkadapter/handler/execute_handler.go b/frontend/pkg/frontend/frontendsdkadapter/handler/execute_handler.go new file mode 100644 index 0000000000000000000000000000000000000000..21d7841ca3e8ce7b1d50bb2e5e94368102e2bce3 --- /dev/null +++ b/frontend/pkg/frontend/frontendsdkadapter/handler/execute_handler.go @@ -0,0 +1,62 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package handler +package handler + +import ( + "fmt" + + "github.com/gin-gonic/gin" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/statuscode" + "frontend/pkg/common/uuid" + "frontend/pkg/frontend/frontendsdkadapter/assembler" + "frontend/pkg/frontend/frontendsdkadapter/models" + "frontend/pkg/frontend/frontendsdkadapter/parser" + "frontend/pkg/frontend/frontendsdkadapter/service" +) + +// ExecuteHandler - +func ExecuteHandler(ctx *gin.Context) { + traceID := ctx.Request.Header.Get(constant.HeaderTraceID) + if traceID == "" { + traceID = uuid.New().String() + } + traceLogger := models.NewTraceLogger("execute", traceID) + + execRequest, err := parser.NewExecuteRequestContext(ctx, traceID) + if err != nil { + msg := fmt.Sprintf("deserialize request failed, err: %s", err.Error()) + errResponse := assembler.NewCommonErrorResponse(statuscode.FrontendStatusBadRequest, msg, + traceLogger.TraceID) + traceLogger.Logger.Errorf(errResponse.ErrorRsp.Message) + assembler.WriteResponse(ctx, nil, errResponse, traceID) + return + } + + traceLogger.With("functionName", execRequest.FunctionName) + traceLogger.TenantID = execRequest.TenantID + response, errResponse := service.ExecuteService(execRequest, traceLogger) + if errResponse != nil { + traceLogger.Logger.Error(errResponse.ErrorRsp.Message) + assembler.WriteResponse(ctx, nil, errResponse, traceID) + return + } + traceLogger.Logger.Info("execute success") + assembler.WriteResponse(ctx, response, errResponse, traceID) +} diff --git a/frontend/pkg/frontend/frontendsdkadapter/handler/execute_handler_test.go b/frontend/pkg/frontend/frontendsdkadapter/handler/execute_handler_test.go new file mode 100644 index 0000000000000000000000000000000000000000..e6d63973ef0dcc684b22adef52062728c12957e8 --- /dev/null +++ b/frontend/pkg/frontend/frontendsdkadapter/handler/execute_handler_test.go @@ -0,0 +1,214 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package handler +package handler + +import ( + "bytes" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/invocation" + "frontend/pkg/frontend/types" +) + +func TestExecuteHandler(t *testing.T) { + gin.SetMode(gin.TestMode) + router := gin.New() + router.POST("/test", ExecuteHandler) + body := createByteArray(1 << 10) + tests := []struct { + name string + header map[string]string + clusterMap map[string]string + payloadInfo string + body []byte + maxDataSize int + maxKeySize int + expectedResponse string + expectedInnerCode string + expectedStatusCode int + expectedTraceID string + mockUploadFunc func() *gomonkey.Patches + mockInvokeFunc func() *gomonkey.Patches + mockDownloadFunc func() *gomonkey.Patches + mockDeleteFunc func() *gomonkey.Patches + }{ + { + name: "execute_success", + header: dataCommonHeader(), + maxKeySize: 5, + payloadInfo: "[{\"dataPrefix\": \"prefix\", \"offset\": 0, \"len\": 512, \"needEncrypt\": false}]", + body: body, + expectedResponse: "a", + expectedInnerCode: "0", + expectedStatusCode: http.StatusOK, + maxDataSize: 1024, + mockUploadFunc: mockDataSystemUploadSuccess, + mockInvokeFunc: mockInvokeHandlerSuccess, + mockDownloadFunc: mockDataSystemDownloadSuccess, + mockDeleteFunc: mockDataSystemDeleteSuccess, + }, + { + name: "payloadInfo_large_than_body", + payloadInfo: "[{\"dataPrefix\": \"prefix\", \"offset\": 0, \"len\": 1048276, \"needEncrypt\": false}]", + header: dataCommonHeader(), + maxKeySize: 5, + body: nil, + expectedResponse: "{\"code\":200400,\"message\":\"deserialize request failed, err: payload len invalid\"}", + expectedInnerCode: "200400", + expectedStatusCode: http.StatusOK, + maxDataSize: 1024, + mockUploadFunc: mockDataSystemUploadSuccess, + mockInvokeFunc: mockInvokeHandlerSuccess, + mockDownloadFunc: mockDataSystemDownloadSuccess, + mockDeleteFunc: mockDataSystemDeleteSuccess, + }, + { + name: "payloadInfo-header-json-invalid", + payloadInfo: `{}`, + header: dataCommonHeader(), + maxKeySize: 5, + body: nil, + expectedResponse: "{\"code\":200400,\"message\":\"deserialize request failed, err: payloadInfo json invalid\"}", + expectedInnerCode: "200400", + expectedStatusCode: http.StatusOK, + maxDataSize: 1024, + mockUploadFunc: mockDataSystemUploadSuccess, + mockInvokeFunc: mockInvokeHandlerSuccess, + mockDownloadFunc: mockDataSystemDownloadSuccess, + mockDeleteFunc: mockDataSystemDeleteSuccess, + }, + { + name: "execute_upload_failed", + header: dataCommonHeader(), + maxKeySize: 5, + payloadInfo: "[{\"dataPrefix\": \"prefix\", \"offset\": 0, \"len\": 512, \"needEncrypt\": false}]", + body: body, + expectedResponse: "{\"code\":200701,\"message\":\"internal upload failed\"}", + expectedInnerCode: "200701", + expectedStatusCode: http.StatusOK, + maxDataSize: 1024, + mockUploadFunc: mockDataSystemUploadFail, + mockInvokeFunc: mockInvokeHandlerSuccess, + mockDownloadFunc: mockDataSystemDownloadSuccess, + mockDeleteFunc: mockDataSystemDeleteSuccess, + }, + { + name: "execute_invoke_fail", + header: dataCommonHeader(), + maxKeySize: 5, + payloadInfo: "[{\"dataPrefix\": \"prefix\", \"offset\": 0, \"len\": 512, \"needEncrypt\": false}]", + body: body, + expectedResponse: "{\"code\":200705,\"message\":\"internal invoke failed\"}", + expectedInnerCode: "200705", + expectedStatusCode: http.StatusOK, + maxDataSize: 1024, + mockUploadFunc: mockDataSystemUploadSuccess, + mockInvokeFunc: mockInvokeHandlerFail, + mockDownloadFunc: mockDataSystemDownloadSuccess, + mockDeleteFunc: mockDataSystemDeleteSuccess, + }, + { + name: "execute_download_fail", + header: dataCommonHeader(), + maxKeySize: 5, + payloadInfo: "[{\"dataPrefix\": \"prefix\", \"offset\": 0, \"len\": 512, \"needEncrypt\": false}]", + body: body, + expectedResponse: "{\"code\":200702,\"message\":\"internal download failed\"}", + expectedInnerCode: "200702", + expectedStatusCode: http.StatusOK, + maxDataSize: 1024, + mockUploadFunc: mockDataSystemUploadSuccess, + mockInvokeFunc: mockInvokeHandlerSuccess, + mockDownloadFunc: mockDataSystemDownloadFail, + mockDeleteFunc: mockDataSystemDeleteSuccess, + }, + { + name: "execute_key_not_match", + header: dataCommonHeader(), + maxKeySize: 5, + payloadInfo: "[{\"dataPrefix\": \"prefix\", \"offset\": 0, \"len\": 512, \"needEncrypt\": false}]", + body: body, + expectedResponse: "{\"code\":200500,\"message\":\"keylen = 1 is not equals to data len = 2\"}", + expectedInnerCode: "200500", + expectedStatusCode: http.StatusOK, + maxDataSize: 1024, + mockUploadFunc: mockDataSystemUploadSuccess, + mockInvokeFunc: mockInvokeHandlerSuccess, + mockDownloadFunc: mockDataSystemDownloadMultipleSuccess, + mockDeleteFunc: mockDataSystemDeleteSuccess, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + req, _ := http.NewRequest(http.MethodPost, "/test", bytes.NewReader(test.body)) + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set(constant.HeaderDataSystemPayloadInfo, test.payloadInfo) + for key, value := range test.header { + req.Header.Set(key, value) + } + + config.GetConfig().HTTPConfig = &types.FrontendHTTP{ + MaxDataSystemMultiDataBodySize: test.maxDataSize, + } + p3 := test.mockUploadFunc() + defer p3.Reset() + p4 := test.mockInvokeFunc() + defer p4.Reset() + p5 := test.mockDownloadFunc() + defer p5.Reset() + p6 := test.mockDeleteFunc() + defer p6.Reset() + + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, test.expectedStatusCode, w.Code) + assert.Equal(t, test.expectedResponse, w.Body.String()) + assert.Equal(t, test.expectedInnerCode, w.Header().Get(constant.HeaderInnerCode)) + assert.NotEmpty(t, w.Header().Get(constant.HeaderTraceID)) + }) + } +} +func mockInvokeHandlerSuccess() *gomonkey.Patches { + return gomonkey.ApplyFunc(invocation.InvokeHandler, func(ctx *types.InvokeProcessContext) error { + ctx.RespHeader[constant.HeaderInnerCode] = "0" + ctx.RespHeader["X-Caas-Data-System-Key"] = "key1" + ctx.StatusCode = http.StatusOK + ctx.RespBody = []byte("internal invoke success") + return nil + }) +} + +func mockInvokeHandlerFail() *gomonkey.Patches { + return gomonkey.ApplyFunc(invocation.InvokeHandler, func(ctx *types.InvokeProcessContext) error { + ctx.RespHeader[constant.HeaderInnerCode] = "400" + ctx.RespBody = []byte("internal invoke failed") + return fmt.Errorf("internal invoke failed") + }) +} diff --git a/frontend/pkg/frontend/frontendsdkadapter/handler/multidata_handler.go b/frontend/pkg/frontend/frontendsdkadapter/handler/multidata_handler.go new file mode 100644 index 0000000000000000000000000000000000000000..be5963e72b042aa4236eb8784ed98f4a8db1701f --- /dev/null +++ b/frontend/pkg/frontend/frontendsdkadapter/handler/multidata_handler.go @@ -0,0 +1,93 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package handler +package handler + +import ( + "fmt" + + "github.com/gin-gonic/gin" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/statuscode" + "frontend/pkg/common/uuid" + "frontend/pkg/frontend/frontendsdkadapter/assembler" + "frontend/pkg/frontend/frontendsdkadapter/models" + "frontend/pkg/frontend/frontendsdkadapter/parser" + "frontend/pkg/frontend/frontendsdkadapter/service" +) + +// MultiGetHandler - +func MultiGetHandler(ctx *gin.Context) { + traceID := ctx.Request.Header.Get(constant.HeaderTraceID) + if traceID == "" { + traceID = uuid.New().String() + } + traceLogger := models.NewTraceLogger("multiget", traceID) + getRequest, err := parser.NewMultiGetRequestContext(ctx, traceID) + if err != nil { + msg := fmt.Sprintf("deserialize request failed, err: %s", err.Error()) + errResponse := assembler.NewCommonErrorResponse(statuscode.FrontendStatusBadRequest, msg, + traceID) + traceLogger.Logger.Error(msg) + assembler.WriteResponse(ctx, nil, errResponse, traceID) + return + } + traceLogger.TenantID = getRequest.TenantID + response, errResponse := service.MultiGetService(getRequest, traceLogger) + assembler.WriteResponse(ctx, response, errResponse, traceID) +} + +// MultiDelHandler - +func MultiDelHandler(ctx *gin.Context) { + traceID := ctx.Request.Header.Get(constant.HeaderTraceID) + if traceID == "" { + traceID = uuid.New().String() + } + traceLogger := models.NewTraceLogger("multidel", traceID) + delRequest, err := parser.NewMultiDelRequestContext(ctx, traceID) + if err != nil { + msg := fmt.Sprintf("deserialize request failed, err: %s", err.Error()) + errResponse := assembler.NewCommonErrorResponse(statuscode.FrontendStatusBadRequest, msg, traceID) + traceLogger.Logger.Errorf(errResponse.ErrorRsp.Message) + assembler.WriteResponse(ctx, nil, errResponse, traceID) + return + } + response, errResponse := service.MultiDelService(delRequest, traceLogger) + assembler.WriteResponse(ctx, response, errResponse, traceID) +} + +// MultiSetHandler - +func MultiSetHandler(ctx *gin.Context) { + traceID := ctx.Request.Header.Get(constant.HeaderTraceID) + if traceID == "" { + traceID = uuid.New().String() + } + traceLogger := models.NewTraceLogger("multiset", traceID) + setRequest, err := parser.NewMultiSetRequestContext(ctx, traceLogger.TraceID) + if err != nil { + msg := fmt.Sprintf("deserialize request failed, err: %s", err.Error()) + errResponse := assembler.NewCommonErrorResponse(statuscode.FrontendStatusBadRequest, msg, + traceID) + traceLogger.Logger.Error(errResponse.ErrorRsp.Message) + assembler.WriteResponse(ctx, nil, errResponse, traceID) + return + } + traceLogger.TenantID = setRequest.TenantID + response, errResponse := service.MultiSetService(setRequest, traceLogger) + assembler.WriteResponse(ctx, response, errResponse, traceID) +} diff --git a/frontend/pkg/frontend/frontendsdkadapter/handler/multidata_handler_test.go b/frontend/pkg/frontend/frontendsdkadapter/handler/multidata_handler_test.go new file mode 100644 index 0000000000000000000000000000000000000000..464aa8509e80d2fab45741582d12d227d2c2ff27 --- /dev/null +++ b/frontend/pkg/frontend/frontendsdkadapter/handler/multidata_handler_test.go @@ -0,0 +1,466 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package handler +package handler + +import ( + "bytes" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/datasystemclient" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/types" +) + +func TestDataSystemMultiDelHandler(t *testing.T) { + gin.SetMode(gin.TestMode) + router := gin.New() + router.POST("/test", MultiDelHandler) + tests := []struct { + name string + header map[string]string + clusterMap map[string]string + payloadData string + dataKeys string + expectedStatusCode int + expectedBody string + expectedInnerCode string + expectedPayloadInfo string + maxBodySize int + maxKeySize int + mockDataSystemFunc func() *gomonkey.Patches + }{ + {name: "del_success", + dataKeys: "key1|key2", + header: dataCommonHeader(), + payloadData: `[{"dataPrefix": "prefix", "offset": 0, "len": 0, "needEncrypt":false},{"dataPrefix": "prefix", "offset": 0, "len": 0, "needEncrypt":false}]`, + expectedStatusCode: http.StatusOK, + expectedBody: "{\"code\":0,\"message\":\"success\"}", + expectedInnerCode: "0", + maxBodySize: 1024, + maxKeySize: 5, + mockDataSystemFunc: mockDataSystemDeleteSuccess, + }, + {name: "del_request_body_too_large", + header: dataCommonHeader(), + dataKeys: "key1|key2", + payloadData: "[]", + expectedStatusCode: http.StatusOK, + expectedBody: "{\"code\":200400,\"message\":\"deserialize request failed, err: body is beyond maximum: 0\"}", + expectedInnerCode: "200400", + expectedPayloadInfo: "", + maxBodySize: 0, + maxKeySize: 5, + mockDataSystemFunc: mockDataSystemDeleteSuccess, + }, + {name: "del_empty_keys", + header: dataCommonHeader(), + payloadData: "[]", + dataKeys: "", + expectedStatusCode: http.StatusOK, + expectedBody: "{\"code\":0,\"message\":\"success\"}", + expectedInnerCode: "0", + maxKeySize: 5, + mockDataSystemFunc: mockDataSystemDeleteSuccess, + }, + {name: "del_dataSystem_fail", + header: dataCommonHeader(), + dataKeys: "key1|key2", + payloadData: `[{"dataPrefix": "prefix", "offset": 0, "len": 0, "needEncrypt":false},{"dataPrefix": "prefix", "offset": 0, "len": 0, "needEncrypt":false}]`, + expectedStatusCode: http.StatusOK, + expectedBody: "{\"code\":200703,\"message\":\"internal delete failed\"}", + expectedInnerCode: "200703", + maxBodySize: 1024, + maxKeySize: 5, + + mockDataSystemFunc: mockDataSystemDeleteFail, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + p3 := test.mockDataSystemFunc() + defer p3.Reset() + req, _ := http.NewRequest(http.MethodPost, "/test", bytes.NewReader([]byte(test.dataKeys))) + req.Header.Set("Content-Type", "application/json") + req.Header.Set(constant.HeaderDataSystemPayloadInfo, test.payloadData) + for key, value := range test.header { + req.Header.Set(key, value) + } + config.GetConfig().HTTPConfig = &types.FrontendHTTP{ + MaxDataSystemMultiDataBodySize: test.maxBodySize, + } + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, test.expectedStatusCode, w.Code) + assert.Equal(t, test.expectedBody, w.Body.String()) + assert.Equal(t, test.expectedInnerCode, w.Header().Get(constant.HeaderInnerCode)) + assert.NotEmpty(t, w.Header().Get(constant.HeaderTraceID)) + assert.Equal(t, test.expectedPayloadInfo, w.Header().Get(constant.HeaderDataSystemPayloadInfo)) + }) + } +} + +func dataCommonHeader() map[string]string { + return map[string]string{ + "Authorization": "value2", + "X-Tenant-Id": "tenantId", + "X-Client-Id": "clientId", + "X-Trace-Id": "traceId", + "X-User-Id": "userId", + "X-Function-Name": "functionname", + } +} + +func mockDataSystemDeleteSuccess() *gomonkey.Patches { + return gomonkey.ApplyFunc(datasystemclient.DeleteArrayRetry, + func(keys []string, config *datasystemclient.Config, traceID string) ([]string, error) { + return nil, nil + }) +} + +func mockDataSystemDeleteFail() *gomonkey.Patches { + return gomonkey.ApplyFunc(datasystemclient.DeleteArrayRetry, + func(keys []string, config *datasystemclient.Config, traceID string) ([]string, error) { + return []string{"key1"}, fmt.Errorf("some keys failed to delete") + }) +} + +func TestDataSystemMultiGetHandler(t *testing.T) { + gin.SetMode(gin.TestMode) + router := gin.New() + router.POST("/test", MultiGetHandler) + tests := []struct { + name string + dataKeys string + header map[string]string + clusterMap map[string]string + expectedStatusCode int + expectedBody string + expectedInnerCode string + expectedPayloadInfo string + maxDataSize int + maxKeySize int + payloadInfo string + mockFunc func() *gomonkey.Patches + }{ + {name: "download_success", + dataKeys: "key1|key2", + expectedStatusCode: http.StatusOK, + header: dataCommonHeader(), + expectedBody: "a", + expectedInnerCode: "0", + maxDataSize: 1024, + maxKeySize: 5, + payloadInfo: `[{"dataKey": "key1", "dataPrefix": "prefix", "offset": 0, "len": 0, "needEncrypt":false}]`, + expectedPayloadInfo: "[{\"dataKey\":\"key1\",\"offset\":0,\"len\":1}]", + mockFunc: mockDataSystemDownloadSuccess, + }, + {name: "download_multiple_success", + dataKeys: "key1|key2", + expectedStatusCode: http.StatusOK, + header: dataCommonHeader(), + expectedBody: "a", + expectedInnerCode: "0", + maxDataSize: 1024, + maxKeySize: 5, + payloadInfo: `[{"dataKey": "key1", "dataPrefix": "prefix", "offset": 0, "len": 0, "needEncrypt":false},{"dataKey": "key2", "dataPrefix": "prefix", "offset": 0, "len": 0, "needEncrypt":false}]`, + expectedPayloadInfo: "[{\"dataKey\":\"key1\",\"offset\":0,\"len\":1},{\"dataKey\":\"key2\",\"offset\":1,\"len\":0}]", + mockFunc: mockDataSystemDownloadMultipleSuccess, + }, + {name: "download_size_not_match", + dataKeys: "key1|key2", + expectedStatusCode: http.StatusOK, + header: dataCommonHeader(), + expectedBody: "{\"code\":200500,\"message\":\"keylen = 2 is not equals to data len = 1\"}", + expectedInnerCode: "200500", + maxDataSize: 1024, + maxKeySize: 5, + payloadInfo: `[{"dataKey": "key1", "dataPrefix": "prefix", "offset": 0, "len": 0, "needEncrypt":false},{"dataKey": "key2", "dataPrefix": "prefix", "offset": 0, "len": 0, "needEncrypt":false}]`, + expectedPayloadInfo: "", + mockFunc: mockDataSystemDownloadSuccess, + }, + {name: "download_request_body_too_large", + dataKeys: string(createByteArray(1<<10 + 1)), + header: dataCommonHeader(), + maxDataSize: 0, + payloadInfo: "[]", + expectedStatusCode: http.StatusOK, + expectedBody: "{\"code\":200400,\"message\":\"deserialize request failed, err: body is beyond maximum: 0\"}", + expectedInnerCode: "200400", + expectedPayloadInfo: "", + mockFunc: mockDataSystemDownloadSuccess, + }, + {name: "download_empty_keys", + dataKeys: "", + header: dataCommonHeader(), + payloadInfo: "[]", + maxDataSize: 1024, + maxKeySize: 5, + expectedStatusCode: http.StatusOK, + expectedBody: "", + expectedInnerCode: "0", + expectedPayloadInfo: "", + mockFunc: mockDataSystemDownloadSuccess, + }, + {name: "download_failed", + dataKeys: "key1|key2", + payloadInfo: `[{"dataPrefix": "prefix", "offset": 0, "len": 0, "needEncrypt":false},{"dataPrefix": "prefix", "offset": 0, "len": 0, "needEncrypt":false}]`, + header: dataCommonHeader(), + maxDataSize: 1024, + maxKeySize: 5, + expectedStatusCode: http.StatusOK, + expectedBody: "{\"code\":200702,\"message\":\"internal download failed\"}", + expectedInnerCode: "200702", + expectedPayloadInfo: "", + mockFunc: mockDataSystemDownloadFail, + }, + {name: "download_failed_parse_err", + dataKeys: "key1|key2", + payloadInfo: `[{"dataPrefix": "prefix", "offset": 0, "len": 0, "needEncrypt":false},{"dataPrefix": "prefix", "offset": 0, "len": 0, "needEncrypt":false}]`, + header: dataCommonHeader(), + maxKeySize: 5, + maxDataSize: 1024, + expectedStatusCode: http.StatusOK, + expectedBody: "{\"code\":200702,\"message\":\"internal download failed\"}", + expectedInnerCode: "200702", + expectedPayloadInfo: "", + mockFunc: mockDataSystemDownloadFail, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + req, _ := http.NewRequest(http.MethodPost, "/test", bytes.NewReader([]byte(test.dataKeys))) + req.Header.Set("Content-Type", "application/json") + req.Header.Set(constant.HeaderDataSystemPayloadInfo, test.payloadInfo) + for key, value := range test.header { + req.Header.Set(key, value) + } + config.GetConfig().HTTPConfig = &types.FrontendHTTP{ + MaxDataSystemMultiDataBodySize: test.maxDataSize, + } + p3 := test.mockFunc() + defer p3.Reset() + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, test.expectedStatusCode, w.Code) + assert.Equal(t, test.expectedBody, w.Body.String()) + assert.Equal(t, test.expectedInnerCode, w.Header().Get(constant.HeaderInnerCode)) + assert.NotEmpty(t, w.Header().Get(constant.HeaderTraceID)) + assert.Equal(t, test.expectedPayloadInfo, w.Header().Get(constant.HeaderDataSystemPayloadInfo)) + }) + } +} + +func mockDataSystemDownloadSuccess() *gomonkey.Patches { + return gomonkey.ApplyFunc(datasystemclient.DownloadArrayRetry, func(keys []string, + config *datasystemclient.Config, traceID string) ([][]byte, error) { + return [][]byte{[]byte("a")}, nil + }) +} + +func mockDataSystemDownloadMultipleSuccess() *gomonkey.Patches { + return gomonkey.ApplyFunc(datasystemclient.DownloadArrayRetry, func(keys []string, + config *datasystemclient.Config, traceID string) ([][]byte, error) { + return [][]byte{[]byte("a"), []byte("")}, nil + }) +} + +func mockDataSystemDownloadFail() *gomonkey.Patches { + return gomonkey.ApplyFunc(datasystemclient.DownloadArrayRetry, func(keys []string, + config *datasystemclient.Config, traceID string) ([][]byte, error) { + return [][]byte{[]byte("a")}, fmt.Errorf("err") + }) +} + +func TestDataSystemMultiSetHandler(t *testing.T) { + gin.SetMode(gin.TestMode) + router := gin.New() + router.POST("/test", MultiSetHandler) + body := createByteArray(1 << 10) + tests := []struct { + name string + header map[string]string + clusterMap map[string]string + payloadInfo string + body []byte + maxDataSize int + maxKeySize int + expectedResponse string + expectedInnerCode string + expectedStatusCode int + expectedTraceID string + mockFunc func() *gomonkey.Patches + }{ + { + name: "empty_request_success", + header: dataCommonHeader(), + maxKeySize: 5, + payloadInfo: "[{\"dataPrefix\": \"prefix\", \"offset\": 0, \"len\": 0, \"needEncrypt\": false}]", + body: nil, + expectedResponse: "{\"dataKeys\":null}", + expectedInnerCode: "0", + expectedStatusCode: http.StatusOK, + maxDataSize: 1024, + mockFunc: mockDataSystemUploadSuccess, + }, + { + name: "payloadInfo_large_than_body", + payloadInfo: "[{\"dataPrefix\": \"prefix\", \"offset\": 0, \"len\": 1048276, \"needEncrypt\": false}]", + header: dataCommonHeader(), + maxKeySize: 5, + body: nil, + expectedResponse: "{\"code\":200400,\"message\":\"deserialize request failed, err: payload len invalid\"}", + expectedInnerCode: "200400", + expectedStatusCode: http.StatusOK, + maxDataSize: 1024, + mockFunc: mockDataSystemUploadSuccess, + }, + { + name: "payloadInfo-header-json-invalid", + payloadInfo: `{}`, + header: dataCommonHeader(), + maxKeySize: 5, + body: nil, + expectedResponse: "{\"code\":200400,\"message\":\"deserialize request failed, err: payloadInfo json invalid\"}", + expectedInnerCode: "200400", + expectedStatusCode: http.StatusOK, + maxDataSize: 1024, + mockFunc: mockDataSystemUploadSuccess, + }, + { + name: "body_len_too_large", + payloadInfo: `[]`, + maxKeySize: 5, + body: append(body, 'a'), + expectedResponse: "{\"code\":200400,\"message\":\"deserialize request failed, err: body is beyond maximum: 0\"}", + expectedInnerCode: "200400", + expectedStatusCode: http.StatusOK, + maxDataSize: 0, + mockFunc: mockDataSystemUploadSuccess, + }, + { + name: "upload_success", + payloadInfo: `[{"dataPrefix": "prefix", "offset": 0, "len": 512, "needEncrypt": false }]`, + header: dataCommonHeader(), + maxKeySize: 5, + body: body, + expectedResponse: "{\"dataKeys\":[\"key1\"]}", + expectedInnerCode: "0", + expectedStatusCode: http.StatusOK, + maxDataSize: 1024, + mockFunc: mockDataSystemUploadSuccess, + }, + { + name: "upload_multiple_success", + payloadInfo: `[{"dataPrefix": "prefix", "offset": 0, "len": 512, "needEncrypt": false }, {"dataPrefix": "prefix", "offset": 512, "len": 512, "needEncrypt": false}]`, + header: dataCommonHeader(), + maxKeySize: 5, + body: body, + expectedResponse: "{\"dataKeys\":[\"key1\",\"key1\"]}", + expectedInnerCode: "0", + expectedStatusCode: http.StatusOK, + maxDataSize: 1024, + mockFunc: mockDataSystemUploadSuccess, + }, + { + name: "upload_fail", + payloadInfo: `[{"dataPrefix": "prefix","offset": 0,"len": 1024,"needEncrypt": false}]`, + header: dataCommonHeader(), + maxKeySize: 5, + body: body, + expectedResponse: "{\"code\":200701,\"message\":\"internal upload failed\"}", + expectedInnerCode: "200701", + expectedStatusCode: http.StatusOK, + maxDataSize: 1024, + mockFunc: mockDataSystemUploadFail, + }, + { + name: "upload_execeed_key_size", + payloadInfo: `[{"dataPrefix": "prefix","offset": 0,"len": 1024,"needEncrypt": false}]`, + header: dataCommonHeader(), + maxKeySize: 1, + body: body, + expectedResponse: "{\"code\":200701,\"message\":\"internal upload failed\"}", + expectedInnerCode: "200701", + expectedStatusCode: http.StatusOK, + maxDataSize: 1024, + mockFunc: mockDataSystemUploadFail, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + req, _ := http.NewRequest(http.MethodPost, "/test", bytes.NewReader(test.body)) + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set(constant.HeaderDataSystemPayloadInfo, test.payloadInfo) + for key, value := range test.header { + req.Header.Set(key, value) + } + config.GetConfig().HTTPConfig = &types.FrontendHTTP{ + MaxDataSystemMultiDataBodySize: test.maxDataSize, + } + p3 := test.mockFunc() + defer p3.Reset() + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, test.expectedStatusCode, w.Code) + assert.Equal(t, test.expectedResponse, w.Body.String()) + assert.Equal(t, test.expectedInnerCode, w.Header().Get(constant.HeaderInnerCode)) + assert.NotEmpty(t, w.Header().Get(constant.HeaderTraceID)) + }) + } +} + +func createByteArray(byteSize int) []byte { + byteArray := make([]byte, byteSize) + for i := 0; i < byteSize; i++ { + byteArray[i] = 'A' + } + return byteArray +} + +func mockDataSystemUploadSuccess() *gomonkey.Patches { + return gomonkey.ApplyFunc(datasystemclient.UploadWithKeyRetry, func(value []byte, + config *datasystemclient.Config, param api.SetParam, traceID string) (string, error) { + return "key1", nil + }) +} + +func mockDataSystemUploadFail() *gomonkey.Patches { + return gomonkey.ApplyFunc(datasystemclient.UploadWithKeyRetry, func(value []byte, + config *datasystemclient.Config, param api.SetParam, traceID string) (string, error) { + return "", fmt.Errorf("dsclient is nil") + }) +} diff --git a/frontend/pkg/frontend/frontendsdkadapter/models/common_req.go b/frontend/pkg/frontend/frontendsdkadapter/models/common_req.go new file mode 100644 index 0000000000000000000000000000000000000000..97b35da9be16980a622d5f548829a9ea72c06194 --- /dev/null +++ b/frontend/pkg/frontend/frontendsdkadapter/models/common_req.go @@ -0,0 +1,48 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package models multidata request response transfer object +package models + +import ( + "github.com/gin-gonic/gin" + + "frontend/pkg/common/faas_common/constant" +) + +// RequestParser - +type RequestParser interface { + ParseRequest(ctx *gin.Context, maxBodySize int64) error +} + +// CommonReqHeader - +type CommonReqHeader struct { + TraceID string + TenantID string + Headers map[string]string +} + +// ParserHeader - +func (reqHeader *CommonReqHeader) ParserHeader(ctx *gin.Context) { + reqHeader.Headers = make(map[string]string) + for key, values := range ctx.Request.Header { + if len(values) > 0 { + reqHeader.Headers[key] = values[0] + } + } + reqHeader.TenantID = reqHeader.Headers[constant.HeaderTenantID] + reqHeader.TraceID = reqHeader.Headers[constant.HeaderTraceID] +} diff --git a/frontend/pkg/frontend/frontendsdkadapter/models/common_rsp.go b/frontend/pkg/frontend/frontendsdkadapter/models/common_rsp.go new file mode 100644 index 0000000000000000000000000000000000000000..07113d88545cd558266f572c77d79c9836e1fa43 --- /dev/null +++ b/frontend/pkg/frontend/frontendsdkadapter/models/common_rsp.go @@ -0,0 +1,79 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package models +package models + +import ( + "encoding/json" + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/frontend/common/httpconstant" +) + +// ResponseWriter - +type ResponseWriter interface { + WriteResponse(ctx *gin.Context) error +} + +// CommonRspHeader - +type CommonRspHeader struct { + InnerCode int32 + TraceID string + Headers map[string]string + ContentType string +} + +// BadResponse HTTP request message that does not return 200 +type BadResponse struct { + Code int `json:"code"` + Message string `json:"message"` +} + +// CommonErrorResponse - +type CommonErrorResponse struct { + CommonRspHeader + ErrorRsp BadResponse +} + +// WriteResponse - +func (rsp *CommonErrorResponse) WriteResponse(ctx *gin.Context) error { + return writeJSONResponse(ctx, http.StatusOK, rsp.InnerCode, rsp.TraceID, rsp.ErrorRsp) +} + +func writeJSONResponse(ctx *gin.Context, statusCode int, innerCode int32, traceID string, payload interface{}) error { + ctx.Writer.WriteHeader(statusCode) + ctx.Writer.Header().Set(constant.HeaderContentType, httpconstant.ApplicationJSON) + ctx.Writer.Header().Set(constant.HeaderInnerCode, fmt.Sprint(innerCode)) + ctx.Writer.Header().Set(constant.HeaderTraceID, fmt.Sprint(traceID)) + + body, err := json.Marshal(payload) + if err != nil { + log.GetLogger().Errorf("failed to marshal response body: %s", err.Error()) + return err + } + _, err = ctx.Writer.Write(body) + if err != nil { + log.GetLogger().Errorf("failed to write response: %s", err.Error()) + return err + } + return nil +} diff --git a/frontend/pkg/frontend/frontendsdkadapter/models/del_req.go b/frontend/pkg/frontend/frontendsdkadapter/models/del_req.go new file mode 100644 index 0000000000000000000000000000000000000000..12e60d52911a0700481aac9cc49215cfc44392fe --- /dev/null +++ b/frontend/pkg/frontend/frontendsdkadapter/models/del_req.go @@ -0,0 +1,42 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package models +package models + +import ( + "github.com/gin-gonic/gin" + + "frontend/pkg/frontend/common/httputil" +) + +// MultiDelRequest - +type MultiDelRequest struct { + CommonReqHeader + DataSystemPayloadInfo *DataSystemPayloadInfo + DataKeys string +} + +// ParseRequest - +func (req *MultiDelRequest) ParseRequest(context *gin.Context, maxBodySize int64) error { + req.CommonReqHeader.ParserHeader(context) + body, err := httputil.ReadLimitedBody(context.Request.Body, maxBodySize) + if err != nil { + return err + } + req.DataKeys = string(body) + return nil +} diff --git a/frontend/pkg/frontend/frontendsdkadapter/models/del_rsp.go b/frontend/pkg/frontend/frontendsdkadapter/models/del_rsp.go new file mode 100644 index 0000000000000000000000000000000000000000..ce9941b8436fc102861abc4ec70ded353ed38d4e --- /dev/null +++ b/frontend/pkg/frontend/frontendsdkadapter/models/del_rsp.go @@ -0,0 +1,65 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package models +package models + +import ( + "encoding/json" + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/frontend/common/httpconstant" +) + +// SuccessResponse HTTP request message +type SuccessResponse struct { + Code int `json:"code"` + Message string `json:"message"` +} + +// MultiDelSuccessResponse - +type MultiDelSuccessResponse struct { + CommonRspHeader + SuccessResponse +} + +// WriteResponse - +func (rsp *MultiDelSuccessResponse) WriteResponse(ctx *gin.Context) error { + ctx.Header(constant.HeaderContentType, httpconstant.ApplicationJSON) + ctx.Header(constant.HeaderInnerCode, fmt.Sprint(rsp.InnerCode)) + ctx.Header(constant.HeaderTraceID, fmt.Sprint(rsp.TraceID)) + + ctx.Writer.WriteHeader(http.StatusOK) + + marshal, err := json.Marshal(rsp.SuccessResponse) + if err != nil { + log.GetLogger().Errorf("failed to write rsp err %s", err.Error()) + return err + } + + _, err = ctx.Writer.Write(marshal) + if err != nil { + log.GetLogger().Errorf("failed to write rsp err %s", err.Error()) + return err + } + + return nil +} diff --git a/frontend/pkg/frontend/frontendsdkadapter/models/execute_req.go b/frontend/pkg/frontend/frontendsdkadapter/models/execute_req.go new file mode 100644 index 0000000000000000000000000000000000000000..c87e199d439a2ea852adbd43460333ac685af026 --- /dev/null +++ b/frontend/pkg/frontend/frontendsdkadapter/models/execute_req.go @@ -0,0 +1,64 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package models +package models + +import ( + "io" + + "github.com/gin-gonic/gin" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/frontend/common/httputil" +) + +// ExecuteRequest - +type ExecuteRequest struct { + CommonReqHeader + DataSystemPayloadInfo *DataSystemPayloadInfo + FunctionName string + PayloadData *PayloadData + RawData []byte +} + +// ParseRequest - +func (req *ExecuteRequest) ParseRequest(ctx *gin.Context, maxBodySize int64) error { + err := req.parseHeader(ctx) + if err != nil { + return err + } + err = req.readBody(ctx.Request.Body, maxBodySize) + if err != nil { + return err + } + return nil +} + +func (req *ExecuteRequest) parseHeader(ctx *gin.Context) error { + req.CommonReqHeader.ParserHeader(ctx) + req.FunctionName = req.Headers[constant.HeaderFunctionName] + return nil +} + +func (req *ExecuteRequest) readBody(inputStream io.ReadCloser, maxBodySize int64) error { + rawData, err := httputil.ReadLimitedBody(inputStream, maxBodySize) + if err != nil { + return err + } + req.RawData = rawData + return nil +} diff --git a/frontend/pkg/frontend/frontendsdkadapter/models/execute_rsp.go b/frontend/pkg/frontend/frontendsdkadapter/models/execute_rsp.go new file mode 100644 index 0000000000000000000000000000000000000000..a32cba1afa4fb85f97520b324bbc5067cfc5584a --- /dev/null +++ b/frontend/pkg/frontend/frontendsdkadapter/models/execute_rsp.go @@ -0,0 +1,46 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + +package models + +import ( + "fmt" + + "github.com/gin-gonic/gin" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/statuscode" + "frontend/pkg/frontend/common/httpconstant" +) + +// ExecuteSuccessResponse - +type ExecuteSuccessResponse struct { + CommonRspHeader + DataSystemPayloadInfo *DataSystemPayloadInfo + RawData [][]byte +} + +// WriteResponse - +func (rsp *ExecuteSuccessResponse) WriteResponse(ctx *gin.Context) error { + if (rsp.DataSystemPayloadInfo == nil && len(rsp.RawData) > 0) || + (rsp.DataSystemPayloadInfo != nil && len(rsp.RawData) == 0) { + return fmt.Errorf("payload incomplete: missing PayloadInfo or RawData") + } + + ctx.Writer.WriteHeader(statuscode.Code(int(rsp.InnerCode))) + ctx.Writer.Header().Set(constant.HeaderContentType, httpconstant.StreamContentType) + if rsp.DataSystemPayloadInfo != nil { + ctx.Writer.Header().Set(constant.HeaderDataSystemPayloadInfo, rsp.DataSystemPayloadInfo.ToJSON()) + } + ctx.Writer.Header().Set(constant.HeaderInnerCode, fmt.Sprint(rsp.InnerCode)) + ctx.Writer.Header().Set(constant.HeaderTraceID, fmt.Sprint(rsp.TraceID)) + + for _, data := range rsp.RawData { + _, err := ctx.Writer.Write(data) + if err != nil { + log.GetLogger().Errorf("failed to write rsp err %s", err.Error()) + return err + } + } + return nil +} diff --git a/frontend/pkg/frontend/frontendsdkadapter/models/get_req.go b/frontend/pkg/frontend/frontendsdkadapter/models/get_req.go new file mode 100644 index 0000000000000000000000000000000000000000..25bdd0cc4528be36135e6fc3500f76304dc5e86a --- /dev/null +++ b/frontend/pkg/frontend/frontendsdkadapter/models/get_req.go @@ -0,0 +1,42 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package models +package models + +import ( + "github.com/gin-gonic/gin" + + "frontend/pkg/frontend/common/httputil" +) + +// MultiGetRequest - +type MultiGetRequest struct { + CommonReqHeader + DataSystemPayloadInfo *DataSystemPayloadInfo + RawData []byte +} + +// ParseRequest - +func (req *MultiGetRequest) ParseRequest(ctx *gin.Context, maxBodySize int64) error { + req.CommonReqHeader.ParserHeader(ctx) + body, err := httputil.ReadLimitedBody(ctx.Request.Body, maxBodySize) + if err != nil { + return err + } + req.RawData = body + return nil +} diff --git a/frontend/pkg/frontend/frontendsdkadapter/models/get_rsp.go b/frontend/pkg/frontend/frontendsdkadapter/models/get_rsp.go new file mode 100644 index 0000000000000000000000000000000000000000..14f2b31c18c4382351bfdb096c1c96639ca2a250 --- /dev/null +++ b/frontend/pkg/frontend/frontendsdkadapter/models/get_rsp.go @@ -0,0 +1,47 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + +// Package models +package models + +import ( + "fmt" + + "github.com/gin-gonic/gin" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/statuscode" + "frontend/pkg/frontend/common/httpconstant" +) + +// MultiGetSuccessResponse - +type MultiGetSuccessResponse struct { + CommonRspHeader + DataSystemPayloadInfo *DataSystemPayloadInfo + RawData [][]byte +} + +// WriteResponse - +func (rsp *MultiGetSuccessResponse) WriteResponse(ctx *gin.Context) error { + if (rsp.DataSystemPayloadInfo == nil && len(rsp.RawData) > 0) || + (rsp.DataSystemPayloadInfo != nil && len(rsp.RawData) == 0) { + return fmt.Errorf("payload incomplete: missing PayloadInfo or RawData") + } + + ctx.Writer.WriteHeader(statuscode.Code(int(rsp.InnerCode))) + ctx.Writer.Header().Set(constant.HeaderContentType, httpconstant.StreamContentType) + if rsp.DataSystemPayloadInfo != nil { + ctx.Writer.Header().Set(constant.HeaderDataSystemPayloadInfo, rsp.DataSystemPayloadInfo.ToJSON()) + } + + ctx.Writer.Header().Set(constant.HeaderInnerCode, fmt.Sprint(rsp.InnerCode)) + ctx.Writer.Header().Set(constant.HeaderTraceID, fmt.Sprint(rsp.TraceID)) + for _, data := range rsp.RawData { + _, err := ctx.Writer.Write(data) + if err != nil { + log.GetLogger().Errorf("failed to write rsp err %s", err.Error()) + return err + } + } + return nil +} diff --git a/frontend/pkg/frontend/frontendsdkadapter/models/multidata_model.go b/frontend/pkg/frontend/frontendsdkadapter/models/multidata_model.go new file mode 100644 index 0000000000000000000000000000000000000000..5bf5ea2669bcf1751793b352a1a5d75d3d373f13 --- /dev/null +++ b/frontend/pkg/frontend/frontendsdkadapter/models/multidata_model.go @@ -0,0 +1,68 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package models +package models + +import ( + "encoding/json" +) + +// PayloadInfo - +type PayloadInfo struct { + DataPrefix string `json:"dataPrefix,omitempty"` + DataKey string `json:"dataKey,omitempty"` + Offset int `json:"offset"` + Length int `json:"len"` + NeedEncrypt bool `json:"needEncrypt,omitempty"` +} + +// DataSystemPayloadInfo - +type DataSystemPayloadInfo struct { + Data []*PayloadInfo `json:"data"` +} + +// ToJSON - +func (d *DataSystemPayloadInfo) ToJSON() string { + headerStr, err := json.Marshal(d.Data) + if err != nil { + return "[]" + } + return string(headerStr) +} + +// MultipartData - +type MultipartData struct { + Data []byte + DataPrefix string + Size int + NeedEncrypt bool +} + +// PayloadData - +type PayloadData struct { + DataList []*MultipartData + Size int +} + +// DataKeyPrefixJoin - +func DataKeyPrefixJoin(d *DataSystemPayloadInfo) []string { + result := make([]string, len(d.Data)) + for i, part := range d.Data { + result[i] = part.DataPrefix + part.DataKey + } + return result +} diff --git a/frontend/pkg/frontend/frontendsdkadapter/models/set_req.go b/frontend/pkg/frontend/frontendsdkadapter/models/set_req.go new file mode 100644 index 0000000000000000000000000000000000000000..abefd28eb80f3128b384b34633bbc36663cd9610 --- /dev/null +++ b/frontend/pkg/frontend/frontendsdkadapter/models/set_req.go @@ -0,0 +1,57 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package models +package models + +import ( + "io" + + "github.com/gin-gonic/gin" + + "frontend/pkg/frontend/common/httputil" +) + +// MultiSetRequest - +type MultiSetRequest struct { + CommonReqHeader + DataSystemPayloadInfo *DataSystemPayloadInfo + PayloadData *PayloadData + RawData []byte +} + +// ParseRequest - +func (req *MultiSetRequest) ParseRequest(ctx *gin.Context, maxBodySize int) error { + req.parseHeader(ctx) + err := req.readBody(ctx.Request.Body, maxBodySize) + if err != nil { + return err + } + return nil +} + +func (req *MultiSetRequest) readBody(inputStream io.ReadCloser, maxBodySize int) error { + rawData, err := httputil.ReadLimitedBody(inputStream, int64(maxBodySize)) + if err != nil { + return err + } + req.RawData = rawData + return nil +} + +func (req *MultiSetRequest) parseHeader(ctx *gin.Context) { + req.CommonReqHeader.ParserHeader(ctx) +} diff --git a/frontend/pkg/frontend/frontendsdkadapter/models/set_rsp.go b/frontend/pkg/frontend/frontendsdkadapter/models/set_rsp.go new file mode 100644 index 0000000000000000000000000000000000000000..514b80397ea93626cefa30f89115ebf9e6621de7 --- /dev/null +++ b/frontend/pkg/frontend/frontendsdkadapter/models/set_rsp.go @@ -0,0 +1,40 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package models +package models + +import ( + "github.com/gin-gonic/gin" + + "frontend/pkg/common/faas_common/statuscode" +) + +// DataKeyList - +type DataKeyList struct { + DataKeys []string `json:"dataKeys"` +} + +// MultiSetSuccessResponse - +type MultiSetSuccessResponse struct { + CommonRspHeader + DataKeyList DataKeyList +} + +// WriteResponse - +func (rsp *MultiSetSuccessResponse) WriteResponse(ctx *gin.Context) error { + return writeJSONResponse(ctx, statuscode.Code(int(rsp.InnerCode)), rsp.InnerCode, rsp.TraceID, rsp.DataKeyList) +} diff --git a/frontend/pkg/frontend/frontendsdkadapter/models/tracelogger.go b/frontend/pkg/frontend/frontendsdkadapter/models/tracelogger.go new file mode 100644 index 0000000000000000000000000000000000000000..0f89e2ab1e5c9937891a25dd6c68f32c571024c8 --- /dev/null +++ b/frontend/pkg/frontend/frontendsdkadapter/models/tracelogger.go @@ -0,0 +1,62 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package models +package models + +import ( + "github.com/google/uuid" + "go.uber.org/zap" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/logger/log" +) + +// TraceInfo - + +// TraceLogger - +type TraceLogger struct { + TenantID string + TraceID string + DataKey string + Logger api.FormatLogger +} + +// NewTraceLogger - +func NewTraceLogger(reqType string, traceID string) *TraceLogger { + if traceID == "" { + traceID = uuid.NewString() + } + return &TraceLogger{ + Logger: log.GetLogger().With(zap.Any("type", reqType)), + TraceID: traceID, + } +} + +// AppendDataKey - +func (t *TraceLogger) AppendDataKey(prefix string, key string) { + if t.DataKey != "" { + t.DataKey = t.DataKey + "&" + prefix + key + return + } + t.DataKey = prefix + key +} + +// With - +func (t *TraceLogger) With(key string, value string) { + t.Logger = t.Logger.With(zap.Any(key, value)) +} diff --git a/frontend/pkg/frontend/frontendsdkadapter/parser/payload_parser.go b/frontend/pkg/frontend/frontendsdkadapter/parser/payload_parser.go new file mode 100644 index 0000000000000000000000000000000000000000..f0dd2c5291e6ce3a51f1e2748413ac699c9d98d4 --- /dev/null +++ b/frontend/pkg/frontend/frontendsdkadapter/parser/payload_parser.go @@ -0,0 +1,66 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package parser +package parser + +import ( + "encoding/json" + "fmt" + + "frontend/pkg/frontend/frontendsdkadapter/models" +) + +// ReadPayloadData - +func ReadPayloadData(body []byte, payloadInfo *models.DataSystemPayloadInfo) (*models.PayloadData, + error) { + return parsePayloadData(payloadInfo, body) +} + +// ParsePayloadHeaderJSON - +func ParsePayloadHeaderJSON(payloadInfoStr string) (*models.DataSystemPayloadInfo, error) { + var payloadInfoList []*models.PayloadInfo + err := json.Unmarshal([]byte(payloadInfoStr), &payloadInfoList) + if err != nil { + return nil, fmt.Errorf("payloadInfo json invalid") + } + payloadInfo := &models.DataSystemPayloadInfo{ + Data: payloadInfoList, + } + return payloadInfo, nil +} + +// parsePayloadData - +func parsePayloadData(payloadInfo *models.DataSystemPayloadInfo, body []byte) (*models.PayloadData, + error) { + payload := &models.PayloadData{} + for _, imgData := range payloadInfo.Data { + if imgData.Offset+imgData.Length > len(body) { + return nil, fmt.Errorf("payload len invalid") + } + + imageBytes := body[imgData.Offset : imgData.Offset+imgData.Length] + + payload.DataList = append(payload.DataList, &models.MultipartData{ + DataPrefix: imgData.DataPrefix, + Size: imgData.Length, + Data: imageBytes, + NeedEncrypt: imgData.NeedEncrypt, + }) + payload.Size += len(imageBytes) + } + return payload, nil +} diff --git a/frontend/pkg/frontend/frontendsdkadapter/parser/request_parser.go b/frontend/pkg/frontend/frontendsdkadapter/parser/request_parser.go new file mode 100644 index 0000000000000000000000000000000000000000..3c23f340786f7bd9af5b1547b5a6929047a84552 --- /dev/null +++ b/frontend/pkg/frontend/frontendsdkadapter/parser/request_parser.go @@ -0,0 +1,121 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package parser +package parser + +import ( + "github.com/gin-gonic/gin" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/frontendsdkadapter/models" +) + +const ( + megabytes = 1024 * 1024 +) + +// NewMultiSetRequestContext - +func NewMultiSetRequestContext(ctx *gin.Context, traceID string) (*models.MultiSetRequest, error) { + setRequest := &models.MultiSetRequest{} + + err := setRequest.ParseRequest(ctx, config.GetConfig().HTTPConfig.MaxDataSystemMultiDataBodySize*megabytes) + if err != nil { + return nil, err + } + setRequest.TraceID = traceID + // header + payloadInfoStr := setRequest.Headers[constant.HeaderDataSystemPayloadInfo] + payloadInfo, err := ParsePayloadHeaderJSON(payloadInfoStr) + if err != nil { + return nil, err + } + + setRequest.DataSystemPayloadInfo = payloadInfo + // body + data, err := ReadPayloadData(setRequest.RawData, setRequest.DataSystemPayloadInfo) + if err != nil { + return nil, err + } + setRequest.PayloadData = data + + return setRequest, nil +} + +// NewMultiGetRequestContext - +func NewMultiGetRequestContext(ctx *gin.Context, traceID string) (*models.MultiGetRequest, error) { + getRequest := &models.MultiGetRequest{} + err := getRequest.ParseRequest(ctx, int64(config.GetConfig().HTTPConfig.MaxDataSystemMultiDataBodySize*megabytes)) + if err != nil { + return nil, err + } + getRequest.TraceID = traceID + // header + payloadInfoStr := getRequest.Headers[constant.HeaderDataSystemPayloadInfo] + payloadInfo, err := ParsePayloadHeaderJSON(payloadInfoStr) + if err != nil { + return nil, err + } + + getRequest.DataSystemPayloadInfo = payloadInfo + return getRequest, nil +} + +// NewMultiDelRequestContext - +func NewMultiDelRequestContext(ctx *gin.Context, traceID string) (*models.MultiDelRequest, error) { + delRequest := &models.MultiDelRequest{} + err := delRequest.ParseRequest(ctx, int64(config.GetConfig().HTTPConfig.MaxDataSystemMultiDataBodySize*megabytes)) + if err != nil { + return nil, err + } + delRequest.TraceID = traceID + // header + payloadInfoStr := delRequest.Headers[constant.HeaderDataSystemPayloadInfo] + payloadInfo, err := ParsePayloadHeaderJSON(payloadInfoStr) + if err != nil { + return nil, err + } + + delRequest.DataSystemPayloadInfo = payloadInfo + return delRequest, nil +} + +// NewExecuteRequestContext - +func NewExecuteRequestContext(ctx *gin.Context, traceID string) (*models.ExecuteRequest, error) { + execRequest := &models.ExecuteRequest{} + err := execRequest.ParseRequest(ctx, int64(config.GetConfig().HTTPConfig.MaxDataSystemMultiDataBodySize*megabytes)) + if err != nil { + return nil, err + } + execRequest.TraceID = traceID + + // header + payloadInfoStr := execRequest.Headers[constant.HeaderDataSystemPayloadInfo] + payloadInfo, err := ParsePayloadHeaderJSON(payloadInfoStr) + if err != nil { + return nil, err + } + + execRequest.DataSystemPayloadInfo = payloadInfo + // body + data, err := ReadPayloadData(execRequest.RawData, execRequest.DataSystemPayloadInfo) + if err != nil { + return nil, err + } + execRequest.PayloadData = data + return execRequest, nil +} diff --git a/frontend/pkg/frontend/frontendsdkadapter/service/delete_service.go b/frontend/pkg/frontend/frontendsdkadapter/service/delete_service.go new file mode 100644 index 0000000000000000000000000000000000000000..b6f61c5586124f202d564203888d77c9473de358 --- /dev/null +++ b/frontend/pkg/frontend/frontendsdkadapter/service/delete_service.go @@ -0,0 +1,89 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package service +package service + +import ( + "strings" + + "frontend/pkg/common/faas_common/datasystemclient" + "frontend/pkg/common/faas_common/snerror" + "frontend/pkg/common/faas_common/statuscode" + "frontend/pkg/common/faas_common/urnutils" + "frontend/pkg/frontend/frontendsdkadapter/assembler" + "frontend/pkg/frontend/frontendsdkadapter/models" +) + +// DeleteFromDataSystemRequest - +type DeleteFromDataSystemRequest struct { + DataKeyList []string + TenantID string + TraceID string + NeedEncrypt bool +} + +// MultiDelService - +func MultiDelService(request *models.MultiDelRequest, + traceLogger *models.TraceLogger) (*models.MultiDelSuccessResponse, + *models.CommonErrorResponse) { + if len(request.DataSystemPayloadInfo.Data) == 0 { + return assembler.NewMultiDelSuccessResponse(request.TraceID), nil + } + + traceLogger.With("tenantId", request.TenantID) + dataKeyList := models.DataKeyPrefixJoin(request.DataSystemPayloadInfo) + traceLogger.DataKey = strings.Join(dataKeyList, "&") + req := &DeleteFromDataSystemRequest{ + DataKeyList: dataKeyList, + TenantID: request.TenantID, + TraceID: traceLogger.TraceID, + NeedEncrypt: false, + } + err := DeleteFromDataSystemWithTrace(req, traceLogger) + if err != nil { + traceLogger.Logger.Errorf("failed to delete from data system %s", err.Error()) + return nil, assembler.NewCommonErrorResponseByError(err, request.TraceID) + } + traceLogger.Logger.Info("delete success") + return assembler.NewMultiDelSuccessResponse(request.TraceID), nil +} + +// DeleteFromDataSystemWithTrace - +func DeleteFromDataSystemWithTrace(req *DeleteFromDataSystemRequest, + trace *models.TraceLogger) error { + return deleteFromDataSystem(req, trace) +} + +func deleteFromDataSystem(req *DeleteFromDataSystemRequest, trace *models.TraceLogger) error { + trace.With("deletekeys", strings.Join(req.DataKeyList, "|")) + if len(req.DataKeyList) == 0 { + return nil + } + dataKeyList := req.DataKeyList + trace.Logger.Info("start delete") + config := &datasystemclient.Config{ + TenantID: req.TenantID, + NeedEncrypt: false, + } + failedKeys, err := datasystemclient.DeleteArrayRetry(dataKeyList, config, trace.TraceID) + if err != nil { + trace.Logger.Errorf("delete data from data system failed failed keys %s ,err: %s", + urnutils.AnonymizeKeys(failedKeys), err) + return snerror.New(statuscode.DsDeleteFailed, "internal delete failed") + } + return nil +} diff --git a/frontend/pkg/frontend/frontendsdkadapter/service/download_service.go b/frontend/pkg/frontend/frontendsdkadapter/service/download_service.go new file mode 100644 index 0000000000000000000000000000000000000000..7611c49b2ef14769829b218de1c7aab3dbe2eea1 --- /dev/null +++ b/frontend/pkg/frontend/frontendsdkadapter/service/download_service.go @@ -0,0 +1,132 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package service +package service + +import ( + "fmt" + "strings" + + "frontend/pkg/common/faas_common/datasystemclient" + "frontend/pkg/common/faas_common/snerror" + "frontend/pkg/common/faas_common/statuscode" + "frontend/pkg/frontend/frontendsdkadapter/assembler" + "frontend/pkg/frontend/frontendsdkadapter/models" +) + +// DownloadFromDataSystemRequest - +type DownloadFromDataSystemRequest struct { + DataKeyList []string + TenantID string + TraceID string + NeedEncrypt bool +} + +// DownloadFromDataSystemResponse - +type DownloadFromDataSystemResponse struct { + DataSystemPayloadInfo *models.DataSystemPayloadInfo + RawData [][]byte +} + +// MultiGetService - +func MultiGetService(request *models.MultiGetRequest, + traceLogger *models.TraceLogger) (*models.MultiGetSuccessResponse, + *models.CommonErrorResponse) { + if len(request.DataSystemPayloadInfo.Data) == 0 { + return assembler.NewMultiGetSuccessResponse(nil, nil, request.TraceID), nil + } + traceLogger.With("tenantId", request.TenantID) + dataKeyList := models.DataKeyPrefixJoin(request.DataSystemPayloadInfo) + traceLogger.DataKey = strings.Join(dataKeyList, "&") + req := &DownloadFromDataSystemRequest{ + DataKeyList: dataKeyList, + TenantID: request.TenantID, + TraceID: traceLogger.TraceID, + NeedEncrypt: request.DataSystemPayloadInfo.Data[0].NeedEncrypt, + } + data, err := DownloadFromDataSystemWithTrace(req, traceLogger) + if err != nil { + traceLogger.Logger.Errorf("failed to download from data system %s", err.Error()) + return nil, assembler.NewCommonErrorResponseByError(err, request.TraceID) + } + payload, err := genGetPayloadInfo(request.DataSystemPayloadInfo, data) + if err != nil { + traceLogger.Logger.Errorf("failed to generate get payload info %s", err) + return nil, assembler.NewCommonErrorResponseByError(err, request.TraceID) + } + traceLogger.Logger.Info("download success") + return assembler.NewMultiGetSuccessResponse(payload, data, request.TraceID), nil +} + +// DownloadFromDataSystemWithTrace - key中如果有某个key不存在会整个报错 +func DownloadFromDataSystemWithTrace(req *DownloadFromDataSystemRequest, + trace *models.TraceLogger) ([][]byte, error) { + return downloadFromDataSystem(req, trace) +} + +func downloadFromDataSystem(req *DownloadFromDataSystemRequest, trace *models.TraceLogger) ([][]byte, error) { + trace.With("downloadkeys", strings.Join(req.DataKeyList, "|")) + if len(req.DataKeyList) == 0 { + return nil, nil + } + var err error + trace.Logger.Info("start download") + + dataConfig := &datasystemclient.Config{ + TenantID: req.TenantID, + NeedEncrypt: false, + DataKey: nil, + } + dataArray, err := datasystemclient.DownloadArrayRetry(req.DataKeyList, dataConfig, trace.TraceID) + if err != nil { + if err.Error() == datasystemclient.ErrKeyNotFound.Error() { + return nil, snerror.New(statuscode.DsKeyNotFound, err.Error()) + } + if err.Error() == datasystemclient.ErrValueSizeExceeded.Error() { + return nil, snerror.New(statuscode.DsDownloadFailed, "download body too large") + } + return nil, snerror.New(statuscode.DsDownloadFailed, "internal download failed") + } + return dataArray, nil +} + +func genGetPayloadInfo(payloadInfo *models.DataSystemPayloadInfo, data [][]byte) (*models.DataSystemPayloadInfo, + error) { + if len(payloadInfo.Data) != len(data) { + return nil, fmt.Errorf("keylen = %d is not equals to data len = %d", len(payloadInfo.Data), len(data)) + } + + if len(payloadInfo.Data) == 0 { + return nil, nil + } + + payloadInfos := make([]*models.PayloadInfo, len(data)) + currentOffset := 0 + for i, item := range data { + payloadInfos[i] = &models.PayloadInfo{ + DataKey: payloadInfo.Data[i].DataKey, + Offset: currentOffset, + Length: len(item), + } + + currentOffset += len(item) + } + + return &models.DataSystemPayloadInfo{ + Data: payloadInfos, + }, nil +} diff --git a/frontend/pkg/frontend/frontendsdkadapter/service/execute_service.go b/frontend/pkg/frontend/frontendsdkadapter/service/execute_service.go new file mode 100644 index 0000000000000000000000000000000000000000..c8fba63f7f91ca792d35ad5c9a06a1ae5a8aeb62 --- /dev/null +++ b/frontend/pkg/frontend/frontendsdkadapter/service/execute_service.go @@ -0,0 +1,231 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package service +package service + +import ( + "fmt" + "net/http" + "strings" + "time" + + "frontend/pkg/common/faas_common/aliasroute" + "frontend/pkg/common/faas_common/statuscode" + "frontend/pkg/common/faas_common/urnutils" + "frontend/pkg/frontend/frontendsdkadapter/assembler" + "frontend/pkg/frontend/frontendsdkadapter/models" + "frontend/pkg/frontend/invocation" + "frontend/pkg/frontend/types" +) + +const ( + acquireTimeout = 55 + caaSHeaderDataSystemKey = "X-Caas-Data-System-Key" +) + +// ExecuteService - +func ExecuteService(request *models.ExecuteRequest, + traceLogger *models.TraceLogger) (*models.ExecuteSuccessResponse, + *models.CommonErrorResponse) { + traceLogger.With("tenantId", request.TenantID) + // build invokeCtx 包括函数名解析及别名路由 + invokeCtx, err := buildProcessContext(request) + if err != nil { + return nil, assembler.NewCommonErrorResponse(statuscode.FrontendStatusBadRequest, err.Error(), + request.TraceID) + } + + traceLogger.Logger.Info("start execute") + // 上传 + dataKeyList, err := uploadData(request, traceLogger) + if err != nil { + return nil, assembler.NewCommonErrorResponseByError(err, request.TraceID) + } + + // 删除文件 + defer func() { + cleanupData(dataKeyList, request.TenantID, traceLogger) + }() + + resDataKeyList, errorResponse := executeInvoke(invokeCtx, dataKeyList, traceLogger) + if errorResponse != nil { + return nil, errorResponse + } + + // 下载 + return downloadAndGenerateResponse(resDataKeyList, request, traceLogger) +} + +// uploadData 上传数据到数据系统 +func uploadData(request *models.ExecuteRequest, traceLogger *models.TraceLogger) ([]string, error) { + uploadReq := &UploadToDataSystemRequest{ + PayloadData: request.PayloadData, + TenantID: request.TenantID, + TraceID: traceLogger.TraceID, + ExecMode: true, + } + + dataKeyList, err := UploadToDataSystemWithTrace(uploadReq, traceLogger) + if err != nil { + traceLogger.Logger.Errorf("failed to upload to data system %s", err) + return nil, err + } + + return dataKeyList, nil +} + +// cleanupData 清理数据系统中的临时文件 +func cleanupData(dataKeyList []string, tenantID string, traceLogger *models.TraceLogger) { + delReq := &DeleteFromDataSystemRequest{ + DataKeyList: dataKeyList, + TenantID: tenantID, + TraceID: traceLogger.TraceID, + NeedEncrypt: false, + } + + if err := DeleteFromDataSystemWithTrace(delReq, traceLogger); err != nil { + traceLogger.Logger.Errorf("failed to delete from data system %s", err) + } +} + +// executeInvoke 执行函数调用 +func executeInvoke(invokeCtx *types.InvokeProcessContext, dataKeyList []string, + traceLogger *models.TraceLogger) ([]string, *models.CommonErrorResponse) { + traceLogger.Logger.Info("start invoke") + + resDataKeyList, errorResponse := invoke(invokeCtx, dataKeyList) + if errorResponse != nil { + traceLogger.Logger.Errorf("failed to invoke %s", errorResponse.ErrorRsp.Message) + return nil, errorResponse + } + + return resDataKeyList, nil +} + +func invoke(ctx *types.InvokeProcessContext, dataKeyList []string) ([]string, *models.CommonErrorResponse) { + if len(dataKeyList) == 0 { + return []string{}, nil + } + // 数据系统key塞到请求头里 + ctx.ReqHeader[caaSHeaderDataSystemKey] = strings.Join(dataKeyList, "|") + // invoke + err := invocation.InvokeHandler(ctx) + if err != nil { + return nil, assembler.NewCommonErrorResponse(statuscode.UserFunctionInvokeError, err.Error(), ctx.TraceID) + } + if ctx.StatusCode != http.StatusOK || ctx.RequestTraceInfo.InnerCode != 0 { + return nil, assembler.NewCommonErrorResponseByBody(ctx.RespBody, "invoke err", ctx.TraceID) + } + keys := ctx.RespHeader[caaSHeaderDataSystemKey] + resDataKeyList := strings.FieldsFunc(keys, func(r rune) bool { + return r == '|' + }) + + return resDataKeyList, nil +} + +// downloadAndGenerateResponse 下载结果数据并生成响应 +func downloadAndGenerateResponse(resDataKeyList []string, request *models.ExecuteRequest, + traceLogger *models.TraceLogger) (*models.ExecuteSuccessResponse, *models.CommonErrorResponse) { + + // 下载结果数据 + data, err := downloadResultData(resDataKeyList, request, traceLogger) + if err != nil { + return nil, assembler.NewCommonErrorResponseByError(err, request.TraceID) + } + + // 生成响应载荷 + payload, err := genExecPayloadInfo(resDataKeyList, data) + if err != nil { + traceLogger.Logger.Errorf("failed to genExecPayloadInfo %s", err) + return nil, assembler.NewCommonErrorResponseByError(err, request.TraceID) + } + + return assembler.NewExecuteSuccessResponse(payload, data, traceLogger.TraceID), nil +} + +// downloadResultData 从数据系统下载结果数据 +func downloadResultData(resDataKeyList []string, request *models.ExecuteRequest, + traceLogger *models.TraceLogger) ([][]byte, error) { + + downloadReq := &DownloadFromDataSystemRequest{ + DataKeyList: resDataKeyList, + TenantID: request.TenantID, + TraceID: request.TraceID, + NeedEncrypt: false, + } + + data, err := DownloadFromDataSystemWithTrace(downloadReq, traceLogger) + if err != nil { + traceLogger.Logger.Errorf("failed to download from data system %s", err) + return nil, err + } + + return data, nil +} + +func buildProcessContext(request *models.ExecuteRequest) (*types.InvokeProcessContext, + error) { + processCtx := &types.InvokeProcessContext{ + ReqHeader: make(map[string]string), + RespHeader: make(map[string]string), + StartTime: time.Now(), + } + processCtx.TraceID = request.TraceID + processCtx.RequestID = request.TraceID + processCtx.ReqHeader = request.Headers + processCtx.NeedReadRespHeader = true + functionURN, err := urnutils.GetFuncInfoWithVersion(extractFunctionURN(request)) + if err != nil { + return nil, err + } + processCtx.FuncKey = urnutils.CombineFunctionKey(functionURN.TenantID, + functionURN.FuncName, functionURN.FuncVersion) + processCtx.RequestTraceInfo = &types.RequestTraceInfo{} + processCtx.AcquireTimeout = acquireTimeout + return processCtx, nil +} + +func extractFunctionURN(request *models.ExecuteRequest) string { + return aliasroute.ResolveAliasedFunctionNameToURN(request.FunctionName, request.TenantID, request.Headers) +} + +func genExecPayloadInfo(keys []string, data [][]byte) (*models.DataSystemPayloadInfo, error) { + if len(keys) != len(data) { + return nil, fmt.Errorf("keylen = %d is not equals to data len = %d", len(keys), len(data)) + } + + if len(keys) == 0 { + return nil, nil + } + + payloadInfos := make([]*models.PayloadInfo, len(data)) + currentOffset := 0 + for i, item := range data { + payloadInfos[i] = &models.PayloadInfo{ + DataKey: keys[i], + Offset: currentOffset, + Length: len(item), + } + + currentOffset += len(item) + } + + return &models.DataSystemPayloadInfo{ + Data: payloadInfos, + }, nil +} diff --git a/frontend/pkg/frontend/frontendsdkadapter/service/upload_service.go b/frontend/pkg/frontend/frontendsdkadapter/service/upload_service.go new file mode 100644 index 0000000000000000000000000000000000000000..05a32ef7c8e044bf06d5ca6877a2d4468baeda2e --- /dev/null +++ b/frontend/pkg/frontend/frontendsdkadapter/service/upload_service.go @@ -0,0 +1,108 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package service +package service + +import ( + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/datasystemclient" + "frontend/pkg/common/faas_common/snerror" + "frontend/pkg/common/faas_common/statuscode" + "frontend/pkg/common/faas_common/utils" + "frontend/pkg/frontend/frontendsdkadapter/assembler" + "frontend/pkg/frontend/frontendsdkadapter/models" +) + +const ( + executeTTLSecond = 1800 + uploadTTLSecond = 86400 +) + +// UploadToDataSystemRequest - +type UploadToDataSystemRequest struct { + PayloadData *models.PayloadData + TenantID string + TraceID string + ExecMode bool +} + +// MultiSetService - +func MultiSetService(request *models.MultiSetRequest, + traceLogger *models.TraceLogger) (*models.MultiSetSuccessResponse, + *models.CommonErrorResponse) { + traceLogger.With("tenantId", request.TenantID) + req := &UploadToDataSystemRequest{ + PayloadData: request.PayloadData, + TenantID: request.TenantID, + TraceID: traceLogger.TraceID, + ExecMode: false, + } + dataKeyList, err := UploadToDataSystemWithTrace(req, traceLogger) + if err != nil { + traceLogger.Logger.Errorf("failed to upload to data system %s", err.Error()) + return nil, assembler.NewCommonErrorResponseByError(err, request.TraceID) + } + traceLogger.Logger.Info("upload success") + return assembler.NewMultiSetSuccessResponse(dataKeyList, request.TraceID), nil +} + +// UploadToDataSystemWithTrace - 返回存入数据系统的拼接后的key +func UploadToDataSystemWithTrace(req *UploadToDataSystemRequest, + trace *models.TraceLogger) ([]string, error) { + dataKeyList, err := uploadToDataSystem(req.PayloadData, req.ExecMode, req.TenantID, trace) + return dataKeyList, err +} + +// uploadToDataSystem - +func uploadToDataSystem(payloadData *models.PayloadData, execMode bool, tenantID string, + trace *models.TraceLogger) ([]string, error) { + if payloadData.Size == 0 { + return nil, nil + } + trace.Logger.Info("start upload") + var dataKeyList []string + var err error + param := api.SetParam{WriteMode: datasystemclient.UploadWriteMode, TTLSecond: uploadTTLSecond} + + if execMode { + param.WriteMode = datasystemclient.ExecuteWriteMode + param.TTLSecond = executeTTLSecond + } + + for _, data := range payloadData.DataList { + prefix := "" + // Do not pass prefix for Execute + if !execMode { + prefix = data.DataPrefix + } + + config := &datasystemclient.Config{TenantID: tenantID, KeyPrefix: prefix, NeedEncrypt: false, DataKey: nil} + // 返回的是数据系统生成的key，不带前缀拼接 + dataKey, err := datasystemclient.UploadWithKeyRetry(data.Data, config, param, trace.TraceID) + utils.ClearByteMemory(data.Data) + if err != nil { + return nil, snerror.New(statuscode.DsUploadFailed, "internal upload failed") + } + trace.Logger.Infof("upload to data system success key %s, len %d", data.DataPrefix+dataKey, "len", + len(data.Data)) + trace.AppendDataKey(data.DataPrefix, dataKey) + dataKeyList = append(dataKeyList, dataKey) + } + trace.With("uploadkeys", trace.DataKey) + return dataKeyList, err +} diff --git a/frontend/pkg/frontend/functionmeta/metadata.go b/frontend/pkg/frontend/functionmeta/metadata.go new file mode 100644 index 0000000000000000000000000000000000000000..28cdc4d157510fc44930a44a734ffdf7a9c102f8 --- /dev/null +++ b/frontend/pkg/frontend/functionmeta/metadata.go @@ -0,0 +1,292 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package functionmeta function metadata sync with etcd +package functionmeta + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "time" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/singleflight" + "frontend/pkg/common/faas_common/trietree" + "frontend/pkg/common/faas_common/types" + "frontend/pkg/common/faas_common/urnutils" + "frontend/pkg/common/faas_common/utils" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/instanceleasemanager" + "frontend/pkg/frontend/schedulerproxy" + "frontend/pkg/frontend/subscriber" +) + +const ( + businessIndex = 4 + 2*iota + tenantIndex + funcNameIndex + versionIndex +) + +const ( + functionEtcdKeyLen = 11 + // set the key timeout interval in singleFlight after etcd access fails so that subsequent requests can be retried + singleFlightKeyTTL = 10 * time.Second +) + +var ( + funcSpecMap sync.Map + funcRouteMap sync.Map + sf = singleflight.NewSingleFlight() + errNotFound = errors.New("not found") + trie = trietree.NewTrie() + + // subject - + subject = subscriber.NewSubject() +) + +// GetFunctionMetaDataSubject - +func GetFunctionMetaDataSubject() *subscriber.Subject { + return subject +} + +type funcKeyInfo struct { + tenantID string + funcName string + version string +} + +// LoadFuncSpecWithPath - +func LoadFuncSpecWithPath(path string, traceID string) (*types.FuncSpec, bool) { + routePrefix := trie.LongestMatch(strings.Split(path, constant.URLSeparator)) + if routePrefix == "" { + log.GetLogger().Errorf("route match failed, path: %s,traceID %s", path, traceID) + return nil, false + } + value, ok := funcRouteMap.Load(routePrefix) + if !ok { + log.GetLogger().Errorf("function not found with path: %s,traceID %s", routePrefix, traceID) + return nil, false + } + funcSpec, ok := value.(*types.FuncSpec) + if !ok { + return nil, false + } + return funcSpec, true +} + +// LoadFuncSpec load funcSpec by function key +func LoadFuncSpec(funcKey string) (*types.FuncSpec, bool) { + value, ok := funcSpecMap.Load(funcKey) + if !ok { + return fetchMetaEtcdWithSingleFlight(funcKey) + } + funcSpec, ok := value.(*types.FuncSpec) + if !ok { + return nil, false + } + return funcSpec, true +} + +// ProcessUpdate process update FuncSpec +func ProcessUpdate(etcdKey string, value []byte, etcdType string) error { + functionKeyInfo, err := getFunctionKeyInfo(etcdKey) + if err != nil { + log.GetLogger().Errorf("get function key by key : %s type: %s with error: %s", etcdKey, etcdType, err.Error()) + return err + } + functionKey := urnutils.CombineFunctionKey(functionKeyInfo.tenantID, + functionKeyInfo.funcName, functionKeyInfo.version) + currFuncSpec, err := buildFuncSpec(functionKey, value, etcdType) + if err != nil { + return err + } + specValue, exist := funcSpecMap.Load(functionKey) + preFuncSpec, ok := specValue.(*types.FuncSpec) + if etcdType == etcd3.CAEMeta && exist && ok && preFuncSpec.ETCDType != etcd3.CAEMeta { + // CAE ETCD 不能覆盖 FAAS ETCD 数据 + log.GetLogger().Infof("function from meta etcd exists, skip update from cae, key %s", functionKey) + return nil + } + log.GetLogger().Infof("store new metadata: %s， etcdType %s", functionKey, etcdType) + funcSpecMap.Store(functionKey, currFuncSpec) + sf.Remove(functionKey) + if currFuncSpec.FuncMetaData.BusinessType == constant.BusinessTypeServe { + updateRoute(preFuncSpec, currFuncSpec) + } + subject.PublishEvent(subscriber.Update, currFuncSpec) + return nil +} + +func updateRoute(preFuncSpec *types.FuncSpec, currFuncSpec *types.FuncSpec) { + var preRoutePrefix string + var currRoutePrefix string + if preFuncSpec != nil && len(preFuncSpec.ExtendedMetaData.ServeDeploySchema.Applications) != 0 { + preRoutePrefix = preFuncSpec.ExtendedMetaData.ServeDeploySchema. + Applications[constant.ApplicationIndex].RoutePrefix + } + if len(currFuncSpec.ExtendedMetaData.ServeDeploySchema.Applications) != 0 { + currRoutePrefix = currFuncSpec.ExtendedMetaData.ServeDeploySchema. + Applications[constant.ApplicationIndex].RoutePrefix + } + if preRoutePrefix != currRoutePrefix { + trie.Delete(strings.Split(preRoutePrefix, constant.URLSeparator)) + funcRouteMap.Delete(preRoutePrefix) + } + funcRouteMap.Store(currRoutePrefix, currFuncSpec) + trie.Insert(strings.Split(currRoutePrefix, constant.URLSeparator)) +} + +func buildFuncSpec(functionKey string, value []byte, etcdType string) (*types.FuncSpec, error) { + funcMeta := &types.FunctionMetaInfo{} + if err := json.Unmarshal(value, funcMeta); err != nil { + log.GetLogger().Errorf("failed to unmarshal the etcd event value, etcdType: %s", etcdType) + return &types.FuncSpec{}, err + } + utils.SetFuncMetaDynamicConfEnable(funcMeta) + funcSpec := &types.FuncSpec{ + ETCDType: etcdType, + FunctionKey: functionKey, + FuncMetaSignature: utils.GetFuncMetaSignature(funcMeta, config.GetConfig().RawStsConfig.StsEnable), + FuncMetaData: funcMeta.FuncMetaData, + S3MetaData: funcMeta.S3MetaData, + EnvMetaData: funcMeta.EnvMetaData, + StsMetaData: funcMeta.StsMetaData, + ResourceMetaData: funcMeta.ResourceMetaData, + InstanceMetaData: funcMeta.InstanceMetaData, + ExtendedMetaData: funcMeta.ExtendedMetaData, + } + return funcSpec, nil +} + +// ProcessDelete process delete FuncSpec +func ProcessDelete(etcdKey string, ETCDType string) error { + functionKeyInfo, err := getFunctionKeyInfo(etcdKey) + if err != nil { + log.GetLogger().Errorf("get function key %s by %s type: with error: %s", etcdKey, ETCDType, err.Error()) + return err + } + functionKey := urnutils.CombineFunctionKey(functionKeyInfo.tenantID, + functionKeyInfo.funcName, functionKeyInfo.version) + + specValue, exist := funcSpecMap.Load(functionKey) + spec, ok := specValue.(*types.FuncSpec) + if exist && ok { + if ETCDType == etcd3.CAEMeta && spec.ETCDType != etcd3.CAEMeta { + log.GetLogger().Infof("function from meta etcd exists, skip delete from cae, key %s", functionKey) + return nil + } + funcSpecMap.Delete(functionKey) + if spec.FuncMetaData.BusinessType == constant.BusinessTypeServe && + len(spec.ExtendedMetaData.ServeDeploySchema.Applications) != 0 { + routePrefix := spec.ExtendedMetaData.ServeDeploySchema.Applications[constant.ApplicationIndex].RoutePrefix + trie.Delete(strings.Split(routePrefix, constant.URLSeparator)) + funcRouteMap.Delete(routePrefix) + } + subject.PublishEvent(subscriber.Delete, spec) + } + sf.Remove(functionKey) + schedulerproxy.Proxy.DeleteBalancer(functionKey) + log.GetLogger().Infof("delete function balancer :%s, type: %s", functionKey, ETCDType) + instanceleasemanager.GetInstanceManager().ClearFuncLeasePools(functionKey) + return nil +} + +func getFunctionKeyInfo(etcdKey string) (funcKeyInfo, error) { + keys := strings.Split(etcdKey, constant.ETCDEventKeySeparator) + if len(keys) != functionEtcdKeyLen { + return funcKeyInfo{}, errors.New("incorrect etcdKey length") + } + return funcKeyInfo{ + tenantID: keys[tenantIndex], + funcName: keys[funcNameIndex], + version: keys[versionIndex], + }, nil +} + +func getFunctionWithVersion(funcName string, version string) string { + name := getNoPrefixFuncName(funcName) + if version == constant.DefaultURNVersion { + return name + } + return fmt.Sprintf("%s:%s", name, version) +} + +func getNoPrefixFuncName(name string) string { + lastIndex := strings.LastIndex(name, "@") + if lastIndex > 0 { + return name[lastIndex+1:] + } + return name +} + +func fetchMetaEtcdWithSingleFlight(funcKey string) (*types.FuncSpec, bool) { + tenantID, funcName, funcVersion := utils.ParseFuncKey(funcKey) + silentEtcdKey := fmt.Sprintf(constant.SilentFuncKey, tenantID, funcName, funcVersion) + metaEtcdKey := fmt.Sprintf(constant.MetaFuncKey, tenantID, funcName, funcVersion) + meta, err := sf.Do(funcKey, func() (interface{}, error) { + metaClient := etcd3.GetMetaEtcdClient() + if metaClient.Client == nil { + log.GetLogger().Warnf("failed to init meta ETCD client") + return nil, errors.New("failed to init meta ETCD client") + } + getRespValue, err := fetchEtcdWithKey(metaClient, silentEtcdKey, funcKey) + if err != nil { + if err != errNotFound { + return nil, err + } + getRespValue, err = fetchEtcdWithKey(metaClient, metaEtcdKey, funcKey) + if err != nil { + return nil, err + } + } + funcSpec, err := buildFuncSpec(funcKey, getRespValue, etcd3.Meta) + if err != nil { + return nil, err + } + log.GetLogger().Infof("fetch new metadata: %s", funcKey) + funcSpecMap.Store(funcKey, funcSpec) + return funcSpec, nil + }) + if err != nil { + return nil, false + } + return meta.(*types.FuncSpec), true +} + +func fetchEtcdWithKey(metaClient *etcd3.EtcdClient, etcdKey string, funcKey string) ([]byte, error) { + defaultEtcdCtx := etcd3.CreateEtcdCtxInfoWithTimeout(context.Background(), etcd3.DurationContextTimeout) + getResp, err := metaClient.GetResponse(defaultEtcdCtx, etcdKey) + if err != nil { + log.GetLogger().Errorf("failed to get function metadata from etcd, err: %s, key: %s", + err.Error(), etcdKey) + time.AfterFunc(singleFlightKeyTTL, func() { + sf.Remove(funcKey) + }) + return nil, err + } + if len(getResp.Kvs) == 0 { + log.GetLogger().Warnf("function key is not exist,key: %s", etcdKey) + return nil, errNotFound + } + return getResp.Kvs[0].Value, nil +} diff --git a/frontend/pkg/frontend/functionmeta/metadata_test.go b/frontend/pkg/frontend/functionmeta/metadata_test.go new file mode 100644 index 0000000000000000000000000000000000000000..31066d05b44cc373a3550c9846aa91437eede77e --- /dev/null +++ b/frontend/pkg/frontend/functionmeta/metadata_test.go @@ -0,0 +1,289 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package function function metadata sync with etcd +package functionmeta + +import ( + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + "go.etcd.io/etcd/api/v3/mvccpb" + "go.etcd.io/etcd/client/v3" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/types" +) + +func TestMetaData(t *testing.T) { + metaInfo := types.FunctionMetaInfo{} + value, _ := json.Marshal(metaInfo) + err := ProcessUpdate("/tenant1//func1//version1", value, "meta") + assert.NotNil(t, err) + + err = ProcessUpdate("//////tenant1//func1//version1", nil, "meta") + assert.NotNil(t, err) + + err = ProcessUpdate("//////tenant1//func1//version1", value, "meta") + assert.Equal(t, nil, err) + + err = ProcessUpdate("//////tenant1//func1//version1", value, "CAEMeta") + assert.Equal(t, nil, err) + + loaded, ok := LoadFuncSpec("tenant1/func1/version1") + assert.Equal(t, true, ok) + assert.NotNil(t, loaded) + + err = ProcessDelete("/tenant1//func1//version1", "meta") + assert.NotNil(t, err) + + err = ProcessDelete("//////tenant1//func1//version1", "CAEMeta") + assert.Equal(t, nil, err) + + err = ProcessDelete("//////tenant1//func1//version1", "meta") + assert.Equal(t, nil, err) + + err = ProcessDelete("//////tenant1//func1//version1", "CAEMeta") + assert.Equal(t, nil, err) + + err = ProcessUpdate("//////tenant1//func2//version2", value, "CAEMeta") + assert.Equal(t, nil, err) + + loaded2, ok := LoadFuncSpec("tenant1/func2/version2") + assert.Equal(t, true, ok) + assert.NotNil(t, loaded2) + + err = ProcessDelete("//////tenant1//func2//version2", "CAEMeta") + assert.Equal(t, nil, err) + + convey.Convey("Test funcSpecMap not exists", t, func() { + convey.Convey("etcd meta not exist", func() { + etcdClient := &etcd3.EtcdClient{ + Client: &clientv3.Client{}, + } + defer gomonkey.ApplyFunc(etcd3.GetMetaEtcdClient, func() *etcd3.EtcdClient { + return etcdClient + }).Reset() + defer gomonkey.ApplyMethod(reflect.TypeOf(&etcd3.EtcdClient{}), "GetResponse", + func(_ *etcd3.EtcdClient, ctxInfo etcd3.EtcdCtxInfo, etcdKey string, + opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + return &clientv3.GetResponse{ + Kvs: []*mvccpb.KeyValue{}, + }, nil + }).Reset() + _, ok = LoadFuncSpec("tenant1/func1/version1") + assert.Equal(t, false, ok) + }) + convey.Convey("etcd meta exist", func() { + etcdClient := &etcd3.EtcdClient{ + Client: &clientv3.Client{}, + } + defer gomonkey.ApplyFunc(etcd3.GetMetaEtcdClient, func() *etcd3.EtcdClient { + return etcdClient + }).Reset() + defer gomonkey.ApplyMethod(reflect.TypeOf(&etcd3.EtcdClient{}), "GetResponse", + func(_ *etcd3.EtcdClient, ctxInfo etcd3.EtcdCtxInfo, etcdKey string, + opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + return &clientv3.GetResponse{ + Kvs: []*mvccpb.KeyValue{ + { + Key: []byte("test ok"), + Value: []byte(`{"resourceMetaData":{"cpu":100,"memory":100}}`), + }, + }, + }, nil + }).Reset() + _, ok = LoadFuncSpec("tenant1/func1/version2") + assert.Equal(t, true, ok) + }) + }) + +} + +func TestFetchMetaEtcdWithSingleFlight(t *testing.T) { + convey.Convey("Test FetchMetaEtcdWithSingleFlight", t, func() { + convey.Convey("etcd client is nil", func() { + etcdClient := &etcd3.EtcdClient{} + defer gomonkey.ApplyFunc(etcd3.GetMetaEtcdClient, func() *etcd3.EtcdClient { + return etcdClient + }).Reset() + funcKey := "123/testFunc/1" + _, ok := fetchMetaEtcdWithSingleFlight(funcKey) + assert.Equal(t, false, ok) + }) + convey.Convey("get values error", func() { + etcdClient := &etcd3.EtcdClient{ + Client: &clientv3.Client{}, + } + defer gomonkey.ApplyFunc(etcd3.GetMetaEtcdClient, func() *etcd3.EtcdClient { + return etcdClient + }).Reset() + defer gomonkey.ApplyMethod(reflect.TypeOf(&etcd3.EtcdClient{}), "GetResponse", + func(_ *etcd3.EtcdClient, ctxInfo etcd3.EtcdCtxInfo, etcdKey string, + opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + return nil, errors.New("error") + }).Reset() + funcKey := "123/testFunc/2" + _, ok := fetchMetaEtcdWithSingleFlight(funcKey) + assert.Equal(t, false, ok) + }) + convey.Convey("value got from etcd is empty", func() { + etcdClient := &etcd3.EtcdClient{ + Client: &clientv3.Client{}, + } + var fetchEtcdTime int + defer gomonkey.ApplyFunc(etcd3.GetMetaEtcdClient, func() *etcd3.EtcdClient { + return etcdClient + }).Reset() + defer gomonkey.ApplyMethod(reflect.TypeOf(&etcd3.EtcdClient{}), "GetResponse", + func(_ *etcd3.EtcdClient, ctxInfo etcd3.EtcdCtxInfo, etcdKey string, + opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + fetchEtcdTime++ + return &clientv3.GetResponse{ + Kvs: []*mvccpb.KeyValue{}, + }, nil + }).Reset() + funcKey := "123/testFunc/3" + _, ok := fetchMetaEtcdWithSingleFlight(funcKey) + assert.Equal(t, false, ok) + assert.Equal(t, 2, fetchEtcdTime) + }) + convey.Convey("fetch success", func() { + etcdClient := &etcd3.EtcdClient{ + Client: &clientv3.Client{}, + } + defer gomonkey.ApplyFunc(etcd3.GetMetaEtcdClient, func() *etcd3.EtcdClient { + return etcdClient + }).Reset() + defer gomonkey.ApplyMethod(reflect.TypeOf(&etcd3.EtcdClient{}), "GetResponse", + func(_ *etcd3.EtcdClient, ctxInfo etcd3.EtcdCtxInfo, etcdKey string, + opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + return &clientv3.GetResponse{ + Kvs: []*mvccpb.KeyValue{ + { + Key: []byte("test ok"), + Value: []byte(`{"resourceMetaData":{"cpu":100,"memory":100}}`), + }, + }, + }, nil + }).Reset() + funcKey := "123/testFunc/4" + funcSpec, ok := fetchMetaEtcdWithSingleFlight(funcKey) + assert.Equal(t, true, ok) + convey.So(funcSpec.ResourceMetaData.CPU, convey.ShouldEqual, 100) + convey.So(funcSpec.ResourceMetaData.Memory, convey.ShouldEqual, 100) + }) + convey.Convey("fetch silent function success", func() { + etcdClient := &etcd3.EtcdClient{ + Client: &clientv3.Client{}, + } + defer gomonkey.ApplyFunc(etcd3.GetMetaEtcdClient, func() *etcd3.EtcdClient { + return etcdClient + }).Reset() + defer gomonkey.ApplyMethod(reflect.TypeOf(&etcd3.EtcdClient{}), "GetResponse", + func(_ *etcd3.EtcdClient, ctxInfo etcd3.EtcdCtxInfo, etcdKey string, + opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + if strings.HasPrefix(etcdKey, "/silent") { + return &clientv3.GetResponse{ + Kvs: []*mvccpb.KeyValue{ + { + Key: []byte("test ok"), + Value: []byte(`{"resourceMetaData":{"cpu":100,"memory":100}}`), + }, + }, + }, nil + } + return &clientv3.GetResponse{Kvs: []*mvccpb.KeyValue{}}, nil + }).Reset() + funcKey := "123/testFunc/5" + funcSpec, ok := fetchMetaEtcdWithSingleFlight(funcKey) + assert.Equal(t, true, ok) + convey.So(funcSpec.ResourceMetaData.CPU, convey.ShouldEqual, 100) + convey.So(funcSpec.ResourceMetaData.Memory, convey.ShouldEqual, 100) + }) + }) +} + +func TestLoadFuncSpecWithPath(t *testing.T) { + routePrefix := "/hello" + trie.Insert(strings.Split(routePrefix, constant.URLSeparator)) + funcRouteMap.Store(routePrefix, &types.FuncSpec{}) + spec, ok := LoadFuncSpecWithPath(routePrefix, "") + assert.NotNil(t, spec) + assert.Equal(t, true, ok) + spec, ok = LoadFuncSpecWithPath("/hellos", "") + assert.Nil(t, spec) + assert.Equal(t, false, ok) +} + +func TestUpdateRoute(t *testing.T) { + currFuncSpec := &types.FuncSpec{ + ExtendedMetaData: types.ExtendedMetaData{ + ServeDeploySchema: types.ServeDeploySchema{ + Applications: []types.ServeApplicationSchema{{ + Name: "testApp", + RoutePrefix: "/hello", + }, + }, + }, + }, + } + updateRoute(nil, currFuncSpec) + _, ok := funcRouteMap.Load("/hello") + assert.Equal(t, true, ok) + ok = trie.Search(strings.Split("/hello", constant.URLSeparator)) + assert.Equal(t, true, ok) + + preFuncSpec := &types.FuncSpec{ + ExtendedMetaData: types.ExtendedMetaData{ + ServeDeploySchema: types.ServeDeploySchema{ + Applications: []types.ServeApplicationSchema{{ + Name: "testApp", + RoutePrefix: "/hello", + }, + }, + }, + }, + } + currFuncSpec = &types.FuncSpec{ + ExtendedMetaData: types.ExtendedMetaData{ + ServeDeploySchema: types.ServeDeploySchema{ + Applications: []types.ServeApplicationSchema{{ + Name: "testApp", + RoutePrefix: "/world", + }, + }, + }, + }, + } + updateRoute(preFuncSpec, currFuncSpec) + _, ok = funcRouteMap.Load("/hello") + assert.Equal(t, false, ok) + ok = trie.Search(strings.Split("/hello", constant.URLSeparator)) + assert.Equal(t, false, ok) + + _, ok = funcRouteMap.Load("/world") + assert.Equal(t, true, ok) + ok = trie.Search(strings.Split("/world", constant.URLSeparator)) + assert.Equal(t, true, ok) +} diff --git a/frontend/pkg/frontend/functiontask/functiontask.go b/frontend/pkg/frontend/functiontask/functiontask.go new file mode 100644 index 0000000000000000000000000000000000000000..6e5e6ecd34f41be9baf8649e2783ac3e8ab52a79 --- /dev/null +++ b/frontend/pkg/frontend/functiontask/functiontask.go @@ -0,0 +1,316 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package functiontask manage Function LB and deal with worker instance +package functiontask + +import ( + "encoding/json" + "fmt" + "sync" + "sync/atomic" + "time" + + "github.com/valyala/fasthttp" + "go.uber.org/zap" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/loadbalance" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/statuscode" + "frontend/pkg/common/faas_common/utils" + "frontend/pkg/frontend/common/httputil" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/types" +) + +const ( + busProxyHealthy = 0 + busProxyUnhealthy = 1 + busHeartBeatLogInternal = 20 +) + +var ( + once sync.Once + + busproxys *BusProxies +) + +// GetBusProxies - test用例使用之前最好使用clearGetProxies清理下 +func GetBusProxies() *BusProxies { + once.Do(func() { + busproxys = &BusProxies{ + logger: log.GetLogger().With(zap.Any("BusProxies", "")), + loadBalance: loadbalance.NewCHGeneric(), + list: make(map[string]*BusProxy), + nodeIPToNodeID: make(map[string]string), + } + }) + return busproxys +} + +// BusProxies - +type BusProxies struct { + list map[string]*BusProxy + nodeIPToNodeID map[string]string + sync.RWMutex + logger api.FormatLogger + loadBalance loadbalance.LoadBalance +} + +// BusProxy - +type BusProxy struct { + ch chan struct{} + NodeID string + NodeIP string + + url string + status *int32 // busProxyHealthy: 0, busProxyUnhealthy: 1 + types.HeartbeatConfig `json:"-"` + m sync.RWMutex + healthyCB func(nodeIP string) + unhealthyCB func(nodeIP string) + logger api.FormatLogger +} + +// Add - +func (fts *BusProxies) Add(nodeID, nodeIP string) { + fts.Lock() + defer fts.Unlock() + if _, ok := fts.list[nodeID]; ok { + fts.logger.Infof("no need add duplicate busproxy: %s", nodeID) + return + } + f := newBusProxy(nodeIP, fts.lbAdd, fts.lbDel) + fts.list[nodeID] = f + fts.nodeIPToNodeID[nodeIP] = nodeID +} + +// Delete - +func (fts *BusProxies) Delete(nodeID string) { + fts.Lock() + defer fts.Unlock() + f, ok := fts.list[nodeID] + if !ok { + fts.logger.Infof("no need Delete BusProxy: %s, not exist", nodeID) + return + } + f.stopMonitor() + fts.logger.Infof("Delete BusProxy: %s", nodeID) + delete(fts.list, nodeID) + delete(fts.nodeIPToNodeID, f.NodeIP) +} + +func (fts *BusProxies) lbAdd(nodeIP string) { + fts.logger.Infof("add busproxy: %s into loadbalance", nodeIP) + fts.loadBalance.Add(nodeIP, 0) +} + +func (fts *BusProxies) lbDel(nodeIP string) { + fts.logger.Infof("del busproxy: %s from loadbalance", nodeIP) + fts.loadBalance.Remove(nodeIP) +} + +// UpdateConfig - +func (fts *BusProxies) UpdateConfig() { + fts.Lock() + defer fts.Unlock() + fts.logger.Infof("BusProxies update config") + defer fts.logger.Infof("BusProxies update config over") + for _, b := range fts.list { + b.updateConfig() + } +} + +// GetNum - +func (fts *BusProxies) GetNum() int { + fts.RLock() + num := len(fts.list) + fts.RUnlock() + return num +} + +// DoRange - +func (fts *BusProxies) DoRange(fn func(nodeID string, nodeIP string) bool) { + fts.RLock() + defer fts.RUnlock() + for nodeID, ft := range fts.list { + if !fn(nodeID, ft.NodeIP) { + return + } + } +} + +// NextWithName - +func (fts *BusProxies) NextWithName(name string, move bool) string { + raw := fts.loadBalance.Next(name, move) + if raw == nil { + return "" + } + nodeIP, ok := raw.(string) + if !ok { + fts.logger.Warnf("node is not string type: %v", raw) + return "" + } + return nodeIP +} + +// IsBusProxyHealthy - +func (fts *BusProxies) IsBusProxyHealthy(nodeIP string, traceID string) bool { + fts.RLock() + defer fts.RUnlock() + nodeID, ok := fts.nodeIPToNodeID[nodeIP] + if !ok { + fts.logger.Warnf("not found the busproxy: %s, traceID: %s", nodeIP, traceID) + return false + } + b, ok := fts.list[nodeID] + if !ok { + fts.logger.Warnf("not found the busproxy: %s, traceID: %s", nodeIP, traceID) + return false + } + return b.IsHealthy() +} + +func newBusProxy(nodeIP string, healthyCB func(nodeIP string), unhealthyCB func(nodeIP string)) *BusProxy { + url := "" + if config.GetConfig().HTTPSConfig.HTTPSEnable { + url = fmt.Sprintf("https://%s:%s/heartbeat", nodeIP, constant.BusProxyHTTPPort) + } else { + url = fmt.Sprintf("http://%s:%s/heartbeat", nodeIP, constant.BusProxyHTTPPort) + } + + status := new(int32) + *status = busProxyUnhealthy + b := &BusProxy{ + ch: make(chan struct{}), + NodeIP: nodeIP, + url: url, + HeartbeatConfig: *config.GetConfig().HeartbeatConfig, + status: status, + m: sync.RWMutex{}, + healthyCB: healthyCB, + unhealthyCB: unhealthyCB, + logger: log.GetLogger().With(zap.Any("busproxy", ""), zap.Any("nodeIP", nodeIP), + zap.Any("url", url), zap.Any("heartbeatConfig", *config.GetConfig().HeartbeatConfig)), + } + go b.startMonitor(b.ch, status) + return b +} + +func (b *BusProxy) updateConfig() { + url := "" + if config.GetConfig().HTTPSConfig.HTTPSEnable { + url = fmt.Sprintf("https://%s:%s/heartbeat", b.NodeIP, constant.BusProxyHTTPPort) + } else { + url = fmt.Sprintf("http://%s:%s/heartbeat", b.NodeIP, constant.BusProxyHTTPPort) + } + heartbeatConfig := config.GetConfig().HeartbeatConfig + bytesNew, err1 := json.Marshal(heartbeatConfig) + bytesOld, err2 := json.Marshal(b.HeartbeatConfig) + if err1 != nil || err2 != nil { + b.logger.Warnf("unmarshal heartbeatConfig failed") + return + } + + if url != b.url || string(bytesOld) != string(bytesNew) { + b.logger.Infof("update heartbeatConfig, newUrl: %s, new heartbeatConfig: %s", url, string(bytesNew)) + defer b.logger.Infof("update heartbeatConfig over") + b.stopMonitor() + b.m.Lock() + b.url = url + b.HeartbeatConfig = *heartbeatConfig + status := new(int32) + *status = busProxyUnhealthy + b.status = status // 我认为心跳检测热更新不是一个太好的功能，因为可能会导致在热更新期间流量受损 + b.ch = make(chan struct{}) + b.logger = log.GetLogger().With(zap.Any("busproxy", ""), zap.Any("nodeIP", b.NodeIP), zap.Any("url", url), + zap.Any("heartbeatConfig", *config.GetConfig().HeartbeatConfig)) + go b.startMonitor(b.ch, status) + b.m.Unlock() + } else { + b.logger.Infof("config not need update") + } +} + +func (b *BusProxy) stopMonitor() { + utils.SafeCloseChannel(b.ch) + b.logger.Warnf("stop monitor") + b.unhealthyCB(b.NodeIP) +} + +func (b *BusProxy) startMonitor(ch chan struct{}, status *int32) { + b.m.RLock() + url := b.url + heartConfig := b.HeartbeatConfig + b.m.RUnlock() + ticker := time.NewTicker(time.Second * time.Duration(b.HeartbeatInterval)) + defer ticker.Stop() + count := new(int32) + *count = 0 + for { + select { + case <-ch: + b.logger.Infof("heartbeat exit") + return + case <-ticker.C: + go b.doHeartBeat(url, time.Duration(heartConfig.HeartbeatTimeout), count, status, + int32(heartConfig.HeartbeatTimeoutThreshold)) + } + } +} + +func (b *BusProxy) doHeartBeat(url string, timeout time.Duration, count *int32, status *int32, threshold int32) { + req := fasthttp.AcquireRequest() + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseRequest(req) + defer fasthttp.ReleaseResponse(resp) + req.SetRequestURI(url) + req.Header.SetMethod(fasthttp.MethodGet) + httputil.AddAuthorizationHeaderForFG(req) + err := httputil.GetHeartbeatClient().DoTimeout(req, resp, time.Second*timeout) + code := resp.StatusCode() + if err != nil || code != statuscode.FrontendStatusOk { + if atomic.LoadInt32(count)%busHeartBeatLogInternal == 0 { + errMsg := fmt.Sprintf("code: %d", code) + if err != nil { + errMsg = fmt.Sprintf("err: %v, %s", err, errMsg) + } + b.logger.Warnf("heartbeat not ok, errMsg: %s, count: %d", errMsg, atomic.LoadInt32(count)) + } + + curCount := atomic.AddInt32(count, 1) + if curCount >= threshold && atomic.LoadInt32(status) == busProxyHealthy { + atomic.StoreInt32(status, busProxyUnhealthy) + b.unhealthyCB(b.NodeIP) + b.logger.Warnf("status from healthy to unhealthy") + } + } else { + atomic.StoreInt32(count, 0) + if atomic.LoadInt32(status) == busProxyUnhealthy { + atomic.StoreInt32(status, busProxyHealthy) + b.healthyCB(b.NodeIP) + b.logger.Warnf("status from unhealthy to healthy") + } + } +} + +// IsHealthy - +func (b *BusProxy) IsHealthy() bool { + return atomic.LoadInt32(b.status) == busProxyHealthy +} diff --git a/frontend/pkg/frontend/functiontask/functiontask_test.go b/frontend/pkg/frontend/functiontask/functiontask_test.go new file mode 100644 index 0000000000000000000000000000000000000000..da50a1f37736d7a3698545c0d69000ae62e33252 --- /dev/null +++ b/frontend/pkg/frontend/functiontask/functiontask_test.go @@ -0,0 +1,480 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package functiontask + +import ( + "fmt" + "frontend/pkg/common/faas_common/statuscode" + "reflect" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + "github.com/valyala/fasthttp" + + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/tls" + "frontend/pkg/frontend/common/httputil" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/types" +) + +func shieldGetConfig() *gomonkey.Patches { + mockConfig := &types.Config{ + HTTPConfig: &types.FrontendHTTP{ + RespTimeOut: 0, + WorkerInstanceReadTimeOut: 1, + MaxRequestBodySize: 0, + }, + HTTPSConfig: &tls.InternalHTTPSConfig{ + HTTPSEnable: false, + }, + HeartbeatConfig: &types.HeartbeatConfig{ + HeartbeatTimeout: 3, + HeartbeatInterval: 1, + HeartbeatTimeoutThreshold: 2, + }, + } + return gomonkey.ApplyFunc(config.GetConfig, func() *types.Config { + return mockConfig + }) +} + +func clearGetProxies() { + clearList := []string{} + GetBusProxies().DoRange(func(nodeID, nodeIP string) bool { + clearList = append(clearList, nodeID) + return true + }) + for _, nodeID := range clearList { + GetBusProxies().Delete(nodeID) + } +} + +func TestBusProxys(t *testing.T) { + defer gomonkey.ApplyPrivateMethod(reflect.TypeOf(&BusProxy{}), "startMonitor", func(_ *BusProxy, _ chan struct{}, _ *int32) { + }).Reset() + defer shieldGetConfig().Reset() + convey.Convey("TestBusProxys_base", t, func() { + clearGetProxies() + defer clearGetProxies() + nodeID1, nodeIP1 := "1", "1.1.1.1" + nodeID2, nodeIP2 := "2", "2.2.2.2" + nodeID3, nodeIP3 := "3", "3.3.3.3" + GetBusProxies().Add(nodeID1, nodeIP1) + GetBusProxies().Add(nodeID2, nodeIP2) + GetBusProxies().Add(nodeID3, nodeIP3) + convey.So(GetBusProxies().GetNum(), convey.ShouldEqual, 3) + + GetBusProxies().Add(nodeID1, "4.4.4.4") // 这个添加失败 + convey.So(GetBusProxies().GetNum(), convey.ShouldEqual, 3) + convey.So(GetBusProxies().list[nodeID1].NodeIP, convey.ShouldEqual, "1.1.1.1") + + GetBusProxies().Delete(nodeID1) + GetBusProxies().Delete(nodeID2) + convey.So(GetBusProxies().GetNum(), convey.ShouldEqual, 1) + + count := 0 + flag := false + f := func(nodeID string, nodeIP string) bool { + if nodeID == "3" { + flag = true + } + count++ + return true + } + + for i := 0; i < 100; i++ { + count = 0 + flag = false + GetBusProxies().DoRange(f) + convey.So(flag, convey.ShouldBeTrue) + convey.So(count == 1, convey.ShouldBeTrue) + } + }) + + convey.Convey("TestBusProxys_parallel", t, func() { + clearGetProxies() + defer clearGetProxies() + nodes := make([]struct { + nodeID string + nodeIP string + }, 150, 150) + for i := 0; i < 150; i++ { + nodes[i].nodeID, nodes[i].nodeIP = strconv.Itoa(i), fmt.Sprintf("%s.%s.%s.%s", strconv.Itoa(i), strconv.Itoa(i), strconv.Itoa(i), strconv.Itoa(i)) + } + + wg := sync.WaitGroup{} + addf := func(index int) { + for i := index; i < index+10; i++ { + GetBusProxies().Add(nodes[i].nodeID, nodes[i].nodeIP) + } + go func() { + for i := index + 10; i < index+20; i++ { + GetBusProxies().Add(nodes[i].nodeID, nodes[i].nodeIP) + } + wg.Done() + }() + } + delf := func(index int) { + for i := index; i < index+12; i++ { + GetBusProxies().Delete(nodes[i].nodeID) + } + go func() { + for i := index + 12; i < index+25; i++ { + GetBusProxies().Delete(nodes[i].nodeID) + } + wg.Done() + }() + } + + for i := 0; i < 100; i += 20 { + wg.Add(1) + go addf(i) + } + wg.Wait() + + convey.So(GetBusProxies().GetNum(), convey.ShouldEqual, 100) + for i := 0; i < 100; i++ { + convey.So(GetBusProxies().list[strconv.Itoa(i)], convey.ShouldNotBeNil) + } + + for i := 100; i < 140; i += 20 { + wg.Add(1) + addf(i) + } + for i := 0; i < 100; i += 25 { + wg.Add(1) + delf(i) + } + wg.Wait() + convey.So(GetBusProxies().GetNum(), convey.ShouldEqual, 40) + for i := 100; i < 140; i++ { + convey.So(GetBusProxies().list[strconv.Itoa(i)], convey.ShouldNotBeNil) + } + }) +} + +func TestBusProxy_startMonitor(t *testing.T) { + defer shieldGetConfig().Reset() + convey.Convey("TestBusProxy_startMonitor", t, func() { + clearGetProxies() + defer clearGetProxies() + // var status *int32 + status := new(int32) + *status = busProxyUnhealthy + healthyCount := 0 + unhealthyCount := 0 + b := &BusProxy{ + ch: make(chan struct{}), + NodeIP: "", + url: "", + status: status, + HeartbeatConfig: types.HeartbeatConfig{ + HeartbeatTimeoutThreshold: 3, + HeartbeatTimeout: 3, + HeartbeatInterval: 1, + }, + m: sync.RWMutex{}, + healthyCB: func(nodeIP string) { + healthyCount++ + unhealthyCount = 0 + }, + unhealthyCB: func(nodeIP string) { + healthyCount = 0 + unhealthyCount++ + }, + logger: log.GetLogger(), + } + + count := 0 + f := func(response *fasthttp.Response) error { + count++ + if count < 4 { + if count%2 == 0 { + response.SetStatusCode(fasthttp.StatusInternalServerError) + return nil + } + return fmt.Errorf("error") + } + response.SetStatusCode(statuscode.FrontendStatusOk) + return nil + } + defer gomonkey.ApplyFunc(httputil.AddAuthorizationHeaderForFG, func(req *fasthttp.Request) { + return + }).Reset() + defer gomonkey.ApplyMethod(reflect.TypeOf(httputil.GetHeartbeatClient()), "DoTimeout", + func(_ *fasthttp.Client, req *fasthttp.Request, resp *fasthttp.Response, timeout time.Duration) error { + return f(resp) + }).Reset() + go b.startMonitor(b.ch, b.status) + time.Sleep(3 * time.Second) + convey.So(b.IsHealthy(), convey.ShouldBeFalse) + + time.Sleep(1*time.Second + 100*time.Millisecond) + convey.So(healthyCount, convey.ShouldEqual, 1) + convey.So(unhealthyCount, convey.ShouldEqual, 0) + convey.So(b.IsHealthy(), convey.ShouldBeTrue) + b.stopMonitor() + convey.So(unhealthyCount, convey.ShouldEqual, 1) + convey.So(healthyCount, convey.ShouldEqual, 0) + }) +} + +func TestBusProxies_UpdateConfig(t *testing.T) { + convey.Convey("TestBusProxies_UpdateConfig", t, func() { + + count := 0 + defer gomonkey.ApplyPrivateMethod(reflect.TypeOf(&BusProxy{}), "startMonitor", func(_ *BusProxy, _ chan struct{}, _ *int32) { + count++ + }).Reset() + + mockConfig := &types.Config{ + HTTPConfig: &types.FrontendHTTP{ + RespTimeOut: 0, + WorkerInstanceReadTimeOut: 1, + MaxRequestBodySize: 0, + }, + HTTPSConfig: &tls.InternalHTTPSConfig{ + HTTPSEnable: false, + }, + HeartbeatConfig: &types.HeartbeatConfig{ + HeartbeatTimeout: 3, + HeartbeatInterval: 1, + HeartbeatTimeoutThreshold: 2, + }, + } + defer gomonkey.ApplyFunc(config.GetConfig, func() *types.Config { + return mockConfig + }).Reset() + + mockCB := func(_ string) { + return + } + b := newBusProxy("1.1.1.1", mockCB, mockCB) + time.Sleep(100 * time.Millisecond) + convey.So(count == 1, convey.ShouldBeTrue) + + b.updateConfig() + time.Sleep(100 * time.Millisecond) + convey.So(count == 1, convey.ShouldBeTrue) + + mockConfig.HTTPSConfig.HTTPSEnable = true + b.updateConfig() + time.Sleep(100 * time.Millisecond) + convey.So(count == 2, convey.ShouldBeTrue) + b.updateConfig() + time.Sleep(100 * time.Millisecond) + convey.So(count == 2, convey.ShouldBeTrue) + + mockConfig.HeartbeatConfig.HeartbeatTimeout = 6 + b.updateConfig() + time.Sleep(100 * time.Millisecond) + convey.So(count == 3, convey.ShouldBeTrue) + b.stopMonitor() + }) +} + +func TestBusProxies_complex(t *testing.T) { + convey.Convey("TestBusProxies_complex", t, func() { + defer gomonkey.ApplyFunc(httputil.AddAuthorizationHeaderForFG, func(req *fasthttp.Request) { + return + }).Reset() + + defer shieldGetConfig().Reset() + clearGetProxies() + defer clearGetProxies() + + IS1111Ok := false + defer gomonkey.ApplyMethod(reflect.TypeOf(httputil.GetHeartbeatClient()), "DoTimeout", + func(_ *fasthttp.Client, req *fasthttp.Request, resp *fasthttp.Response, timeout time.Duration) error { + if strings.Contains(req.URI().String(), "1.1.1.1") { + if IS1111Ok { + resp.SetStatusCode(statuscode.FrontendStatusOk) + return nil + } else { + resp.SetStatusCode(fasthttp.StatusInternalServerError) + return nil + } + + } + resp.SetStatusCode(statuscode.FrontendStatusOk) + return nil + }).Reset() + + nodes := make([]struct { + nodeID string + nodeIP string + }, 11, 11) + for i := 1; i <= 10; i++ { + nodes[i].nodeID, nodes[i].nodeIP = strconv.Itoa(i), strconv.Itoa(i)+"."+strconv.Itoa(i)+"."+strconv.Itoa(i)+"."+strconv.Itoa(i) + GetBusProxies().Add(nodes[i].nodeID, nodes[i].nodeIP) + } + time.Sleep(1*time.Second + time.Millisecond*100) + + m := map[string]int{ + "2.2.2.2": 0, + "3.3.3.3": 0, + "4.4.4.4": 0, + "5.5.5.5": 0, + "6.6.6.6": 0, + "7.7.7.7": 0, + "8.8.8.8": 0, + "9.9.9.9": 0, + "10.10.10.10": 0, + } + for i := 0; i < 10; i++ { + nodeIP := GetBusProxies().NextWithName("test", true) + _, ok := m[nodeIP] + convey.So(ok, convey.ShouldBeTrue) + m[nodeIP]++ + } + count1 := 0 + count2 := 0 + for k, v := range m { + t.Logf("choose node: %s, count: %d\n", k, v) + if v == 1 { + count1++ + } + if v == 2 { + count2++ + } + convey.So(v == 1 || v == 2, convey.ShouldBeTrue) + } + convey.So(count1, convey.ShouldEqual, 8) + convey.So(count2, convey.ShouldEqual, 1) + convey.So(GetBusProxies().list["1"], convey.ShouldNotBeNil) + convey.So(GetBusProxies().IsBusProxyHealthy("1.1.1.1", ""), convey.ShouldBeFalse) + IS1111Ok = true + + time.Sleep(1*time.Second + 500*time.Millisecond) + convey.So(GetBusProxies().IsBusProxyHealthy("1.1.1.1", ""), convey.ShouldBeTrue) + m = map[string]int{ + "1.1.1.1": 0, + "2.2.2.2": 0, + "3.3.3.3": 0, + "4.4.4.4": 0, + "5.5.5.5": 0, + "6.6.6.6": 0, + "7.7.7.7": 0, + "8.8.8.8": 0, + "9.9.9.9": 0, + "10.10.10.10": 0, + } + for i := 0; i < 10; i++ { + nodeIP := GetBusProxies().NextWithName("test", true) + _, ok := m[nodeIP] + convey.So(ok, convey.ShouldBeTrue) + m[nodeIP]++ + } + for k, v := range m { + t.Logf("choose node: %s, count: %d\n", k, v) + convey.So(v == 1, convey.ShouldBeTrue) + } + + GetBusProxies().Delete(nodes[10].nodeID) + GetBusProxies().Delete(nodes[8].nodeID) + for k, v := range GetBusProxies().list { + t.Logf("print proxy in map, k: %s, v: %s", k, v.NodeIP) // TODO + } + convey.So(len(GetBusProxies().list) == 8, convey.ShouldBeTrue) + m = map[string]int{ + "1.1.1.1": 0, + "2.2.2.2": 0, + "3.3.3.3": 0, + "4.4.4.4": 0, + "5.5.5.5": 0, + "6.6.6.6": 0, + "7.7.7.7": 0, + "9.9.9.9": 0, + } + + for i := 0; i < 8; i++ { + nodeIP := GetBusProxies().NextWithName("test", true) + _, ok := m[nodeIP] + convey.So(ok, convey.ShouldBeTrue) + m[nodeIP]++ + } + for k, v := range m { + t.Logf("choose node: %s, count: %d\n", k, v) + convey.So(v == 1, convey.ShouldBeTrue) + } + + // 清理 + for i := 1; i <= 10; i++ { + GetBusProxies().Delete(nodes[i].nodeID) + } + }) +} + +func TestBusProxies_UpdateConfig2(t *testing.T) { + convey.Convey("TestBusProxies_UpdateConfig2", t, func() { + clearGetProxies() + defer clearGetProxies() + m := make(map[string]struct{}) + defer gomonkey.ApplyPrivateMethod(reflect.TypeOf(&BusProxy{}), "updateConfig", func(b *BusProxy) { + m[b.NodeIP] = struct{}{} + }).Reset() + + mockCB := func(nodeIP string) {} + GetBusProxies().list["1.1.1.1"] = &BusProxy{ + NodeIP: "1.1.1.1", + logger: log.GetLogger(), + unhealthyCB: mockCB, + } + GetBusProxies().list["2.2.2.2"] = &BusProxy{ + NodeIP: "2.2.2.2", + logger: log.GetLogger(), + unhealthyCB: mockCB, + } + GetBusProxies().list["3.3.3.3"] = &BusProxy{ + NodeIP: "3.3.3.3", + logger: log.GetLogger(), + unhealthyCB: mockCB, + } + mockConfig := &types.Config{ + HTTPConfig: &types.FrontendHTTP{ + RespTimeOut: 0, + WorkerInstanceReadTimeOut: 1, + MaxRequestBodySize: 0, + }, + HTTPSConfig: &tls.InternalHTTPSConfig{ + HTTPSEnable: false, + }, + HeartbeatConfig: &types.HeartbeatConfig{ + HeartbeatTimeout: 3, + HeartbeatInterval: 1, + HeartbeatTimeoutThreshold: 2, + }, + } + defer gomonkey.ApplyFunc(config.GetConfig, func() *types.Config { + return mockConfig + }).Reset() + GetBusProxies().UpdateConfig() + + _, ok1 := m["1.1.1.1"] + _, ok2 := m["2.2.2.2"] + _, ok3 := m["3.3.3.3"] + + convey.So(len(m) == 3, convey.ShouldBeTrue) + convey.So(ok1 && ok2 && ok3, convey.ShouldBeTrue) + }) +} diff --git a/frontend/pkg/frontend/go.mod b/frontend/pkg/frontend/go.mod new file mode 100644 index 0000000000000000000000000000000000000000..b0920365044fe8fa429803a731a7aff814c10585 --- /dev/null +++ b/frontend/pkg/frontend/go.mod @@ -0,0 +1,61 @@ +module frontend/pkg/frontend + +go 1.24.1 + +require ( + frontend/pkg/common v1.0.0 + github.com/agiledragon/gomonkey/v2 v2.11.0 + github.com/asaskevich/govalidator/v11 v11.0.1-0.20250122183457-e11347878e23 + github.com/gin-gonic/gin v1.10.0 + github.com/google/uuid v1.6.0 + github.com/magiconair/properties v1.8.7 + github.com/smartystreets/goconvey v1.8.1 + github.com/stretchr/testify v1.10.0 + github.com/valyala/fasthttp v1.58.0 + go.etcd.io/etcd/api/v3 v3.5.11 + go.etcd.io/etcd/client/v3 v3.5.11 + go.uber.org/zap v1.27.0 + golang.org/x/time v0.10.0 + google.golang.org/protobuf v1.36.6 + yuanrong.org/kernel/runtime v1.0.0 +) + +replace ( + frontend/pkg/common => ../common + github.com/agiledragon/gomonkey => github.com/agiledragon/gomonkey v2.0.1+incompatible + github.com/fsnotify/fsnotify => github.com/fsnotify/fsnotify v1.7.0 + // for test or internal use + github.com/gin-gonic/gin => github.com/gin-gonic/gin v1.10.0 + github.com/olekukonko/tablewriter => github.com/olekukonko/tablewriter v0.0.5 + github.com/operator-framework/operator-lib => github.com/operator-framework/operator-lib v0.4.0 + github.com/prashantv/gostub => github.com/prashantv/gostub v1.0.0 + github.com/robfig/cron/v3 => github.com/robfig/cron/v3 v3.0.1 + github.com/smartystreets/goconvey => github.com/smartystreets/goconvey v1.6.4 + github.com/stretchr/testify => github.com/stretchr/testify v1.5.1 + github.com/valyala/fasthttp => github.com/valyala/fasthttp v1.58.0 + go.etcd.io/etcd/api/v3 => go.etcd.io/etcd/api/v3 v3.5.11 + go.etcd.io/etcd/client/v3 => go.etcd.io/etcd/client/v3 v3.5.11 + go.opentelemetry.io/otel => go.opentelemetry.io/otel v1.24.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace => go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc => go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 + go.opentelemetry.io/otel/metric => go.opentelemetry.io/otel/metric v1.24.0 + go.opentelemetry.io/otel/sdk => go.opentelemetry.io/otel/sdk v1.24.0 + go.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace v1.24.0 + go.uber.org/zap => go.uber.org/zap v1.27.0 + golang.org/x/crypto => golang.org/x/crypto v0.24.0 + // affects VPC plugin building, will cause error if not pinned + golang.org/x/net => golang.org/x/net v0.26.0 + golang.org/x/sync => golang.org/x/sync v0.0.0-20190423024810-112230192c58 + golang.org/x/sys => golang.org/x/sys v0.21.0 + golang.org/x/text => golang.org/x/text v0.16.0 + golang.org/x/time => golang.org/x/time v0.10.0 + google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d + google.golang.org/grpc => google.golang.org/grpc v1.67.0 + google.golang.org/protobuf => google.golang.org/protobuf v1.36.6 + gopkg.in/yaml.v3 => gopkg.in/yaml.v3 v3.0.1 + yuanrong.org/kernel/runtime => ../../../runtime/api/go + k8s.io/api => k8s.io/api v0.31.2 + k8s.io/apimachinery => k8s.io/apimachinery v0.31.2 + k8s.io/client-go => k8s.io/client-go v0.31.2 + github.com/asaskevich/govalidator/v11 => github.com/asaskevich/govalidator/v11 v11.0.1-0.20250122183457-e11347878e23 +) diff --git a/frontend/pkg/frontend/instanceconfigmanager/eventhandler.go b/frontend/pkg/frontend/instanceconfigmanager/eventhandler.go new file mode 100644 index 0000000000000000000000000000000000000000..bd9882c95ef73f2c951c8f70926068bcfc21805e --- /dev/null +++ b/frontend/pkg/frontend/instanceconfigmanager/eventhandler.go @@ -0,0 +1,118 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package instanceconfigmanager - +package instanceconfigmanager + +import ( + "sync" + + "go.uber.org/zap" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/instanceconfig" + "frontend/pkg/frontend/subscriber" +) + +// manager - +var manager = &Manager{ + lock: sync.RWMutex{}, + instanceConfigMaps: make(map[string]map[string]*instanceconfig.Configuration), +} + +// subject - +var subject = subscriber.NewSubject() + +// GetInstanceConfigSubject - +func GetInstanceConfigSubject() *subscriber.Subject { + return subject +} + +// Manager - +type Manager struct { + lock sync.RWMutex + instanceConfigMaps map[string]map[string]*instanceconfig.Configuration +} + +// Load - +func Load(funcKey, invokeLabel string) (*instanceconfig.Configuration, bool) { + manager.lock.RLock() + defer manager.lock.RUnlock() + + insConfigs, ok := manager.instanceConfigMaps[funcKey] + if !ok { + return nil, false + } + + insConfig, ok := insConfigs[invokeLabel] + if !ok { + return nil, false + } + return insConfig, true +} + +// ProcessUpdate - +func ProcessUpdate(event *etcd3.Event, logger api.FormatLogger) { + instanceConfig, err := instanceconfig.ParseInstanceConfigFromEtcdEvent(event.Key, event.Value) + if err != nil { + logger.Warnf("ParseInstanceConfigFromEtcdEvent failed, err: %s", err.Error()) + return + } + logger = logger.With(zap.Any("funcKey", instanceConfig.FuncKey), zap.Any("label", instanceConfig.InstanceLabel)) + + manager.lock.Lock() + defer manager.lock.Unlock() + instanceConfigMap, ok := manager.instanceConfigMaps[instanceConfig.FuncKey] + if !ok { + instanceConfigMap = make(map[string]*instanceconfig.Configuration) + manager.instanceConfigMaps[instanceConfig.FuncKey] = instanceConfigMap + } + instanceConfigMap[instanceConfig.InstanceLabel] = instanceConfig + subject.PublishEvent(subscriber.Update, instanceConfig) + logger.Infof("add instanceConfig ok") +} + +// ProcessDelete - +func ProcessDelete(event *etcd3.Event, logger api.FormatLogger) { + instanceConfig, err := instanceconfig.ParseInstanceConfigFromEtcdEvent(event.Key, event.PrevValue) + if err != nil { + logger.Warnf("ParseInstanceConfigFromEtcdEvent failed, err: %s", err.Error()) + return + } + logger = logger.With(zap.Any("funcKey", instanceConfig.FuncKey), zap.Any("label", instanceConfig.InstanceLabel)) + + manager.lock.Lock() + defer manager.lock.Unlock() + instanceConfigMap, ok := manager.instanceConfigMaps[instanceConfig.FuncKey] + if !ok { + logger.Infof("funcKey not exist") + return + } + + insConfig, ok := instanceConfigMap[instanceConfig.InstanceLabel] + if ok { + delete(instanceConfigMap, instanceConfig.InstanceLabel) + logger.Infof("delete instanceConfig ok") + } else { + logger.Infof("delete duplicates") + } + if len(instanceConfigMap) == 0 { + delete(manager.instanceConfigMaps, instanceConfig.FuncKey) + } + subject.PublishEvent(subscriber.Delete, insConfig) +} diff --git a/frontend/pkg/frontend/instanceconfigmanager/eventhandler_test.go b/frontend/pkg/frontend/instanceconfigmanager/eventhandler_test.go new file mode 100644 index 0000000000000000000000000000000000000000..d6b95da8f5edcef89bc0ce94f568dac482a95825 --- /dev/null +++ b/frontend/pkg/frontend/instanceconfigmanager/eventhandler_test.go @@ -0,0 +1,176 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package instanceconfigmanager + +import ( + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/stretchr/testify/assert" + + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/instanceconfig" + "frontend/pkg/common/faas_common/logger/log" +) + +func TestLoad(t *testing.T) { + // Prepare test data + funcKey := "test-func" + invokeLabel := "test-label" + mockConfig := &instanceconfig.Configuration{ + FuncKey: funcKey, + InstanceLabel: invokeLabel, + } + + // Preload data + manager.instanceConfigMaps[funcKey] = map[string]*instanceconfig.Configuration{ + invokeLabel: mockConfig, + } + defer func() { + manager.instanceConfigMaps = make(map[string]map[string]*instanceconfig.Configuration) + }() + + t.Run("Successfully load existing config", func(t *testing.T) { + config, ok := Load(funcKey, invokeLabel) + assert.True(t, ok) + assert.Equal(t, mockConfig, config) + }) + + t.Run("Load non-existent funcKey", func(t *testing.T) { + config, ok := Load("nonexistent", invokeLabel) + assert.False(t, ok) + assert.Nil(t, config) + }) + + t.Run("Load non-existent label", func(t *testing.T) { + config, ok := Load(funcKey, "nonexistent") + assert.False(t, ok) + assert.Nil(t, config) + }) +} + +func TestProcessUpdate(t *testing.T) { + // Reset global state + manager.instanceConfigMaps = make(map[string]map[string]*instanceconfig.Configuration) + + t.Run("Process new config addition", func(t *testing.T) { + patches := gomonkey.NewPatches() + defer patches.Reset() + // Mock event publishing + var publishCalled int + patches.ApplyMethodFunc(subject, "PublishEvent", func(string, interface{}) { + publishCalled++ + }) + + // Execute test + ProcessUpdate(&etcd3.Event{ + Key: "/instances/business/yrk/cluster/cluster001/tenant/12345678901234561234567890123456/function/0@test111@yrfunc111/version/latest", + Value: []byte("{\"instanceMetaData\":{\"maxInstance\":100,\"minInstance\":1,\"concurrentNum\":100,\"instanceType\":\"\",\"idleMode\":false,\"poolLabel\":\"\",\"poolId\":\"\"}}"), + }, log.GetLogger()) + ProcessUpdate(&etcd3.Event{ + Key: "/instances/business/yrk/cluster/cluster001/tenant/12345678901234561234567890123456/function/0@test111@yrfunc111/version/latest/label/aaa", + Value: []byte("{\"instanceMetaData\":{\"maxInstance\":100,\"minInstance\":1,\"concurrentNum\":100,\"instanceType\":\"\",\"idleMode\":false,\"poolLabel\":\"\",\"poolId\":\"\"}}"), + }, log.GetLogger()) + + // Verify results + assert.Equal(t, 1, len(manager.instanceConfigMaps)) + assert.Equal(t, 2, len(manager.instanceConfigMaps["12345678901234561234567890123456/0@test111@yrfunc111/latest"])) + assert.Equal(t, 1, int(manager.instanceConfigMaps["12345678901234561234567890123456/0@test111@yrfunc111/latest"][""].InstanceMetaData.MinInstance)) + assert.Equal(t, 1, int(manager.instanceConfigMaps["12345678901234561234567890123456/0@test111@yrfunc111/latest"]["aaa"].InstanceMetaData.MinInstance)) + assert.Equal(t, publishCalled, 2) + }) + + t.Run("Config parse failure", func(t *testing.T) { + manager.instanceConfigMaps = make(map[string]map[string]*instanceconfig.Configuration) + ProcessUpdate(&etcd3.Event{ + Key: "/instances/business/yrk/cluster/cluster001/tenant/12345678901234561234567890123456/function/0@test111@yrfunc111/version/latest/label", + Value: []byte("{\"instanceMetaData\":{\"maxInstance\":100,\"minInstance\":1,\"concurrentNum\":100,\"instanceType\":\"\",\"idleMode\":false,\"poolLabel\":\"\",\"poolId\":\"\"}}"), + }, log.GetLogger()) + ProcessUpdate(&etcd3.Event{ + Key: "/instances/business/yrk/cluster/cluster001/tenant/12345678901234561234567890123456/function/0@test111@yrfunc111/version", + Value: []byte("{\"instanceMetaData\":{\"maxInstance\":100,\"minInstance\":1,\"concurrentNum\":100,\"instanceType\":\"\",\"idleMode\":false,\"poolLabel\":\"\",\"poolId\":\"\"}}"), + }, log.GetLogger()) + + // Verify no new config was added + assert.Equal(t, 0, len(manager.instanceConfigMaps)) + }) +} + +func TestProcessDelete(t *testing.T) { + // Prepare test data + funcKey := "12345678901234561234567890123456/0@test111@yrfunc111/latest" + invokeLabel := "aaa" + mockConfig := &instanceconfig.Configuration{ + FuncKey: funcKey, + InstanceLabel: invokeLabel, + } + mockConfigEmptyLabel := &instanceconfig.Configuration{ + FuncKey: funcKey, + InstanceLabel: "", + } + + // Preload data + manager.instanceConfigMaps[funcKey] = map[string]*instanceconfig.Configuration{ + invokeLabel: mockConfig, + "": mockConfigEmptyLabel, + } + + t.Run("Successfully delete config", func(t *testing.T) { + // Mock parse result + patches := gomonkey.NewPatches() + defer patches.Reset() + // Mock event publishing + var publishCalled bool + patches.ApplyMethodFunc(subject, "PublishEvent", func(string, interface{}) { + publishCalled = true + }) + + // Execute test + ProcessDelete(&etcd3.Event{ + Key: "/instances/business/yrk/cluster/cluster001/tenant/12345678901234561234567890123456/function/0@test111@yrfunc111/version/latest/label/aaa", + Value: []byte("{\"instanceMetaData\":{\"maxInstance\":100,\"minInstance\":1,\"concurrentNum\":100,\"instanceType\":\"\",\"idleMode\":false,\"poolLabel\":\"\",\"poolId\":\"\"}}"), + PrevValue: []byte("{\"instanceMetaData\":{\"maxInstance\":100,\"minInstance\":1,\"concurrentNum\":100,\"instanceType\":\"\",\"idleMode\":false,\"poolLabel\":\"\",\"poolId\":\"\"}}"), + }, log.GetLogger()) + + // Verify results + assert.Equal(t, 1, len(manager.instanceConfigMaps[funcKey])) + assert.True(t, publishCalled) + }) + + t.Run("Delete non-existent config", func(t *testing.T) { + manager.instanceConfigMaps[funcKey] = map[string]*instanceconfig.Configuration{ + invokeLabel: mockConfig, + "": mockConfigEmptyLabel, + } + // Mock parse returns non-existent funcKey + patches := gomonkey.NewPatches() + defer patches.Reset() + + ProcessDelete(&etcd3.Event{ + Key: "/nonexistent/path", + PrevValue: []byte("invalid-data"), + }, log.GetLogger()) + + // Verify original data remains unchanged + assert.Equal(t, 2, len(manager.instanceConfigMaps[funcKey])) + }) +} + +func TestGlobalInstanceConfigManager(t *testing.T) { + // Test global singleton + assert.NotNil(t, manager) +} diff --git a/frontend/pkg/frontend/instanceleasemanager/instance_report.go b/frontend/pkg/frontend/instanceleasemanager/instance_report.go new file mode 100644 index 0000000000000000000000000000000000000000..f9c914544b4a1ff74fd6458c8b8361bb72e7b7da --- /dev/null +++ b/frontend/pkg/frontend/instanceleasemanager/instance_report.go @@ -0,0 +1,81 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package instancleaseemanager for message process +package instanceleasemanager + +import ( + "sync" + "time" +) + +// InstanceReport contains the necessary metric info +type InstanceReport struct { + ProcReqNum int64 `json:"procReqNum"` + AvgProcTime int64 `json:"avgProcTime"` + MaxProcTime int64 `json:"maxProcTime"` + IsAbnormal bool `json:"isAbnormal"` +} + +// ReportRecord is a counter to calculate the metric +type ReportRecord struct { + // these two field will be accessed by only one go routine + // the requests completed at the current report period + requestsCount int64 + // the total time spent by the requests completed at the current report period + totalDuration int64 + // the max of the time spent by all the requests yet + maxDuration int64 + isAbnormal bool + sync.RWMutex +} + +func (mc *ReportRecord) recordAbnormal() { + mc.Lock() + mc.isAbnormal = true + mc.Unlock() +} + +func (mc *ReportRecord) recordRequest(duration time.Duration) { + mc.Lock() + mc.requestsCount++ + durationInMill := duration.Milliseconds() + mc.totalDuration += durationInMill + if durationInMill > mc.maxDuration { + mc.maxDuration = durationInMill + } + mc.Unlock() +} + +func (mc *ReportRecord) report(reset bool) InstanceReport { + mc.Lock() + report := InstanceReport{ + ProcReqNum: mc.requestsCount, + MaxProcTime: mc.maxDuration, + IsAbnormal: mc.isAbnormal, + } + if mc.requestsCount == 0 { + report.AvgProcTime = -1 + } else { + report.AvgProcTime = mc.totalDuration / mc.requestsCount + } + if reset { + mc.requestsCount = 0 + mc.totalDuration = 0 + } + mc.Unlock() + return report +} diff --git a/frontend/pkg/frontend/instanceleasemanager/lease_manager.go b/frontend/pkg/frontend/instanceleasemanager/lease_manager.go new file mode 100644 index 0000000000000000000000000000000000000000..f1cd080a5bbfa91703bae3b98fb7786847e7692f --- /dev/null +++ b/frontend/pkg/frontend/instanceleasemanager/lease_manager.go @@ -0,0 +1,552 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package instanceleasemanager for message process +package instanceleasemanager + +import ( + "container/list" + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + "sync" + "time" + + "github.com/valyala/fasthttp" + "go.uber.org/zap" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/snerror" + "frontend/pkg/common/faas_common/tls" + "frontend/pkg/common/faas_common/types" + "frontend/pkg/frontend/common/httputil" + "frontend/pkg/frontend/common/util" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/schedulerproxy" +) + +const ( + renewAction = "retain" + releaseAction = "release" + idleHoldTime = 100 // millisecond + defaultAcquireLeaseTimeout = 120 // second + beforeRetainTime = 100 // millisecond + defaultMapSize = 16 + callSchedulerPath = "/invoke" +) + +var ( + instanceManager *Manager + once sync.Once +) + +// InstanceLease holds a lease of an invokable instanceID acquired from instance scheduler +type InstanceLease struct { + types.InstanceAllocationInfo + stateID string + reportRecord *ReportRecord + acquireOption util.AcquireOption + claimTime time.Time + available bool + releaseCh chan struct{} + releaseTimer *time.Timer + sync.RWMutex +} + +func (il *InstanceLease) report(reset bool) InstanceReport { + return il.reportRecord.report(reset) +} + +func (il *InstanceLease) claim() bool { + il.Lock() + if !il.available { + il.Unlock() + return false + } + il.available = false + il.claimTime = time.Now() + if il.releaseTimer != nil { + il.releaseTimer.Stop() + il.releaseTimer = nil + } + il.Unlock() + return true +} + +func (il *InstanceLease) free(abnormal bool, record bool) { + if abnormal { + il.reportRecord.recordAbnormal() + il.releaseCh <- struct{}{} + return + } + var claimTime time.Time + il.Lock() + // if claimTime is zero then it's already freed and should not record request + if !il.claimTime.IsZero() { + claimTime = il.claimTime + il.claimTime = time.Time{} + } + il.available = true + il.Unlock() + if record && !claimTime.IsZero() { + il.reportRecord.recordRequest(time.Now().Sub(claimTime)) + } + il.Lock() + if il.releaseTimer == nil { + il.releaseTimer = time.AfterFunc(idleHoldTime*time.Millisecond, func() { + if il.claim() { + il.releaseCh <- struct{}{} + } + }) + } + il.Unlock() +} + +type instanceLeaseRecord struct { + lease *InstanceLease + element *list.Element +} + +// InstanceLeasePool stores instance leases +type InstanceLeasePool struct { + funcKey string + idleLeaseList *list.List + leaseRecord map[string]*instanceLeaseRecord + sync.RWMutex +} + +// GetInstanceManager creates Manager +func GetInstanceManager() *Manager { + once.Do(func() { + instanceManager = &Manager{ + leasePools: make(map[string]*InstanceLeasePool, defaultMapSize), + } + }) + return instanceManager +} + +func newInstanceLeasePool(funcKey string) *InstanceLeasePool { + return &InstanceLeasePool{ + funcKey: funcKey, + idleLeaseList: list.New(), + leaseRecord: make(map[string]*instanceLeaseRecord, defaultMapSize), + } +} + +func (ip *InstanceLeasePool) invokeHandler(schedulerID, traceID string, + args []*api.Arg, timeout int64) (string, error) { + scheduler, err := schedulerproxy.Proxy.GetSchedulerByInstanceName(schedulerID, traceID) + if err != nil { + return "", err + } + if len(scheduler.Address) == 0 { + return "", errors.New("scheduler address is empty") + } + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(resp) + err = prepareSchedulerRequest(req, scheduler.Address, args, traceID) + if err != nil { + return "", err + } + err = httputil.GetSchedulerClient().DoTimeout(req, resp, time.Duration(timeout)*time.Second) + if err != nil { + return "", err + } + if resp.StatusCode() != http.StatusOK { + return "", fmt.Errorf("call scheduler failed,http code %d", resp.StatusCode()) + } + return string(resp.Body()), nil +} +func prepareSchedulerRequest(schedulerReq *fasthttp.Request, dstHost string, + args []*api.Arg, traceID string) error { + schedulerReq.SetRequestURI(callSchedulerPath) + schedulerReq.Header.SetMethod(http.MethodPost) + schedulerReq.Header.ResetConnectionClose() + schedulerReq.SetHost(dstHost) + schedulerReq.URI().SetScheme(tls.GetURLScheme(config.GetConfig().HTTPSConfig.HTTPSEnable)) + schedulerReq.Header.Set(constant.HeaderTraceID, traceID) + httputil.AddAuthorizationHeaderForFG(schedulerReq) + argsData, err := json.Marshal(args) + if err != nil { + return err + } + schedulerReq.SetBody(argsData) + return nil +} + +func (ip *InstanceLeasePool) invokeScheduler(schedulerID, traceID string, args []*api.Arg, + timeout int64) (*types.InstanceResponse, error) { + if timeout <= 0 { + timeout = defaultAcquireLeaseTimeout + } + responseData, err := ip.invokeHandler(schedulerID, traceID, args, timeout) + if err != nil { + log.GetLogger().Errorf("invoke to instance scheduler %s encounters error %s "+ + "traceID %s", schedulerID, err.Error(), traceID) + return nil, err + } + instanceResponse := &types.InstanceResponse{} + err = json.Unmarshal([]byte(responseData), instanceResponse) + if err != nil { + log.GetLogger().Errorf("failed to marshal instance response from scheduler %s error %s traceID %s", + schedulerID, err.Error(), traceID) + return nil, err + } + return instanceResponse, nil +} + +func (ip *InstanceLeasePool) acquireHandler(funcKey string, stateID string, option util.AcquireOption) (*InstanceLease, + snerror.SNError) { + logger := log.GetLogger().With(zap.Any("traceID", option.TraceID), zap.Any("stateID", stateID)) + logger.Infof("acquireHandler for %s, stateID %s, faasscheduler: %s-%s", + funcKey, stateID, option.SchedulerFuncKey, option.SchedulerID) + args := createInvokeArgs(option, funcKey, stateID) + response, err := ip.invokeScheduler(option.SchedulerID, option.TraceID, args, option.Timeout) + if err != nil { + logger.Errorf("failed to acquire instance for function %s from scheduler %s error %s traceID %s", + funcKey, option.SchedulerID, err.Error(), option.TraceID) + return nil, snerror.NewWithError(constant.InsAcquireFailedErrorCode, err) + } + if response.ErrorCode != constant.InsReqSuccessCode { + logger.Errorf("failed to acquire instance for function %s from scheduler %s error %s traceID %s", + funcKey, option.SchedulerID, response.ErrorMessage, option.TraceID) + return nil, snerror.New(response.ErrorCode, response.ErrorMessage) + } + return &InstanceLease{ + InstanceAllocationInfo: response.InstanceAllocationInfo, + stateID: stateID, + reportRecord: &ReportRecord{}, + acquireOption: option, + available: true, + releaseCh: make(chan struct{}, 1), + }, nil +} + +func createInvokeArgs(option util.AcquireOption, funcKey string, stateID string) []*api.Arg { + var invokeArgs []*api.Arg + var acquireOps []byte + instanceRequirement := make(map[string][]byte, 3) + if stateID == "" { + acquireOps = []byte(fmt.Sprintf("acquire#%s", funcKey)) + } else { + acquireOps = []byte(fmt.Sprintf("acquire#%s;%s", funcKey, stateID)) + } + + if option.DesignateInstanceID == "" { + resourcesData, err := json.Marshal(option.ResourceSpecs) + instanceRequirement[constant.InstanceRequirementResourcesKey] = resourcesData + if err != nil { + log.GetLogger().Errorf("failed to marshal resource when acquire %s instance, error %s", + funcKey, err.Error()) + } + } else { + log.GetLogger().Infof("acquire specified instance[%s] lease, traceID %s", option.DesignateInstanceID, + option.TraceID) + instanceRequirement[constant.InstanceRequirementInsIDKey] = []byte(option.DesignateInstanceID) + } + + callerPodName := getPodName() + log.GetLogger().Infof("caller pod name is %s", callerPodName) + if callerPodName != "" { + instanceRequirement[constant.InstanceCallerPodName] = []byte(callerPodName) + } + + if option.TrafficLimited { + instanceRequirement[constant.InstanceTrafficLimited] = []byte("true") + } + + insRequirementBytes, err := json.Marshal(instanceRequirement) + if err != nil { + log.GetLogger().Errorf("failed to marshal resource when acquire %s instance, error %s", + funcKey, err.Error()) + } + acquireArg := &api.Arg{Type: api.Value, Data: acquireOps} + instanceArg := &api.Arg{Type: api.Value, Data: insRequirementBytes} + traceID := &api.Arg{Type: api.Value, Data: []byte(option.TraceID)} + invokeArgs = []*api.Arg{acquireArg, instanceArg, traceID} + return invokeArgs +} + +func getPodName() string { + podName := os.Getenv(constant.HostNameEnvKey) + if os.Getenv(constant.PodNameEnvKey) != "" { + podName = os.Getenv(constant.PodNameEnvKey) + } + return podName +} + +func (ip *InstanceLeasePool) processRenewAndRelease(action, leaseID string, option util.AcquireOption, + report InstanceReport) (*types.InstanceResponse, error) { + reportData, err := json.Marshal(report) + if err != nil { + return nil, err + } + actionArg := &api.Arg{ + Type: api.Value, + Data: []byte(fmt.Sprintf("%s#%s", action, leaseID)), + } + reportArg := &api.Arg{ + Type: api.Value, + Data: reportData, + } + traceID := &api.Arg{Type: api.Value, Data: []byte(option.TraceID)} + response, err := ip.invokeScheduler(option.SchedulerID, option.TraceID, + []*api.Arg{actionArg, reportArg, traceID}, option.Timeout) + if err == nil && response.ErrorCode != constant.InsReqSuccessCode { + err = fmt.Errorf("code %d, message %s", response.ErrorCode, response.ErrorMessage) + } + return response, err +} + +func (ip *InstanceLeasePool) renewHandler(leaseID string, option util.AcquireOption, + report InstanceReport) (*types.InstanceResponse, error) { + rsp, err := ip.processRenewAndRelease(renewAction, leaseID, option, report) + if err != nil { + log.GetLogger().Errorf("failed to renew instance lease %s from scheduler %s for function %s error %s "+ + "traceID %s", leaseID, option.SchedulerID, ip.funcKey, err.Error(), option.TraceID) + return nil, err + } + return rsp, nil +} + +func (ip *InstanceLeasePool) releaseHandler(leaseID string, option util.AcquireOption, + report InstanceReport) error { + _, err := ip.processRenewAndRelease(releaseAction, leaseID, option, report) + if err != nil { + log.GetLogger().Errorf("failed to release instance lease %s from scheduler %s for function %s error %s "+ + "traceID %s", leaseID, option.SchedulerID, ip.funcKey, err.Error(), option.TraceID) + return err + } + return nil +} + +func (ip *InstanceLeasePool) removeLease(leaseID string) { + ip.Lock() + record, exist := ip.leaseRecord[leaseID] + if !exist { + ip.Unlock() + return + } + delete(ip.leaseRecord, leaseID) + if ip.idleLeaseList.Len() != 0 && record.element != nil { + ip.idleLeaseList.Remove(record.element) + } + ip.Unlock() +} + +func (ip *InstanceLeasePool) handleLeaseLifeCycle(lease *InstanceLease, timer *time.Timer) { + if timer == nil { + log.GetLogger().Warnf("timer is nil,not need to start Lease life cycle for lease %s", lease.ThreadID) + return + } + defer func() { + timer.Stop() + }() + for { + release := false + select { + case _, ok := <-timer.C: + if !ok { + log.GetLogger().Warnf("release timer is closed for lease %s", lease.ThreadID) + } + timer.Reset(time.Duration(lease.LeaseInterval)*time.Millisecond - beforeRetainTime*time.Millisecond) + if !ok { + log.GetLogger().Warnf("timer is closed for lease %s", lease.ThreadID) + } + if lease.claim() { + release = true + } + case _, ok := <-lease.releaseCh: + if !ok { + log.GetLogger().Warnf("release channel is closed for lease %s", lease.ThreadID) + } + release = true + } + if release { + ip.removeLease(lease.ThreadID) + if err := ip.releaseHandler(lease.ThreadID, lease.acquireOption, lease.report(true)); err != nil { + log.GetLogger().Errorf("failed to release lease %s for function %s", lease.ThreadID, ip.funcKey) + } + return + } + rsp, err := ip.renewHandler(lease.ThreadID, lease.acquireOption, lease.report(false)) + if err != nil { + log.GetLogger().Warnf("renew failed lease %s for function %s", lease.ThreadID, ip.funcKey) + ip.removeLease(lease.ThreadID) + return + } + lease.LeaseInterval = rsp.LeaseInterval + } +} + +func (ip *InstanceLeasePool) traverseIdleLeaseList(stateID string, option util.AcquireOption) *InstanceLease { + ip.RLock() + idleLeaseList := ip.idleLeaseList + ip.RUnlock() + var lease *InstanceLease + for i := idleLeaseList.Front(); i != nil; i = i.Next() { + l, ok := i.Value.(*InstanceLease) + if !ok { + ip.Lock() + ip.idleLeaseList.Remove(i) + ip.Unlock() + continue + } + // If the function signature has changed, the lease is not reused. + if l.acquireOption.FuncSig != option.FuncSig { + log.GetLogger().Warnf("lease %s has a different signature %s which should be %s for function %s,traceID %s", + l.ThreadID, l.acquireOption.FuncSig, option.FuncSig, ip.funcKey, option.TraceID) + ip.Lock() + ip.idleLeaseList = list.New() + ip.Unlock() + break + } + if stateID != l.stateID { + continue + } + if option.DesignateInstanceID != "" && l.InstanceID != option.DesignateInstanceID { + continue + } + if l.claim() { + lease = l + ip.Lock() + ip.idleLeaseList.Remove(i) + ip.Unlock() + break + } + } + return lease +} + +func (ip *InstanceLeasePool) acquireInstanceLease(stateID string, option util.AcquireOption) (*InstanceLease, + snerror.SNError) { + lease := ip.traverseIdleLeaseList(stateID, option) + if lease != nil { + return lease, nil + } + lease, snError := ip.acquireHandler(ip.funcKey, stateID, option) + if snError != nil { + return nil, snError + } + lease.claim() + ip.RLock() + _, exist := ip.leaseRecord[lease.ThreadID] + ip.RUnlock() + if exist { + log.GetLogger().Errorf("acquired lease %s already exist for function %s traceID %s", lease.ThreadID, + ip.funcKey, option.TraceID) + // acquired a repeated lease, should acquire a new lease + return nil, snerror.New(constant.InsAcquireLeaseExistErrorCode, "lease already exist") + } + ip.Lock() + ip.leaseRecord[lease.ThreadID] = &instanceLeaseRecord{ + lease: lease, + } + ip.Unlock() + timer := time.NewTimer(time.Duration(lease.LeaseInterval)*time.Millisecond - beforeRetainTime*time.Millisecond) + go ip.handleLeaseLifeCycle(lease, timer) + log.GetLogger().Infof("succeed to acquire lease %s for function %s from scheduler %s", + lease.ThreadID, ip.funcKey, option.SchedulerID) + return lease, nil +} + +func (ip *InstanceLeasePool) releaseInstanceLease(leaseID string, abnormal bool) { + ip.Lock() + record, exist := ip.leaseRecord[leaseID] + if !exist { + ip.Unlock() + return + } + if !abnormal { + record.element = ip.idleLeaseList.PushBack(record.lease) + } + ip.Unlock() + record.lease.free(abnormal, true) +} + +// Manager manges +type Manager struct { + leasePools map[string]*InstanceLeasePool + sync.RWMutex +} + +// ClearFuncLeasePools - +func (im *Manager) ClearFuncLeasePools(funcKey string) { + log.GetLogger().Infof("function %s is delete,clean lease pools", funcKey) + im.Lock() + defer im.Unlock() + leasePool, exist := im.leasePools[funcKey] + if !exist { + log.GetLogger().Infof("function %s leasePool is not exist,no need to delete", funcKey) + return + } + leasePool.Lock() + for _, leaseRecord := range leasePool.leaseRecord { + leaseRecord.lease.releaseCh <- struct{}{} + } + leasePool.Unlock() + delete(im.leasePools, funcKey) +} + +// AcquireInstanceLease - +func (im *Manager) AcquireInstanceLease(funcKey, stateID string, + option util.AcquireOption) (*InstanceLease, snerror.SNError) { + log.GetLogger().Infof("acquire instance lease for function %s state %s from instance "+ + "scheduler %s traceID %s", funcKey, stateID, option.SchedulerID, option.TraceID) + im.Lock() + leasePool, exist := im.leasePools[funcKey] + if !exist { + leasePool = newInstanceLeasePool(funcKey) + im.leasePools[funcKey] = leasePool + } + im.Unlock() + return leasePool.acquireInstanceLease(stateID, option) +} + +// AcquireInstanceAllocation - +func (im *Manager) AcquireInstanceAllocation(funcKey string, stateID string, + option util.AcquireOption) (*types.InstanceAllocationInfo, snerror.SNError) { + lease, err := im.AcquireInstanceLease(funcKey, stateID, option) + if err != nil { + return &types.InstanceAllocationInfo{}, err + } + return &lease.InstanceAllocationInfo, nil +} + +// ReleaseInstanceAllocation - +func (im *Manager) ReleaseInstanceAllocation(allocation *types.InstanceAllocationInfo, abnormal bool, traceID string) { + if allocation == nil || allocation.ThreadID == "" { + return + } + log.GetLogger().Infof("release instance lease %s for function %s,abnormal %t,traceID %s", + allocation.ThreadID, allocation.FuncKey, abnormal, traceID) + im.RLock() + leasePool, exist := im.leasePools[allocation.FuncKey] + if !exist { + log.GetLogger().Errorf("funcKey %s is not in lease pools!", allocation.FuncKey) + im.RUnlock() + return + } + im.RUnlock() + leasePool.releaseInstanceLease(allocation.ThreadID, abnormal) +} diff --git a/frontend/pkg/frontend/instanceleasemanager/lease_manager_test.go b/frontend/pkg/frontend/instanceleasemanager/lease_manager_test.go new file mode 100644 index 0000000000000000000000000000000000000000..f4532f8e092193479988bf10fb016a7582abedda --- /dev/null +++ b/frontend/pkg/frontend/instanceleasemanager/lease_manager_test.go @@ -0,0 +1,229 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package instanceleasemanager + +import ( + "encoding/json" + "fmt" + "reflect" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + "github.com/valyala/fasthttp" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/localauth" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/tls" + commType "frontend/pkg/common/faas_common/types" + "frontend/pkg/frontend/common/util" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/schedulerproxy" + "frontend/pkg/frontend/types" +) + +func TestInstanceLeasePool_releaseInstanceLease(t *testing.T) { + convey.Convey("release instance lease test", t, func() { + convey.Convey("baseline", func() { + pool := newInstanceLeasePool("func1") + freeCalled := 0 + p := gomonkey.ApplyFunc((*InstanceLease).free, func(_ *InstanceLease, abnormal bool, record bool) { + freeCalled++ + }) + defer p.Reset() + pool.leaseRecord["aaa"] = &instanceLeaseRecord{ + lease: &InstanceLease{}, + element: nil, + } + pool.releaseInstanceLease("aaa", true) + convey.So(freeCalled, convey.ShouldEqual, 1) + }) + }) +} + +func TestInstanceManager_ReleaseInstanceAllocation(t *testing.T) { + convey.Convey("release instance allocation test", t, func() { + convey.Convey("baseline", func() { + im := GetInstanceManager() + im.leasePools["func1"] = &InstanceLeasePool{} + im.ReleaseInstanceAllocation(&commType.InstanceAllocationInfo{ + FuncKey: "func1", + FuncSig: "", + InstanceID: "aaa", + ThreadID: "func1", + LeaseInterval: 0, + }, true, "") + }) + convey.Convey("not exsit", func() { + im := GetInstanceManager() + im.ReleaseInstanceAllocation(&commType.InstanceAllocationInfo{ + FuncKey: "func1", + FuncSig: "", + InstanceID: "aaa", + ThreadID: "func1", + LeaseInterval: 0, + }, true, "") + }) + }) +} + +func Test_AcquireRepeatedLease(t *testing.T) { + convey.Convey("AcquireRepeatedLease", t, func() { + leasePool := newInstanceLeasePool("test-function") + schedulerproxy.Proxy.Add(&commType.InstanceInfo{ + FunctionName: "test-scheduler", + InstanceName: "test-schedulerID", + Address: "127.0.0.1", + }, log.GetLogger()) + resp := &commType.InstanceResponse{ + InstanceAllocationInfo: commType.InstanceAllocationInfo{ThreadID: "lease1-1", + InstanceID: "lease1", LeaseInterval: 100000}, + ErrorCode: constant.InsReqSuccessCode, + ErrorMessage: "", + SchedulerTime: 0, + } + body, _ := json.Marshal(resp) + c := &fasthttp.Client{} + defer gomonkey.ApplyMethod(reflect.TypeOf(c), + "DoTimeout", func(c *fasthttp.Client, req *fasthttp.Request, + resp *fasthttp.Response, timeout time.Duration) error { + resp.SetBody(body) + resp.SetStatusCode(200) + return nil + }).Reset() + defer gomonkey.ApplyFunc(config.GetConfig, func() *types.Config { + return &types.Config{ + HTTPSConfig: &tls.InternalHTTPSConfig{}, + LocalAuth: &localauth.AuthConfig{}, + } + }).Reset() + lease1, snError := leasePool.acquireInstanceLease("", util.AcquireOption{ + DesignateInstanceID: "", + SchedulerFuncKey: "test-scheduler", + SchedulerID: "test-schedulerID", + RequestID: "123456789", + TraceID: "123456", + ResourceSpecs: nil, + Timeout: 30, + FuncSig: "", + }) + convey.So(leasePool.idleLeaseList.Len(), convey.ShouldEqual, 0) + convey.So(lease1.ThreadID, convey.ShouldEqual, "lease1-1") + convey.So(snError, convey.ShouldBeNil) + + // releaseLease and add to idleLeaseList + leasePool.releaseInstanceLease(lease1.ThreadID, false) + convey.So(leasePool.idleLeaseList.Len(), convey.ShouldEqual, 1) + // acquire again within 100 ms,get lease from idleLeaseList + lease1, snError = leasePool.acquireInstanceLease("", util.AcquireOption{ + DesignateInstanceID: "", + SchedulerFuncKey: "test-scheduler", + SchedulerID: "test-schedulerID", + RequestID: "123456789", + TraceID: "123456", + ResourceSpecs: nil, + Timeout: 30, + FuncSig: "", + }) + convey.So(leasePool.idleLeaseList.Len(), convey.ShouldEqual, 0) + convey.So(lease1.ThreadID, convey.ShouldEqual, "lease1-1") + convey.So(snError, convey.ShouldBeNil) + + // releaseLease and add to idleLeaseList + leasePool.releaseInstanceLease(lease1.ThreadID, false) + convey.So(leasePool.idleLeaseList.Len(), convey.ShouldEqual, 1) + + // 100ms no acquire, delete + time.Sleep(120 * time.Millisecond) + convey.So(len(leasePool.leaseRecord), convey.ShouldEqual, 0) + convey.So(leasePool.idleLeaseList.Len(), convey.ShouldEqual, 0) + + }) +} + +func TestFree(t *testing.T) { + convey.Convey("Test Free", t, func() { + il := &InstanceLease{ + releaseCh: make(chan struct{}, 1), + reportRecord: &ReportRecord{}, + claimTime: time.Now(), + } + convey.Convey("Free Abnormal", func() { + + il.free(true, true) + if !il.reportRecord.isAbnormal { + t.Error("recordAbnormal was not called") + } + <-il.releaseCh + convey.So(il.reportRecord.isAbnormal, convey.ShouldBeTrue) + }) + convey.Convey("Free Normal", func() { + il.free(false, true) + <-il.releaseCh + convey.So(il.available, convey.ShouldBeFalse) + }) + }) +} + +func TestInstanceLeasePool_handleLeaseLifeCycle(t *testing.T) { + convey.Convey("test handleLeaseLifeCycle", t, func() { + convey.Convey("renew failed", func() { + pool := &InstanceLeasePool{leaseRecord: map[string]*instanceLeaseRecord{}} + il := &InstanceLease{ + available: false, + releaseCh: make(chan struct{}), + InstanceAllocationInfo: commType.InstanceAllocationInfo{ThreadID: "aaa"}, + reportRecord: &ReportRecord{}, + } + pool.handleLeaseLifeCycle(il, nil) + pool.handleLeaseLifeCycle(il, time.NewTimer(100*time.Millisecond)) + p := gomonkey.ApplyFunc((*InstanceLeasePool).renewHandler, func(_ *InstanceLeasePool, leaseID string, option util.AcquireOption, + report InstanceReport) (*commType.InstanceResponse, error) { + return nil, fmt.Errorf("error") + }) + defer p.Reset() + convey.So(len(pool.leaseRecord), convey.ShouldEqual, 0) + }) + }) +} + +func TestInstanceLeasePool_renewHandler(t *testing.T) { + convey.Convey("test renewHandler", t, func() { + convey.Convey("baseline", func() { + pool := &InstanceLeasePool{leaseRecord: map[string]*instanceLeaseRecord{}} + gomonkey.ApplyFunc((*InstanceLeasePool).processRenewAndRelease, func(_ *InstanceLeasePool, action, leaseID string, option util.AcquireOption, + report InstanceReport) (*commType.InstanceResponse, error) { + return &commType.InstanceResponse{}, nil + }) + response, err := pool.renewHandler("aaa", util.AcquireOption{}, InstanceReport{}) + convey.So(err, convey.ShouldBeNil) + convey.So(response, convey.ShouldNotBeNil) + }) + convey.Convey("renew failed", func() { + pool := &InstanceLeasePool{leaseRecord: map[string]*instanceLeaseRecord{}} + gomonkey.ApplyFunc((*InstanceLeasePool).processRenewAndRelease, func(_ *InstanceLeasePool, action, leaseID string, option util.AcquireOption, + report InstanceReport) (*commType.InstanceResponse, error) { + return nil, fmt.Errorf("error") + }) + response, err := pool.renewHandler("aaa", util.AcquireOption{}, InstanceReport{}) + convey.So(response, convey.ShouldBeNil) + convey.So(err, convey.ShouldNotBeNil) + }) + }) +} diff --git a/frontend/pkg/frontend/instancemanager/alarm.go b/frontend/pkg/frontend/instancemanager/alarm.go new file mode 100644 index 0000000000000000000000000000000000000000..05057a3d012f0eaf943d6f58a3002690f81e132d --- /dev/null +++ b/frontend/pkg/frontend/instancemanager/alarm.go @@ -0,0 +1,69 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package instancemanager - +package instancemanager + +import ( + "fmt" + "os" + "time" + + "frontend/pkg/common/faas_common/alarm" + "frontend/pkg/common/faas_common/constant" +) + +func reportNoAvailableSchedulerInstAlarm() { + alarmInfo := &alarm.LogAlarmInfo{ + AlarmID: alarm.NoAvailableSchedulerInstance00001, + AlarmName: "NoAvailableSchedulerInstance", + AlarmLevel: alarm.Level2, + } + alarmDetail := &alarm.Detail{ + OpType: alarm.GenerateAlarmLog, + SourceTag: os.Getenv(constant.PodNameEnvKey) + "|" + os.Getenv(constant.PodIPEnvKey) + + "|" + os.Getenv(constant.ClusterName) + "|NoAvailableSchedulerInstance", + Details: fmt.Sprintf("There is no available scheduler instance"), + StartTimestamp: int(time.Now().Unix()), + EndTimestamp: 0, + } + if os.Getenv(constant.CloudMapId) != "" { + alarmDetail.Details += fmt.Sprintf(", environment name: %s", os.Getenv(constant.CloudMapId)) + alarmDetail.SourceTag = os.Getenv(constant.CloudMapId) + "|" + alarmDetail.SourceTag + } + alarm.ReportOrClearAlarm(alarmInfo, alarmDetail) +} + +func clearNoAvailableSchedulerInstAlarm() { + alarmInfo := &alarm.LogAlarmInfo{ + AlarmID: alarm.NoAvailableSchedulerInstance00001, + AlarmName: "NoAvailableSchedulerInstance", + AlarmLevel: alarm.Level2, + } + alarmDetail := &alarm.Detail{ + OpType: alarm.ClearAlarmLog, + SourceTag: os.Getenv(constant.PodNameEnvKey) + "|" + os.Getenv(constant.PodIPEnvKey) + + "|" + os.Getenv(constant.ClusterName) + "|NoAvailableSchedulerInstance", + Details: fmt.Sprintf("There is available scheduler instance"), + StartTimestamp: 0, + EndTimestamp: int(time.Now().Unix()), + } + if os.Getenv(constant.CloudMapId) != "" { + alarmDetail.Details += fmt.Sprintf(", environment name: %s", os.Getenv(constant.CloudMapId)) + alarmDetail.SourceTag = os.Getenv(constant.CloudMapId) + "|" + alarmDetail.SourceTag + } + alarm.ReportOrClearAlarm(alarmInfo, alarmDetail) +} diff --git a/frontend/pkg/frontend/instancemanager/alarm_test.go b/frontend/pkg/frontend/instancemanager/alarm_test.go new file mode 100644 index 0000000000000000000000000000000000000000..2e90bb07f323d0047eadfbe3ff92fdb2f1009de0 --- /dev/null +++ b/frontend/pkg/frontend/instancemanager/alarm_test.go @@ -0,0 +1,80 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package instancemanager + +import ( + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/common/faas_common/alarm" +) + +func TestReportOrClearNoAvailableSchedulerInstAlarm(t *testing.T) { + convey.Convey("report or clear no available scheduler instance alarm", t, func() { + var alarmInfoArgs alarm.LogAlarmInfo + var detailArgs alarm.Detail + patch := gomonkey.ApplyFunc(alarm.ReportOrClearAlarm, func(alarmInfo *alarm.LogAlarmInfo, detail *alarm.Detail) { + alarmInfoArgs = *alarmInfo + detailArgs = *detail + }) + defer patch.Reset() + + convey.Convey("report no available scheduler instance alarm", func() { + reportNoAvailableSchedulerInstAlarm() + + alarmInfoExpected := alarm.LogAlarmInfo{ + AlarmID: alarm.NoAvailableSchedulerInstance00001, + AlarmName: "NoAvailableSchedulerInstance", + AlarmLevel: alarm.Level2, + } + alarmDetailExpected := alarm.Detail{ + OpType: alarm.GenerateAlarmLog, + SourceTag: "|" + "|" + "|NoAvailableSchedulerInstance", + Details: "There is no available scheduler instance", + EndTimestamp: 0, + } + convey.So(alarmInfoArgs, convey.ShouldResemble, alarmInfoExpected) + convey.So(detailArgs.OpType, convey.ShouldEqual, alarmDetailExpected.OpType) + convey.So(detailArgs.SourceTag, convey.ShouldEqual, alarmDetailExpected.SourceTag) + convey.So(detailArgs.Details, convey.ShouldEqual, alarmDetailExpected.Details) + convey.So(detailArgs.EndTimestamp, convey.ShouldEqual, alarmDetailExpected.EndTimestamp) + }) + + convey.Convey("clear no available scheduler instance alarm", func() { + clearNoAvailableSchedulerInstAlarm() + + alarmInfoExpected := alarm.LogAlarmInfo{ + AlarmID: alarm.NoAvailableSchedulerInstance00001, + AlarmName: "NoAvailableSchedulerInstance", + AlarmLevel: alarm.Level2, + } + alarmDetailExpected := alarm.Detail{ + OpType: alarm.ClearAlarmLog, + SourceTag: "|" + "|" + "|NoAvailableSchedulerInstance", + Details: "There is available scheduler instance", + StartTimestamp: 0, + } + convey.So(alarmInfoArgs, convey.ShouldResemble, alarmInfoExpected) + convey.So(detailArgs.OpType, convey.ShouldEqual, alarmDetailExpected.OpType) + convey.So(detailArgs.SourceTag, convey.ShouldEqual, alarmDetailExpected.SourceTag) + convey.So(detailArgs.Details, convey.ShouldEqual, alarmDetailExpected.Details) + convey.So(detailArgs.StartTimestamp, convey.ShouldEqual, alarmDetailExpected.StartTimestamp) + }) + }) +} diff --git a/frontend/pkg/frontend/instancemanager/appinfo.go b/frontend/pkg/frontend/instancemanager/appinfo.go new file mode 100644 index 0000000000000000000000000000000000000000..87c34defe41544dbfcdee8a4ad645411ff9ef8a6 --- /dev/null +++ b/frontend/pkg/frontend/instancemanager/appinfo.go @@ -0,0 +1,189 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package instancemanager - +package instancemanager + +import ( + "encoding/json" + "fmt" + "strings" + "sync" + + "go.uber.org/zap" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/types" +) + +// key is instance_id/app_id/submission_id, value is *AppInfo +var appsInfo sync.Map + +// GetAppDetailsByID - +func GetAppDetailsByID(submissionID string) (*constant.AppInfo, error) { + logger := log.GetLogger().With(zap.Any("SubmissionId", submissionID)) + value, ok := appsInfo.Load(submissionID) + if ok { + appInfo, _ := value.(*constant.AppInfo) + return appInfo, nil + } + err := fmt.Errorf("failed to get appjobInfo, submissionID: %s", submissionID) + logger.Errorf(err.Error()) + return nil, err +} + +// GetAppStatusByID - +func GetAppStatusByID(submissionID string) string { + logger := log.GetLogger().With(zap.Any("SubmissionId", submissionID)) + value, ok := appsInfo.Load(submissionID) + if ok { + appInfo, ok := value.(*constant.AppInfo) + if !ok { + logger.Errorf("failed to Get AppDetails") + return "" + } + return appInfo.Status + } + logger.Errorf("the submissionID not exist") + return "" +} + +// ListAppsInfo - +func ListAppsInfo() ([]*constant.AppInfo, error) { + listAppsInfo := make([]*constant.AppInfo, 0) + appsInfo.Range(func(k, v interface{}) bool { + appInfo, ok := v.(*constant.AppInfo) + if !ok { + log.GetLogger().Errorf("list appsInfo failed!") + } + listAppsInfo = append(listAppsInfo, appInfo) + return true + }) + return listAppsInfo, nil +} + +// ProcessAppInfoUpdate - +func ProcessAppInfoUpdate(event *etcd3.Event) { + logger := log.GetLogger().With(zap.Any("etcdKey", event.Key)) + appInfo := &types.InstanceSpecification{} + err := json.Unmarshal(event.Value, appInfo) + if err != nil { + logger.Errorf("failed to unmarshal app event, err: %s", err.Error()) + return + } + StoreAppInfo(event.Key, appInfo) + return +} + +// ProcessAppInfoDelete - +func ProcessAppInfoDelete(event *etcd3.Event) { + keyParts := strings.Split(event.Key, constant.ETCDEventKeySeparator) + if len(keyParts) != constant.ValidEtcdKeyLenForInstance { + log.GetLogger().Warnf("failed to delete app %s", event.Key) + return + } + appsInfo.Delete(keyParts[constant.InstanceIDIndexForInstance]) + return +} + +func switch2AppStatus(statsCode constant.InstanceStatus, statusType constant.InstanceStatusType) string { + appStatus := "" + switch statsCode { + case constant.KernelInstanceStatusScheduling, constant.KernelInstanceStatusCreating: + appStatus = constant.AppStatusPending + case constant.KernelInstanceStatusRunning, constant.KernelInstanceStatusExiting, + constant.KernelInstanceStatusSubHealth: + appStatus = constant.AppStatusRunning + case constant.KernelInstanceStatusFailed, constant.KernelInstanceStatusScheduleFailed, + constant.KernelInstanceStatusEvicting, constant.KernelInstanceStatusEvicted: + appStatus = constant.AppStatusFailed + case constant.KernelInstanceStatusFatal: + if statusType == constant.KernelInstanceStatusTypeReturn { + appStatus = constant.AppStatusSucceeded + } else if statusType == constant.KernelInstanceStatusTypeUserKillInfo { + appStatus = constant.AppStatusStopped + } else { + appStatus = constant.AppStatusFailed + } + default: + log.GetLogger().Warnf("invalid appStatusCode", statsCode) + } + return appStatus +} + +// StoreAppInfo - +func StoreAppInfo(key string, value *types.InstanceSpecification) { + if !strings.HasPrefix(value.InstanceID, constant.FunctionNameApp) { + return + } + appStatusCode := constant.InstanceStatus(value.InstanceStatus.Code) + appStatusType := constant.InstanceStatusType(value.InstanceStatus.Type) + logger := log.GetLogger().With(zap.Any("etcdKey", key)) + appStatus := switch2AppStatus(appStatusCode, appStatusType) + var delegateDownload types.LocalMetaData + var workingDir, errType, endTime string + err := json.Unmarshal([]byte(value.CreateOptions[constant.DelegateDownloadKey]), &delegateDownload) + if err != nil { + logger.Warnf("marshal CreateOptions[DELEGATE_DOWNLOAD] failed") + } else { + workingDir = delegateDownload.CodePath + } + runtimeEnv := map[string]interface{}{ + "pip": value.CreateOptions[constant.PostStartExec], + constant.WorkingDirType: workingDir, + "envVars": value.CreateOptions[constant.DelegateEnvVar], + } + driverInfo := constant.DriverInfo{ + ID: value.InstanceID, + PID: value.Extensions.PID, + NodeIPAddress: strings.Split(value.RuntimeAddress, ":")[0], + } + if appStatus == constant.AppStatusFailed || appStatus == constant.AppStatusStopped { + errType = value.InstanceStatus.Msg + } + if appStatus == constant.AppStatusFailed || appStatus == constant.AppStatusStopped || + appStatus == constant.AppStatusSucceeded && &value.Extensions != nil { + endTime = value.Extensions.UpdateTimestamp + } + appInfo := &constant.AppInfo{ // app_id, instance_id & submission_id values are the same + Key: key, + Type: constant.AppType, + Entrypoint: value.CreateOptions[constant.EntryPointKey], + SubmissionID: value.InstanceID, + Status: appStatus, + RuntimeEnv: runtimeEnv, + Metadata: str2map(value.CreateOptions[constant.UserMetadataKey]), + DriverExitCode: value.InstanceStatus.ExitCode, + DriverInfo: driverInfo, + ErrorType: errType, + StartTime: value.Extensions.CreateTimestamp, + EndTime: endTime, + DriverNodeID: value.FunctionProxyID, + } + appsInfo.Store(value.InstanceID, appInfo) +} + +func str2map(jsonStr string) map[string]string { + var result map[string]string + err := json.Unmarshal([]byte(jsonStr), &result) + if err != nil { + log.GetLogger().Warnf("failed to unmarshal %s, err: %s", jsonStr, err.Error()) + return nil + } + return result +} diff --git a/frontend/pkg/frontend/instancemanager/appinfo_test.go b/frontend/pkg/frontend/instancemanager/appinfo_test.go new file mode 100644 index 0000000000000000000000000000000000000000..f54e3070588cacfcede487f95fc4e2afeb363854 --- /dev/null +++ b/frontend/pkg/frontend/instancemanager/appinfo_test.go @@ -0,0 +1,178 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package instancemanager + +import ( + "encoding/json" + "testing" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/types" +) + +func TestGetAppStatusByID(t *testing.T) { + StoreAppInfo("app-123456", &types.InstanceSpecification{ + InstanceID: "app-123456", + StartTime: "", + InstanceStatus: types.InstanceStatus{ + Code: 3, + ExitCode: 1, + }, + CreateOptions: map[string]string{ + "POST_START_EXEC": "pip3.9 install numpy=1.24 scipy=1.25", + "DELEGATE_DOWNLOAD": "{\"code_path\":\"file:///home/ray/codeNew/god_gt_factory/file/lidar_seg_042801.zip\",\"storage_type\":\"working_dir\"}", + "ENTRYPOINT": "sleep 200", + "DELEGATE_ENV_VAR": "{\"DEPLOY_REGION\":\"suzhou_std\",\"SOURCE_REGION\":\"suzhou_std\"}", + "USER_PROVIDED_METADATA": "{\"autoscenes_ids\":\"auto_1-test\",\"task_type\":\"task_1\",\"ttl\":\"1250\"}", + }, + }) + + type args struct { + submissionID string + } + + tests := []struct { + name string + args args + want string + }{ + {"case1", args{submissionID: "app-123456"}, "RUNNING"}, + {"case2", args{submissionID: "app-654321"}, ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GetAppStatusByID(tt.args.submissionID); got != tt.want { + t.Errorf("GetNodeIPFromInstanceInfo() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGetAppDetailsByID(t *testing.T) { + StoreAppInfo("app-123456", &types.InstanceSpecification{ + InstanceID: "app-123456", + StartTime: "", + InstanceStatus: types.InstanceStatus{ + Code: 3, + ExitCode: 0, + }, + CreateOptions: map[string]string{ + "POST_START_EXEC": "pip3.9 install numpy==1.24 scipy==1.25 && pip3.9 check", + "DELEGATE_DOWNLOAD": "{\"code_path\":\"file:///home/ray/codeNew/god_gt_factory/file/lidar_seg_042801.zip\",\"storage_type\":\"working_dir\"}", + "ENTRYPOINT": "sleep 200", + "DELEGATE_ENV_VAR": "{\"DEPLOY_REGION\":\"suzhou_std\",\"SOURCE_REGION\":\"suzhou_std\"}", + "USER_PROVIDED_METADATA": "{\"autoscenes_ids\":\"auto_1-test\",\"task_type\":\"task_1\",\"ttl\":\"1250\"}", + }, + }) + + type args struct { + submissionID string + } + + tests := []struct { + name string + args args + want *constant.AppInfo + }{ + {"case1", args{submissionID: "app-123456"}, &constant.AppInfo{ + Key: "app-123456", + Type: "SUBMISSION", + Entrypoint: "sleep 200", + SubmissionID: "app-123456", + Status: "RUNNING", + StartTime: "", + EndTime: "", + Metadata: map[string]string{ + "autoscenes_ids": "suzhou_std", + "task_type": "task_1", + "ttl": "1250", + }, + RuntimeEnv: map[string]interface{}{ + "envVars": "{\"DEPLOY_REGION\":\"suzhou_std\",\"SOURCE_REGION\":\"suzhou_std\"}", + "pip": "pip3.9 install numpy==1.24 scipy==1.11.0 \u0026\u0026 pip3.9 check", + "workingDir": "file:///usr1/deploy/file.zip", + }, + DriverAgentHttpAddress: "", + DriverNodeID: "", + DriverExitCode: 0, + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, _ := GetAppDetailsByID(tt.args.submissionID) + if got.Status != tt.want.Status { + t.Errorf("GetAppDetailsByID().Status = %v, want.Status %v", got.Status, tt.want.Status) + } + if got.Type != tt.want.Type { + t.Errorf("GetAppDetailsByID().Type = %v, want.Type %v", got.Type, tt.want.Type) + } + if got.Entrypoint != tt.want.Entrypoint { + t.Errorf("GetAppDetailsByID().entrypoint = %v, want.entrypoint %v", got.Entrypoint, tt.want.Entrypoint) + } + if got.DriverExitCode != tt.want.DriverExitCode { + t.Errorf("GetAppDetailsByID().driverExitCode = %v, want.driverExitCode %v", got.DriverExitCode, tt.want.DriverExitCode) + } + }) + } + +} + +func TestProcessUpdateAndDelete(t *testing.T) { + appInfo := &types.InstanceSpecification{ + InstanceID: "app-666666", + StartTime: "", + InstanceStatus: types.InstanceStatus{ + Code: 3, + ExitCode: 1, + }, + CreateOptions: map[string]string{ + "POST_START_EXEC": "pip3.9 install numpy=1.24 scipy=1.25", + "DELEGATE_DOWNLOAD": "{\"code_path\":\"file:///home/ray/codeNew/god_gt_factory/file/lidar_seg_042801.zip\",\"storage_type\":\"working_dir\"}", + "ENTRYPOINT": "sleep 200", + "DELEGATE_ENV_VAR": "{\"DEPLOY_REGION\":\"suzhou_std\",\"SOURCE_REGION\":\"suzhou_std\"}", + "USER_PROVIDED_METADATA": "{\"autoscenes_ids\":\"auto_1-test\",\"task_type\":\"task_1\",\"ttl\":\"1250\"}", + }, + } + info, _ := json.Marshal(appInfo) + event := &etcd3.Event{ + Key: "x/x/x/x/x/x/x/x/x/x/x/x/x/app-666666", + Value: info, + } + ProcessAppInfoUpdate(event) + + type args struct { + submissionID string + event *etcd3.Event + } + + tests := []struct { + name string + args args + want string + }{ + {"case1", args{submissionID: "app-666666"}, "RUNNING"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GetAppStatusByID(tt.args.submissionID); got != tt.want { + t.Errorf("GetNodeIPFromInstanceInfo() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/frontend/pkg/frontend/instancemanager/eventhandler.go b/frontend/pkg/frontend/instancemanager/eventhandler.go new file mode 100644 index 0000000000000000000000000000000000000000..4ac6968f443cfe08ef8b516cfd02dbbbac28674e --- /dev/null +++ b/frontend/pkg/frontend/instancemanager/eventhandler.go @@ -0,0 +1,144 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package instancemanager - +package instancemanager + +import ( + "strings" + + "go.uber.org/zap" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/instance" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/utils" +) + +const ( + funcKeyDelimiter = "/" + validFuncKeyLen = 3 + innerFuncKeyDelimiter = "-" + funcNameIndexInFuncKey = 2 + faasManagerFuncName = "faasmanager" + + functionKeyNote = "FUNCTION_KEY_NOTE" +) + +// IsFaaSManager checks if a funcKey is t +func IsFaaSManager(funcKey string) bool { + items := strings.Split(funcKey, innerFuncKeyDelimiter) + if len(items) != validFuncKeyLen { + return false + } + return items[funcNameIndexInFuncKey] == faasManagerFuncName +} + +// isFaaSScheduler used to filter the etcd event which stands for a faas scheduler +func isFaaSScheduler(etcdPath string) bool { + info, err := utils.GetFunctionInstanceInfoFromEtcdKey(etcdPath) + if err != nil { + return false + } + return strings.Contains(info.FunctionName, "faasscheduler") +} + +// ProcessInstanceUpdate - +func ProcessInstanceUpdate(event *etcd3.Event) { + logger := log.GetLogger().With(zap.Any("etcdType", event.Type), + zap.Any("revisionId", event.Rev)) + instanceId := instance.GetInstanceIDFromEtcdKey(event.Key) + insSpec := instance.GetInsSpecFromEtcdValue(event.Key, event.Value) + if len(instanceId) == 0 || insSpec == nil { + logger.Warnf("ignoring invalid etcd key, key: %s", event.Key) + return + } + logger = logger.With(zap.Any("instanceId", instanceId)) + ProcessAppInfoUpdate(event) + + items := strings.Split(insSpec.Function, funcKeyDelimiter) + if len(items) != validFuncKeyLen { + return + } + insSpec.InstanceID = instanceId + + if IsFaaSManager(items[1]) { + return + } + + if isFaaSScheduler(event.Key) { + if insSpec.InstanceStatus.Code == int32(constant.KernelInstanceStatusRunning) { + GetFaaSSchedulerInstanceManager().addInstance(instanceId, insSpec, logger) + } else { + GetFaaSSchedulerInstanceManager().delInstance(instanceId, logger) + } + return + } + + functionKey := insSpec.CreateOptions[functionKeyNote] + if functionKey == "" { + logger.Warnf("ignoring invalid instance meta data, function is empty, eventKey: %s", event.Key) + return + } + + if insSpec.InstanceStatus.Code != int32(constant.KernelInstanceStatusRunning) { + GetGlobalInstanceScheduler().delInstance(functionKey, insSpec, logger) + } else { + GetGlobalInstanceScheduler().addInstance(functionKey, insSpec, logger) + } +} + +// ProcessInstanceDelete - +func ProcessInstanceDelete(event *etcd3.Event) { + logger := log.GetLogger().With(zap.Any("etcdKey", event.Key), zap.Any("etcdType", event.Type), + zap.Any("revisionId", event.Rev)) + instanceId := instance.GetInstanceIDFromEtcdKey(event.Key) + insSpec := instance.GetInsSpecFromEtcdValue(event.Key, event.PrevValue) + if len(instanceId) == 0 || insSpec == nil { + logger.Warnf("ignoring invalid etcd key") + return + } + logger = logger.With(zap.Any("instanceId", instanceId)) + ProcessAppInfoDelete(event) + items := strings.Split(insSpec.Function, funcKeyDelimiter) + if len(items) != validFuncKeyLen { + return + } + if IsFaaSManager(items[1]) { + return + } + + if isFaaSScheduler(event.Key) { + GetFaaSSchedulerInstanceManager().delInstance(instanceId, logger) + return + } + + functionKey := insSpec.CreateOptions[functionKeyNote] + if functionKey == "" { + logger.Warnf("ignoring invalid instance meta data, function is IsEmpty") + return + } + insSpec.InstanceID = instanceId + GetGlobalInstanceScheduler().delInstance(functionKey, insSpec, logger) +} + +// ProcessInstanceSync - +func ProcessInstanceSync(event *etcd3.Event) { + logger := log.GetLogger().With(zap.Any("etcdKey", event.Key), zap.Any("etcdType", event.Type), + zap.Any("revisionId", event.Rev)) + GetFaaSSchedulerInstanceManager().sync(logger) +} diff --git a/frontend/pkg/frontend/instancemanager/eventhandler_test.go b/frontend/pkg/frontend/instancemanager/eventhandler_test.go new file mode 100644 index 0000000000000000000000000000000000000000..269a661f369f4f146f4d8fd1da5181dedb9459a8 --- /dev/null +++ b/frontend/pkg/frontend/instancemanager/eventhandler_test.go @@ -0,0 +1,165 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package instancemanager + +import ( + "encoding/json" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/types" +) + +func getCommonInsSpec() *types.InstanceSpecification { + bytes := []byte("{\"instanceID\":\"5f000000-0000-4000-824c-75b4b7dae0a3\",\"requestID\":\"787b900780b2d80600\",\"runtimeID\":\"runtime-5f000000-0000-4000-824c-75b4b7dae0a3-0000000074dd\",\"runtimeAddress\":\"127.0.0.1:32568\",\"functionAgentID\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt\",\"functionProxyID\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt\",\"function\":\"12345678901234561234567890123456/0-system-faasExecutorGo1.x/$latest\",\"resources\":{\"resources\":{\"CPU\":{\"name\":\"CPU\",\"scalar\":{\"value\":500}},\"Memory\":{\"name\":\"Memory\",\"scalar\":{\"value\":500}}}},\"scheduleOption\":{\"schedPolicyName\":\"monopoly\",\"affinity\":{\"instanceAffinity\":{},\"resource\":{},\"instance\":{\"scope\":\"NODE\"}},\"initCallTimeOut\":305,\"resourceSelector\":{\"resource.owner\":\"1c50bc05-0000-4000-8000-00ed778a549c\"},\"extension\":{\"schedule_policy\":\"monopoly\"},\"range\":{},\"scheduleTimeoutMs\":\"5000\"},\"createOptions\":{\"INSTANCE_LABEL_NOTE\":\"\",\"DELEGATE_DECRYPT\":\"{\\\"accessKey\\\":\\\"\\\",\\\"authToken\\\":\\\"\\\",\\\"cryptoAlgorithm\\\":\\\"\\\",\\\"encrypted_user_data\\\":\\\"\\\",\\\"envKey\\\":\\\"\\\",\\\"environment\\\":\\\"\\\",\\\"secretKey\\\":\\\"\\\",\\\"securityAk\\\":\\\"\\\",\\\"securitySk\\\":\\\"\\\",\\\"securityToken\\\":\\\"\\\"}\",\"lifecycle\":\"detached\",\"resource.owner\":\"static_function\",\"FUNCTION_KEY_NOTE\":\"8d86c63b22e24d9ab650878b75408ea6/0@default@func6ac6741a01334320809dfb7dc1e98049/latest\",\"ConcurrentNum\":\"1000\",\"tenantId\":\"8d86c63b22e24d9ab650878b75408ea6\",\"INSTANCE_TYPE_NOTE\":\"reserved\",\"init_call_timeout\":\"305\",\"call_timeout\":\"60\",\"RESOURCE_SPEC_NOTE\":\"{\\\"cpu\\\":500,\\\"invokeLabels\\\":\\\"\\\",\\\"memory\\\":500}\",\"DELEGATE_DIRECTORY_QUOTA\":\"512\",\"GRACEFUL_SHUTDOWN_TIME\":\"900\",\"DELEGATE_DIRECTORY_INFO\":\"/tmp\"},\"instanceStatus\":{\"code\":3,\"msg\":\"running\"},\"schedulerChain\":[\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt\"],\"parentID\":\"static_function\",\"parentFunctionProxyAID\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt-LocalSchedInstanceCtrlActor@10.158.98.238:22423\",\"storageType\":\"local\",\"scheduleTimes\":1,\"deployTimes\":1,\"args\":[{\"value\":\"EkdAAVpDMTIzNDU2Nzg5MDEyMzQ1NjEyMzQ1Njc4OTAxMjM0NTYvMC1zeXN0ZW0tZmFhc0V4ZWN1dG9yR28xLngvJGxhdGVzdBplEgASBy9pbnZva2UYAiD///////////8BKGQwAUJHCAMSQzEyMzQ1Njc4OTAxMjM0NTYxMjM0NTY3ODkwMTIzNDU2LzAtc3lzdGVtLWZhYXNFeGVjdXRvckdvMS54LyRsYXRlc3Q=\"},{\"value\":\"AAAAAAAAAAAAAAAAAAAAAHsiZnVuY01ldGFEYXRhIjp7Im5hbWUiOiIwQGRlZmF1bHRAZnVuYzZhYzY3NDFhMDEzMzQzMjA4MDlkZmI3ZGMxZTk4MDQ5IiwiZnVuY3Rpb25Vcm4iOiJzbjpjbjp5cms6OGQ4NmM2M2IyMmUyNGQ5YWI2NTA4NzhiNzU0MDhlYTY6ZnVuY3Rpb246MEBkZWZhdWx0QGZ1bmM2YWM2NzQxYTAxMzM0MzIwODA5ZGZiN2RjMWU5ODA0OSIsImZ1bmN0aW9uVmVyc2lvblVybiI6InNuOmNuOnlyazo4ZDg2YzYzYjIyZTI0ZDlhYjY1MDg3OGI3NTQwOGVhNjpmdW5jdGlvbjowQGRlZmF1bHRAZnVuYzZhYzY3NDFhMDEzMzQzMjA4MDlkZmI3ZGMxZTk4MDQ5OmxhdGVzdCIsInZlcnNpb24iOiJsYXRlc3QiLCJmdW5jdGlvblVwZGF0ZVRpbWUiOiIyMDI1LTA2LTIzIDIzOjQ0OjIyLjAwMCIsInJ1bnRpbWUiOiJjdXN0b20gaW1hZ2UiLCJoYW5kbGVyIjoiL2ludm9rZSIsInRpbWVvdXQiOjYwLCJzZXJ2aWNlIjoiZGVmYXVsdCIsInRlbmFudElkIjoiOGQ4NmM2M2IyMmUyNGQ5YWI2NTA4NzhiNzU0MDhlYTYiLCJidXNpbmVzc0lkIjoieXJrIiwicmV2aXNpb25JZCI6IjIwMjUwNjIzMTU0NDIyMDEyIiwiZnVuY19uYW1lIjoiZnVuYzZhYzY3NDFhMDEzMzQzMjA4MDlkZmI3ZGMxZTk4MDQ5In0sImVudk1ldGFEYXRhIjp7ImVudmlyb25tZW50IjoiIn0sImluc3RhbmNlTWV0YURhdGEiOnsibWF4SW5zdGFuY2UiOjEwMCwibWluSW5zdGFuY2UiOjEsImNvbmN1cnJlbnROdW0iOjEwMDAsInNjYWxlUG9saWN5Ijoic3RhdGljRnVuY3Rpb24ifSwicmVzb3VyY2VNZXRhRGF0YSI6eyJjcHUiOjUwMCwibWVtb3J5Ijo1MDB9LCJjb2RlTWV0YURhdGEiOnsic3RvcmFnZV90eXBlIjoiIn0sImV4dGVuZGVkTWV0YURhdGEiOnsiaW5pdGlhbGl6ZXIiOnsiaW5pdGlhbGl6ZXJfdGltZW91dCI6MzAwLCJpbml0aWFsaXplcl9oYW5kbGVyIjoiIn0sImN1c3RvbV9jb250YWluZXJfY29uZmlnIjp7ImltYWdlIjoic3dyLmNuLXNvdXRod2VzdC0yLm15aHVhd2VpY2xvdWQuY29tL3dpc2VmdW5jdGlvbi9jdXN0b20taW1hZ2U6MS4xLjEzLjIwMjUwNTA2MTczNDEyIn19fQ==\"},{\"value\":\"AAAAAAAAAAAAAAAAAAAAAHsiY2FsbFJvdXRlIjoiaW52b2tlIiwicG9ydCI6ODAwMH0=\"},{\"value\":\"AAAAAAAAAAAAAAAAAAAAAHsic2NoZWR1bGVyRnVuY0tleSI6IiIsInNjaGVkdWxlcklETGlzdCI6W119\"},{\"value\":\"AAAAAAAAAAAAAAAAAAAAAHt9\"},{\"value\":\"AAAAAAAAAAAAAAAAAAAAAHt9\"}],\"version\":\"3\",\"dataSystemHost\":\"10.158.97.96\",\"gracefulShutdownTime\":\"600\",\"tenantID\":\"8d86c63b22e24d9ab650878b75408ea6\",\"extensions\":{\"receivedTimestamp\":\"1750782213307\",\"podDeploymentName\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz\",\"podName\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt\",\"pid\":\"71\",\"podNamespace\":\"wisefunctionservice-495f57a3-09ee-44d2-87e5-a109cda4dc40\",\"createTimestamp\":\"1750782213\",\"updateTimestamp\":\"1750782231\"},\"unitID\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt\"}") + + insSpec := &types.InstanceSpecification{} + err := json.Unmarshal(bytes, insSpec) + if err != nil { + // 不会发生 + } + return insSpec +} + +func getCommonEventKey() string { + return "/sn/instance/business/yrk/tenant/12345678901234561234567890123456/function/0-system-faasExecutorGo1.x/version/$latest/defaultaz/787b900780b2d80600/5f000000-0000-4000-824c-75b4b7dae0a3" +} + +func getInsSpecBytes(insSpec *types.InstanceSpecification) []byte { + bytes, _ := json.Marshal(insSpec) + return bytes +} + +func Test_ProcessInstanceUpdate(t *testing.T) { + convey.Convey("Test_ProcessInstanceUpdate simple", t, func() { + insSpec := getCommonInsSpec() + event := &etcd3.Event{ + Key: getCommonEventKey(), + Value: getInsSpecBytes(insSpec), + PrevValue: nil, + Rev: 0, + ETCDType: "", + } + + processAppInfoUpdateTrigger := false + defer gomonkey.ApplyFunc(ProcessAppInfoUpdate, func(event2 *etcd3.Event) { + processAppInfoUpdateTrigger = true + }).Reset() + + ProcessInstanceUpdate(event) + convey.So(processAppInfoUpdateTrigger, convey.ShouldBeTrue) + processAppInfoUpdateTrigger = false + + event.Key = "/sn/instance/business/yrk/tenant/12345678901234561234567890123456/function/0-system-faasExecutorGo1.x/version/$latest/defaultaz/787b900780b2d80600" + ProcessInstanceUpdate(event) + convey.So(processAppInfoUpdateTrigger, convey.ShouldBeFalse) + + event.Key = getCommonEventKey() + + insSpec = getCommonInsSpec() + insSpec.Function = "0-0-faasmanager" + event.Value = getInsSpecBytes(insSpec) + convey.So(IsFaaSManager(insSpec.Function), convey.ShouldBeTrue) + + key := "/sn/instance/business/yrk/tenant/0/function/0-system-faasscheduler/version/$latest/defaultaz//scheduler-efunctionschedulerservicecn-perf-gy-scheduler-green-ytvj7-svkpr" + convey.So(isFaaSScheduler(key), convey.ShouldBeTrue) + + delFuncKey := "" + delInstanceTrigger := false + delInstanceId := "" + defer gomonkey.ApplyPrivateMethod(reflect.TypeOf(GetGlobalInstanceScheduler()), "delInstance", func(_ *FunctionInstancesMap, funcKey string, insSpec *types.InstanceSpecification) { + delFuncKey = funcKey + delInstanceTrigger = true + delInstanceId = insSpec.InstanceID + }).Reset() + addFuncKey := "" + addInstanceTrigger := false + addInsntaceId := "" + defer gomonkey.ApplyPrivateMethod(reflect.TypeOf(GetGlobalInstanceScheduler()), "addInstance", func(_ *FunctionInstancesMap, funcKey string, insSpec *types.InstanceSpecification) { + addFuncKey = funcKey + addInstanceTrigger = true + addInsntaceId = insSpec.InstanceID + }).Reset() + event.Value = getInsSpecBytes(getCommonInsSpec()) + ProcessInstanceUpdate(event) + convey.So(addFuncKey, convey.ShouldEqual, "8d86c63b22e24d9ab650878b75408ea6/0@default@func6ac6741a01334320809dfb7dc1e98049/latest") + convey.So(addInstanceTrigger, convey.ShouldBeTrue) + convey.So(addInsntaceId, convey.ShouldEqual, "5f000000-0000-4000-824c-75b4b7dae0a3") + convey.So(delInstanceTrigger, convey.ShouldBeFalse) + addInstanceTrigger = false + + insSpec = getCommonInsSpec() + insSpec.InstanceStatus.Code = 2 + event.Value = getInsSpecBytes(insSpec) + ProcessInstanceUpdate(event) + convey.So(delFuncKey, convey.ShouldEqual, "8d86c63b22e24d9ab650878b75408ea6/0@default@func6ac6741a01334320809dfb7dc1e98049/latest") + convey.So(delInstanceTrigger, convey.ShouldBeTrue) + convey.So(delInstanceId, convey.ShouldEqual, "5f000000-0000-4000-824c-75b4b7dae0a3") + convey.So(addInstanceTrigger, convey.ShouldBeFalse) + }) +} + +func Test_ProcessInstanceDelete(t *testing.T) { + convey.Convey("Test_ProcessInstanceUpdate simple", t, func() { + insSpec := getCommonInsSpec() + event := &etcd3.Event{ + Key: getCommonEventKey(), + PrevValue: getInsSpecBytes(insSpec), + Rev: 0, + ETCDType: "", + } + + processAppInfoDeleteTrigger := false + defer gomonkey.ApplyFunc(ProcessAppInfoDelete, func(event2 *etcd3.Event) { + processAppInfoDeleteTrigger = true + }).Reset() + + ProcessInstanceDelete(event) + convey.So(processAppInfoDeleteTrigger, convey.ShouldBeTrue) + processAppInfoDeleteTrigger = false + + event.Key = "/sn/instance/business/yrk/tenant/12345678901234561234567890123456/function/0-system-faasExecutorGo1.x/version/$latest/defaultaz/787b900780b2d80600" + ProcessInstanceDelete(event) + convey.So(processAppInfoDeleteTrigger, convey.ShouldBeFalse) + + delFuncKey := "" + delInstanceTrigger := false + delInsntaceId := "" + defer gomonkey.ApplyPrivateMethod(reflect.TypeOf(GetGlobalInstanceScheduler()), "delInstance", func(_ *FunctionInstancesMap, funcKey string, insSpec *types.InstanceSpecification) { + delFuncKey = funcKey + delInstanceTrigger = true + delInsntaceId = insSpec.InstanceID + }).Reset() + + event.Key = getCommonEventKey() + event.PrevValue = getInsSpecBytes(getCommonInsSpec()) + ProcessInstanceDelete(event) + convey.So(delFuncKey, convey.ShouldEqual, "8d86c63b22e24d9ab650878b75408ea6/0@default@func6ac6741a01334320809dfb7dc1e98049/latest") + convey.So(delInstanceTrigger, convey.ShouldBeTrue) + convey.So(delInsntaceId, convey.ShouldEqual, "5f000000-0000-4000-824c-75b4b7dae0a3") + delInstanceTrigger = false + + insSpec.CreateOptions[functionKeyNote] = "" + event.PrevValue = getInsSpecBytes(insSpec) + ProcessInstanceDelete(event) + convey.So(delInstanceTrigger, convey.ShouldBeFalse) + }) +} diff --git a/frontend/pkg/frontend/instancemanager/faasscheduler.go b/frontend/pkg/frontend/instancemanager/faasscheduler.go new file mode 100644 index 0000000000000000000000000000000000000000..e5dbf5c1ca33667f36bbbf6be11b8d50b03c4577 --- /dev/null +++ b/frontend/pkg/frontend/instancemanager/faasscheduler.go @@ -0,0 +1,120 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package instancemanager - +package instancemanager + +import ( + "sync" + "sync/atomic" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/types" +) + +var globalFaaSSchedulerInstanceManager = &FaaSSchedulerInstanceManager{ + lock: sync.RWMutex{}, + faaSSchedulerInstanceMap: make(map[string]*types.InstanceSpecification), +} + +// GetFaaSSchedulerInstanceManager - +func GetFaaSSchedulerInstanceManager() *FaaSSchedulerInstanceManager { + return globalFaaSSchedulerInstanceManager +} + +// FaaSSchedulerInstanceManager - +type FaaSSchedulerInstanceManager struct { + lock sync.RWMutex + synced atomic.Bool + faaSSchedulerInstanceMap map[string]*types.InstanceSpecification +} + +func (f *FaaSSchedulerInstanceManager) sync(logger api.FormatLogger) { + f.synced.Store(true) + if f.IsEmpty() { + logger.Warnf("trigger no scheduler instances alarm") + reportNoAvailableSchedulerInstAlarm() + } + logger.Infof("sync scheduler instance event over") +} + +func (f *FaaSSchedulerInstanceManager) addInstance(instanceId string, instance *types.InstanceSpecification, + logger api.FormatLogger) { + f.lock.Lock() + if _, ok := f.faaSSchedulerInstanceMap[instanceId]; ok { + f.lock.Unlock() + return + } + f.faaSSchedulerInstanceMap[instanceId] = instance + f.lock.Unlock() + logger.Infof("add instance to faaSSchedulerInstanceMap") + if f.size() == 1 && f.synced.Load() { + f.lock.Lock() + if len(f.faaSSchedulerInstanceMap) == 1 { + logger.Infof("clear no scheduler instances alarm") + clearNoAvailableSchedulerInstAlarm() + } + f.lock.Unlock() + } +} + +func (f *FaaSSchedulerInstanceManager) delInstance(instanceId string, logger api.FormatLogger) { + f.lock.Lock() + if _, ok := f.faaSSchedulerInstanceMap[instanceId]; !ok { + f.lock.Unlock() + logger.Infof("no need delete, %s not in faaSSchedulerInstanceManager", instanceId) + return + } + delete(f.faaSSchedulerInstanceMap, instanceId) + f.lock.Unlock() + logger.Infof("delete instance from faaSSchedulerInstanceMap") + if f.IsEmpty() { + f.lock.Lock() + if len(f.faaSSchedulerInstanceMap) == 0 { + logger.Warnf("trigger no scheduler instances alarm") + reportNoAvailableSchedulerInstAlarm() + } + f.lock.Unlock() + } +} + +func (f *FaaSSchedulerInstanceManager) size() int { + f.lock.RLock() + defer f.lock.RUnlock() + return len(f.faaSSchedulerInstanceMap) +} + +// IsEmpty - +func (f *FaaSSchedulerInstanceManager) IsEmpty() bool { + return f.size() == 0 +} + +// Reset - just for test +func (f *FaaSSchedulerInstanceManager) Reset() { + f.lock.Lock() + defer f.lock.Unlock() + f.faaSSchedulerInstanceMap = make(map[string]*types.InstanceSpecification) + f.synced.Store(false) +} + +// IsExist - +func (f *FaaSSchedulerInstanceManager) IsExist(instanceId string) bool { + f.lock.RLock() + defer f.lock.RUnlock() + _, ok := f.faaSSchedulerInstanceMap[instanceId] + return ok +} diff --git a/frontend/pkg/frontend/instancemanager/faasscheduler_test.go b/frontend/pkg/frontend/instancemanager/faasscheduler_test.go new file mode 100644 index 0000000000000000000000000000000000000000..21032c6e7ea6ab2d0b57b2197c5b29baf2468d57 --- /dev/null +++ b/frontend/pkg/frontend/instancemanager/faasscheduler_test.go @@ -0,0 +1,100 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package instancemanager + +import ( + "fmt" + "sync" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/common/faas_common/logger/log" + commontype "frontend/pkg/common/faas_common/types" +) + +func clearTest() { + GetFaaSSchedulerInstanceManager().faaSSchedulerInstanceMap = make(map[string]*commontype.InstanceSpecification) + GetFaaSSchedulerInstanceManager().synced.Store(false) +} + +func Test_faaSSchedulerInstanceManager_complex(t *testing.T) { + convey.Convey("faaSSchedulerInstanceManager_complex", t, func() { + clearTest() + GetFaaSSchedulerInstanceManager().addInstance("0", nil, log.GetLogger()) + convey.So(GetFaaSSchedulerInstanceManager().size(), convey.ShouldEqual, 1) + + wg := sync.WaitGroup{} + + for i := 0; i < 10; i++ { + wg.Add(1) + go func(i int) { + for j := 0; j < 10; j++ { + GetFaaSSchedulerInstanceManager().addInstance(fmt.Sprintf("%d%d", i, j), nil, log.GetLogger()) + GetFaaSSchedulerInstanceManager().addInstance(fmt.Sprintf("%d%d", i, j), nil, log.GetLogger()) + } + wg.Done() + }(i) + } + wg.Wait() + convey.So(GetFaaSSchedulerInstanceManager().size(), convey.ShouldEqual, 10*10+1) + + GetFaaSSchedulerInstanceManager().delInstance("0", log.GetLogger()) + convey.So(GetFaaSSchedulerInstanceManager().size(), convey.ShouldEqual, 10*10) + for i := 0; i < 10; i++ { + wg.Add(1) + go func(i int) { + for j := 0; j < 10; j++ { + GetFaaSSchedulerInstanceManager().delInstance(fmt.Sprintf("%d%d", i, j), log.GetLogger()) + } + wg.Done() + }(i) + } + wg.Wait() + convey.So(GetFaaSSchedulerInstanceManager().IsEmpty(), convey.ShouldBeTrue) + }) +} + +func Test_faaSSchedulerInstanceManager_alarm(t *testing.T) { + convey.Convey("Test_faaSSchedulerInstanceManager_alarm", t, func() { + clearTest() + reportTrigger := false + clearTrigger := false + defer gomonkey.ApplyFunc(reportNoAvailableSchedulerInstAlarm, func() { + reportTrigger = true + }).Reset() + defer gomonkey.ApplyFunc(clearNoAvailableSchedulerInstAlarm, func() { + clearTrigger = true + }).Reset() + GetFaaSSchedulerInstanceManager().addInstance("0", nil, log.GetLogger()) + convey.So(clearTrigger, convey.ShouldBeFalse) + convey.So(reportTrigger, convey.ShouldBeFalse) + + GetFaaSSchedulerInstanceManager().delInstance("0", log.GetLogger()) + convey.So(reportTrigger, convey.ShouldBeTrue) + reportTrigger = false + + GetFaaSSchedulerInstanceManager().sync(log.GetLogger()) + convey.So(reportTrigger, convey.ShouldBeTrue) + convey.So(clearTrigger, convey.ShouldBeFalse) + + GetFaaSSchedulerInstanceManager().addInstance("0", nil, log.GetLogger()) + convey.So(clearTrigger, convey.ShouldBeTrue) + clearTest() + }) +} diff --git a/frontend/pkg/frontend/instancemanager/instance.go b/frontend/pkg/frontend/instancemanager/instance.go new file mode 100644 index 0000000000000000000000000000000000000000..fff5efe4ba08d7c9c901e122ab193dbee1e20dc9 --- /dev/null +++ b/frontend/pkg/frontend/instancemanager/instance.go @@ -0,0 +1,254 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package instancemanager - +package instancemanager + +import ( + "sync" + + "go.uber.org/zap" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/resspeckey" + "frontend/pkg/common/faas_common/types" + "frontend/pkg/frontend/subscriber" +) + +var gInstanceScheduler = &FunctionInstancesMap{ + lock: sync.RWMutex{}, + instancesMap: make(map[string]*functionInstanceMap), +} + +var subject = subscriber.NewSubject() + +// GetInstanceSubject - +func GetInstanceSubject() *subscriber.Subject { + return subject +} + +// GetGlobalInstanceScheduler - +func GetGlobalInstanceScheduler() *FunctionInstancesMap { + return gInstanceScheduler +} + +// FunctionInstancesMap - +type FunctionInstancesMap struct { + lock sync.RWMutex + instancesMap map[string]*functionInstanceMap +} + +type functionInstanceMap struct { + lock sync.RWMutex + instanceQueues map[string]*functionInstanceQueue // key: resKey, value: *functionInstanceQueue +} + +type functionInstanceQueue struct { + lock sync.RWMutex + instances map[string]*types.InstanceSpecification // key: instanceId, value: *instance +} + +// GetInstance - +func (g *FunctionInstancesMap) GetInstance(funcKey, resSpecKey, instanceId string) *types.InstanceSpecification { + g.lock.RLock() + defer g.lock.RUnlock() + resInstancesMap, ok := g.instancesMap[funcKey] + if !ok { + return nil + } + return resInstancesMap.getInstance(resSpecKey, instanceId) +} + +func (g *FunctionInstancesMap) addInstance(funcKey string, instance *types.InstanceSpecification, + logger api.FormatLogger) { + g.lock.Lock() + resInstancesMap, ok := g.instancesMap[funcKey] + if !ok { + resInstancesMap = &functionInstanceMap{ + lock: sync.RWMutex{}, + instanceQueues: make(map[string]*functionInstanceQueue), + } + g.instancesMap[funcKey] = resInstancesMap + } + logger = logger.With(zap.Any("funcKey", funcKey)) + g.lock.Unlock() + resInstancesMap.addInstance(instance, logger) +} + +func (g *FunctionInstancesMap) delInstance(funcKey string, instance *types.InstanceSpecification, + logger api.FormatLogger) { + g.lock.Lock() + resInstancesMap, ok := g.instancesMap[funcKey] + logger = logger.With(zap.Any("funcKey", funcKey)) + if !ok { + g.lock.Unlock() + return + } + g.lock.Unlock() + resInstancesMap.delInstance(instance, logger) + if resInstancesMap.size() == 0 { + g.lock.Lock() + if resInstancesMap.size() == 0 { + delete(g.instancesMap, funcKey) + logger.Infof("no instances in funcKey, delete this funcKey map") + } + g.lock.Unlock() + } +} + +// GetRandomInstanceWithoutUnexpectedInstance - +func (g *FunctionInstancesMap) GetRandomInstanceWithoutUnexpectedInstance( + funcKey string, resKey string, unexpectInstanceIds []string, logger api.FormatLogger) *types.InstanceSpecification { + logger = logger.With(zap.Any("resKey", resKey)) + g.lock.RLock() + resInstanceMap, ok := g.instancesMap[funcKey] + g.lock.RUnlock() + if !ok { + logger.Errorf("the funcKey has no instance in instancesMap") + return nil + } + resInstanceMap.lock.RLock() + queue, ok := resInstanceMap.instanceQueues[resKey] + resInstanceMap.lock.RUnlock() + if !ok { + logger.Errorf("the resKey has no instance in funcKey instanceMap") + return nil + } + + queue.lock.RLock() + defer queue.lock.RUnlock() + + instances := make(map[string]*types.InstanceSpecification, len(queue.instances)) + for k, v := range queue.instances { + instances[k] = v + } + for _, unexpectedInstanceId := range unexpectInstanceIds { + delete(instances, unexpectedInstanceId) + } + + for _, v := range instances { + logger.Infof("choose instance: %s, total instance num: %d, unexpected instance num: %d", v.InstanceID, + len(queue.instances), len(unexpectInstanceIds)) + return v + } + logger.Errorf("the resKey has no available instance in funcKey instanceMap, total instance num: %d, "+ + "unexpected instance num: %d", len(queue.instances), len(unexpectInstanceIds)) + return nil +} + +func (f *functionInstanceMap) size() int { + f.lock.RLock() + defer f.lock.RUnlock() + return len(f.instanceQueues) +} + +func (f *functionInstanceMap) getInstance(resSpecKey, instanceId string) *types.InstanceSpecification { + f.lock.RLock() + defer f.lock.RUnlock() + q, ok := f.instanceQueues[resSpecKey] + if !ok { + return nil + } + return q.getInstance(instanceId) +} + +func (f *functionInstanceMap) addInstance(instance *types.InstanceSpecification, logger api.FormatLogger) { + f.lock.Lock() + resKey, err := resspeckey.GetResKeyFromStr(instance.CreateOptions[constant.ResourceSpecNote]) + if err != nil { + f.lock.Unlock() + logger.Warnf("add instance failed, parse resKey failed, err: %s, resKeyStr: %s", err.Error(), + instance.CreateOptions[constant.ResourceSpecNote]) + return + } + logger = logger.With(zap.Any("resKey", resKey.String())) + q, ok := f.instanceQueues[resKey.String()] + if !ok { + q = &functionInstanceQueue{ + lock: sync.RWMutex{}, + instances: make(map[string]*types.InstanceSpecification), + } + f.instanceQueues[resKey.String()] = q + } + f.lock.Unlock() + q.addInstance(instance, logger) +} + +func (f *functionInstanceMap) delInstance(instance *types.InstanceSpecification, logger api.FormatLogger) { + f.lock.Lock() + resKey, err := resspeckey.GetResKeyFromStr(instance.CreateOptions[constant.ResourceSpecNote]) + if err != nil { + f.lock.Unlock() + logger.Warnf("no need delete instance, parse resKey failed, err: %s, resKeyStr: %s", err.Error(), + instance.CreateOptions[constant.ResourceSpecNote]) + return + } + logger = logger.With(zap.Any("resKey", resKey)) + q, ok := f.instanceQueues[resKey.String()] + if !ok { + f.lock.Unlock() + return + } + f.lock.Unlock() + q.delInstance(instance, logger) + if q.size() == 0 { + f.lock.Lock() + if q.size() == 0 { + delete(f.instanceQueues, resKey.String()) + logger.Infof("no instances, delete this resKey map") + } + f.lock.Unlock() + } +} + +func (i *functionInstanceQueue) addInstance(instance *types.InstanceSpecification, logger api.FormatLogger) { + i.lock.Lock() + defer i.lock.Unlock() + i.instances[instance.InstanceID] = instance + subject.PublishEvent(subscriber.Update, instance) + logger.Infof("add instance ok") +} + +func (i *functionInstanceQueue) size() int { + i.lock.RLock() + defer i.lock.RUnlock() + return len(i.instances) +} + +func (i *functionInstanceQueue) delInstance(instance *types.InstanceSpecification, logger api.FormatLogger) { + i.lock.Lock() + defer i.lock.Unlock() + _, ok := i.instances[instance.InstanceID] + if !ok { + logger.Infof("no need delete unexist instance") + return + } + delete(i.instances, instance.InstanceID) + subject.PublishEvent(subscriber.Delete, instance) + logger.Infof("delete instance ok") +} + +func (i *functionInstanceQueue) getInstance(instanceId string) *types.InstanceSpecification { + i.lock.RLock() + defer i.lock.RUnlock() + instance, ok := i.instances[instanceId] + if !ok { + return nil + } + return instance +} diff --git a/frontend/pkg/frontend/instancemanager/instance_test.go b/frontend/pkg/frontend/instancemanager/instance_test.go new file mode 100644 index 0000000000000000000000000000000000000000..c96ba3a9507068ef6f4451c806797a7e28e81f47 --- /dev/null +++ b/frontend/pkg/frontend/instancemanager/instance_test.go @@ -0,0 +1,464 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package instancemanager + +import ( + "strconv" + "sync" + "testing" + + "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/common/faas_common/logger/log" + commontype "frontend/pkg/common/faas_common/types" +) + +func TestInstanceQueueAddInstance(t *testing.T) { + convey.Convey("test functionInstanceQueue addInstance", t, func() { + var wg sync.WaitGroup + expectedInstanceQueueSize := 5 + instanceTestQueue := &functionInstanceQueue{ + lock: sync.RWMutex{}, + instances: make(map[string]*commontype.InstanceSpecification, expectedInstanceQueueSize), + } + + for i := 0; i < expectedInstanceQueueSize; i++ { + wg.Add(1) + instance := &commontype.InstanceSpecification{ + InstanceID: "functionInstanceQueue" + strconv.Itoa(i), + } + go func(instance *commontype.InstanceSpecification) { + defer wg.Done() + instanceTestQueue.addInstance(instance, log.GetLogger()) + }(instance) + } + + wg.Wait() + convey.So(instanceTestQueue.size(), convey.ShouldEqual, expectedInstanceQueueSize) + }) +} + +func TestInstanceQueueDelInstance(t *testing.T) { + convey.Convey("test functionInstanceQueue delInstance", t, func() { + var wg sync.WaitGroup + expectedInstanceQueueSize := 5 + instanceMap := make(map[string]*commontype.InstanceSpecification, expectedInstanceQueueSize) + for i := 0; i < expectedInstanceQueueSize; i++ { + instanceID := "functionInstanceQueue" + strconv.Itoa(i) + instanceMap[instanceID] = &commontype.InstanceSpecification{ + InstanceID: instanceID, + } + } + instanceTestQueue := &functionInstanceQueue{ + lock: sync.RWMutex{}, + instances: instanceMap, + } + + for _, v := range instanceMap { + wg.Add(1) + go func(instance *commontype.InstanceSpecification) { + defer wg.Done() + instanceTestQueue.delInstance(instance, log.GetLogger()) + }(v) + } + + wg.Wait() + convey.So(instanceTestQueue.size(), convey.ShouldEqual, 0) + }) +} + +func TestInstanceQueueGetSize(t *testing.T) { + convey.Convey("test functionInstanceQueue size", t, func() { + instanceSliceAddSize := 5 + instanceSliceDelSize := 3 + instanceTestQueue := &functionInstanceQueue{ + lock: sync.RWMutex{}, + instances: make(map[string]*commontype.InstanceSpecification, instanceSliceAddSize), + } + instanceSlice := make([]*commontype.InstanceSpecification, instanceSliceAddSize) + for i := 0; i < instanceSliceAddSize; i++ { + instanceID := "functionInstanceQueue" + strconv.Itoa(i) + instanceSlice[i] = &commontype.InstanceSpecification{ + InstanceID: instanceID, + } + } + + var wg sync.WaitGroup + for _, v := range instanceSlice { + wg.Add(1) + go func(instance *commontype.InstanceSpecification) { + defer wg.Done() + instanceTestQueue.addInstance(instance, log.GetLogger()) + }(v) + } + wg.Wait() + + for i := 0; i < instanceSliceDelSize; i++ { + wg.Add(1) + go func(instance *commontype.InstanceSpecification) { + defer wg.Done() + instanceTestQueue.delInstance(instance, log.GetLogger()) + }(instanceSlice[i]) + } + wg.Wait() + convey.So(instanceTestQueue.size(), convey.ShouldEqual, instanceSliceAddSize-instanceSliceDelSize) + }) +} + +func TestFunctionInstanceMapAddInstance(t *testing.T) { + convey.Convey("test functionInstanceMap addInstance", t, func() { + convey.Convey("multiple resKeys, check if there are corresponding multiple instancequeues.", func() { + instanceSlice := []*commontype.InstanceSpecification{ + { + InstanceID: "functionInstanceQueue", + CreateOptions: map[string]string{ + "RESOURCE_SPEC_NOTE": "{\"cpu\":100,\"memory\":100}", + }, + }, + { + InstanceID: "functionInstanceQueue", + CreateOptions: map[string]string{ + "RESOURCE_SPEC_NOTE": "{\"cpu\":110,\"memory\":110}", + }, + }, + { + InstanceID: "functionInstanceQueue", + CreateOptions: map[string]string{ + "RESOURCE_SPEC_NOTE": "{\"cpu\":120,\"memory\":120}", + }, + }, + } + expectedFuncInstMapSize := len(instanceSlice) + functionInstanceTestQueue := &functionInstanceMap{ + lock: sync.RWMutex{}, + instanceQueues: make(map[string]*functionInstanceQueue, expectedFuncInstMapSize), + } + + var wg sync.WaitGroup + for _, v := range instanceSlice { + wg.Add(1) + go func(instance *commontype.InstanceSpecification) { + defer wg.Done() + functionInstanceTestQueue.addInstance(instance, log.GetLogger()) + }(v) + } + + wg.Wait() + convey.So(functionInstanceTestQueue.size(), convey.ShouldEqual, expectedFuncInstMapSize) + }) + + convey.Convey("resKey format is incorrect, add instance failed", func() { + instanceSlice := []*commontype.InstanceSpecification{ + { + InstanceID: "functionInstanceQueue", + CreateOptions: map[string]string{ + "RESOURCE_SPEC_NOTE": "{\"cpu\":\"abc\",\"memory\":\"100\"}", + }, + }, + } + expectedFuncInstMapSize := len(instanceSlice) + functionInstanceTestQueue := &functionInstanceMap{ + lock: sync.RWMutex{}, + instanceQueues: make(map[string]*functionInstanceQueue, expectedFuncInstMapSize), + } + + var wg sync.WaitGroup + for _, v := range instanceSlice { + wg.Add(1) + go func(instance *commontype.InstanceSpecification) { + defer wg.Done() + functionInstanceTestQueue.addInstance(instance, log.GetLogger()) + }(v) + } + + wg.Wait() + convey.So(functionInstanceTestQueue.size(), convey.ShouldEqual, 0) + }) + + convey.Convey("For the same resKey, functionInstanceMap instanceQueues size meets the expectation.", func() { + instanceSlice := []*commontype.InstanceSpecification{ + { + InstanceID: "instanceQueue0", + CreateOptions: map[string]string{ + "RESOURCE_SPEC_NOTE": "{\"cpu\":100,\"memory\":100}", + }, + }, + { + InstanceID: "instanceQueue1", + CreateOptions: map[string]string{ + "RESOURCE_SPEC_NOTE": "{\"cpu\":100,\"memory\":100}", + }, + }, + } + functionInstanceTestQueue := &functionInstanceMap{ + lock: sync.RWMutex{}, + instanceQueues: make(map[string]*functionInstanceQueue, 1), + } + + for _, v := range instanceSlice { + functionInstanceTestQueue.addInstance(v, log.GetLogger()) + } + + // InstanceID不同，RESOURCE_SPEC_NOTE相同，预期结果如下 + convey.So(functionInstanceTestQueue.size(), convey.ShouldEqual, 1) + convey.So(functionInstanceTestQueue.instanceQueues["cpu-100-mem-100-storage-0-cstRes--cstResSpec--invokeLabel-"].size(), convey.ShouldEqual, 2) + }) + }) +} + +func TestFunctionInstanceMapDelInstance(t *testing.T) { + convey.Convey("test functionInstanceMap delInstance", t, func() { + instance1 := &commontype.InstanceSpecification{ + InstanceID: "instanceQueue1", + CreateOptions: map[string]string{ + "RESOURCE_SPEC_NOTE": "{\"cpu\":100,\"memory\":100}", + }, + } + instance2 := &commontype.InstanceSpecification{ + InstanceID: "instanceQueue2", + CreateOptions: map[string]string{ + "RESOURCE_SPEC_NOTE": "{\"cpu\":100,\"memory\":100}", + }, + } + + resKey := "cpu-100-mem-100-storage-0-cstRes--cstResSpec--invokeLabel-" + functionInstanceTestQueue := &functionInstanceMap{ + lock: sync.RWMutex{}, + instanceQueues: make(map[string]*functionInstanceQueue, 1), + } + functionInstanceTestQueue.addInstance(instance1, log.GetLogger()) + functionInstanceTestQueue.addInstance(instance2, log.GetLogger()) + + convey.Convey("delete the instanceQueues of resKey correctly ", func() { + functionInstanceTestQueue.delInstance(instance1, log.GetLogger()) + convey.So(functionInstanceTestQueue.size(), convey.ShouldEqual, 1) + convey.So(functionInstanceTestQueue.instanceQueues[resKey].size(), convey.ShouldEqual, 1) + }) + + convey.Convey("resKey is not in instanceQueues, skip delete", func() { + otherInstance := &commontype.InstanceSpecification{ + InstanceID: "functionInstanceQueue", + CreateOptions: map[string]string{ + "RESOURCE_SPEC_NOTE": "{\"cpu\":110,\"memory\":110}", + }, + } + functionInstanceTestQueue.delInstance(otherInstance, log.GetLogger()) + convey.So(functionInstanceTestQueue.size(), convey.ShouldEqual, 1) + }) + + convey.Convey("", func() { + functionInstanceTestQueue.delInstance(instance1, log.GetLogger()) + functionInstanceTestQueue.delInstance(instance2, log.GetLogger()) + convey.So(functionInstanceTestQueue.size(), convey.ShouldEqual, 0) + }) + }) +} + +func TestGlobalInstancesMapAddInstance(t *testing.T) { + convey.Convey("test globalInstancesMap addInstance", t, func() { + funcKey := "c53626012ba84727b938ca8bf03108ef/0@default@zscaetest/latest" + instanceSlice := []*commontype.InstanceSpecification{ + { + InstanceID: "functionInstanceQueue", + CreateOptions: map[string]string{ + "RESOURCE_SPEC_NOTE": "{\"cpu\":100,\"memory\":100}", + }, + }, + { + InstanceID: "functionInstanceQueue", + CreateOptions: map[string]string{ + "RESOURCE_SPEC_NOTE": "{\"cpu\":110,\"memory\":110}", + }, + }, + { + InstanceID: "functionInstanceQueue", + CreateOptions: map[string]string{ + "RESOURCE_SPEC_NOTE": "{\"cpu\":120,\"memory\":120}", + }, + }, + } + expectedFuncInstMapSize := len(instanceSlice) + + convey.Convey("add instance success", func() { + globalInstanceTestQueue := &FunctionInstancesMap{ + lock: sync.RWMutex{}, + instancesMap: make(map[string]*functionInstanceMap, 1), + } + for _, v := range instanceSlice { + globalInstanceTestQueue.addInstance(funcKey, v, log.GetLogger()) + } + + convey.So(len(globalInstanceTestQueue.instancesMap), convey.ShouldEqual, 1) + convey.So(globalInstanceTestQueue.instancesMap[funcKey].size(), convey.ShouldEqual, expectedFuncInstMapSize) + }) + + convey.Convey("add instance success in the concurrent scenario", func() { + globalInstanceTestQueue := &FunctionInstancesMap{ + lock: sync.RWMutex{}, + instancesMap: make(map[string]*functionInstanceMap, 1), + } + var wg sync.WaitGroup + for _, v := range instanceSlice { + wg.Add(1) + go func(v *commontype.InstanceSpecification) { + defer wg.Done() + globalInstanceTestQueue.addInstance(funcKey, v, log.GetLogger()) + }(v) + } + + wg.Wait() + convey.So(len(globalInstanceTestQueue.instancesMap), convey.ShouldEqual, 1) + convey.So(globalInstanceTestQueue.instancesMap[funcKey].size(), convey.ShouldEqual, expectedFuncInstMapSize) + }) + }) +} + +func TestGlobalInstancesMapDelInstance(t *testing.T) { + convey.Convey("test globalInstancesMap delInstance", t, func() { + funcKey := "c53626012ba84727b938ca8bf03108ef/0@default@zscaetest/latest" + instanceSlice := []*commontype.InstanceSpecification{ + { + InstanceID: "functionInstanceQueue", + CreateOptions: map[string]string{ + "RESOURCE_SPEC_NOTE": "{\"cpu\":100,\"memory\":100}", + }, + }, + { + InstanceID: "functionInstanceQueue", + CreateOptions: map[string]string{ + "RESOURCE_SPEC_NOTE": "{\"cpu\":110,\"memory\":110}", + }, + }, + } + globalInstanceTestQueue := &FunctionInstancesMap{ + lock: sync.RWMutex{}, + instancesMap: make(map[string]*functionInstanceMap, 1), + } + for _, v := range instanceSlice { + globalInstanceTestQueue.addInstance(funcKey, v, log.GetLogger()) + } + + convey.Convey("delete instance success", func() { + globalInstanceTestQueue.delInstance(funcKey, instanceSlice[0], log.GetLogger()) + + convey.So(len(globalInstanceTestQueue.instancesMap), convey.ShouldEqual, 1) + convey.So(globalInstanceTestQueue.instancesMap[funcKey].size(), convey.ShouldEqual, 1) + }) + + convey.Convey("delete duplicate instance success", func() { + globalInstanceTestQueue.delInstance(funcKey, instanceSlice[0], log.GetLogger()) + globalInstanceTestQueue.delInstance(funcKey, instanceSlice[0], log.GetLogger()) + + convey.So(len(globalInstanceTestQueue.instancesMap), convey.ShouldEqual, 1) + convey.So(globalInstanceTestQueue.instancesMap[funcKey].size(), convey.ShouldEqual, 1) + }) + + convey.Convey("delete all instance success", func() { + for _, v := range instanceSlice { + globalInstanceTestQueue.delInstance(funcKey, v, log.GetLogger()) + } + + convey.So(globalInstanceTestQueue.instancesMap, convey.ShouldBeEmpty) + }) + + convey.Convey("delete instance success in the concurrent scenario", func() { + var wg sync.WaitGroup + for _, v := range instanceSlice { + wg.Add(1) + go func(v *commontype.InstanceSpecification) { + defer wg.Done() + globalInstanceTestQueue.delInstance(funcKey, v, log.GetLogger()) + }(v) + } + + wg.Wait() + convey.So(globalInstanceTestQueue.instancesMap, convey.ShouldBeEmpty) + }) + }) +} + +func TestGetRandomInstanceWithoutUnexpectedInstance(t *testing.T) { + convey.Convey("When testing GetRandomInstanceWithoutUnexpectedInstance", t, func() { + // Setup test data + setupTestEnv := func() *FunctionInstancesMap { + testMap := &FunctionInstancesMap{ + instancesMap: make(map[string]*functionInstanceMap), + } + testMap.instancesMap["funcKey1"] = &functionInstanceMap{ + instanceQueues: make(map[string]*functionInstanceQueue), + } + testMap.instancesMap["funcKey1"].instanceQueues["resKey1"] = &functionInstanceQueue{ + instances: map[string]*commontype.InstanceSpecification{ + "inst001": {InstanceID: "inst001"}, + "inst002": {InstanceID: "inst002"}, + "inst003": {InstanceID: "inst003"}, + }, + } + return testMap + } + + convey.Convey("With valid funcKey and resKey", func() { + testMap := setupTestEnv() + + convey.Convey("Should return random instance when no instances excluded", func() { + instance := testMap.GetRandomInstanceWithoutUnexpectedInstance( + "funcKey1", "resKey1", []string{}, log.GetLogger()) + convey.So(instance, convey.ShouldNotBeNil) + convey.So([]string{"inst001", "inst002", "inst003"}, convey.ShouldContain, instance.InstanceID) + }) + + convey.Convey("Should return remaining instance after excluding some", func() { + instance := testMap.GetRandomInstanceWithoutUnexpectedInstance( + "funcKey1", "resKey1", []string{"inst001", "inst004"}, log.GetLogger()) + convey.So(instance, convey.ShouldNotBeNil) + convey.So([]string{"inst002", "inst003"}, convey.ShouldContain, instance.InstanceID) + }) + + convey.Convey("Should return nil when all instances excluded", func() { + instance := testMap.GetRandomInstanceWithoutUnexpectedInstance( + "funcKey1", "resKey1", + []string{"inst001", "inst002", "inst003"}, + log.GetLogger()) + convey.So(instance, convey.ShouldBeNil) + }) + }) + + convey.Convey("With invalid parameters", func() { + testMap := setupTestEnv() + + convey.Convey("Should return nil with invalid funcKey", func() { + instance := testMap.GetRandomInstanceWithoutUnexpectedInstance( + "invalidFunc", "resKey1", []string{}, log.GetLogger()) + convey.So(instance, convey.ShouldBeNil) + }) + + convey.Convey("Should return nil with invalid resKey", func() { + instance := testMap.GetRandomInstanceWithoutUnexpectedInstance( + "funcKey1", "invalidRes", []string{}, log.GetLogger()) + convey.So(instance, convey.ShouldBeNil) + }) + + convey.Convey("Should return nil when instance queue is empty", func() { + testMap.instancesMap["funcKey1"].instanceQueues["resKey1"].instances = + make(map[string]*commontype.InstanceSpecification) + instance := testMap.GetRandomInstanceWithoutUnexpectedInstance( + "funcKey1", "resKey1", []string{}, log.GetLogger()) + convey.So(instance, convey.ShouldBeNil) + }) + }) + }) +} diff --git a/frontend/pkg/frontend/invocation/doinvoke.go b/frontend/pkg/frontend/invocation/doinvoke.go new file mode 100644 index 0000000000000000000000000000000000000000..1d5cb816e57ba787b1a8531bf47fe301801d48a1 --- /dev/null +++ b/frontend/pkg/frontend/invocation/doinvoke.go @@ -0,0 +1,94 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package invocation + +import ( + "errors" + "time" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/statuscode" + commontype "frontend/pkg/common/faas_common/types" + "frontend/pkg/frontend/common/httpconstant" + "frontend/pkg/frontend/common/httputil" + "frontend/pkg/frontend/common/util" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/functionmeta" + "frontend/pkg/frontend/responsehandler" + "frontend/pkg/frontend/schedulerproxy" + "frontend/pkg/frontend/types" +) + +const ( + maxInvokeRetries = 5 + retrySleepTime = 1000 * time.Millisecond + baseTen = 10 + bitSize = 64 +) + +// InvokeHandler the handler of invoke +func InvokeHandler(ctx *types.InvokeProcessContext) error { + var err error + traceID := ctx.TraceID + funcKey := ctx.FuncKey + funcSpec, exist := functionmeta.LoadFuncSpec(funcKey) + if !exist { + responsehandler.SetErrorInContext(ctx, statuscode.FuncMetaNotFound, "function metadata not found") + log.GetLogger().Errorf("function %s doesn't exist in cache", funcKey) + return errors.New("function doesn't exist") + } + sessionId := ctx.ReqHeader[httpconstant.HeaderInstanceSession] + instanceLabel := ctx.ReqHeader[httpconstant.HeaderInstanceLabel] + log.GetLogger().Infof("invoking function %s, signature %s, traceID %s, sessionId %s, instanceLabel %s", + funcSpec.FunctionKey, funcSpec.FuncMetaSignature, traceID, sessionId, instanceLabel) + err = doInvokeWithRetry(ctx, funcSpec) + if err != nil { + log.GetLogger().Errorf("failed to finish the request, traceID %s, error: %s", traceID, err.Error()) + httputil.HandleInvokeError(ctx, err) + } + log.GetLogger().Debugf("invoke function %s success, traceID %s, sessionId %s, instanceLabel %s", + funcSpec.FunctionKey, traceID, sessionId, instanceLabel) + return err +} + +func doInvokeWithRetry(ctx *types.InvokeProcessContext, funcSpec *commontype.FuncSpec) error { + execute := func() error { + ctx.ShouldRetry = false + return doInvoke(ctx, funcSpec) + } + shouldRetry := func() bool { + log.GetLogger().Warnf("invoke will be retried is %t, traceID: %s", ctx.ShouldRetry, ctx.TraceID) + return ctx.ShouldRetry + } + return util.Retry(execute, shouldRetry, maxInvokeRetries, retrySleepTime) +} + +func doInvoke(ctx *types.InvokeProcessContext, funcSpec *commontype.FuncSpec) error { + if config.GetConfig().FunctionInvokeBackend == constant.BackendTypeFG { + return functionInvokeForFG(ctx, funcSpec) + } + kernelReqHandler := newKernelRequestHandler(ctx, funcSpec) + return kernelReqHandler.invoke() +} + +func resetSchedulerProxy(ctx *types.InvokeProcessContext) { + if ctx.TrafficLimited { + schedulerproxy.Proxy.Reset() + ctx.TrafficLimited = false + } +} diff --git a/frontend/pkg/frontend/invocation/doinvoke_test.go b/frontend/pkg/frontend/invocation/doinvoke_test.go new file mode 100644 index 0000000000000000000000000000000000000000..203e78206df97f53c86ec6a9914e05d9a0f70fe9 --- /dev/null +++ b/frontend/pkg/frontend/invocation/doinvoke_test.go @@ -0,0 +1,136 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package invocation + +import ( + "errors" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/statuscode" + commonTypes "frontend/pkg/common/faas_common/types" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/functionmeta" + "frontend/pkg/frontend/responsehandler" + "frontend/pkg/frontend/types" +) + +func Test_doInnerCodeString(t *testing.T) { + msg := innerCode(-1).String() + assert.Equal(t, msg, "invalidCode") + msg = innerCode(0).String() + assert.Equal(t, msg, "sysOK") + msg = innerCode(100).String() + assert.Equal(t, msg, "usrErr") + msg = innerCode(10001).String() + assert.Equal(t, msg, "sysErr") +} + +func Test_prepareDynamicResource(t *testing.T) { + type args struct { + ctx *types.InvokeProcessContext + } + tests := []struct { + name string + args args + want map[string]int64 + wantErr bool + }{ + {"case1", args{ctx: &types.InvokeProcessContext{ReqHeader: map[string]string{constant.HeaderCPUSize: "600", constant.HeaderMemorySize: "512"}}}, + map[string]int64{constant.ResourceCPUName: 600, constant.ResourceMemoryName: 512}, false}, + {"case2", args{ctx: &types.InvokeProcessContext{}}, + map[string]int64{}, false}, + {"case3", args{ctx: &types.InvokeProcessContext{ReqHeader: map[string]string{ + constant.HeaderCustomResource: "{\"CPU\":1234567890}"}}}, + map[string]int64{constant.ResourceCPUName: 1234567890}, false}, + {"case4", args{ctx: &types.InvokeProcessContext{ReqHeader: map[string]string{ + constant.HeaderCustomResourceNew: "{\"CPU\":1234567890}"}}}, + map[string]int64{constant.ResourceCPUName: 1234567890}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := prepareDynamicResource(tt.args.ctx) + if (err != nil) != tt.wantErr { + t.Errorf("prepareDynamicResource() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("prepareDynamicResource() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getTimeout(t *testing.T) { + funcSpecTimeout := int64(300) + ctxTimeout := int64(60) + timeout := getTimeout(funcSpecTimeout, ctxTimeout) + assert.Equal(t, ctxTimeout, timeout) + funcSpecTimeout = int64(300) + ctxTimeout = int64(0) + timeout = getTimeout(funcSpecTimeout, ctxTimeout) + assert.Equal(t, funcSpecTimeout, timeout) +} + +func TestInvokeHandler(t *testing.T) { + setCode := 0 + defer gomonkey.ApplyFunc(responsehandler.SetErrorInContext, func(ctx *types.InvokeProcessContext, innerCode int, + message interface{}) { + setCode = innerCode + }).Reset() + defer gomonkey.ApplyFunc(config.GetConfig, func() *types.Config { + return &types.Config{} + }).Reset() + convey.Convey("TestInvokeHandler", t, func() { + convey.Convey("funcMeta not found", func() { + defer gomonkey.ApplyFunc(functionmeta.LoadFuncSpec, func(funcKey string) (*commonTypes.FuncSpec, bool) { + return nil, false + }).Reset() + ctx := &types.InvokeProcessContext{} + err := InvokeHandler(ctx) + convey.So(err, convey.ShouldNotBeEmpty) + convey.So(setCode, convey.ShouldEqual, statuscode.FuncMetaNotFound) + }) + convey.Convey("invoke error", func() { + defer gomonkey.ApplyFunc(functionmeta.LoadFuncSpec, func(funcKey string) (*commonTypes.FuncSpec, bool) { + return &commonTypes.FuncSpec{}, true + }).Reset() + defer gomonkey.ApplyPrivateMethod(reflect.TypeOf(&kernelRequestHandler{}), "invoke", func(_ *kernelRequestHandler) error { + return errors.New("some error") + }).Reset() + ctx := &types.InvokeProcessContext{} + err := InvokeHandler(ctx) + convey.So(err, convey.ShouldNotBeEmpty) + }) + convey.Convey("invoke success", func() { + defer gomonkey.ApplyFunc(functionmeta.LoadFuncSpec, func(funcKey string) (*commonTypes.FuncSpec, bool) { + return &commonTypes.FuncSpec{}, true + }).Reset() + defer gomonkey.ApplyPrivateMethod(reflect.TypeOf(&kernelRequestHandler{}), "invoke", func(_ *kernelRequestHandler) error { + return nil + }).Reset() + ctx := &types.InvokeProcessContext{} + err := InvokeHandler(ctx) + convey.So(err, convey.ShouldBeNil) + }) + }) +} diff --git a/frontend/pkg/frontend/invocation/fgadapter.go b/frontend/pkg/frontend/invocation/fgadapter.go new file mode 100644 index 0000000000000000000000000000000000000000..1dafb11b816968405f2ed6054073c716ba06a61d --- /dev/null +++ b/frontend/pkg/frontend/invocation/fgadapter.go @@ -0,0 +1,43 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package invocation - +package invocation + +import ( + "frontend/pkg/frontend/middleware" + "frontend/pkg/frontend/responsehandler" +) + +// FGAdapter - +type FGAdapter struct{} + +// MakeInvoker - +func (f *FGAdapter) MakeInvoker() middleware.HandlerChain { + invoker := middleware.NewBaseHandler(InvokeHandler) + invoker.Use( + middleware.GraceExitFilter, + middleware.BodySizeChecker, + middleware.TrafficLimiter, + middleware.RequestAuthCheck, + ) + return invoker +} + +// MakeResponseHandler - +func (f *FGAdapter) MakeResponseHandler() responsehandler.HandlerInterface { + return &responsehandler.FGResponseHandler{} +} diff --git a/frontend/pkg/frontend/invocation/fgadapter_test.go b/frontend/pkg/frontend/invocation/fgadapter_test.go new file mode 100644 index 0000000000000000000000000000000000000000..80e0360f1bcd11bedba2e6cc88034fd7a3db3f62 --- /dev/null +++ b/frontend/pkg/frontend/invocation/fgadapter_test.go @@ -0,0 +1,170 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package invocation - +package invocation + +import ( + "encoding/json" + "errors" + "net/http" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + "github.com/valyala/fasthttp" + + "frontend/pkg/common/faas_common/snerror" + "frontend/pkg/common/faas_common/statuscode" + "frontend/pkg/frontend/common/httputil" + "frontend/pkg/frontend/responsehandler" + "frontend/pkg/frontend/types" +) + +func TestMakeInvoker(t *testing.T) { + adapter := &FGAdapter{} + invoker := adapter.MakeInvoker() + assert.NotNil(t, invoker) +} + +func TestSetResponseFromInvocation(t *testing.T) { + message := []byte(`{"billingDuration":"xxx","innerCode": "0", "body": "test"}`) + ctx := &types.InvokeProcessContext{ + ReqHeader: map[string]string{}, + RespHeader: map[string]string{}, + } + responsehandler.Handler = (&FGAdapter{}).MakeResponseHandler() + want := []byte(`"test"`) + type args struct { + ctx *types.InvokeProcessContext + message []byte + } + tests := []struct { + name string + args args + }{ + { + name: "TestSetResponseFromInvocation", + args: args{ + ctx: ctx, + message: message, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + responsehandler.SetResponseInContext(tt.args.ctx, tt.args.message) + assert.Equal(t, tt.args.ctx.RespBody, want) + }) + } + convey.Convey("TestSetResponseFromInvocation", t, func() { + convey.Convey("do nothing and return", func() { + ctx = &types.InvokeProcessContext{ + ReqHeader: map[string]string{}, + RespHeader: map[string]string{}, + } + ctx.RespHeader["123"] = "zhangsan" + responsehandler.SetResponseInContext(ctx, []byte("message")) + convey.So(len(ctx.RespBody), convey.ShouldEqual, 0) + }) + convey.Convey("failed to unmarshal call response data", func() { + ctx = &types.InvokeProcessContext{ + ReqHeader: map[string]string{}, + RespHeader: map[string]string{}, + } + responsehandler.SetResponseInContext(ctx, []byte("message")) + convey.So(len(ctx.RespBody), convey.ShouldEqual, 0) + }) + convey.Convey("failed to get the innerCode", func() { + ctx = &types.InvokeProcessContext{ + ReqHeader: map[string]string{}, + RespHeader: map[string]string{}, + } + responsehandler.SetResponseInContext(ctx, []byte(`{"billingDuration":"xxx","innerCode": "0xx", "body": "test"}`)) + convey.So(len(ctx.RespBody), convey.ShouldEqual, 0) + }) + }) +} + +func TestHandleInvokeError(t *testing.T) { + responsehandler.Handler = (&FGAdapter{}).MakeResponseHandler() + convey.Convey("HandleInvokeError", t, func() { + convey.Convey("false SetResponseFromFrontend", func() { + err := errors.New("failed to create, code: \"4\", message: {\"errorCode\":\"6001\",\"message\":\"initError\"}") + ctx := &types.InvokeProcessContext{ + ReqHeader: map[string]string{}, + RespHeader: map[string]string{}, + } + httputil.HandleInvokeError(ctx, err) + convey.So(ctx.StatusCode, convey.ShouldEqual, fasthttp.StatusInternalServerError) + }) + convey.Convey("true", func() { + err := errors.New(`failed to finish the request error failed to create, code: 2002, message: {"errorCode":"4211","message":"init failed bootstrap timed out after 5s"}`) + ctx := &types.InvokeProcessContext{ + ReqHeader: map[string]string{}, + RespHeader: map[string]string{}, + } + httputil.HandleInvokeError(ctx, err) + convey.So(ctx.StatusCode, convey.ShouldEqual, fasthttp.StatusInternalServerError) + }) + convey.Convey("temp true", func() { + err := errors.New(`failed to finish the request error failed to create, code: 2002, message: [{"errorCode":"4211","message":"init failed bootstrap timed out after 5s"}]`) + ctx := &types.InvokeProcessContext{ + ReqHeader: map[string]string{}, + RespHeader: map[string]string{}, + } + httputil.HandleInvokeError(ctx, err) + convey.So(ctx.StatusCode, convey.ShouldEqual, fasthttp.StatusInternalServerError) + }) + }) +} + +func TestHandleCreateInstanceError(t *testing.T) { + responsehandler.Handler = (&FGAdapter{}).MakeResponseHandler() + convey.Convey("HandleCreateInstanceError", t, func() { + convey.Convey("failed to handle create error", func() { + ctx := &types.InvokeProcessContext{ + ReqHeader: map[string]string{}, + RespHeader: map[string]string{}, + } + err := snerror.New(4201, "") + instanceError := httputil.HandleCreateInstanceError(ctx, err) + convey.So(instanceError, convey.ShouldBeTrue) + convey.So(ctx.StatusCode, convey.ShouldEqual, http.StatusInternalServerError) + + err = snerror.New(statuscode.InternalErrorCode, `[{"errorCode": "wrong code", "message":"failed to init function"}]`) + httputil.HandleCreateInstanceError(ctx, err) + convey.So(instanceError, convey.ShouldBeTrue) + convey.So(ctx.StatusCode, convey.ShouldEqual, http.StatusInternalServerError) + }) + }) +} + +func TestSetResponseFromFrontend(t *testing.T) { + responsehandler.Handler = (&FGAdapter{}).MakeResponseHandler() + convey.Convey("SetResponseFromFrontend", t, func() { + defer gomonkey.ApplyFunc(json.Marshal, func(v any) ([]byte, error) { + return nil, errors.New("marshal error") + }).Reset() + ctx := &types.InvokeProcessContext{ + ReqHeader: map[string]string{}, + RespHeader: map[string]string{}, + } + responsehandler.SetErrorInContext(ctx, 4211, "failed") + convey.So(ctx.RespBody, convey.ShouldBeNil) + }) +} diff --git a/frontend/pkg/frontend/invocation/function_invoke_for_fg.go b/frontend/pkg/frontend/invocation/function_invoke_for_fg.go new file mode 100644 index 0000000000000000000000000000000000000000..63b24662dd77f869c6749f15c422b1e52df055bd --- /dev/null +++ b/frontend/pkg/frontend/invocation/function_invoke_for_fg.go @@ -0,0 +1,521 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package invocation + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "github.com/valyala/fasthttp" + "go.uber.org/zap" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/snerror" + "frontend/pkg/common/faas_common/statuscode" + "frontend/pkg/common/faas_common/tls" + commontype "frontend/pkg/common/faas_common/types" + "frontend/pkg/common/faas_common/utils" + "frontend/pkg/frontend/common/httpconstant" + "frontend/pkg/frontend/common/httputil" + "frontend/pkg/frontend/common/util" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/functiontask" + "frontend/pkg/frontend/instanceleasemanager" + "frontend/pkg/frontend/responsehandler" + "frontend/pkg/frontend/schedulerproxy" + "frontend/pkg/frontend/stream" + "frontend/pkg/frontend/types" +) + +const ( + invokePath = "/invoke" + defaultBodyMap = 100 + requestTimeout = "timeout" + retryInterval = 1000 +) + +var ( + // ErrServiceNotAvailable - + ErrServiceNotAvailable = errors.New("worker service is not available") +) + +type innerCode int + +// String convert innerCode to error type string +func (ic innerCode) String() string { + if ic < 0 { + return "invalidCode" + } + if ic == 0 { + return "sysOK" + } + if ic < snerror.UserErrorMax { + return "usrErr" + } + return "sysErr" +} + +func functionInvokeForFG(ctx *types.InvokeProcessContext, funcSpec *commontype.FuncSpec) error { + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + readBodyStart := time.Now() + transferHTTPRequest(ctx, req) + readBodyTotal := time.Since(readBodyStart) + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(resp) + prepareRequest(ctx, req, funcSpec) + defer stream.ReleaseResponse(ctx.StreamInfo.ResponseStreamName) + err := invokeFunction(ctx, funcSpec, req, resp) + if err != nil { + setErrorResponse(ctx, err) + return nil + } + writeRspStart := time.Now() + processResp(ctx, resp) + writeRspCost := time.Since(writeRspStart) + setTraceInfo(ctx, resp) + fillAfterInvokeHeader(ctx) + setLogAndPublishMetrics(ctx, readBodyTotal, writeRspCost) + return nil +} + +func processResp(ctx *types.InvokeProcessContext, resp *fasthttp.Response) { + logger := log.GetLogger().With(zap.Any("traceID", ctx.TraceID)) + if ctx.StreamInfo.ResponseStopChan != nil && stream.CheckIsResponseStream(ctx.StreamInfo.ResponseStreamName) { + logger.Infof("start wait download stream rsp") + timeout := ctx.RequestTraceInfo.Deadline.Sub(time.Now()) + writeTimoutTimer := time.NewTimer(timeout) + select { + case _, ok := <-ctx.StreamInfo.ResponseStopChan.C: + // wait until the response stream is written. + if !ok { + logger.Infof("received response stream stop signal, return") + } + case <-writeTimoutTimer.C: + logger.Warnf("wait for response stream timeout, timeout: %v", timeout) + setErrorResponse(ctx, snerror.New(statuscode.InternalErrorCode, "wait for response stream timeout")) + } + } else if ctx.IsHTTPUploadStream && ctx.StreamInfo.GetRequestStreamErrorCode() != 0 { + // 上传流场景优先使用上传协程中的异常码，否则才使用http的影响返回 + logger.Warnf("upload stream async err code %d", ctx.StreamInfo.GetRequestStreamErrorCode()) + setErrorResponse(ctx, snerror.New(int(ctx.StreamInfo.GetRequestStreamErrorCode()), "internal error")) + } else { + transferHTTPResponse(ctx, resp) + } +} + +func setErrorResponse(ctx *types.InvokeProcessContext, err error) { + errorCode := statuscode.FrontendStatusInternalError + responsehandler.SetErrorInContextWithDefault(ctx, err, errorCode, err.Error()) + log.GetLogger().Infof("%s | %s | %s | %f | | | | innerCode %d | %s | invoke end in frontend", + ctx.RequestTraceInfo.FuncName, ctx.RequestTraceInfo.TenantID, ctx.TraceID, + time.Since(ctx.StartTime).Seconds(), + ctx.RequestTraceInfo.InnerCode, innerCode(ctx.RequestTraceInfo.InnerCode).String()) +} + +func setTraceInfo(ctx *types.InvokeProcessContext, resp *fasthttp.Response) { + ctx.RequestTraceInfo.CallNode = string(resp.Header.Peek(constant.HeaderCallNode)) + ctx.RequestTraceInfo.CallInstance = string(resp.Header.Peek(constant.HeaderCallInstance)) + + ctx.RequestTraceInfo.TotalCost = time.Since(ctx.StartTime) + ctx.RequestTraceInfo.FrontendCost = ctx.RequestTraceInfo.TotalCost - ctx.RequestTraceInfo.AllBusCost + workerCost := string(resp.Header.Peek(constant.HeaderWorkerCost)) + if len(workerCost) != 0 { + v, err := strconv.ParseInt(workerCost, baseTen, bitSize) + if err != nil { + log.GetLogger().Errorf("failed to ParseInt, workerCost %s,traceID:%s", workerCost, + ctx.TraceID) + } else { + ctx.RequestTraceInfo.WorkerCost = time.Duration(v) * time.Nanosecond + } + } + ctx.RequestTraceInfo.BusCost = ctx.RequestTraceInfo.AllBusCost - ctx.RequestTraceInfo.WorkerCost +} + +func fillAfterInvokeHeader(ctx *types.InvokeProcessContext) { + ctx.RespHeader[constant.HeaderCallNode] = ctx.RequestTraceInfo.CallNode + ctx.RespHeader[constant.HeaderCallInstance] = ctx.RequestTraceInfo.CallInstance +} + +func setLogAndPublishMetrics(ctx *types.InvokeProcessContext, + readBodyTotal time.Duration, writeRspCost time.Duration) { + // funcName|tenantID|traceID|cost|frontendCost:busCost:workerCost:readBodyCost|contentLength|nodeIP|workerPod| + // innerCode|ErrMsg + log.GetLogger().Infof("%s | %s | %s | %f | %f:%f:%f:%f:%f | %d |%s | %s | innerCode %d | %s | "+ + "invoke end in frontend", ctx.RequestTraceInfo.FuncName, ctx.RequestTraceInfo.TenantID, ctx.TraceID, + ctx.RequestTraceInfo.TotalCost.Seconds(), ctx.RequestTraceInfo.FrontendCost.Seconds(), + ctx.RequestTraceInfo.BusCost.Seconds(), ctx.RequestTraceInfo.WorkerCost.Seconds(), readBodyTotal.Seconds(), + writeRspCost.Seconds(), len(ctx.ReqBody), ctx.RequestTraceInfo.CallNode, ctx.RequestTraceInfo.CallInstance, + ctx.RequestTraceInfo.InnerCode, innerCode(ctx.RequestTraceInfo.InnerCode).String()) +} + +func invokeFunction(ctx *types.InvokeProcessContext, funcSpec *commontype.FuncSpec, + req *fasthttp.Request, resp *fasthttp.Response) error { + var ( + instanceAllocationInfo *commontype.InstanceAllocationInfo + err error + ) + for { + instanceAllocationInfo, err = acquireInstance(ctx, funcSpec) + if err != nil { + return err + } + if !functiontask.GetBusProxies().IsBusProxyHealthy(instanceAllocationInfo.NodeIP, ctx.TraceID) { + instanceAllocationInfo, err = selectBusForInstance(ctx) + if err != nil { + return err + } + } + initProxyRequest(req, instanceAllocationInfo, funcSpec) + needBreak, needTry, err := invokeBus(ctx, req, resp, funcSpec) + if err != nil { + instanceleasemanager.GetInstanceManager().ReleaseInstanceAllocation(instanceAllocationInfo, + true, ctx.TraceID) + if needTry { + time.Sleep(retryInterval * time.Millisecond) + continue + } + return err + } + if needBreak { + instanceleasemanager.GetInstanceManager().ReleaseInstanceAllocation(instanceAllocationInfo, + false, ctx.TraceID) + return nil + } + if needTry { + instanceleasemanager.GetInstanceManager().ReleaseInstanceAllocation(instanceAllocationInfo, + false, ctx.TraceID) + time.Sleep(retryInterval * time.Millisecond) + } + } +} + +func selectBusForInstance(ctx *types.InvokeProcessContext) (*commontype.InstanceAllocationInfo, error) { + nodeIP := functiontask.GetBusProxies().NextWithName(ctx.FuncKey, true) + if nodeIP == "" { + log.GetLogger().Errorf("select bus failed not found busProxy,traceID:%s", ctx.TraceID) + return nil, fmt.Errorf("not found busProxy") + } + log.GetLogger().Infof("select bus for function:%s,"+ + "busIP:%s,traceID:%s", ctx.RequestTraceInfo.AnonymizeURN, nodeIP, ctx.TraceID) + return &commontype.InstanceAllocationInfo{ + NodeIP: nodeIP, + NodePort: constant.BusProxyHTTPPort, + }, nil +} + +func prepareRequest(ctx *types.InvokeProcessContext, req *fasthttp.Request, funcSpec *commontype.FuncSpec) { + insResource := getInsResource(funcSpec, req) + setHeader(req, ctx.RequestTraceInfo.URN, insResource) + setRequestDeadline(ctx.RequestTraceInfo, funcSpec) + prepareStreamResponse(ctx, req) +} + +func transferHTTPRequest(ctx *types.InvokeProcessContext, req *fasthttp.Request) { + for key, value := range ctx.ReqHeader { + req.Header.Set(key, value) + } + streamName := ctx.ReqHeader[constant.HeaderRequestStreamName] + streamEvent := ctx.ReqHeader[constant.HeaderStreamAPIGEvent] + if streamName != "" && streamEvent != "" { + body := []byte(streamEvent) + + req.Header.Set(httpconstant.ContentType, httpconstant.ApplicationJSON) + req.Header.Set(constant.HeaderContentLength, strconv.Itoa(len(body))) + req.SetBody(body) + } else { + req.SetBody(ctx.ReqBody) + } +} + +func transferHTTPResponse(ctx *types.InvokeProcessContext, resp *fasthttp.Response) { + ctx.RespHeader[httpconstant.ContentType] = httpconstant.ApplicationJSON + if ctx.RequestTraceInfo.InnerCode == statuscode.HeavyLoadCode { + ctx.RespHeader[constant.HeaderInnerCode] = strconv.Itoa(statuscode.FrontendStatusInternalError) + } else { + ctx.RespHeader[constant.HeaderInnerCode] = string(resp.Header.Peek(constant.HeaderInnerCode)) + } + ctx.RespHeader[constant.HeaderLogResult] = string(resp.Header.Peek(constant.HeaderLogResult)) + ctx.RespHeader[constant.HeaderInvokeSummary] = string(resp.Header.Peek(constant.HeaderInvokeSummary)) + ctx.RespHeader[constant.HeaderBillingDuration] = string(resp.Header.Peek(constant.HeaderBillingDuration)) + ctx.StatusCode = resp.StatusCode() + ctx.RespBody = resp.Body() +} + +func getInsResource(funcSpec *commontype.FuncSpec, req *fasthttp.Request) commontype.InstanceResource { + if funcSpec.FuncMetaData.BusinessType == constant.BusinessTypeCAE { + return commontype.InstanceResource{ + CPU: "0", + Memory: "0", + } + } + insResource := commontype.InstanceResource{ + CPU: string(req.Header.Peek(constant.HeaderCPUSize)), + Memory: string(req.Header.Peek(constant.HeaderMemorySize)), + } + if insResource.CPU == "" && insResource.Memory == "" { + insResource.CPU, insResource.Memory = getCPUAndMemory(&funcSpec.ResourceMetaData) + } + return insResource +} + +func getCPUAndMemory(metaResource *commontype.ResourceMetaData) (string, string) { + return strconv.Itoa(int(metaResource.CPU)), strconv.Itoa(int(metaResource.Memory)) +} + +func setHeader(req *fasthttp.Request, urn string, insResource commontype.InstanceResource) { + req.Header.Set(constant.HeaderInvokeURN, urn) + if req.Header.Peek(constant.HeaderLogType) == nil { + req.Header.Set(constant.HeaderLogType, constant.DefaultLogFlag) + } + req.Header.Set(constant.HeaderCPUSize, insResource.CPU) + req.Header.Set(constant.HeaderMemorySize, insResource.Memory) +} + +func initProxyRequest(proxyReq *fasthttp.Request, instanceInfo *commontype.InstanceAllocationInfo, + funcSpec *commontype.FuncSpec) { + proxyReq.SetRequestURI(invokePath) + setLubanBody(proxyReq) + proxyReq.Header.SetMethod("POST") + proxyReq.SetHost(instanceInfo.NodeIP + ":" + instanceInfo.NodePort) + proxyReq.URI().SetScheme(tls.GetURLScheme(config.GetConfig().HTTPSConfig.HTTPSEnable)) + proxyReq.Header.ResetConnectionClose() + // defaultaz-#- will Spliced by bus + proxyReq.Header.Set(constant.HeaderInstanceID, strings.TrimPrefix(instanceInfo.InstanceID, "defaultaz-#-")) + proxyReq.Header.Set(constant.HeaderInstanceIP, instanceInfo.InstanceIP) + if instanceInfo.CPU != 0 && instanceInfo.Memory != 0 && + funcSpec.FuncMetaData.BusinessType != constant.BusinessTypeCAE { + proxyReq.Header.Set(constant.HeaderCPUSize, strconv.Itoa(int(instanceInfo.CPU))) + proxyReq.Header.Set(constant.HeaderMemorySize, strconv.Itoa(int(instanceInfo.Memory))) + } + httputil.AddAuthorizationHeaderForFG(proxyReq) +} + +func setLubanBody(req *fasthttp.Request) { + lubanID := req.Header.Peek(httpconstant.HeaderLuBanGTraceID) + if len(lubanID) == 0 { + return + } + bodyMap := make(map[string]interface{}, defaultBodyMap) + if err := json.Unmarshal(req.Body(), &bodyMap); err != nil { + return + } + bodyMap[httpconstant.HeaderLuBanNTraceID] = string(req.Header.Peek(httpconstant.HeaderLuBanNTraceID)) + bodyMap[httpconstant.HeaderLuBanGTraceID] = string(lubanID) + bodyMap[httpconstant.HeaderLuBanSpanID] = string(req.Header.Peek(httpconstant.HeaderLuBanSpanID)) + bodyMap[httpconstant.HeaderLuBanEvnID] = string(req.Header.Peek(httpconstant.HeaderLuBanEvnID)) + bodyMap[httpconstant.HeaderLuBanEventID] = string(req.Header.Peek(httpconstant.HeaderLuBanEventID)) + bodyMap[httpconstant.HeaderLuBanDomainID] = string(req.Header.Peek(httpconstant.HeaderLuBanDomainID)) + rsp, err := json.Marshal(bodyMap) + if err != nil { + return + } + req.SetBody(rsp) +} + +func setRequestDeadline(requestTraceInfo *types.RequestTraceInfo, funcSpec *commontype.FuncSpec) { + requestTraceInfo.Deadline = time.Now().Add(getRequestDeadline(funcSpec)) +} + +func prepareStreamResponse(ctx *types.InvokeProcessContext, req *fasthttp.Request) { + if stream.RegisterResponse(ctx) { + req.Header.Set(constant.HeaderResponseStreamName, ctx.StreamInfo.ResponseStreamName) + req.Header.Set(constant.HeaderFrontendResponseStreamName, stream.GetFrontendResponseStreamName()) + } +} + +func getRequestDeadline(funcSpec *commontype.FuncSpec) time.Duration { + var timeout time.Duration + if funcSpec.FuncMetaData.Runtime == constant.CustomContainerRuntimeType { + timeout = time.Duration(config.GetConfig().HTTPConfig.WorkerInstanceReadTimeOut) * time.Second + } else { + funcTimeout := funcSpec.FuncMetaData.Timeout + if funcTimeout > httputil.GetSyncRequestTimeout() { + funcTimeout = httputil.GetSyncRequestTimeout() + } + timeout = time.Duration(config.GetConfig().E2EMaxDelayTime+ + funcSpec.ExtendedMetaData.Initializer.Timeout+funcTimeout) * time.Second + } + return timeout +} + +func getInnerCode(ctx *types.InvokeProcessContext, resp *fasthttp.Response) int { + Code := resp.Header.Peek(constant.HeaderInnerCode) + innerCodeStr, err := strconv.Atoi(string(Code)) + if err != nil { + log.GetLogger().Warnf("failed to parse inner code <%s>, error %s,traceID:%s", + string(Code), err.Error(), ctx.TraceID) + innerCodeStr = 0 + } + return innerCodeStr +} + +// invokeBus return (needBreak, needTry, error) +func invokeBus(ctx *types.InvokeProcessContext, req *fasthttp.Request, + resp *fasthttp.Response, funcSpec *commontype.FuncSpec) (bool, bool, error) { + if ctx.RequestTraceInfo.TryCount >= config.GetConfig().InvokeMaxRetryTimes { + log.GetLogger().Warnf("failed to request bus for %d times,traceID:%s", + config.GetConfig().InvokeMaxRetryTimes, ctx.TraceID) + return false, false, ErrServiceNotAvailable + } + log.GetLogger().Infof("send request to %s,functionURN: %s,traceID:%s", req.Host(), + ctx.RequestTraceInfo.AnonymizeURN, ctx.TraceID) + err := doRequest(ctx, req, resp) + if err != nil && config.GetConfig().RetryConfig != nil && config.GetConfig().RetryConfig.InstanceExceptionRetry { + log.GetLogger().Errorf("retry %d, connection of worker instance %s is not healthy: %s,traceID:%s", + ctx.RequestTraceInfo.TryCount, req.Host(), err.Error(), ctx.TraceID) + if err.Error() == requestTimeout { + return false, false, err + } + ctx.RequestTraceInfo.TryCount++ + req.Header.Set(constant.HeaderRetryFlag, constant.TrueStr) + if ctx.RequestTraceInfo.TryCount >= config.GetConfig().InvokeMaxRetryTimes { + return false, false, err + } + return false, true, nil + } + ctx.RequestTraceInfo.InnerCode = getInnerCode(ctx, resp) + if shouldRetry(ctx.RequestTraceInfo.InnerCode, funcSpec.FuncMetaData.BusinessType) { + log.GetLogger().Warnf("request in proxy %s for function %s failed: %d,retry count: %d,traceID:%s", + req.Host(), ctx.RequestTraceInfo.AnonymizeURN, ctx.RequestTraceInfo.InnerCode, + ctx.RequestTraceInfo.TryCount, ctx.TraceID) + ctx.RequestTraceInfo.TryCount++ + return false, true, nil + } + // instance is not health should release abnormal try + if ctx.RequestTraceInfo.InnerCode == statuscode.SpecificInstanceNotFound { + log.GetLogger().Errorf("function %s request in proxy %s failed should release abnormal,"+ + "response: %d, cost: %s,traceID:%s", ctx.RequestTraceInfo.AnonymizeURN, req.Host(), + ctx.RequestTraceInfo.InnerCode, ctx.RequestTraceInfo.LastBusCost, ctx.TraceID) + return false, true, ErrServiceNotAvailable + } + if ctx.RequestTraceInfo.InnerCode != statuscode.InnerResponseSuccessCode { + log.GetLogger().Errorf("function %s request in proxy %s failed, response: %d, cost: %s,traceID:%s", + ctx.RequestTraceInfo.AnonymizeURN, req.Host(), ctx.RequestTraceInfo.InnerCode, + ctx.RequestTraceInfo.LastBusCost, ctx.TraceID) + } + return true, false, nil +} + +func doRequest(ctx *types.InvokeProcessContext, req *fasthttp.Request, + resp *fasthttp.Response) error { + start := time.Now() + timeout := ctx.RequestTraceInfo.Deadline.Sub(start) + if timeout <= 0 { + log.GetLogger().Errorf("request has reached deadline,traceID:%s", ctx.TraceID) + return errors.New("timeout") + } + err := httputil.GetGlobalClient().DoTimeout(req, resp, timeout) + if err != nil { + resp.Header.Set(constant.HeaderInnerCode, strconv.Itoa(statuscode.FrontendStatusInternalError)) + resp.SetStatusCode(http.StatusInternalServerError) + return err + } + cost := time.Since(start) + ctx.RequestTraceInfo.LastBusCost = cost + ctx.RequestTraceInfo.AllBusCost += cost + log.GetLogger().Debugf("response from %s with http status code %d,traceID:%s", + req.Host(), resp.StatusCode(), ctx.TraceID) + return nil +} + +// If retry is needed, false is returned. Otherwise, true is returned. +func shouldRetry(innerCode int, businessType string) bool { + switch innerCode { + case statuscode.WorkerExitErrCode, statuscode.UserFuncIsUpdatedCode, + statuscode.RefreshSilentFunc, statuscode.ClientExitErrCode, + // bus return + statuscode.BackpressureCode, statuscode.InstanceExceedConcurrency, + // scheduler return + constant.InsAcquireLeaseExistErrorCode: + return true + case statuscode.SendReqErrCode: + if utils.IsCAEFunc(businessType) { + return true + } + default: + return false + } + return false +} + +func acquireInstance(ctx *types.InvokeProcessContext, + funcSpec *commontype.FuncSpec) (*commontype.InstanceAllocationInfo, error) { + defer resetSchedulerProxy(ctx) + resourceSpecs, err := util.ConvertResourceSpecs(ctx, funcSpec) + if err != nil { + return nil, err + } + var instanceAllocationInfo *commontype.InstanceAllocationInfo + var snError snerror.SNError + logger := log.GetLogger().With(zap.Any("traceId", ctx.TraceID), zap.Any("function", ctx.FuncKey)) + for { + scheduler, err := schedulerproxy.Proxy.Get(ctx.FuncKey, logger) + if err != nil || scheduler == nil { + logger.Errorf("failed to get scheduler, error:%s", err.Error()) + responsehandler.SetErrorInContext(ctx, statuscode.FrontendStatusInternalError, err.Error()) + return nil, err + } + acquireOption := util.AcquireOption{ + SchedulerID: scheduler.InstanceName, + SchedulerFuncKey: scheduler.FunctionName, + TraceID: ctx.TraceID, + ResourceSpecs: resourceSpecs, + Timeout: util.GetAcquireTimeout(funcSpec), + FuncSig: funcSpec.FuncMetaSignature, + TrafficLimited: ctx.TrafficLimited, + } + instanceAllocationInfo, snError = instanceleasemanager.GetInstanceManager().AcquireInstanceAllocation( + ctx.FuncKey, "", acquireOption) + if snError != nil { + logger.Errorf("failed to acquire lease, error:%+v", err) + if snError.Code() == constant.AcquireLeaseTrafficLimitErrorCode { + schedulerproxy.Proxy.SetStain(funcSpec.FunctionKey, scheduler.InstanceName) + ctx.TrafficLimited = true + continue + } + if shouldRetry(snError.Code(), funcSpec.FuncMetaData.BusinessType) && + ctx.RequestTraceInfo.TryCount < (config.GetConfig().InvokeMaxRetryTimes-1) { + logger.Warnf("failed to acquire lease for function %s failed: %d, retry count: %d", + ctx.RequestTraceInfo.AnonymizeURN, ctx.RequestTraceInfo.InnerCode, ctx.RequestTraceInfo.TryCount) + ctx.RequestTraceInfo.TryCount++ + time.Sleep(retryInterval * time.Millisecond) + continue + } + if snError.Code() == statuscode.NoInstanceAvailableErrCode && + utils.IsCAEFunc(funcSpec.FuncMetaData.BusinessType) { + instanceAllocationInfo, err = selectBusForInstance(ctx) + if err != nil { + return nil, err + } + logger.Infof("no instance available success to select bus for cae function:%s, busIP:%s", + ctx.RequestTraceInfo.AnonymizeURN, instanceAllocationInfo.NodeIP) + return instanceAllocationInfo, nil + } + return nil, snError + } + logger.Infof("success to acquire lease:%s, instanceID:%s", + instanceAllocationInfo.ThreadID, instanceAllocationInfo.InstanceID) + return instanceAllocationInfo, nil + } +} diff --git a/frontend/pkg/frontend/invocation/function_invoke_for_fg_test.go b/frontend/pkg/frontend/invocation/function_invoke_for_fg_test.go new file mode 100644 index 0000000000000000000000000000000000000000..94932443f203f4ef014a8f86bd9c073596b2e503 --- /dev/null +++ b/frontend/pkg/frontend/invocation/function_invoke_for_fg_test.go @@ -0,0 +1,411 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package invocation + +import ( + "encoding/json" + "errors" + "reflect" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + "github.com/valyala/fasthttp" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/snerror" + "frontend/pkg/common/faas_common/statuscode" + "frontend/pkg/common/faas_common/tls" + commontype "frontend/pkg/common/faas_common/types" + "frontend/pkg/frontend/common/httpconstant" + "frontend/pkg/frontend/common/util" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/functiontask" + "frontend/pkg/frontend/instanceleasemanager" + "frontend/pkg/frontend/responsehandler" + "frontend/pkg/frontend/schedulerproxy" + "frontend/pkg/frontend/stream" + "frontend/pkg/frontend/types" +) + +func TestProcessResp(t *testing.T) { + type testCase struct { + name string + ctx *types.InvokeProcessContext + resp *fasthttp.Response + setupMocks func() *gomonkey.Patches + expectedError bool + } + responsehandler.Handler = (&FGAdapter{}).MakeResponseHandler() + cases := []testCase{ + { + name: "HTTP upload stream with error code", + ctx: &types.InvokeProcessContext{ + TraceID: "trace-2", + IsHTTPUploadStream: true, + StreamInfo: &types.StreamInvokeInfo{ + RequestStreamErrorCode: 1, + }, + RespHeader: make(map[string]string), + RequestTraceInfo: &types.RequestTraceInfo{}, + }, + resp: &fasthttp.Response{}, + expectedError: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + processResp(tc.ctx, tc.resp) + }) + } + + t.Run("response stop channel closed channel", func(t *testing.T) { + stopChan := &types.StreamStopChan{C: make(chan struct{})} + close(stopChan.C) + + ctx := &types.InvokeProcessContext{ + TraceID: "trace-closed-chan", + IsHTTPUploadStream: false, + StreamInfo: &types.StreamInvokeInfo{ + RequestStreamErrorCode: 0, + ResponseStopChan: stopChan, + ResponseStreamName: "test-stream", + }, + RespHeader: make(map[string]string), + RequestTraceInfo: &types.RequestTraceInfo{ + Deadline: time.Now().Add(1 * time.Millisecond), + }, + } + resp := &fasthttp.Response{} + + patch := gomonkey.ApplyFunc(stream.CheckIsResponseStream, func(streamName string) bool { + return true + }) + defer patch.Reset() + + processResp(ctx, resp) + + assert.Equal(t, 0, ctx.RequestTraceInfo.InnerCode, "Expected InnerCode to remain 0") + }) + + t.Run("response stream timeout", func(t *testing.T) { + stopChan := &types.StreamStopChan{C: make(chan struct{})} + + ctx := &types.InvokeProcessContext{ + TraceID: "trace-timeout", + IsHTTPUploadStream: false, + StreamInfo: &types.StreamInvokeInfo{ + RequestStreamErrorCode: 0, + ResponseStopChan: stopChan, + ResponseStreamName: "test-stream", + }, + RespHeader: make(map[string]string), + RequestTraceInfo: &types.RequestTraceInfo{ + Deadline: time.Now().Add(1 * time.Millisecond), + }, + } + resp := &fasthttp.Response{} + + patch := gomonkey.ApplyFunc(stream.CheckIsResponseStream, func(streamName string) bool { + return true + }) + defer patch.Reset() + + var capturedError error + patch2 := gomonkey.ApplyFunc(setErrorResponse, func(invokeCtx *types.InvokeProcessContext, err error) { + capturedError = err + }) + defer patch2.Reset() + + processResp(ctx, resp) + + assert.NotNil(t, capturedError, "Expected error to be set on timeout") + var snerr snerror.SNError + ok := errors.As(capturedError, &snerr) + assert.True(t, ok, "Expected SNError type") + assert.Equal(t, statuscode.InternalErrorCode, snerr.Code(), "Expected internal error code") + assert.Contains(t, snerr.Error(), "wait for response stream timeout", "Expected timeout error message") + }) +} + +func TestAcquireInstance(t *testing.T) { + var patches *gomonkey.Patches + + type testCase struct { + name string + setupMocks func() + expectedResult *commontype.InstanceAllocationInfo + expectedError error + } + + schedulerproxy.Proxy.Add(&commontype.InstanceInfo{InstanceName: "instance1"}, log.GetLogger()) + + cases := []testCase{ + { + name: "No Instance Available", + setupMocks: func() { + patches = gomonkey.ApplyMethod(reflect.TypeOf(instanceleasemanager.GetInstanceManager()), + "AcquireInstanceAllocation", + func(im *instanceleasemanager.Manager, funcKey, version string, + option util.AcquireOption) (*commontype.InstanceAllocationInfo, snerror.SNError) { + return nil, snerror.New(statuscode.NoInstanceAvailableErrCode, "no instance available") + }) + patches.ApplyFunc(time.Sleep, func(d time.Duration) {}) + gomonkey.ApplyMethod(reflect.TypeOf(functiontask.GetBusProxies()), "NextWithName", + func(_ *functiontask.BusProxies, + FuncKey string, move bool) string { + return "192.168.1.1" + }) + }, + expectedResult: &commontype.InstanceAllocationInfo{NodeIP: "192.168.1.1", NodePort: "22423"}, + expectedError: nil, + }, + { + name: "Should retry error", + setupMocks: func() { + patches = gomonkey.ApplyMethod(reflect.TypeOf(instanceleasemanager.GetInstanceManager()), + "AcquireInstanceAllocation", + func(im *instanceleasemanager.Manager, funcKey, version string, + option util.AcquireOption) (*commontype.InstanceAllocationInfo, snerror.SNError) { + return nil, snerror.New(statuscode.SendReqErrCode, "send req error") + }) + patches.ApplyFunc(time.Sleep, func(d time.Duration) {}) + patches.ApplyFunc(selectBusForInstance, + func(ctx *types.InvokeProcessContext) (*commontype.InstanceAllocationInfo, error) { + return &commontype.InstanceAllocationInfo{NodeIP: "192.168.1.1"}, nil + }) + }, + expectedResult: nil, + expectedError: snerror.New(statuscode.SendReqErrCode, "send req error"), + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tc.setupMocks() + defer patches.Reset() + + ctx := &types.InvokeProcessContext{ + FuncKey: "test-func", + TraceID: "test-trace", + RequestTraceInfo: &types.RequestTraceInfo{ + TryCount: -2, + }, + } + funcSpec := &commontype.FuncSpec{ + FuncMetaData: commontype.FuncMetaData{ + BusinessType: constant.BusinessTypeCAE, + }, + } + + result, err := acquireInstance(ctx, funcSpec) + + assert.Equal(t, tc.expectedResult, result) + assert.Equal(t, tc.expectedError, err) + }) + } +} + +func TestInvokeBus(t *testing.T) { + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(resp) + convey.Convey("test invoke bus error", t, func() { + convey.Convey("retry max time", func() { + defer gomonkey.ApplyFunc(config.GetConfig, func() *types.Config { + return &types.Config{ + InvokeMaxRetryTimes: 1, + } + }).Reset() + ctx := &types.InvokeProcessContext{ + RequestTraceInfo: &types.RequestTraceInfo{ + TryCount: 2, + }, + } + _, _, err := invokeBus(ctx, req, resp, &commontype.FuncSpec{}) + convey.So(err, convey.ShouldEqual, ErrServiceNotAvailable) + }) + convey.Convey("retry error timeout", func() { + defer gomonkey.ApplyFunc(config.GetConfig, func() *types.Config { + return &types.Config{ + InvokeMaxRetryTimes: 5, + RetryConfig: &types.RetryConfig{ + InstanceExceptionRetry: true, + }, + HTTPSConfig: &tls.InternalHTTPSConfig{}, + HTTPConfig: &types.FrontendHTTP{}, + } + }).Reset() + c := &fasthttp.Client{} + defer gomonkey.ApplyMethod(reflect.TypeOf(c), + "DoTimeout", func(c *fasthttp.Client, req *fasthttp.Request, + resp *fasthttp.Response, timeout time.Duration) error { + return errors.New(requestTimeout) + }).Reset() + ctx := &types.InvokeProcessContext{ + RequestTraceInfo: &types.RequestTraceInfo{ + Deadline: time.Now().Add(5 * time.Second), + }, + } + _, _, err := invokeBus(ctx, req, resp, &commontype.FuncSpec{}) + convey.So(err.Error(), convey.ShouldEqual, requestTimeout) + }) + convey.Convey("retry count", func() { + defer gomonkey.ApplyFunc(config.GetConfig, func() *types.Config { + return &types.Config{ + InvokeMaxRetryTimes: 1, + RetryConfig: &types.RetryConfig{ + InstanceExceptionRetry: true, + }, + HTTPSConfig: &tls.InternalHTTPSConfig{}, + HTTPConfig: &types.FrontendHTTP{}, + } + }).Reset() + c := &fasthttp.Client{} + defer gomonkey.ApplyMethod(reflect.TypeOf(c), + "DoTimeout", func(c *fasthttp.Client, req *fasthttp.Request, + resp *fasthttp.Response, timeout time.Duration) error { + return errors.New("request failed") + }).Reset() + ctx := &types.InvokeProcessContext{ + RequestTraceInfo: &types.RequestTraceInfo{ + Deadline: time.Now().Add(5 * time.Second), + }, + } + _, _, err := invokeBus(ctx, req, resp, &commontype.FuncSpec{}) + convey.So(err.Error(), convey.ShouldEqual, "request failed") + }) + convey.Convey("should retry", func() { + defer gomonkey.ApplyFunc(config.GetConfig, func() *types.Config { + return &types.Config{ + InvokeMaxRetryTimes: 1, + RetryConfig: &types.RetryConfig{ + InstanceExceptionRetry: true, + }, + HTTPSConfig: &tls.InternalHTTPSConfig{}, + HTTPConfig: &types.FrontendHTTP{}, + } + }).Reset() + c := &fasthttp.Client{} + defer gomonkey.ApplyMethod(reflect.TypeOf(c), + "DoTimeout", func(c *fasthttp.Client, req *fasthttp.Request, + resp *fasthttp.Response, timeout time.Duration) error { + resp.Header.Set(constant.HeaderInnerCode, "150461") + return nil + }).Reset() + ctx := &types.InvokeProcessContext{ + RequestTraceInfo: &types.RequestTraceInfo{ + Deadline: time.Now().Add(5 * time.Second), + }, + } + _, needTry, err := invokeBus(ctx, req, resp, + &commontype.FuncSpec{FuncMetaData: commontype.FuncMetaData{}}) + convey.So(needTry, convey.ShouldBeTrue) + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("should retry and delete", func() { + defer gomonkey.ApplyFunc(config.GetConfig, func() *types.Config { + return &types.Config{ + InvokeMaxRetryTimes: 1, + RetryConfig: &types.RetryConfig{ + InstanceExceptionRetry: true, + }, + HTTPSConfig: &tls.InternalHTTPSConfig{}, + HTTPConfig: &types.FrontendHTTP{}, + } + }).Reset() + c := &fasthttp.Client{} + defer gomonkey.ApplyMethod(reflect.TypeOf(c), + "DoTimeout", func(c *fasthttp.Client, req *fasthttp.Request, + resp *fasthttp.Response, timeout time.Duration) error { + resp.Header.Set(constant.HeaderInnerCode, "150460") + return nil + }).Reset() + ctx := &types.InvokeProcessContext{ + RequestTraceInfo: &types.RequestTraceInfo{ + Deadline: time.Now().Add(5 * time.Second), + }, + } + _, needTry, err := invokeBus(ctx, req, resp, + &commontype.FuncSpec{FuncMetaData: commontype.FuncMetaData{}}) + convey.So(needTry, convey.ShouldBeTrue) + convey.So(err, convey.ShouldNotBeNil) + }) + }) +} + +func TestSetLubanBody(t *testing.T) { + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + req.Header.Set(httpconstant.HeaderLuBanGTraceID, "123") + req.Header.Set(httpconstant.HeaderLuBanNTraceID, "123") + req.Header.Set(httpconstant.HeaderLuBanSpanID, "123") + req.Header.Set(httpconstant.HeaderLuBanEvnID, "123") + req.Header.Set(httpconstant.HeaderLuBanEventID, "123") + req.Header.Set(httpconstant.HeaderLuBanDomainID, "123") + rawData := map[string]string{"aaa": "bbb"} + data, _ := json.Marshal(rawData) + req.SetBody(data) + setLubanBody(req) + bodyMap := make(map[string]interface{}, defaultBodyMap) + _ = json.Unmarshal(req.Body(), &bodyMap) + assert.Equal(t, "123", bodyMap[httpconstant.HeaderLuBanGTraceID]) +} + +func TestPrepareStreamResponse(t *testing.T) { + type testCase struct { + name string + ctx *types.InvokeProcessContext + req *fasthttp.Request + expectedStreamName string + expectedFrontendStreamName string + } + + cases := []testCase{ + { + name: "RegisterResponse returns true", + ctx: &types.InvokeProcessContext{ + StreamInfo: &types.StreamInvokeInfo{ + ResponseStreamName: "responseStreamName", + }, + }, + req: &fasthttp.Request{ + Header: fasthttp.RequestHeader{}, + }, + expectedStreamName: "responseStreamName", + expectedFrontendStreamName: stream.GetFrontendResponseStreamName(), + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + patch := gomonkey.ApplyFunc(stream.RegisterResponse, func(ctx interface{}) bool { + return true + }) + defer patch.Reset() + + prepareStreamResponse(tc.ctx, tc.req) + assert.Equal(t, tc.expectedStreamName, string(tc.req.Header.Peek(constant.HeaderResponseStreamName))) + assert.Equal(t, tc.expectedFrontendStreamName, + string(tc.req.Header.Peek(constant.HeaderFrontendResponseStreamName))) + }) + } +} diff --git a/frontend/pkg/frontend/invocation/function_invoke_for_kernel.go b/frontend/pkg/frontend/invocation/function_invoke_for_kernel.go new file mode 100644 index 0000000000000000000000000000000000000000..538b4295f896ab7d285df3e343f461a3a16ad593 --- /dev/null +++ b/frontend/pkg/frontend/invocation/function_invoke_for_kernel.go @@ -0,0 +1,460 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package invocation - +package invocation + +import ( + "encoding/json" + "fmt" + "math" + "strconv" + "time" + + "go.uber.org/zap" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/resspeckey" + "frontend/pkg/common/faas_common/snerror" + "frontend/pkg/common/faas_common/statuscode" + commontype "frontend/pkg/common/faas_common/types" + "frontend/pkg/common/uuid" + "frontend/pkg/frontend/common/httpconstant" + "frontend/pkg/frontend/common/httputil" + "frontend/pkg/frontend/common/util" + "frontend/pkg/frontend/instancemanager" + "frontend/pkg/frontend/responsehandler" + "frontend/pkg/frontend/schedulerproxy" + "frontend/pkg/frontend/types" + "frontend/pkg/frontend/wisecloud" +) + +func computeTimeout(originTimeout int64, beginTime time.Time) int64 { + costTime := time.Now().Sub(beginTime) + costTimeSecond := int64(math.Trunc(costTime.Seconds())) + return originTimeout - costTimeSecond +} + +type kernelRequestHandler struct { + ctx *types.InvokeProcessContext + funcSpec *commontype.FuncSpec + funcKey string + resSpecKeyStr string + resSpecKey *resspeckey.ResSpecKey + logger api.FormatLogger + + startTime time.Time + invokeWithOutScheduler bool + timeout int64 + + currentSchedulerInfo *commontype.InstanceInfo + unexpectedInstances []string +} + +func newKernelRequestHandler(ctx *types.InvokeProcessContext, funcSpec *commontype.FuncSpec) *kernelRequestHandler { + if ctx.InvokeTimeout == 0 { + ctx.InvokeTimeout = funcSpec.FuncMetaData.Timeout + } + resSpecKey := convertResSpecKey(ctx, funcSpec) + return &kernelRequestHandler{ + ctx: ctx, + funcSpec: funcSpec, + funcKey: funcSpec.FunctionKey, + resSpecKey: &resSpecKey, + resSpecKeyStr: resSpecKey.String(), + logger: log.GetLogger().With(zap.Any("traceId", ctx.TraceID), zap.Any("function", funcSpec.FunctionKey), + zap.Any("timeout", ctx.InvokeTimeout), zap.Any("acquireTimeout", ctx.AcquireTimeout)), + startTime: time.Now(), + invokeWithOutScheduler: ctx.InvokeWithoutScheduler, + unexpectedInstances: make([]string, 0), + timeout: ctx.InvokeTimeout, + } +} + +func (k *kernelRequestHandler) makeReq(logger api.FormatLogger) (*util.InvokeRequest, error) { + var err error + k.currentSchedulerInfo = nil + if !k.invokeWithOutScheduler { + k.currentSchedulerInfo, err = schedulerproxy.Proxy.Get(k.funcKey, logger) + if err != nil { + logger.Warnf("failed to get scheduler, err: %s", err.Error()) + } + } + + var instanceId string + if needDownGrade(k.currentSchedulerInfo) { + k.invokeWithOutScheduler = true // 这里要处理的情况是，当无可用scheduler时，该请求后续都不走租约机制，直接选择实例调用 + instance := instancemanager.GetGlobalInstanceScheduler().GetRandomInstanceWithoutUnexpectedInstance( + k.funcKey, k.resSpecKeyStr, k.unexpectedInstances, logger) + + if instance == nil { + pendingRequest := &wisecloud.PendingRequest{ + CreatedTime: time.Now(), + ScheduleTimeout: time.Duration(k.ctx.AcquireTimeout) * time.Second, + ResultChan: make(chan *wisecloud.PendingResponse, 1), + } + wisecloud.GetQueueManager().AddPendingRequest(k.funcKey, k.resSpecKey, pendingRequest) + pendingResponse := <-pendingRequest.ResultChan + if pendingResponse.Error != nil { + return nil, pendingResponse.Error + } + if pendingResponse.Instance == nil { + return nil, fmt.Errorf("no available instance, no available scheduler") + } + instance = pendingResponse.Instance + } + instanceId = instance.InstanceID + } + + req, err := convert(k.ctx, k.currentSchedulerInfo, k.funcSpec, instanceId) + if err != nil { + logger.Errorf("failed to convert request, err: %s", err.Error()) + return nil, err + } + return req, nil +} + +func (k *kernelRequestHandler) invoke() error { + defer resetSchedulerProxy(k.ctx) + count := 0 + for { + count++ + k.ctx.RequestID = uuid.New().String() + k.ctx.InvokeTimeout = computeTimeout(k.ctx.InvokeTimeout, k.startTime) + if k.ctx.InvokeTimeout <= 0 { + return fmt.Errorf("do invoke failed, timeout") + } + + logger := k.logger.With(zap.Any("requestId", k.ctx.RequestID), zap.Any("timeLeft", k.ctx.InvokeTimeout), + zap.Any("count", count)) + req, err := k.makeReq(logger) + if err != nil { + logger.Errorf("make req failed: %s", err.Error()) + httputil.HandleInvokeError(k.ctx, err) + return err + } + + if k.invokeWithOutScheduler { + wisecloud.GetMetricsManager().InvokeStart(k.funcKey, k.resSpecKeyStr, req.InstanceID) + } + + snError := invokeByClient(k.ctx, *req, logger) + if k.invokeWithOutScheduler { + wisecloud.GetMetricsManager().InvokeEnd(k.funcKey, k.resSpecKeyStr, req.InstanceID) + } + if snError != nil { + if snError.Code() == constant.AcquireLeaseTrafficLimitErrorCode && k.currentSchedulerInfo != nil { + schedulerproxy.Proxy.SetStain(k.funcKey, k.currentSchedulerInfo.InstanceName) + k.ctx.TrafficLimited = true + continue + } else if snError.Code() == statuscode.FrontendStatusInternalError { + logger.Errorf("failed to invoke name by client, err: %s", snError.Error()) + responsehandler.SetErrorInContext(k.ctx, statuscode.FrontendStatusInternalError, snError.Error()) + return snError + } else if snError.Code() == statuscode.ErrAllSchedulerUnavailable { + logger.Warnf("all schedulers are unavailable") + k.invokeWithOutScheduler = true // 这里要处理的情况是，当无可用scheduler时，该请求后续都不走租约机制，直接选择实例调用 + continue + } else if k.invokeWithOutScheduler && invokeInstanceNeedRetry(snError.Code()) { + logger.Warnf("do invokeByInstanceId failed, retry, code: %d, message: %s", + snError.Code(), snError.Error()) + k.unexpectedInstances = append(k.unexpectedInstances, req.InstanceID) + continue + } else { + httputil.HandleInvokeError(k.ctx, snError) + } + } + return nil + } +} + +func invokeInstanceNeedRetry(code int) bool { + // 暂时不考虑区分 同实例重试和不同实例重试的错误码 + needRetryCode := map[int]struct{}{ + statuscode.ErrInstanceNotFound: {}, // 1003 + statuscode.ErrInstanceExitedCode: {}, // 1007 + statuscode.ErrInstanceCircuitCode: {}, // 1009 + statuscode.ErrInstanceEvicted: {}, // 1013 + + // 参考libruntime写法, runtime\src\libruntime\invokeadaptor\task_submitter.cpp + statuscode.ErrRequestBetweenRuntimeBusCode: {}, // 3001 + statuscode.ErrInnerCommunication: {}, // 3002 + statuscode.ErrRequestBetweenRuntimeFrontendCode: {}, // 3008 + + statuscode.ErrSharedMemoryLimited: {}, // 4202 + statuscode.ErrOperateDiskFailed: {}, // 4203 + statuscode.ErrInsufficientDiskSpace: {}, // 4204 + statuscode.ErrFinalized: {}, // 9000 + } + _, ok := needRetryCode[code] + return ok +} + +func getAcquireReqCPUAndMemory(ctx *types.InvokeProcessContext, funcSpec *commontype.FuncSpec) (int64, int64) { + cpu := funcSpec.ResourceMetaData.CPU + memory := funcSpec.ResourceMetaData.Memory + if ctx == nil || ctx.ReqHeader == nil { + return cpu, memory + } + if cpuString := util.PeekIgnoreCase(ctx.ReqHeader, constant.HeaderCPUSize); cpuString != "" { + cpuInt, err := strconv.Atoi(cpuString) + if err != nil || cpuInt <= 0 { + log.GetLogger().Warnf("invalid value %s from request header", constant.HeaderCPUSize) + } else { + cpu = int64(cpuInt) + } + } + + if memoryString := util.PeekIgnoreCase(ctx.ReqHeader, constant.HeaderMemorySize); memoryString != "" { + memoryInt, err := strconv.Atoi(memoryString) + if err != nil || memoryInt <= 0 { + log.GetLogger().Warnf("invalid value %s from request header", constant.ResourceMemoryName) + } else { + memory = int64(memoryInt) + } + } + return cpu, memory +} + +func convertResSpecKey(ctx *types.InvokeProcessContext, funcSpec *commontype.FuncSpec) resspeckey.ResSpecKey { + invokeLabel := util.PeekIgnoreCase(ctx.ReqHeader, httpconstant.HeaderInstanceLabel) + cpu, memory := getAcquireReqCPUAndMemory(ctx, funcSpec) + + resSpec := resspeckey.ConvertResourceMetaDataToResSpec(funcSpec.ResourceMetaData) + resSpec.InvokeLabel = invokeLabel + resSpec.CPU = cpu + resSpec.Memory = memory + + return resspeckey.ConvertToResSpecKey(resSpec) +} + +func needDownGrade(schedulerInfo *commontype.InstanceInfo) bool { + if schedulerInfo == nil || schedulerproxy.Proxy.IsEmpty() { + return true + } + return false +} + +func invokeByClient(ctx *types.InvokeProcessContext, request util.InvokeRequest, + logger api.FormatLogger) snerror.SNError { + logger.Infof("send request %v to grpc", request) + + invokeStart := time.Now() + var notifyMsg []byte + var err error + if request.InstanceID != "" { + notifyMsg, err = util.NewClient().Invoke(request) + } else { + notifyMsg, err = util.NewClient().InvokeByName(request) + } + + invokeTotalTime := time.Since(invokeStart) + logger.Debugf("get response %s, err: %v", string(notifyMsg), err) + + if err != nil { + if rtErr, ok := err.(api.ErrorInfo); ok { + logger.Errorf("invoke request, errCode: %d, error: %s, totalTime: %v", + rtErr.Code, rtErr.Error(), invokeTotalTime.Seconds()) + if snErr := checkErrorMsg(rtErr.Error()); snErr != nil { + return snErr + } + return snerror.New(rtErr.Code, rtErr.Error()) + } + if snError := checkInstanceResp(notifyMsg); snError != nil { + return snError + } + logger.Errorf("invoke GRPC request error: %s, totalTime: %v", err.Error(), invokeTotalTime.Seconds()) + errMsg := fmt.Sprintf("invoke GRPC request error: %s", err.Error()) + httputil.JudgeRetry(err, ctx) + return snerror.New(statuscode.FrontendStatusInternalError, errMsg) + } + respMsg, snErr := responsehandler.SetResponseInContext(ctx, notifyMsg) + if snErr != nil { + return snErr + } + if ctx.RequestTraceInfo != nil { + ctx.RequestTraceInfo.FrontendCost = invokeTotalTime + ctx.RequestTraceInfo.WorkerCost = httputil.GetTimeFromResp(respMsg.UserFuncTime) + } + logger.Infof("invoke end, totalTime: %f, executorTime: %f, userTime: %f", invokeTotalTime.Seconds(), + httputil.GetTimeFromResp(respMsg.ExecutorTime).Seconds(), httputil.GetTimeFromResp(respMsg.UserFuncTime).Seconds()) + return nil +} + +// Convert an http request to a POSIX invoke request +func convert(ctx *types.InvokeProcessContext, schedulerInfo *commontype.InstanceInfo, funcSpec *commontype.FuncSpec, + instanceId string) (*util.InvokeRequest, error) { + resourceSpecs, err := util.ConvertResourceSpecs(ctx, funcSpec) + if err != nil { + return nil, err + } + + req := &util.InvokeRequest{ + Function: ctx.FuncKey, + TraceID: ctx.TraceID, + RequestID: ctx.RequestID, + ReturnObjectIDs: []string{}, + ResourceSpecs: resourceSpecs, + PoolLabel: util.PeekIgnoreCase(ctx.ReqHeader, httpconstant.HeaderPoolLabel), + InvokeTag: convertInvokeTag(ctx), + InstanceLabel: util.PeekIgnoreCase(ctx.ReqHeader, httpconstant.HeaderInstanceLabel), + AcquireTimeout: getTimeout(util.GetAcquireTimeout(funcSpec), ctx.AcquireTimeout), + InvokeTimeout: ctx.InvokeTimeout, + FuncSig: funcSpec.FuncMetaSignature, + TrafficLimited: ctx.TrafficLimited, + BusinessType: funcSpec.FuncMetaData.BusinessType, + TenantID: funcSpec.FuncMetaData.TenantID, + InstanceID: instanceId, + } + + if schedulerInfo != nil { + req.SchedulerID = schedulerInfo.InstanceID + req.SchedulerFuncKey = schedulerInfo.FunctionName + } + + instanceSession := util.PeekIgnoreCase(ctx.ReqHeader, httpconstant.HeaderInstanceSession) + if instanceSession != "" { + session := &commontype.InstanceSessionConfig{} + err = json.Unmarshal([]byte(instanceSession), &session) + if err != nil { + return nil, err + } + req.InstanceSession = session + } + body, err := httputil.TranslateInvokeMsgToCallReq(ctx) + if err != nil { + responsehandler.SetErrorInContext(ctx, statuscode.FrontendStatusInternalError, err.Error()) + return req, err + } + dynamicResourceSpecs, err := prepareDynamicResource(ctx) + if err != nil { + responsehandler.SetErrorInContext(ctx, statuscode.FrontendStatusInternalError, err.Error()) + return req, err + } + if dynamicResourceSpecs[constant.ResourceCPUName] > 0 && dynamicResourceSpecs[constant.ResourceMemoryName] > 0 { + req.ResourceSpecs = dynamicResourceSpecs + } + + req.Args = newArgList([]byte(ctx.TraceID), body) + return req, nil +} + +func getTimeout(funcSpecTimeout int64, ctxTimeout int64) int64 { + if ctxTimeout != 0 { + return ctxTimeout + } + return funcSpecTimeout +} + +func checkErrorMsg(msg string) snerror.SNError { + if len(msg) != 0 { + var errInfo struct { + ErrorCode int `json:"code"` + ErrorMessage string `json:"message"` + } + if unMarshalErr := json.Unmarshal([]byte(msg), &errInfo); unMarshalErr != nil { + log.GetLogger().Debugf("unmarshal notifyMsg error : %s", unMarshalErr.Error()) + return nil + } + if errInfo.ErrorCode != 0 && errInfo.ErrorMessage != "" { + // current faasscheduler has reached instance limit, should retry and chose another faasscheduler + return snerror.New(errInfo.ErrorCode, errInfo.ErrorMessage) + } + } + return nil +} + +func checkInstanceResp(notifyMsg []byte) snerror.SNError { + if notifyMsg != nil && len(notifyMsg) != 0 { + var insResponse struct { + ErrorCode int `json:"errorCode"` + ErrorMessage string `json:"errorMessage"` + } + if unMarshalErr := json.Unmarshal(notifyMsg, &insResponse); unMarshalErr != nil { + log.GetLogger().Errorf("unmarshal notifyMsg error : %s", unMarshalErr.Error()) + } + if insResponse.ErrorCode != 0 && insResponse.ErrorMessage != "" { + // current faasscheduler has reached instance limit, should retry and chose another faasscheduler + return snerror.New(insResponse.ErrorCode, insResponse.ErrorMessage) + } + } + return nil +} + +func newArgList(payloads ...[]byte) []*api.Arg { + var result []*api.Arg + for _, p := range payloads { + result = append(result, &api.Arg{Type: api.Value, Data: p}) + } + return result +} + +func convertInvokeTag(ctx *types.InvokeProcessContext) map[string]string { + m := make(map[string]string) + headerValue, ok := ctx.ReqHeader[httpconstant.HeaderInvokeTag] + if !ok || headerValue == "" { + return m + } + err := json.Unmarshal([]byte(headerValue), &m) + if err != nil { + log.GetLogger().Errorf("convert invoke tag failed, traceId: %s, err: %s", ctx.TraceID, err.Error()) + return m + } + return m +} + +func prepareDynamicResource(ctx *types.InvokeProcessContext) (map[string]int64, error) { + dynamicResourcesRoute := make(map[string]int64) + cpuBytes := ctx.ReqHeader[httpconstant.HeaderCPUSize] + memoryBytes := ctx.ReqHeader[httpconstant.HeaderMemorySize] + customResourcesString := httputil.GetCompatibleHeader(ctx.ReqHeader, constant.HeaderCustomResourceNew, + constant.HeaderCustomResource) + + logger := log.GetLogger().With(zap.Any("traceId", ctx.TraceID), zap.Any("funcKey", ctx.FuncKey)) + if cpuBytes != "" && memoryBytes != "" { + cpu, err := strconv.ParseInt(cpuBytes, baseTen, bitSize) + if err != nil { + return dynamicResourcesRoute, err + } + memory, err := strconv.ParseInt(memoryBytes, baseTen, bitSize) + if err != nil { + return dynamicResourcesRoute, err + } + dynamicResourcesRoute[constant.ResourceCPUName] = cpu + dynamicResourcesRoute[constant.ResourceMemoryName] = memory + } + if customResourcesString != "" { + var customResources map[string]int64 + if err := json.Unmarshal([]byte(customResourcesString), &customResources); err != nil { + logger.Errorf("failed to unmarshal custom resources %s", err.Error()) + return dynamicResourcesRoute, err + } + for resourceType, resource := range customResources { + if resource > constant.MinCustomResourcesSize { + dynamicResourcesRoute[resourceType] = resource + } else { + logger.Warnf("ignore invalid value %f of custom resource %s", resource, resourceType) + } + } + } + if len(dynamicResourcesRoute) != 0 { + logger.Infof("dynamicResourcesRoute is %v", dynamicResourcesRoute) + } + return dynamicResourcesRoute, nil +} diff --git a/frontend/pkg/frontend/invocation/function_invoke_for_kernel_test.go b/frontend/pkg/frontend/invocation/function_invoke_for_kernel_test.go new file mode 100644 index 0000000000000000000000000000000000000000..2b50ed984c681e39a5341d4cc8a62c0e267c8f0a --- /dev/null +++ b/frontend/pkg/frontend/invocation/function_invoke_for_kernel_test.go @@ -0,0 +1,566 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package invocation + +import ( + "fmt" + "reflect" + "strconv" + "strings" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/resspeckey" + "frontend/pkg/common/faas_common/snerror" + "frontend/pkg/common/faas_common/statuscode" + "frontend/pkg/common/faas_common/types" + "frontend/pkg/frontend/common/httpconstant" + "frontend/pkg/frontend/common/util" + "frontend/pkg/frontend/instancemanager" + "frontend/pkg/frontend/schedulerproxy" + types2 "frontend/pkg/frontend/types" + "frontend/pkg/frontend/wisecloud" +) + +func Test_getAcquireReqCPUAndMemory(t *testing.T) { + convey.Convey("Test_getAcquireReqCPUAndMemory", t, func() { + funcSpec := &types.FuncSpec{ + ResourceMetaData: types.ResourceMetaData{ + CPU: 100, + Memory: 100, + }, + } + + ctx := &types2.InvokeProcessContext{} + + cpu, memory := getAcquireReqCPUAndMemory(ctx, funcSpec) + convey.So(cpu, convey.ShouldEqual, 100) + convey.So(memory, convey.ShouldEqual, 100) + + ctx.ReqHeader = make(map[string]string) + ctx.ReqHeader[constant.HeaderCPUSize] = "200" + ctx.ReqHeader[constant.HeaderMemorySize] = "200" + + cpu, memory = getAcquireReqCPUAndMemory(ctx, funcSpec) + convey.So(cpu, convey.ShouldEqual, 200) + convey.So(memory, convey.ShouldEqual, 200) + + ctx.ReqHeader[constant.HeaderCPUSize] = "200dfa" + ctx.ReqHeader[constant.HeaderMemorySize] = "200dafadsf" + cpu, memory = getAcquireReqCPUAndMemory(ctx, funcSpec) + convey.So(cpu, convey.ShouldEqual, 100) + convey.So(memory, convey.ShouldEqual, 100) + }) +} + +func Test_convertResSpecKey(t *testing.T) { + convey.Convey("Test_convertResSpecKey", t, func() { + funcSpec := &types.FuncSpec{ + ResourceMetaData: types.ResourceMetaData{ + CPU: 100, + Memory: 100, + CustomResources: "{}", + CustomResourcesSpec: "{}", + }, + } + + ctx := &types2.InvokeProcessContext{ + ReqHeader: make(map[string]string), + } + + resKey := convertResSpecKey(ctx, funcSpec) + convey.So(resKey.String(), convey.ShouldEqual, "cpu-100-mem-100-storage-0-cstRes--cstResSpec--invokeLabel-") + + ctx.ReqHeader[constant.HeaderCPUSize] = "200" + ctx.ReqHeader[constant.HeaderMemorySize] = "200" + + resKey = convertResSpecKey(ctx, funcSpec) + convey.So(resKey.String(), convey.ShouldEqual, "cpu-200-mem-200-storage-0-cstRes--cstResSpec--invokeLabel-") + + ctx.ReqHeader[httpconstant.HeaderInstanceLabel] = "labeltest" + funcSpec.ResourceMetaData.CustomResourcesSpec = "crspec000" + funcSpec.ResourceMetaData.CustomResources = "cr000" + funcSpec.ResourceMetaData.EphemeralStorage = 321 + resKey = convertResSpecKey(ctx, funcSpec) + convey.So(resKey.String(), convey.ShouldEqual, "cpu-200-mem-200-storage-321-cstRes--cstResSpec--invokeLabel-labeltest") + }) +} + +func clearSchedulerProxy() { + defer schedulerproxy.Proxy.Reset() + for { + schedulerInfo, err := schedulerproxy.Proxy.Get("0/0/0", log.GetLogger()) + if err != nil { + return + } + schedulerproxy.Proxy.Remove(schedulerInfo, log.GetLogger()) + } +} + +func mockSchedulerProxyAdd(id string) { + schedulerproxy.Proxy.Add(&types.InstanceInfo{ + TenantID: id, + FunctionName: id, + Version: id, + InstanceName: id, + InstanceID: id, + Address: id, + }, log.GetLogger()) +} + +func mockSchedulerProxyRemove(id string) { + schedulerproxy.Proxy.Remove(&types.InstanceInfo{ + TenantID: id, + FunctionName: id, + Version: id, + InstanceName: id, + InstanceID: id, + Address: id, + }, log.GetLogger()) +} + +func mockFunctionInstanceAdd(instanceId string) { + event := &etcd3.Event{} + event.Key = "/sn/instance/business/yrk/tenant/12345678901234561234567890123456/function/0-system-faasExecutorGo1.x/version/$latest/defaultaz/787b900780b2d80600/" + instanceId + event.Value = []byte("{\"instanceID\":\"" + instanceId + "\",\"requestID\":\"787b900780b2d80600\",\"runtimeID\":\"runtime-5f000000-0000-4000-824c-75b4b7dae0a3-0000000074dd\",\"runtimeAddress\":\"127.0.0.1:32568\",\"functionAgentID\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt\",\"functionProxyID\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt\",\"function\":\"12345678901234561234567890123456/0-system-faasExecutorGo1.x/$latest\",\"resources\":{\"resources\":{\"CPU\":{\"name\":\"CPU\",\"scalar\":{\"value\":500}},\"Memory\":{\"name\":\"Memory\",\"scalar\":{\"value\":500}}}},\"scheduleOption\":{\"schedPolicyName\":\"monopoly\",\"affinity\":{\"instanceAffinity\":{},\"resource\":{},\"instance\":{\"scope\":\"NODE\"}},\"initCallTimeOut\":305,\"resourceSelector\":{\"resource.owner\":\"1c50bc05-0000-4000-8000-00ed778a549c\"},\"extension\":{\"schedule_policy\":\"monopoly\"},\"range\":{},\"scheduleTimeoutMs\":\"5000\"},\"createOptions\":{\"INSTANCE_LABEL_NOTE\":\"\",\"DELEGATE_DECRYPT\":\"{\\\"accessKey\\\":\\\"\\\",\\\"authToken\\\":\\\"\\\",\\\"cryptoAlgorithm\\\":\\\"\\\",\\\"encrypted_user_data\\\":\\\"\\\",\\\"envKey\\\":\\\"\\\",\\\"environment\\\":\\\"\\\",\\\"secretKey\\\":\\\"\\\",\\\"securityAk\\\":\\\"\\\",\\\"securitySk\\\":\\\"\\\",\\\"securityToken\\\":\\\"\\\"}\",\"lifecycle\":\"detached\",\"resource.owner\":\"static_function\",\"FUNCTION_KEY_NOTE\":\"8d86c63b22e24d9ab650878b75408ea6/0@default@func6ac6741a01334320809dfb7dc1e98049/latest\",\"ConcurrentNum\":\"1000\",\"tenantId\":\"8d86c63b22e24d9ab650878b75408ea6\",\"INSTANCE_TYPE_NOTE\":\"reserved\",\"init_call_timeout\":\"305\",\"call_timeout\":\"60\",\"RESOURCE_SPEC_NOTE\":\"{\\\"cpu\\\":500,\\\"invokeLabels\\\":\\\"\\\",\\\"memory\\\":500}\",\"DELEGATE_DIRECTORY_QUOTA\":\"512\",\"GRACEFUL_SHUTDOWN_TIME\":\"900\",\"DELEGATE_DIRECTORY_INFO\":\"/tmp\"},\"instanceStatus\":{\"code\":3,\"msg\":\"running\"},\"schedulerChain\":[\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt\"],\"parentID\":\"static_function\",\"parentFunctionProxyAID\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt-LocalSchedInstanceCtrlActor@10.158.98.238:22423\",\"storageType\":\"local\",\"scheduleTimes\":1,\"deployTimes\":1,\"args\":[{\"value\":\"EkdAAVpDMTIzNDU2Nzg5MDEyMzQ1NjEyMzQ1Njc4OTAxMjM0NTYvMC1zeXN0ZW0tZmFhc0V4ZWN1dG9yR28xLngvJGxhdGVzdBplEgASBy9pbnZva2UYAiD///////////8BKGQwAUJHCAMSQzEyMzQ1Njc4OTAxMjM0NTYxMjM0NTY3ODkwMTIzNDU2LzAtc3lzdGVtLWZhYXNFeGVjdXRvckdvMS54LyRsYXRlc3Q=\"},{\"value\":\"AAAAAAAAAAAAAAAAAAAAAHsiZnVuY01ldGFEYXRhIjp7Im5hbWUiOiIwQGRlZmF1bHRAZnVuYzZhYzY3NDFhMDEzMzQzMjA4MDlkZmI3ZGMxZTk4MDQ5IiwiZnVuY3Rpb25Vcm4iOiJzbjpjbjp5cms6OGQ4NmM2M2IyMmUyNGQ5YWI2NTA4NzhiNzU0MDhlYTY6ZnVuY3Rpb246MEBkZWZhdWx0QGZ1bmM2YWM2NzQxYTAxMzM0MzIwODA5ZGZiN2RjMWU5ODA0OSIsImZ1bmN0aW9uVmVyc2lvblVybiI6InNuOmNuOnlyazo4ZDg2YzYzYjIyZTI0ZDlhYjY1MDg3OGI3NTQwOGVhNjpmdW5jdGlvbjowQGRlZmF1bHRAZnVuYzZhYzY3NDFhMDEzMzQzMjA4MDlkZmI3ZGMxZTk4MDQ5OmxhdGVzdCIsInZlcnNpb24iOiJsYXRlc3QiLCJmdW5jdGlvblVwZGF0ZVRpbWUiOiIyMDI1LTA2LTIzIDIzOjQ0OjIyLjAwMCIsInJ1bnRpbWUiOiJjdXN0b20gaW1hZ2UiLCJoYW5kbGVyIjoiL2ludm9rZSIsInRpbWVvdXQiOjYwLCJzZXJ2aWNlIjoiZGVmYXVsdCIsInRlbmFudElkIjoiOGQ4NmM2M2IyMmUyNGQ5YWI2NTA4NzhiNzU0MDhlYTYiLCJidXNpbmVzc0lkIjoieXJrIiwicmV2aXNpb25JZCI6IjIwMjUwNjIzMTU0NDIyMDEyIiwiZnVuY19uYW1lIjoiZnVuYzZhYzY3NDFhMDEzMzQzMjA4MDlkZmI3ZGMxZTk4MDQ5In0sImVudk1ldGFEYXRhIjp7ImVudmlyb25tZW50IjoiIn0sImluc3RhbmNlTWV0YURhdGEiOnsibWF4SW5zdGFuY2UiOjEwMCwibWluSW5zdGFuY2UiOjEsImNvbmN1cnJlbnROdW0iOjEwMDAsInNjYWxlUG9saWN5Ijoic3RhdGljRnVuY3Rpb24ifSwicmVzb3VyY2VNZXRhRGF0YSI6eyJjcHUiOjUwMCwibWVtb3J5Ijo1MDB9LCJjb2RlTWV0YURhdGEiOnsic3RvcmFnZV90eXBlIjoiIn0sImV4dGVuZGVkTWV0YURhdGEiOnsiaW5pdGlhbGl6ZXIiOnsiaW5pdGlhbGl6ZXJfdGltZW91dCI6MzAwLCJpbml0aWFsaXplcl9oYW5kbGVyIjoiIn0sImN1c3RvbV9jb250YWluZXJfY29uZmlnIjp7ImltYWdlIjoic3dyLmNuLXNvdXRod2VzdC0yLm15aHVhd2VpY2xvdWQuY29tL3dpc2VmdW5jdGlvbi9jdXN0b20taW1hZ2U6MS4xLjEzLjIwMjUwNTA2MTczNDEyIn19fQ==\"},{\"value\":\"AAAAAAAAAAAAAAAAAAAAAHsiY2FsbFJvdXRlIjoiaW52b2tlIiwicG9ydCI6ODAwMH0=\"},{\"value\":\"AAAAAAAAAAAAAAAAAAAAAHsic2NoZWR1bGVyRnVuY0tleSI6IiIsInNjaGVkdWxlcklETGlzdCI6W119\"},{\"value\":\"AAAAAAAAAAAAAAAAAAAAAHt9\"},{\"value\":\"AAAAAAAAAAAAAAAAAAAAAHt9\"}],\"version\":\"3\",\"dataSystemHost\":\"10.158.97.96\",\"gracefulShutdownTime\":\"600\",\"tenantID\":\"8d86c63b22e24d9ab650878b75408ea6\",\"extensions\":{\"receivedTimestamp\":\"1750782213307\",\"podDeploymentName\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz\",\"podName\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt\",\"pid\":\"71\",\"podNamespace\":\"wisefunctionservice-495f57a3-09ee-44d2-87e5-a109cda4dc40\",\"createTimestamp\":\"1750782213\",\"updateTimestamp\":\"1750782231\"},\"unitID\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt\"}") + + instancemanager.ProcessInstanceUpdate(event) +} +func mockFunctionInstanceRemove(instanceId string) { + event := &etcd3.Event{} + event.Key = "/sn/instance/business/yrk/tenant/12345678901234561234567890123456/function/0-system-faasExecutorGo1.x/version/$latest/defaultaz/787b900780b2d80600/" + instanceId + event.PrevValue = []byte("{\"instanceID\":\"" + instanceId + "\",\"requestID\":\"787b900780b2d80600\",\"runtimeID\":\"runtime-5f000000-0000-4000-824c-75b4b7dae0a3-0000000074dd\",\"runtimeAddress\":\"127.0.0.1:32568\",\"functionAgentID\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt\",\"functionProxyID\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt\",\"function\":\"12345678901234561234567890123456/0-system-faasExecutorGo1.x/$latest\",\"resources\":{\"resources\":{\"CPU\":{\"name\":\"CPU\",\"scalar\":{\"value\":500}},\"Memory\":{\"name\":\"Memory\",\"scalar\":{\"value\":500}}}},\"scheduleOption\":{\"schedPolicyName\":\"monopoly\",\"affinity\":{\"instanceAffinity\":{},\"resource\":{},\"instance\":{\"scope\":\"NODE\"}},\"initCallTimeOut\":305,\"resourceSelector\":{\"resource.owner\":\"1c50bc05-0000-4000-8000-00ed778a549c\"},\"extension\":{\"schedule_policy\":\"monopoly\"},\"range\":{},\"scheduleTimeoutMs\":\"5000\"},\"createOptions\":{\"INSTANCE_LABEL_NOTE\":\"\",\"DELEGATE_DECRYPT\":\"{\\\"accessKey\\\":\\\"\\\",\\\"authToken\\\":\\\"\\\",\\\"cryptoAlgorithm\\\":\\\"\\\",\\\"encrypted_user_data\\\":\\\"\\\",\\\"envKey\\\":\\\"\\\",\\\"environment\\\":\\\"\\\",\\\"secretKey\\\":\\\"\\\",\\\"securityAk\\\":\\\"\\\",\\\"securitySk\\\":\\\"\\\",\\\"securityToken\\\":\\\"\\\"}\",\"lifecycle\":\"detached\",\"resource.owner\":\"static_function\",\"FUNCTION_KEY_NOTE\":\"8d86c63b22e24d9ab650878b75408ea6/0@default@func6ac6741a01334320809dfb7dc1e98049/latest\",\"ConcurrentNum\":\"1000\",\"tenantId\":\"8d86c63b22e24d9ab650878b75408ea6\",\"INSTANCE_TYPE_NOTE\":\"reserved\",\"init_call_timeout\":\"305\",\"call_timeout\":\"60\",\"RESOURCE_SPEC_NOTE\":\"{\\\"cpu\\\":500,\\\"invokeLabels\\\":\\\"\\\",\\\"memory\\\":500}\",\"DELEGATE_DIRECTORY_QUOTA\":\"512\",\"GRACEFUL_SHUTDOWN_TIME\":\"900\",\"DELEGATE_DIRECTORY_INFO\":\"/tmp\"},\"instanceStatus\":{\"code\":3,\"msg\":\"running\"},\"schedulerChain\":[\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt\"],\"parentID\":\"static_function\",\"parentFunctionProxyAID\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt-LocalSchedInstanceCtrlActor@10.158.98.238:22423\",\"storageType\":\"local\",\"scheduleTimes\":1,\"deployTimes\":1,\"args\":[{\"value\":\"EkdAAVpDMTIzNDU2Nzg5MDEyMzQ1NjEyMzQ1Njc4OTAxMjM0NTYvMC1zeXN0ZW0tZmFhc0V4ZWN1dG9yR28xLngvJGxhdGVzdBplEgASBy9pbnZva2UYAiD///////////8BKGQwAUJHCAMSQzEyMzQ1Njc4OTAxMjM0NTYxMjM0NTY3ODkwMTIzNDU2LzAtc3lzdGVtLWZhYXNFeGVjdXRvckdvMS54LyRsYXRlc3Q=\"},{\"value\":\"AAAAAAAAAAAAAAAAAAAAAHsiZnVuY01ldGFEYXRhIjp7Im5hbWUiOiIwQGRlZmF1bHRAZnVuYzZhYzY3NDFhMDEzMzQzMjA4MDlkZmI3ZGMxZTk4MDQ5IiwiZnVuY3Rpb25Vcm4iOiJzbjpjbjp5cms6OGQ4NmM2M2IyMmUyNGQ5YWI2NTA4NzhiNzU0MDhlYTY6ZnVuY3Rpb246MEBkZWZhdWx0QGZ1bmM2YWM2NzQxYTAxMzM0MzIwODA5ZGZiN2RjMWU5ODA0OSIsImZ1bmN0aW9uVmVyc2lvblVybiI6InNuOmNuOnlyazo4ZDg2YzYzYjIyZTI0ZDlhYjY1MDg3OGI3NTQwOGVhNjpmdW5jdGlvbjowQGRlZmF1bHRAZnVuYzZhYzY3NDFhMDEzMzQzMjA4MDlkZmI3ZGMxZTk4MDQ5OmxhdGVzdCIsInZlcnNpb24iOiJsYXRlc3QiLCJmdW5jdGlvblVwZGF0ZVRpbWUiOiIyMDI1LTA2LTIzIDIzOjQ0OjIyLjAwMCIsInJ1bnRpbWUiOiJjdXN0b20gaW1hZ2UiLCJoYW5kbGVyIjoiL2ludm9rZSIsInRpbWVvdXQiOjYwLCJzZXJ2aWNlIjoiZGVmYXVsdCIsInRlbmFudElkIjoiOGQ4NmM2M2IyMmUyNGQ5YWI2NTA4NzhiNzU0MDhlYTYiLCJidXNpbmVzc0lkIjoieXJrIiwicmV2aXNpb25JZCI6IjIwMjUwNjIzMTU0NDIyMDEyIiwiZnVuY19uYW1lIjoiZnVuYzZhYzY3NDFhMDEzMzQzMjA4MDlkZmI3ZGMxZTk4MDQ5In0sImVudk1ldGFEYXRhIjp7ImVudmlyb25tZW50IjoiIn0sImluc3RhbmNlTWV0YURhdGEiOnsibWF4SW5zdGFuY2UiOjEwMCwibWluSW5zdGFuY2UiOjEsImNvbmN1cnJlbnROdW0iOjEwMDAsInNjYWxlUG9saWN5Ijoic3RhdGljRnVuY3Rpb24ifSwicmVzb3VyY2VNZXRhRGF0YSI6eyJjcHUiOjUwMCwibWVtb3J5Ijo1MDB9LCJjb2RlTWV0YURhdGEiOnsic3RvcmFnZV90eXBlIjoiIn0sImV4dGVuZGVkTWV0YURhdGEiOnsiaW5pdGlhbGl6ZXIiOnsiaW5pdGlhbGl6ZXJfdGltZW91dCI6MzAwLCJpbml0aWFsaXplcl9oYW5kbGVyIjoiIn0sImN1c3RvbV9jb250YWluZXJfY29uZmlnIjp7ImltYWdlIjoic3dyLmNuLXNvdXRod2VzdC0yLm15aHVhd2VpY2xvdWQuY29tL3dpc2VmdW5jdGlvbi9jdXN0b20taW1hZ2U6MS4xLjEzLjIwMjUwNTA2MTczNDEyIn19fQ==\"},{\"value\":\"AAAAAAAAAAAAAAAAAAAAAHsiY2FsbFJvdXRlIjoiaW52b2tlIiwicG9ydCI6ODAwMH0=\"},{\"value\":\"AAAAAAAAAAAAAAAAAAAAAHsic2NoZWR1bGVyRnVuY0tleSI6IiIsInNjaGVkdWxlcklETGlzdCI6W119\"},{\"value\":\"AAAAAAAAAAAAAAAAAAAAAHt9\"},{\"value\":\"AAAAAAAAAAAAAAAAAAAAAHt9\"}],\"version\":\"3\",\"dataSystemHost\":\"10.158.97.96\",\"gracefulShutdownTime\":\"600\",\"tenantID\":\"8d86c63b22e24d9ab650878b75408ea6\",\"extensions\":{\"receivedTimestamp\":\"1750782213307\",\"podDeploymentName\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz\",\"podName\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt\",\"pid\":\"71\",\"podNamespace\":\"wisefunctionservice-495f57a3-09ee-44d2-87e5-a109cda4dc40\",\"createTimestamp\":\"1750782213\",\"updateTimestamp\":\"1750782231\"},\"unitID\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt\"}") + + instancemanager.ProcessInstanceDelete(event) +} + +func mockSchedulerInstanceAdd(key string) { + event := &etcd3.Event{} + event.Key = fmt.Sprintf("/sn/instance/business/yrk/tenant/0/function/0-system-faasscheduler/version/$latest/defaultaz//%s", key) + event.Value = []byte("{\"instanceID\":\"5f000000-0000-4000-824c-75b4b7dae0a3\",\"requestID\":\"787b900780b2d80600\",\"runtimeID\":\"runtime-5f000000-0000-4000-824c-75b4b7dae0a3-0000000074dd\",\"runtimeAddress\":\"127.0.0.1:32568\",\"functionAgentID\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt\",\"functionProxyID\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt\",\"function\":\"12345678901234561234567890123456/0-system-faasExecutorGo1.x/$latest\",\"resources\":{\"resources\":{\"CPU\":{\"name\":\"CPU\",\"scalar\":{\"value\":500}},\"Memory\":{\"name\":\"Memory\",\"scalar\":{\"value\":500}}}},\"scheduleOption\":{\"schedPolicyName\":\"monopoly\",\"affinity\":{\"instanceAffinity\":{},\"resource\":{},\"instance\":{\"scope\":\"NODE\"}},\"initCallTimeOut\":305,\"resourceSelector\":{\"resource.owner\":\"1c50bc05-0000-4000-8000-00ed778a549c\"},\"extension\":{\"schedule_policy\":\"monopoly\"},\"range\":{},\"scheduleTimeoutMs\":\"5000\"},\"createOptions\":{\"INSTANCE_LABEL_NOTE\":\"\",\"DELEGATE_DECRYPT\":\"{\\\"accessKey\\\":\\\"\\\",\\\"authToken\\\":\\\"\\\",\\\"cryptoAlgorithm\\\":\\\"\\\",\\\"encrypted_user_data\\\":\\\"\\\",\\\"envKey\\\":\\\"\\\",\\\"environment\\\":\\\"\\\",\\\"secretKey\\\":\\\"\\\",\\\"securityAk\\\":\\\"\\\",\\\"securitySk\\\":\\\"\\\",\\\"securityToken\\\":\\\"\\\"}\",\"lifecycle\":\"detached\",\"resource.owner\":\"static_function\",\"FUNCTION_KEY_NOTE\":\"8d86c63b22e24d9ab650878b75408ea6/0@default@func6ac6741a01334320809dfb7dc1e98049/latest\",\"ConcurrentNum\":\"1000\",\"tenantId\":\"8d86c63b22e24d9ab650878b75408ea6\",\"INSTANCE_TYPE_NOTE\":\"reserved\",\"init_call_timeout\":\"305\",\"call_timeout\":\"60\",\"RESOURCE_SPEC_NOTE\":\"{\\\"cpu\\\":500,\\\"invokeLabels\\\":\\\"\\\",\\\"memory\\\":500}\",\"DELEGATE_DIRECTORY_QUOTA\":\"512\",\"GRACEFUL_SHUTDOWN_TIME\":\"900\",\"DELEGATE_DIRECTORY_INFO\":\"/tmp\"},\"instanceStatus\":{\"code\":3,\"msg\":\"running\"},\"schedulerChain\":[\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt\"],\"parentID\":\"static_function\",\"parentFunctionProxyAID\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt-LocalSchedInstanceCtrlActor@10.158.98.238:22423\",\"storageType\":\"local\",\"scheduleTimes\":1,\"deployTimes\":1,\"args\":[{\"value\":\"EkdAAVpDMTIzNDU2Nzg5MDEyMzQ1NjEyMzQ1Njc4OTAxMjM0NTYvMC1zeXN0ZW0tZmFhc0V4ZWN1dG9yR28xLngvJGxhdGVzdBplEgASBy9pbnZva2UYAiD///////////8BKGQwAUJHCAMSQzEyMzQ1Njc4OTAxMjM0NTYxMjM0NTY3ODkwMTIzNDU2LzAtc3lzdGVtLWZhYXNFeGVjdXRvckdvMS54LyRsYXRlc3Q=\"},{\"value\":\"AAAAAAAAAAAAAAAAAAAAAHsiZnVuY01ldGFEYXRhIjp7Im5hbWUiOiIwQGRlZmF1bHRAZnVuYzZhYzY3NDFhMDEzMzQzMjA4MDlkZmI3ZGMxZTk4MDQ5IiwiZnVuY3Rpb25Vcm4iOiJzbjpjbjp5cms6OGQ4NmM2M2IyMmUyNGQ5YWI2NTA4NzhiNzU0MDhlYTY6ZnVuY3Rpb246MEBkZWZhdWx0QGZ1bmM2YWM2NzQxYTAxMzM0MzIwODA5ZGZiN2RjMWU5ODA0OSIsImZ1bmN0aW9uVmVyc2lvblVybiI6InNuOmNuOnlyazo4ZDg2YzYzYjIyZTI0ZDlhYjY1MDg3OGI3NTQwOGVhNjpmdW5jdGlvbjowQGRlZmF1bHRAZnVuYzZhYzY3NDFhMDEzMzQzMjA4MDlkZmI3ZGMxZTk4MDQ5OmxhdGVzdCIsInZlcnNpb24iOiJsYXRlc3QiLCJmdW5jdGlvblVwZGF0ZVRpbWUiOiIyMDI1LTA2LTIzIDIzOjQ0OjIyLjAwMCIsInJ1bnRpbWUiOiJjdXN0b20gaW1hZ2UiLCJoYW5kbGVyIjoiL2ludm9rZSIsInRpbWVvdXQiOjYwLCJzZXJ2aWNlIjoiZGVmYXVsdCIsInRlbmFudElkIjoiOGQ4NmM2M2IyMmUyNGQ5YWI2NTA4NzhiNzU0MDhlYTYiLCJidXNpbmVzc0lkIjoieXJrIiwicmV2aXNpb25JZCI6IjIwMjUwNjIzMTU0NDIyMDEyIiwiZnVuY19uYW1lIjoiZnVuYzZhYzY3NDFhMDEzMzQzMjA4MDlkZmI3ZGMxZTk4MDQ5In0sImVudk1ldGFEYXRhIjp7ImVudmlyb25tZW50IjoiIn0sImluc3RhbmNlTWV0YURhdGEiOnsibWF4SW5zdGFuY2UiOjEwMCwibWluSW5zdGFuY2UiOjEsImNvbmN1cnJlbnROdW0iOjEwMDAsInNjYWxlUG9saWN5Ijoic3RhdGljRnVuY3Rpb24ifSwicmVzb3VyY2VNZXRhRGF0YSI6eyJjcHUiOjUwMCwibWVtb3J5Ijo1MDB9LCJjb2RlTWV0YURhdGEiOnsic3RvcmFnZV90eXBlIjoiIn0sImV4dGVuZGVkTWV0YURhdGEiOnsiaW5pdGlhbGl6ZXIiOnsiaW5pdGlhbGl6ZXJfdGltZW91dCI6MzAwLCJpbml0aWFsaXplcl9oYW5kbGVyIjoiIn0sImN1c3RvbV9jb250YWluZXJfY29uZmlnIjp7ImltYWdlIjoic3dyLmNuLXNvdXRod2VzdC0yLm15aHVhd2VpY2xvdWQuY29tL3dpc2VmdW5jdGlvbi9jdXN0b20taW1hZ2U6MS4xLjEzLjIwMjUwNTA2MTczNDEyIn19fQ==\"},{\"value\":\"AAAAAAAAAAAAAAAAAAAAAHsiY2FsbFJvdXRlIjoiaW52b2tlIiwicG9ydCI6ODAwMH0=\"},{\"value\":\"AAAAAAAAAAAAAAAAAAAAAHsic2NoZWR1bGVyRnVuY0tleSI6IiIsInNjaGVkdWxlcklETGlzdCI6W119\"},{\"value\":\"AAAAAAAAAAAAAAAAAAAAAHt9\"},{\"value\":\"AAAAAAAAAAAAAAAAAAAAAHt9\"}],\"version\":\"3\",\"dataSystemHost\":\"10.158.97.96\",\"gracefulShutdownTime\":\"600\",\"tenantID\":\"8d86c63b22e24d9ab650878b75408ea6\",\"extensions\":{\"receivedTimestamp\":\"1750782213307\",\"podDeploymentName\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz\",\"podName\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt\",\"pid\":\"71\",\"podNamespace\":\"wisefunctionservice-495f57a3-09ee-44d2-87e5-a109cda4dc40\",\"createTimestamp\":\"1750782213\",\"updateTimestamp\":\"1750782231\"},\"unitID\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt\"}") + + instancemanager.ProcessInstanceUpdate(event) +} + +func mockSchedulerInstanceRemove(key string) { + event := &etcd3.Event{} + event.Key = fmt.Sprintf("/sn/instance/business/yrk/tenant/0/function/0-system-faasscheduler/version/$latest/defaultaz//%s", key) + event.PrevValue = []byte("{\"instanceID\":\"5f000000-0000-4000-824c-75b4b7dae0a3\",\"requestID\":\"787b900780b2d80600\",\"runtimeID\":\"runtime-5f000000-0000-4000-824c-75b4b7dae0a3-0000000074dd\",\"runtimeAddress\":\"127.0.0.1:32568\",\"functionAgentID\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt\",\"functionProxyID\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt\",\"function\":\"12345678901234561234567890123456/0-system-faasExecutorGo1.x/$latest\",\"resources\":{\"resources\":{\"CPU\":{\"name\":\"CPU\",\"scalar\":{\"value\":500}},\"Memory\":{\"name\":\"Memory\",\"scalar\":{\"value\":500}}}},\"scheduleOption\":{\"schedPolicyName\":\"monopoly\",\"affinity\":{\"instanceAffinity\":{},\"resource\":{},\"instance\":{\"scope\":\"NODE\"}},\"initCallTimeOut\":305,\"resourceSelector\":{\"resource.owner\":\"1c50bc05-0000-4000-8000-00ed778a549c\"},\"extension\":{\"schedule_policy\":\"monopoly\"},\"range\":{},\"scheduleTimeoutMs\":\"5000\"},\"createOptions\":{\"INSTANCE_LABEL_NOTE\":\"\",\"DELEGATE_DECRYPT\":\"{\\\"accessKey\\\":\\\"\\\",\\\"authToken\\\":\\\"\\\",\\\"cryptoAlgorithm\\\":\\\"\\\",\\\"encrypted_user_data\\\":\\\"\\\",\\\"envKey\\\":\\\"\\\",\\\"environment\\\":\\\"\\\",\\\"secretKey\\\":\\\"\\\",\\\"securityAk\\\":\\\"\\\",\\\"securitySk\\\":\\\"\\\",\\\"securityToken\\\":\\\"\\\"}\",\"lifecycle\":\"detached\",\"resource.owner\":\"static_function\",\"FUNCTION_KEY_NOTE\":\"8d86c63b22e24d9ab650878b75408ea6/0@default@func6ac6741a01334320809dfb7dc1e98049/latest\",\"ConcurrentNum\":\"1000\",\"tenantId\":\"8d86c63b22e24d9ab650878b75408ea6\",\"INSTANCE_TYPE_NOTE\":\"reserved\",\"init_call_timeout\":\"305\",\"call_timeout\":\"60\",\"RESOURCE_SPEC_NOTE\":\"{\\\"cpu\\\":500,\\\"invokeLabels\\\":\\\"\\\",\\\"memory\\\":500}\",\"DELEGATE_DIRECTORY_QUOTA\":\"512\",\"GRACEFUL_SHUTDOWN_TIME\":\"900\",\"DELEGATE_DIRECTORY_INFO\":\"/tmp\"},\"instanceStatus\":{\"code\":3,\"msg\":\"running\"},\"schedulerChain\":[\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt\"],\"parentID\":\"static_function\",\"parentFunctionProxyAID\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt-LocalSchedInstanceCtrlActor@10.158.98.238:22423\",\"storageType\":\"local\",\"scheduleTimes\":1,\"deployTimes\":1,\"args\":[{\"value\":\"EkdAAVpDMTIzNDU2Nzg5MDEyMzQ1NjEyMzQ1Njc4OTAxMjM0NTYvMC1zeXN0ZW0tZmFhc0V4ZWN1dG9yR28xLngvJGxhdGVzdBplEgASBy9pbnZva2UYAiD///////////8BKGQwAUJHCAMSQzEyMzQ1Njc4OTAxMjM0NTYxMjM0NTY3ODkwMTIzNDU2LzAtc3lzdGVtLWZhYXNFeGVjdXRvckdvMS54LyRsYXRlc3Q=\"},{\"value\":\"AAAAAAAAAAAAAAAAAAAAAHsiZnVuY01ldGFEYXRhIjp7Im5hbWUiOiIwQGRlZmF1bHRAZnVuYzZhYzY3NDFhMDEzMzQzMjA4MDlkZmI3ZGMxZTk4MDQ5IiwiZnVuY3Rpb25Vcm4iOiJzbjpjbjp5cms6OGQ4NmM2M2IyMmUyNGQ5YWI2NTA4NzhiNzU0MDhlYTY6ZnVuY3Rpb246MEBkZWZhdWx0QGZ1bmM2YWM2NzQxYTAxMzM0MzIwODA5ZGZiN2RjMWU5ODA0OSIsImZ1bmN0aW9uVmVyc2lvblVybiI6InNuOmNuOnlyazo4ZDg2YzYzYjIyZTI0ZDlhYjY1MDg3OGI3NTQwOGVhNjpmdW5jdGlvbjowQGRlZmF1bHRAZnVuYzZhYzY3NDFhMDEzMzQzMjA4MDlkZmI3ZGMxZTk4MDQ5OmxhdGVzdCIsInZlcnNpb24iOiJsYXRlc3QiLCJmdW5jdGlvblVwZGF0ZVRpbWUiOiIyMDI1LTA2LTIzIDIzOjQ0OjIyLjAwMCIsInJ1bnRpbWUiOiJjdXN0b20gaW1hZ2UiLCJoYW5kbGVyIjoiL2ludm9rZSIsInRpbWVvdXQiOjYwLCJzZXJ2aWNlIjoiZGVmYXVsdCIsInRlbmFudElkIjoiOGQ4NmM2M2IyMmUyNGQ5YWI2NTA4NzhiNzU0MDhlYTYiLCJidXNpbmVzc0lkIjoieXJrIiwicmV2aXNpb25JZCI6IjIwMjUwNjIzMTU0NDIyMDEyIiwiZnVuY19uYW1lIjoiZnVuYzZhYzY3NDFhMDEzMzQzMjA4MDlkZmI3ZGMxZTk4MDQ5In0sImVudk1ldGFEYXRhIjp7ImVudmlyb25tZW50IjoiIn0sImluc3RhbmNlTWV0YURhdGEiOnsibWF4SW5zdGFuY2UiOjEwMCwibWluSW5zdGFuY2UiOjEsImNvbmN1cnJlbnROdW0iOjEwMDAsInNjYWxlUG9saWN5Ijoic3RhdGljRnVuY3Rpb24ifSwicmVzb3VyY2VNZXRhRGF0YSI6eyJjcHUiOjUwMCwibWVtb3J5Ijo1MDB9LCJjb2RlTWV0YURhdGEiOnsic3RvcmFnZV90eXBlIjoiIn0sImV4dGVuZGVkTWV0YURhdGEiOnsiaW5pdGlhbGl6ZXIiOnsiaW5pdGlhbGl6ZXJfdGltZW91dCI6MzAwLCJpbml0aWFsaXplcl9oYW5kbGVyIjoiIn0sImN1c3RvbV9jb250YWluZXJfY29uZmlnIjp7ImltYWdlIjoic3dyLmNuLXNvdXRod2VzdC0yLm15aHVhd2VpY2xvdWQuY29tL3dpc2VmdW5jdGlvbi9jdXN0b20taW1hZ2U6MS4xLjEzLjIwMjUwNTA2MTczNDEyIn19fQ==\"},{\"value\":\"AAAAAAAAAAAAAAAAAAAAAHsiY2FsbFJvdXRlIjoiaW52b2tlIiwicG9ydCI6ODAwMH0=\"},{\"value\":\"AAAAAAAAAAAAAAAAAAAAAHsic2NoZWR1bGVyRnVuY0tleSI6IiIsInNjaGVkdWxlcklETGlzdCI6W119\"},{\"value\":\"AAAAAAAAAAAAAAAAAAAAAHt9\"},{\"value\":\"AAAAAAAAAAAAAAAAAAAAAHt9\"}],\"version\":\"3\",\"dataSystemHost\":\"10.158.97.96\",\"gracefulShutdownTime\":\"600\",\"tenantID\":\"8d86c63b22e24d9ab650878b75408ea6\",\"extensions\":{\"receivedTimestamp\":\"1750782213307\",\"podDeploymentName\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz\",\"podName\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt\",\"pid\":\"71\",\"podNamespace\":\"wisefunctionservice-495f57a3-09ee-44d2-87e5-a109cda4dc40\",\"createTimestamp\":\"1750782213\",\"updateTimestamp\":\"1750782231\"},\"unitID\":\"func6ac6741a01334320809dfb7dc1e98049-latest-yrhjz-pqjvt\"}") + + instancemanager.ProcessInstanceDelete(event) +} + +func Test_needDownGrade(t *testing.T) { + convey.Convey("Test_needDownGrade", t, func() { + clearSchedulerProxy() + instancemanager.GetFaaSSchedulerInstanceManager().Reset() + defer clearSchedulerProxy() + defer instancemanager.GetFaaSSchedulerInstanceManager().Reset() + + schedulerInfo := &types.InstanceInfo{} + convey.So(needDownGrade(schedulerInfo), convey.ShouldBeTrue) + + mockSchedulerProxyAdd("0") + mockSchedulerInstanceAdd("1") + convey.So(needDownGrade(schedulerInfo), convey.ShouldBeTrue) + + convey.So(needDownGrade(nil), convey.ShouldBeTrue) + + mockSchedulerInstanceAdd("0") + convey.So(needDownGrade(schedulerInfo), convey.ShouldBeFalse) + mockSchedulerProxyRemove("0") + mockSchedulerInstanceRemove("0") + mockSchedulerInstanceRemove("1") + }) +} + +type fakeClient struct { +} + +func (f *fakeClient) AcquireInstance(functionKey string, req util.AcquireOption) (*types.InstanceAllocationInfo, error) { + //TODO implement me + panic("implement me") +} + +func (f *fakeClient) ReleaseInstance(allocation *types.InstanceAllocationInfo, abnormal bool) { + //TODO implement me + panic("implement me") +} + +func (f *fakeClient) Invoke(req util.InvokeRequest) ([]byte, error) { + //TODO implement me + panic("implement me") +} + +func (f *fakeClient) CreateInstanceRaw(createReq []byte) ([]byte, error) { + return nil, nil +} +func (f *fakeClient) InvokeInstanceRaw(invokeReq []byte) ([]byte, error) { + return nil, nil +} +func (f *fakeClient) KillRaw(killReq []byte) ([]byte, error) { + return nil, nil +} +func (c *fakeClient) CreateInstanceByLibRt(funcMeta api.FunctionMeta, args []api.Arg, invokeOpt api.InvokeOptions) (instanceID string, err error) { + InstanceID := "" + return InstanceID, nil +} +func (c *fakeClient) KillByLibRt(instanceID string, signal int, payload []byte) error { + return nil +} + +// InvokeByName copy from faasinvoker_test.go +func (f *fakeClient) InvokeByName(request util.InvokeRequest) ([]byte, error) { + return nil, nil +} + +func (f *fakeClient) IsHealth() bool { + return true +} + +func (f *fakeClient) IsDsHealth() bool { + return true +} + +func Test_invokeByClient(t *testing.T) { + convey.Convey("Test_invokeByClient", t, func() { + c := &fakeClient{} + defer gomonkey.ApplyFunc(util.NewClient, func() util.Client { + return c + }).Reset() + + invokeTrigger := false + invoekInstance := "" + defer gomonkey.ApplyMethod(reflect.TypeOf(c), "Invoke", func(_ *fakeClient, req util.InvokeRequest) ([]byte, error) { + invokeTrigger = true + invoekInstance = req.InstanceID + return nil, fmt.Errorf("") + }).Reset() + + invokeByNameTrigger := false + defer gomonkey.ApplyMethod(reflect.TypeOf(c), "InvokeByName", func(_ *fakeClient, req util.InvokeRequest) ([]byte, error) { + invokeByNameTrigger = true + return nil, fmt.Errorf("") + }).Reset() + + ctx := &types2.InvokeProcessContext{ + InvokeTimeout: 10, + } + req := util.InvokeRequest{ + InstanceID: "0", + } + + invokeByClient(ctx, req, log.GetLogger()) + convey.So(invokeTrigger, convey.ShouldBeTrue) + convey.So(invoekInstance, convey.ShouldEqual, "0") + convey.So(invokeByNameTrigger, convey.ShouldBeFalse) + + invokeTrigger = false + req.InstanceID = "" + invokeByClient(ctx, req, log.GetLogger()) + convey.So(invokeTrigger, convey.ShouldBeFalse) + convey.So(invokeByNameTrigger, convey.ShouldBeTrue) + }) +} + +func Test_functionInvokeForKernel(t *testing.T) { + convey.Convey("Test_functionInvokeForKernel", t, func() { + ctx := &types2.InvokeProcessContext{ + InvokeTimeout: 10, + RespHeader: make(map[string]string), + } + funcSpec := &types.FuncSpec{} + funcSpec.ResourceMetaData.CPU = 500 + funcSpec.ResourceMetaData.Memory = 500 + + clearSchedulerProxy() + instancemanager.GetFaaSSchedulerInstanceManager().Reset() + defer clearSchedulerProxy() + defer instancemanager.GetFaaSSchedulerInstanceManager().Reset() + defer gomonkey.ApplyMethodFunc(wisecloud.GetQueueManager(), "AddPendingRequest", func(funcKey string, resSpec *resspeckey.ResSpecKey, pendingReq *wisecloud.PendingRequest) { + log.GetLogger().Infof("[zhouyu] debug show addpending request") + pendingReq.ResultChan <- &wisecloud.PendingResponse{Instance: nil} + }).Reset() + + err := newKernelRequestHandler(ctx, funcSpec).invoke() + convey.So(strings.Contains(err.Error(), "no available instance, no available scheduler"), convey.ShouldBeTrue) + + mockSchedulerProxyAdd("0") + mockSchedulerInstanceAdd("0") + var getreq util.InvokeRequest + defer gomonkey.ApplyFunc(invokeByClient, func(_ *types2.InvokeProcessContext, req util.InvokeRequest) snerror.SNError { + getreq = req + return nil + }).Reset() + + p := gomonkey.ApplyFunc(needDownGrade, func() bool { + return false + }) + newKernelRequestHandler(ctx, funcSpec).invoke() + p.Reset() + convey.So(getreq.InstanceID, convey.ShouldEqual, "") + getreq.SchedulerID = "" + defer gomonkey.ApplyFunc(needDownGrade, func() bool { + return true + }).Reset() + + mockFunctionInstanceAdd("111") + funcSpec.FunctionKey = "8d86c63b22e24d9ab650878b75408ea6/0@default@func6ac6741a01334320809dfb7dc1e98049/latest" + newKernelRequestHandler(ctx, funcSpec).invoke() + convey.So(getreq.InstanceID, convey.ShouldEqual, "111") + convey.So(getreq.SchedulerID, convey.ShouldEqual, "0") // hash环上有节点，但是，没有scheduler实例 + }) +} + +func Test_functionInvokeForKernel_retry(t *testing.T) { + convey.Convey("Test_functionInvokeForKernel_retry", t, func() { + clearSchedulerProxy() + instancemanager.GetFaaSSchedulerInstanceManager().Reset() + defer clearSchedulerProxy() + defer instancemanager.GetFaaSSchedulerInstanceManager().Reset() + + ctx := &types2.InvokeProcessContext{ + InvokeTimeout: 10, + } + funcSpec := &types.FuncSpec{} + funcSpec.ResourceMetaData.CPU = 500 + funcSpec.ResourceMetaData.Memory = 500 + funcSpec.FunctionKey = "8d86c63b22e24d9ab650878b75408ea6/0@default@func6ac6741a01334320809dfb7dc1e98049/latest" + ctx.InvokeWithoutScheduler = true + mockFunctionInstanceAdd("111") + + mockSchedulerProxyAdd("0") + var getreq util.InvokeRequest + p := gomonkey.ApplyFunc(invokeByClient, func(_ *types2.InvokeProcessContext, req util.InvokeRequest) snerror.SNError { + getreq = req + return nil + }) + metricsInvokeStartFlag := false + metricsInvokeStartInstance := "" + defer gomonkey.ApplyMethodFunc(reflect.TypeOf(wisecloud.GetMetricsManager()), "InvokeStart", func(funcKey string, resSpecKeyStr string, instanceId string) { + metricsInvokeStartFlag = true + metricsInvokeStartInstance = instanceId + }).Reset() + metricsInvokeEndFlag := false + metricsInvokeEndInstance := "" + defer gomonkey.ApplyMethodFunc(reflect.TypeOf(wisecloud.GetMetricsManager()), "InvokeEnd", func(funcKey string, resSpecKeyStr string, instanceId string) { + metricsInvokeEndFlag = true + metricsInvokeEndInstance = instanceId + }).Reset() + newKernelRequestHandler(ctx, funcSpec).invoke() + convey.So(getreq.InstanceID, convey.ShouldEqual, "111") + convey.So(metricsInvokeStartFlag, convey.ShouldBeTrue) + convey.So(metricsInvokeStartInstance, convey.ShouldEqual, "111") + convey.So(metricsInvokeEndFlag, convey.ShouldBeTrue) + convey.So(metricsInvokeEndInstance, convey.ShouldEqual, "111") + p.Reset() + + // 比较复杂的用例 + // 构造只有一个scheduler + // 首先，调用的请求，是走租约体系，则其schedulerId不为空，且instanceId为空。然后我们构造返回9009 + // 然后，重试调用请求，是走降级，则其schedulerId为空，且instanceId不为空。然后我们构造1003 + // 最后，再次重试调用请求，走降级，则其schedulerId为空，且instanceId不为空且不是上一次重试的instanceId。然后我们构造成功。 + times := 0 + var getreq1 util.InvokeRequest + var getreq2 util.InvokeRequest + var getreq3 util.InvokeRequest + mockSchedulerInstanceAdd("0") + mockFunctionInstanceAdd("222") + defer mockFunctionInstanceRemove("222") + p = gomonkey.ApplyFunc(invokeByClient, func(_ *types2.InvokeProcessContext, req util.InvokeRequest) snerror.SNError { + times++ + if times == 1 { + getreq1 = req + time.Sleep(1*time.Second + 200*time.Millisecond) + return snerror.New(statuscode.ErrAllSchedulerUnavailable, "") + } + if times == 2 { + getreq2 = req + return snerror.New(statuscode.ErrInstanceExitedCode, "") + } + if times == 3 { + getreq3 = req + } + return nil + }) + ctx.InvokeWithoutScheduler = false + newKernelRequestHandler(ctx, funcSpec).invoke() + convey.So(times, convey.ShouldEqual, 3) + convey.So(getreq1.SchedulerID, convey.ShouldEqual, "0") + convey.So(getreq2.SchedulerID, convey.ShouldBeEmpty) + convey.So([]string{"111", "222"}, convey.ShouldContain, getreq2.InstanceID) + convey.So(getreq3.SchedulerID, convey.ShouldBeEmpty) + convey.So([]string{"111", "222"}, convey.ShouldContain, getreq3.InstanceID) + convey.So(getreq2.InstanceID, convey.ShouldNotEqual, getreq3.InstanceID) + convey.So(getreq3.InvokeTimeout, convey.ShouldBeLessThan, 10) + + ctx.InvokeTimeout = 1 + times = 0 + err := newKernelRequestHandler(ctx, funcSpec).invoke() + convey.So(strings.Contains(err.Error(), "do invoke failed, timeout"), convey.ShouldBeTrue) + convey.So(times, convey.ShouldEqual, 1) + p.Reset() + }) +} + +func TestInvokeInstanceNeedRetry(t *testing.T) { + convey.Convey("Test invokeInstanceNeedRetry function", t, func() { + successStatusCodes := []int{ + statuscode.DsDeleteFailed, + statuscode.DsDownloadFailed, + statuscode.DsKeyNotFound, + } + + retryErrorCodes := []int{ + statuscode.ErrInstanceNotFound, + statuscode.ErrInstanceExitedCode, + statuscode.ErrInstanceCircuitCode, + statuscode.ErrInstanceEvicted, + statuscode.ErrRequestBetweenRuntimeBusCode, + statuscode.ErrInnerCommunication, + statuscode.ErrSharedMemoryLimited, + statuscode.ErrOperateDiskFailed, + statuscode.ErrInsufficientDiskSpace, + statuscode.ErrFinalized, + } + + convey.Convey("When passing retry-required error codes", func() { + for _, errCode := range retryErrorCodes { + convey.Convey("Should return true for error code "+strconv.Itoa(errCode), func() { + convey.So(invokeInstanceNeedRetry(errCode), convey.ShouldBeTrue) + }) + } + }) + + convey.Convey("When passing non-retry status codes", func() { + for _, statusCode := range successStatusCodes { + convey.Convey("Should return false for status code "+strconv.Itoa(statusCode), func() { + convey.So(invokeInstanceNeedRetry(statusCode), convey.ShouldBeFalse) + }) + } + }) + + convey.Convey("When passing undefined status codes", func() { + undefinedCodes := []int{9999, -1, 10000} + for _, unknownCode := range undefinedCodes { + convey.Convey("Should return false for unknown code "+strconv.Itoa(unknownCode), func() { + convey.So(invokeInstanceNeedRetry(unknownCode), convey.ShouldBeFalse) + }) + } + }) + }) +} + +func TestKernelRequestHandler_makeReq(t *testing.T) { + convey.Convey("Test makeReq method", t, func() { + ctx := &types2.InvokeProcessContext{ + InvokeTimeout: 10, + } + funcSpec := &types.FuncSpec{} + funcSpec.ResourceMetaData.CPU = 500 + funcSpec.ResourceMetaData.Memory = 500 + funcSpec.FunctionKey = "8d86c63b22e24d9ab650878b75408ea6/0@default@func6ac6741a01334320809dfb7dc1e98049/latest" + ctx.InvokeWithoutScheduler = true + + handler := newKernelRequestHandler(ctx, funcSpec) + + // Mock补丁集合 + var patches *gomonkey.Patches + + convey.Convey("降级情况 - 无scheduler但能获取实例", func() { + clearSchedulerProxy() + mockFunctionInstanceAdd("111") + defer mockFunctionInstanceRemove("111") + req, err := handler.makeReq(log.GetLogger()) + + convey.So(err, convey.ShouldBeNil) + convey.So(req.InstanceID, convey.ShouldEqual, "111") + convey.So(handler.invokeWithOutScheduler, convey.ShouldBeTrue) + }) + + convey.Convey("降级情况 - 需要排队获取实例", func() { + patches = gomonkey.NewPatches() + defer patches.Reset() + + clearSchedulerProxy() + + // Mock 排队返回结果 + mockResponse := &wisecloud.PendingResponse{ + Instance: &types.InstanceSpecification{InstanceID: "222"}, + Error: nil, + } + patches.ApplyMethodFunc(wisecloud.GetQueueManager(), "AddPendingRequest", func(_ string, _ *resspeckey.ResSpecKey, req *wisecloud.PendingRequest) { + req.ResultChan <- mockResponse + }) + + // Mock convert 函数 + patches.ApplyFunc(convert, func(_ *types2.InvokeProcessContext, schedulerInfo *types.InstanceInfo, _ *types.FuncSpec, instanceId string) (*util.InvokeRequest, error) { + return &util.InvokeRequest{InstanceID: instanceId}, nil + }) + + req, err := handler.makeReq(log.GetLogger()) + + convey.So(err, convey.ShouldBeNil) + convey.So(req.InstanceID, convey.ShouldEqual, "222") + convey.So(handler.invokeWithOutScheduler, convey.ShouldBeTrue) + }) + + convey.Convey("异常情况 - 排队获取实例失败", func() { + patches = gomonkey.NewPatches() + defer patches.Reset() + + clearSchedulerProxy() + + // Mock 排队返回错误 + patches.ApplyMethodFunc(wisecloud.GetQueueManager(), "AddPendingRequest", func(_ string, _ *resspeckey.ResSpecKey, req *wisecloud.PendingRequest) { + req.ResultChan <- &wisecloud.PendingResponse{ + Error: fmt.Errorf("queue error"), + } + }) + + req, err := handler.makeReq(log.GetLogger()) + + convey.So(err, convey.ShouldNotBeNil) + convey.So(req, convey.ShouldBeNil) + }) + }) +} + +func TestCheckErrorMsg(t *testing.T) { + convey.Convey("test checkErrorMsg", t, func() { + convey.Convey("msg is nil", func() { + convey.So(checkErrorMsg(""), convey.ShouldBeNil) + }) + convey.Convey("msg format err", func() { + convey.So(checkErrorMsg("dddsaas"), convey.ShouldBeNil) + }) + convey.Convey("msg with correct format", func() { + msg := `{ "code": 123, "message": "123 msg"}` + err := checkErrorMsg(msg) + convey.So(err.Code(), convey.ShouldEqual, 123) + convey.So(err.Error(), convey.ShouldEqual, "123 msg") + }) + }) +} diff --git a/frontend/pkg/frontend/log/healthlog/healthlog.go b/frontend/pkg/frontend/log/healthlog/healthlog.go new file mode 100644 index 0000000000000000000000000000000000000000..bcf88fc66a9ea6a504dfb37688be901fb1c3de35 --- /dev/null +++ b/frontend/pkg/frontend/log/healthlog/healthlog.go @@ -0,0 +1,46 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package healthlog is for printing health logs +package healthlog + +import ( + "time" + + "frontend/pkg/common/faas_common/logger/log" +) + +const logInterval = 5 * time.Minute + +// PrintHealthLog prints timing health logs of components +func PrintHealthLog(stopCh <-chan struct{}, inputLog func(), name string) { + if stopCh == nil { + log.GetLogger().Errorf("stop channel is nil") + return + } + ticker := time.NewTicker(logInterval) + defer ticker.Stop() + time.After(logInterval) + for { + select { + case <-ticker.C: + inputLog() + case <-stopCh: + log.GetLogger().Warnf("%s receives a terminating signal", name) + return + } + } +} diff --git a/frontend/pkg/frontend/log/healthlog/healthlog_test.go b/frontend/pkg/frontend/log/healthlog/healthlog_test.go new file mode 100644 index 0000000000000000000000000000000000000000..bca8a005cbc008f13f09ccf113115093d4cb0258 --- /dev/null +++ b/frontend/pkg/frontend/log/healthlog/healthlog_test.go @@ -0,0 +1,65 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package healthlog + +import "testing" + +func TestPrintHealthLog(t *testing.T) { + type args struct { + stopCh chan struct{} + inputLog func() + name string + } + var a args + a.stopCh = nil + a.inputLog = func() { + return + } + + tests := []struct { + name string + args args + }{ + { + name: "case1", + args: args{ + stopCh: nil, + inputLog: func() { + return + }, + }, + }, + { + name: "case2", + args: args{ + stopCh: make(chan struct{}), + inputLog: func() { + return + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.args.stopCh != nil { + close(tt.args.stopCh) + } + PrintHealthLog(tt.args.stopCh, tt.args.inputLog, tt.args.name) + }) + } +} diff --git a/frontend/pkg/frontend/log/interface.go b/frontend/pkg/frontend/log/interface.go new file mode 100644 index 0000000000000000000000000000000000000000..9440cdc0848102a876e4fce9ed1508be3db03216 --- /dev/null +++ b/frontend/pkg/frontend/log/interface.go @@ -0,0 +1,64 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package log for frontend interface log +package log + +import ( + "sync" + + "frontend/pkg/common/faas_common/logger" + "frontend/pkg/common/faas_common/logger/config" + "frontend/pkg/common/faas_common/logger/log" +) + +var ( + interfaceLog *logger.InterfaceLogger + once sync.Once +) + +// CreateInterLogger Create Inter Logger +func CreateInterLogger(logPath string) error { + cfg := logger.InterfaceEncoderConfig{ModuleName: "frontend"} + + var err error + if interfaceLog, err = logger.NewInterfaceLogger(logPath, "frontend-interface", cfg); err != nil { + return err + } + return nil +} + +// Write Write log String +func Write(msg string) { + coreInfo, _ := config.GetCoreInfoFromEnv() + var err error + once.Do(func() { + if interfaceLog == nil { + if err = CreateInterLogger(coreInfo.FilePath); err != nil { + log.GetLogger().Errorf("failed to create interface logger with error %s", err.Error()) + } + } + }) + if err != nil { + return + } + + if interfaceLog == nil { + return + } + + interfaceLog.Write(msg) +} diff --git a/frontend/pkg/frontend/metrics/metrics.go b/frontend/pkg/frontend/metrics/metrics.go new file mode 100644 index 0000000000000000000000000000000000000000..a1117623503691f94c38ff1a95556672b65f180e --- /dev/null +++ b/frontend/pkg/frontend/metrics/metrics.go @@ -0,0 +1,235 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package metrics report to redis for the monitor market +package metrics + +import ( + "encoding/json" + "fmt" + "os" + "sync" + "time" + + "frontend/pkg/common/faas_common/localauth" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/redisclient" + "frontend/pkg/frontend/config" +) + +const ( + reportTickerTime = 1 * time.Minute + metricsKeyTTL = 60 * time.Minute + + redisRetryTimes = 3 + redisRetryInterval = 100 * time.Millisecond + + bufferSize = 10000 + + listLimit = 3 + + logLimitFrequency = 1000000 +) + +var ( + metricsChan = make(chan RequestStatistics, bufferSize) + + requestInfoRedisKey = fmt.Sprintf("/sn/metrics/requestinfo/%s/%s", + os.Getenv("CLUSTER_ID"), os.Getenv("POD_NAME")) + + statistics = &totalStatistics{} + + reachMaxChanCount = 0 +) + +// RequestStatistics - +type RequestStatistics struct { + ErrorFlag bool + TotalDelay float64 + FrontendDelay float64 + BusDelay float64 + RuntimeDelay float64 +} + +// totalStatistics - +type totalStatistics struct { + lock sync.RWMutex + count int64 + errCount int64 + totalDelay float64 + frontendDelay float64 + busDelay float64 + runtimeDelay float64 +} + +// RequestInfoMetrics request info metrics +type RequestInfoMetrics struct { + TimeStamp int64 `json:"timeStamp,omitempty"` + Count int64 `json:"count,omitempty"` + ErrCount int64 `json:"errCount,omitempty"` + TotalDelay float64 `json:"totalDelay,omitempty"` + FrontendDelay float64 `json:"frontendDelay,omitempty"` + BusDelay float64 `json:"busDelay,omitempty"` + RuntimeDelay float64 `json:"runtimeDelay,omitempty"` +} + +// PublishMetrics - +func PublishMetrics(metrics RequestStatistics) { + if len(metricsChan) < bufferSize { + metricsChan <- metrics + return + } + if reachMaxChanCount%logLimitFrequency == 0 { + reachMaxChanCount = 1 + log.GetLogger().Warnf("metricsChan reaches capacity and will discard the metric statistics :%v", metrics) + return + } + reachMaxChanCount++ +} + +// subscribeMetrics - +func subscribeMetrics(stopChan <-chan struct{}) { + if stopChan == nil { + log.GetLogger().Warnf("stopChan is nil") + return + } + for { + select { + case msg, ok := <-metricsChan: + if !ok { + log.GetLogger().Errorf("metrics channel is closed") + return + } + statistics.lock.Lock() + statistics.count = statistics.count + 1 + if msg.ErrorFlag { + statistics.errCount = statistics.errCount + 1 + } + statistics.totalDelay += msg.TotalDelay + statistics.frontendDelay += msg.FrontendDelay + statistics.busDelay += msg.BusDelay + statistics.runtimeDelay += msg.RuntimeDelay + statistics.lock.Unlock() + case <-stopChan: + log.GetLogger().Warnf("Received signal to quit") + return + } + } +} + +// StartReportMetrics - +func StartReportMetrics(stopChan <-chan struct{}) { + if stopChan == nil { + log.GetLogger().Warnf("stopChan is nil") + return + } + err := initRedisClient(stopChan) + if err != nil { + log.GetLogger().Errorf("failed to new redis client and skip to report metrics, err: %s", err.Error()) + return + } + log.GetLogger().Infof("start to report metrics to server") + go subscribeMetrics(stopChan) + reportTick := time.NewTicker(reportTickerTime) + for { + select { + case <-reportTick.C: + if statistics.count <= 0 { + log.GetLogger().Infof("no request info and skip reporting the metrics") + continue + } + reportRequestInfoMetrics(requestInfoRedisKey, resetMetrics(), metricsKeyTTL) + case _, ok := <-stopChan: + if !ok { + reportTick.Stop() + log.GetLogger().Infof("stop report metrics to redis server") + return + } + } + } +} + +func resetMetrics() []byte { + now := time.Now() + statistics.lock.Lock() + metrics := RequestInfoMetrics{ + TimeStamp: now.Unix() - int64(now.Second()), + Count: statistics.count, + ErrCount: statistics.errCount, + TotalDelay: statistics.totalDelay / float64(statistics.count), + FrontendDelay: statistics.frontendDelay / float64(statistics.count), + BusDelay: statistics.busDelay / float64(statistics.count), + RuntimeDelay: statistics.runtimeDelay / float64(statistics.count), + } + statistics.count = 0 + statistics.errCount = 0 + statistics.totalDelay = 0 + statistics.frontendDelay = 0 + statistics.busDelay = 0 + statistics.runtimeDelay = 0 + statistics.lock.Unlock() + value, err := json.Marshal(metrics) + if err != nil { + log.GetLogger().Errorf("failed to marshal requestInfo metrics, err: %s", err.Error()) + } + return value +} + +func initRedisClient(stopCh <-chan struct{}) error { + var err error + var pwd []byte + pwd, err = localauth.Decrypt(config.GetConfig().RedisConfig.Password) + if err != nil { + log.GetLogger().Errorf("failed to decrypt redis password, %s", err.Error()) + return err + } + redisConf := redisclient.NewRedisClientParam{ + ServerMode: config.GetConfig().RedisConfig.ServerMode, + ServerAddr: config.GetConfig().RedisConfig.ServerAddr, + Password: string(pwd), + Timeout: config.GetConfig().RedisConfig.TimeoutConf, + } + redisCmd, err := redisclient.New(redisclient.NewRedisClientParam{ + ServerMode: config.GetConfig().RedisConfig.ServerMode, + ServerAddr: config.GetConfig().RedisConfig.ServerAddr, + Password: string(pwd), + Timeout: config.GetConfig().RedisConfig.TimeoutConf, + }, stopCh, redisclient.SetEnableTLS(config.GetConfig().RedisConfig.EnableTLS)) + if err != nil { + log.GetLogger().Errorf("failed to new a redis client and will "+ + "retry to reconnect later, err: %s", err.Error()) + } else { + redisclient.SetRedisCmd(redisCmd) + } + go redisclient.CheckRedisConnectivity(&redisConf, redisclient.GetRedisCmd(), stopCh) + return nil +} + +func reportRequestInfoMetrics(redisKey string, redisValue []byte, expireTime time.Duration) { + for i := 0; i < redisRetryTimes; i++ { + if redisclient.GetRedisCmd() == nil { + log.GetLogger().Warnf("[reportCPUFlagsMatchingRatio]redis is not ready") + continue + } + err := redisclient.ZADDMetricsToRedis(redisKey, redisValue, listLimit, expireTime) + if err == nil { + log.GetLogger().Infof("succeed to report metrics key: %s, value:%s", redisKey, string(redisValue)) + return + } + log.GetLogger().Errorf("failed to set key: %s, err: %s, retry time %d", redisKey, err.Error(), i) + time.Sleep(redisRetryInterval) + } +} diff --git a/frontend/pkg/frontend/metrics/metrics_test.go b/frontend/pkg/frontend/metrics/metrics_test.go new file mode 100644 index 0000000000000000000000000000000000000000..3c8675280e4309f880ea0ee4c7941ecd202bf8e1 --- /dev/null +++ b/frontend/pkg/frontend/metrics/metrics_test.go @@ -0,0 +1,175 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package metrics + +import ( + "encoding/json" + "fmt" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/stretchr/testify/assert" + + "frontend/pkg/common/faas_common/localauth" + "frontend/pkg/common/faas_common/redisclient" +) + +func TestMetrics(t *testing.T) { + t.Run("TestStartReportMetrics", testStartReportMetrics) + resetMetrics() + t.Run("TestMetricsChan", testMetricsChan) +} +func testStartReportMetrics(t *testing.T) { + var redisValue string + now := time.Now() + patches := [...]*gomonkey.Patches{ + gomonkey.ApplyFunc(localauth.Decrypt, + func(src string) ([]byte, error) { + return []byte("test"), nil + }), + gomonkey.ApplyFunc(redisclient.New, func(param redisclient.NewRedisClientParam, stopCh <-chan struct{}, + options ...redisclient.Option) (*redisclient.Client, error) { + return &redisclient.Client{}, nil + }), + gomonkey.ApplyFunc(redisclient.ZADDMetricsToRedis, + func(key string, metrics interface{}, limit int64, expireTime time.Duration) error { + redisValue = string(metrics.([]byte)) + return nil + }), + gomonkey.ApplyFunc(time.NewTicker, func(d time.Duration) *time.Ticker { + c := make(chan time.Time, 1) + t := &time.Ticker{ + C: c, + } + time.Sleep(10 * time.Millisecond) + c <- time.Now() + return t + }), + gomonkey.ApplyFunc(redisclient.CheckRedisConnectivity, func(clientRedisConfig *redisclient.NewRedisClientParam, + client *redisclient.Client, stopCh <-chan struct{}) { + }), + gomonkey.ApplyFunc(time.Now, func() time.Time { + return now + }), + } + defer func() { + for idx := range patches { + patches[idx].Reset() + } + }() + mockMetrics := RequestStatistics{ + TotalDelay: 0.1, + BusDelay: 0.02, + FrontendDelay: 0.01, + RuntimeDelay: 0.07, + } + PublishMetrics(mockMetrics) + time.Sleep(10 * time.Millisecond) + assert.Equal(t, int64(0), statistics.count) + // + ch := make(chan struct{}) + go StartReportMetrics(ch) + time.Sleep(100 * time.Millisecond) + timestamp := now.Unix() - int64(now.Second()) + expect := fmt.Sprintf(`{"timeStamp":%d,"count":1,"totalDelay":0.1,"frontendDelay":0.01,"busDelay":0.02,"runtimeDelay":0.07}`, timestamp) + assert.Equal(t, expect, redisValue) + + mockMetrics = RequestStatistics{ + ErrorFlag: true, + TotalDelay: 0.1, + BusDelay: 0.02, + FrontendDelay: 0.05, + RuntimeDelay: 0.03, + } + PublishMetrics(mockMetrics) + time.Sleep(100 * time.Millisecond) + assert.Equal(t, statistics.count, int64(1)) + assert.Equal(t, statistics.errCount, int64(1)) + assert.Equal(t, statistics.totalDelay, 0.1) + assert.Equal(t, statistics.runtimeDelay, 0.03) + + close(ch) + time.Sleep(10 * time.Millisecond) + StartReportMetrics(nil) + subscribeMetrics(nil) +} + +func testMetricsChan(t *testing.T) { + go func() { + for i := 0; i < 100; i++ { + PublishMetrics(RequestStatistics{ + TotalDelay: float64(i) / 100.0, + }) + } + }() + time.Sleep(10 * time.Millisecond) + assert.Equal(t, 100, len(metricsChan)) + + go func() { + for i := 0; i < bufferSize+50; i++ { + PublishMetrics(RequestStatistics{ + TotalDelay: float64(i) / 100.0, + }) + } + }() + time.Sleep(50 * time.Millisecond) + assert.Equal(t, bufferSize, len(metricsChan)) + ch := make(chan struct{}) + go subscribeMetrics(ch) + time.Sleep(10 * time.Millisecond) + assert.Equal(t, int64(bufferSize), statistics.count) + assert.Equal(t, 0, len(metricsChan)) + resetMetrics() + go func() { + for i := 0; i < bufferSize; i++ { + PublishMetrics(RequestStatistics{ + TotalDelay: float64(i) / 100.0, + }) + } + time.Sleep(10 * time.Millisecond) + for i := 0; i < bufferSize/2; i++ { + PublishMetrics(RequestStatistics{ + TotalDelay: float64(i) / 100.0, + }) + } + }() + time.Sleep(30 * time.Millisecond) + assert.Equal(t, int64(bufferSize/2+bufferSize), statistics.count) + + go func() { + for i := 0; i < bufferSize/2; i++ { + PublishMetrics(RequestStatistics{ + TotalDelay: float64(i) / 100.0, + }) + } + time.Sleep(50 * time.Millisecond) + for i := 0; i < bufferSize; i++ { + PublishMetrics(RequestStatistics{ + TotalDelay: float64(i) / 100.0, + }) + } + }() + time.Sleep(1 * time.Millisecond) + b := resetMetrics() + time.Sleep(100 * time.Millisecond) + + var request RequestInfoMetrics + err := json.Unmarshal(b, &request) + assert.Nil(t, err) + assert.Equal(t, int64(3*bufferSize), statistics.count+request.Count) +} diff --git a/frontend/pkg/frontend/middleware/body_size_check.go b/frontend/pkg/frontend/middleware/body_size_check.go new file mode 100644 index 0000000000000000000000000000000000000000..59bcb6eab5e3e35930025c40f18296c7e00db368 --- /dev/null +++ b/frontend/pkg/frontend/middleware/body_size_check.go @@ -0,0 +1,105 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package middleware - +package middleware + +import ( + "errors" + "fmt" + "strconv" + + "go.uber.org/zap" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/statuscode" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/responsehandler" + "frontend/pkg/frontend/types" +) + +const ( + maxRequestBodySizeMsg = "the size of request body is beyond maximum:%d, body size:%d" + contentLengthInvalid = "Content-Length is invalid" + megabytes = 1024 * 1024 + + baseTen = 10 + bitSize = 64 +) + +// BodySizeChecker - +func BodySizeChecker(next Handler) Handler { + return func(ctx *types.InvokeProcessContext) error { + var err error + if ctx.IsHTTPUploadStream { + err = checkStreamRequestContentLength(ctx) + } else { + err = checkRequestBodySize(ctx) + } + if err != nil { + responsehandler.SetErrorInContext(ctx, statuscode.FrontendStatusMaxRequestBodySize, err.Error()) + log.GetLogger().With(zap.Any("traceID", ctx.TraceID)).Errorf("the size of request body is out of range %s", + err.Error()) + return err + } + return next(ctx) + } +} + +// checkRequestBodySize function check whether the body of the function is greater than the configured value. +func checkRequestBodySize(ctx *types.InvokeProcessContext) error { + bodyLength := len(ctx.ReqBody) + maxRequestBodySize := config.GetConfig().HTTPConfig.MaxRequestBodySize * megabytes + if bodyLength > maxRequestBodySize { + errMsg := fmt.Sprintf(maxRequestBodySizeMsg, maxRequestBodySize, bodyLength) + log.GetLogger().Errorf(errMsg) + return errors.New(errMsg) + } + return nil +} + +func checkStreamRequestContentLength(ctx *types.InvokeProcessContext) error { + bodyLength, err := getContentLength(ctx) + if err != nil { + return err + } + + maxRequestBodySize := config.GetConfig().HTTPConfig.MaxStreamRequestBodySize * megabytes + + if bodyLength > int64(maxRequestBodySize) { + errMsg := fmt.Sprintf(maxRequestBodySizeMsg, maxRequestBodySize, bodyLength) + log.GetLogger().Errorf(errMsg) + return errors.New(errMsg) + } + return nil +} + +func getContentLength(ctx *types.InvokeProcessContext) (int64, error) { + logger := log.GetLogger().With(zap.Any("traceID", ctx.TraceID)) + contentLengthStr, ok := ctx.ReqHeader[constant.HeaderContentLength] + if !ok { + logger.Errorf("Content-Length header not found") + return 0, errors.New(contentLengthInvalid) + } + + contentLength, err := strconv.ParseInt(contentLengthStr, baseTen, bitSize) + if err != nil || contentLength < 0 { + logger.Errorf("Content-Length is invalid") + return 0, errors.New(contentLengthInvalid) + } + return contentLength, nil +} diff --git a/frontend/pkg/frontend/middleware/body_size_check_test.go b/frontend/pkg/frontend/middleware/body_size_check_test.go new file mode 100644 index 0000000000000000000000000000000000000000..41fc315e238997cf66984cd1b8146fda25636851 --- /dev/null +++ b/frontend/pkg/frontend/middleware/body_size_check_test.go @@ -0,0 +1,132 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package middleware + +import ( + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/responsehandler" + "frontend/pkg/frontend/types" +) + +func TestBodySizeChecker(t *testing.T) { + tests := []struct { + name string + ctx *types.InvokeProcessContext + maxRequestBodySize int + maxStreamRequestBodySize int + shouldFail bool + }{ + { + name: "not exceeds MaxRequestBodySize", + ctx: &types.InvokeProcessContext{ + ReqBody: make([]byte, 1*megabytes), + }, + maxRequestBodySize: 1, + shouldFail: false, + }, + { + name: "exceeds MaxRequestBodySize", + ctx: &types.InvokeProcessContext{ + ReqBody: make([]byte, 1*megabytes+1), + }, + maxRequestBodySize: 1, + shouldFail: true, + }, + { + name: "not exceeds MaxStreamRequestBodySize", + ctx: &types.InvokeProcessContext{ + ReqBody: []byte("test body"), + ReqHeader: map[string]string{constant.HeaderContentLength: "1048576"}, + IsHTTPUploadStream: true, + }, + maxStreamRequestBodySize: 1, + shouldFail: false, + }, + { + name: "exceeds MaxStreamRequestBodySize", + ctx: &types.InvokeProcessContext{ + ReqBody: []byte("test body"), + ReqHeader: map[string]string{constant.HeaderContentLength: "1048577"}, + IsHTTPUploadStream: true, + }, + maxStreamRequestBodySize: 1, + shouldFail: true, + }, + { + name: "exceeds default 1GB MaxStreamRequestBodySize", + ctx: &types.InvokeProcessContext{ + ReqBody: []byte("test body"), + ReqHeader: map[string]string{constant.HeaderContentLength: "1073741824"}, + IsHTTPUploadStream: true, + }, + maxStreamRequestBodySize: 1024, + shouldFail: false, + }, + { + name: "Content-Length header not found", + ctx: &types.InvokeProcessContext{ + ReqBody: []byte("test body"), + IsHTTPUploadStream: true, + }, + shouldFail: true, + }, + { + name: "Content-Length is invalid", + ctx: &types.InvokeProcessContext{ + ReqBody: []byte("test body"), + ReqHeader: map[string]string{constant.HeaderContentLength: "-1"}, + IsHTTPUploadStream: true, + }, + shouldFail: true, + }, + } + patch := gomonkey.ApplyFunc(responsehandler.SetErrorInContext, func(ctx *types.InvokeProcessContext, innerCode int, message interface{}) { + }) + defer patch.Reset() + convey.Convey("Test BodySizeChecker", t, func() { + for _, tt := range tests { + convey.Convey(tt.name, func() { + conf := types.Config{ + HTTPConfig: &types.FrontendHTTP{ + MaxRequestBodySize: tt.maxRequestBodySize, + MaxStreamRequestBodySize: tt.maxStreamRequestBodySize, + }} + config.SetConfig(conf) + nextHandler := func(ctx *types.InvokeProcessContext) error { return nil } + checker := BodySizeChecker(nextHandler) + err := checker(tt.ctx) + if tt.shouldFail { + convey.So(err, convey.ShouldNotBeNil) + if err == nil { + t.Errorf("Expected error but got none for test: %s", tt.name) + } + } else { + convey.So(err, convey.ShouldBeNil) + if err != nil { + t.Errorf("Did not expect error but got: %v for test: %s", err, tt.name) + } + } + }) + } + }) +} diff --git a/frontend/pkg/frontend/middleware/graceexit.go b/frontend/pkg/frontend/middleware/graceexit.go new file mode 100644 index 0000000000000000000000000000000000000000..10d2b3538c9be01987b34a9c4d6b0382ca390a56 --- /dev/null +++ b/frontend/pkg/frontend/middleware/graceexit.go @@ -0,0 +1,85 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package middleware - +package middleware + +import ( + "errors" + "sync" + + "github.com/gin-gonic/gin" + "github.com/valyala/fasthttp" + + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/snerror" + "frontend/pkg/common/faas_common/statuscode" + "frontend/pkg/frontend/types" +) + +// graceExitFlag flag for grace exit , true means grace exit +var graceExitFlag = false + +// Wg is waitGroup for graceful shutdown +var Wg sync.WaitGroup + +// GraceExitFilter - +func GraceExitFilter(next Handler) Handler { + return func(ctx *types.InvokeProcessContext) error { + if graceExitFlag { + ctx.StatusCode = fasthttp.StatusBadRequest + ctx.RespBody = []byte("frontend exiting") + return errors.New("exiting") + } + + Wg.Add(1) + defer Wg.Done() + return next(ctx) + } +} + +// ErrorWriter - +type ErrorWriter interface { + WriteErrorToGinResponse(ctx *gin.Context, err error) +} + +// GraceExitGinFilter - +func GraceExitGinFilter(errWriter ErrorWriter) gin.HandlerFunc { + return func(ctx *gin.Context) { + if graceExitFlag { + err := snerror.New(statuscode.FrontendStatusBadRequest, "frontend exiting") + errWriter.WriteErrorToGinResponse(ctx, err) + ctx.Abort() + return + } + ctx.Next() + } +} + +// GraceExit is used to exit from the system gracefully, +// and after the received message is processed, the interface returns the following information. +// Sleep operation is added in preStop. Before the SIGTERM signal is sent, the k8s removes the endpoint. +// Add a preStop lifecycle hook that exec's sleep 60 or something similar. +// That will be triggered BEFORE you get SIGTERM +// LBs will be able to observe the deletionTimestamp +// see https://github.com/kubernetes/kubernetes/issues/88236 +func GraceExit() { + log.GetLogger().Infof("begin to call GraceExit") + graceExitFlag = true + // wait until all received messages are processed + Wg.Wait() + log.GetLogger().Infof("end to call GraceExit") +} diff --git a/frontend/pkg/frontend/middleware/graceexit_test.go b/frontend/pkg/frontend/middleware/graceexit_test.go new file mode 100644 index 0000000000000000000000000000000000000000..fea1e4c332dfe0f3b05ef5431163508e912058b7 --- /dev/null +++ b/frontend/pkg/frontend/middleware/graceexit_test.go @@ -0,0 +1,56 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package middleware + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" +) + +func SetGraceExit(flag bool) { + graceExitFlag = flag +} + +func TestGraceExitFilter(t *testing.T) { + router := gin.New() + router.POST("/serverless/caas/v1/execute", GraceExitGinFilter(NewCommonErrorWriter()), + nil) + SetGraceExit(true) + defer func() { + SetGraceExit(false) + }() + req, err := http.NewRequest(http.MethodPost, "/serverless/caas/v1/execute", nil) + assert.NoError(t, err) + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + assert.Equal(t, "frontend exiting", w.Body.String()) +} + +type CommonErrorWriter struct{} + +func NewCommonErrorWriter() *CommonErrorWriter { + return &CommonErrorWriter{} +} + +func (d *CommonErrorWriter) WriteErrorToGinResponse(ctx *gin.Context, err error) { + ctx.Writer.Write([]byte(err.Error())) +} diff --git a/frontend/pkg/frontend/middleware/handlerchain.go b/frontend/pkg/frontend/middleware/handlerchain.go new file mode 100644 index 0000000000000000000000000000000000000000..d076cccb47968c1a23dc7627601e75280efe8c2f --- /dev/null +++ b/frontend/pkg/frontend/middleware/handlerchain.go @@ -0,0 +1,49 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package middleware + +import "frontend/pkg/frontend/types" + +// HandlerChain - +type HandlerChain interface { + Use(mws ...Middleware) + Handle(ctx *types.InvokeProcessContext) error +} + +// baseInvoker - +type baseHandler struct { + handler Handler +} + +// NewBaseHandler - +func NewBaseHandler(handler Handler) HandlerChain { + return &baseHandler{ + handler: handler, + } +} + +// Use middlewares +func (bi *baseHandler) Use(mws ...Middleware) { + for i := len(mws) - 1; i >= 0; i-- { + bi.handler = mws[i](bi.handler) + } +} + +// Handle - +func (bi *baseHandler) Handle(ctx *types.InvokeProcessContext) error { + return bi.handler(ctx) +} diff --git a/frontend/pkg/frontend/middleware/middleware.go b/frontend/pkg/frontend/middleware/middleware.go new file mode 100644 index 0000000000000000000000000000000000000000..dc7a445a6d3bf1d903a7bf6043f0350c1d5dde62 --- /dev/null +++ b/frontend/pkg/frontend/middleware/middleware.go @@ -0,0 +1,33 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package middleware - +package middleware + +import ( + "frontend/pkg/frontend/types" +) + +// Handler - +type Handler func(ctx *types.InvokeProcessContext) error + +// Middleware - +type Middleware func(next Handler) Handler + +var ( + // Invoker - + Invoker HandlerChain +) diff --git a/frontend/pkg/frontend/middleware/requestauthcheck.go b/frontend/pkg/frontend/middleware/requestauthcheck.go new file mode 100644 index 0000000000000000000000000000000000000000..e63bf13e1da86ba6248ff6a17fb1bf685d51b837 --- /dev/null +++ b/frontend/pkg/frontend/middleware/requestauthcheck.go @@ -0,0 +1,68 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package middleware - +package middleware + +import ( + "fmt" + "strings" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/localauth" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/statuscode" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/responsehandler" + "frontend/pkg/frontend/types" + "frontend/pkg/frontend/wisecloud" +) + +// RequestAuthCheck check auth for request +func RequestAuthCheck(next Handler) Handler { + return func(ctx *types.InvokeProcessContext) error { + err := authCheck(ctx) + if err != nil { + responsehandler.SetErrorInContext(ctx, statuscode.FrontendStatusUnAuthorized, err.Error()) + log.GetLogger().Errorf("failed to authenticate request, traceID %s: %s", ctx.TraceID, err.Error()) + return err + } + return next(ctx) + } +} + +func authCheck(c *types.InvokeProcessContext) error { + if !config.GetConfig().AuthenticationEnable { + return nil + } + requestSign := c.ReqHeader[constant.HeaderAuthorization] + if strings.HasPrefix(requestSign, "HMAC-SHA256 ") { + if !wisecloud.AuthDownGradeFunctionCall(c.ReqPath, c.RespHeader, c.ReqBody, config.GetConfig().LocalAuth.AKey, + []byte(config.GetConfig().LocalAuth.SKey)) { + log.GetLogger().Errorf("failed to check authorization for downgrade functioncall") + return fmt.Errorf("auth check failed") + } + return nil + } + timestamp := c.ReqHeader[constant.HeaderAuthTimestamp] + err := localauth.AuthCheckLocally(config.GetConfig().LocalAuth.AKey, config.GetConfig().LocalAuth.SKey, + requestSign, timestamp, config.GetConfig().LocalAuth.Duration) + if err != nil { + log.GetLogger().Errorf("failed to check authorization of URL locally, error: %s", err.Error()) + return err + } + return nil +} diff --git a/frontend/pkg/frontend/middleware/requestauthcheck_test.go b/frontend/pkg/frontend/middleware/requestauthcheck_test.go new file mode 100644 index 0000000000000000000000000000000000000000..aeb4cee3316e1a3eb560e087e54f30c8cd2d6969 --- /dev/null +++ b/frontend/pkg/frontend/middleware/requestauthcheck_test.go @@ -0,0 +1,85 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package middleware + +import ( + "errors" + "frontend/pkg/common/faas_common/localauth" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/types" + "github.com/agiledragon/gomonkey/v2" + "github.com/stretchr/testify/assert" + "strconv" + "testing" + "time" +) + +func Test_authCheck(t *testing.T) { + config.GetConfig().LocalAuth = &localauth.AuthConfig{} + // constant.FConfig.AuthenticationEnable = true + config.GetConfig().AuthenticationEnable = true + defer func() { + config.GetConfig().AuthenticationEnable = false + }() + patched := []*gomonkey.Patches{ + gomonkey.ApplyFunc(localauth.AuthCheckLocally, func(ak string, sk string, requestSign string, timestamp string, duration int) error { + if timestamp == "" { + return errors.New("no auth check info") + } + return nil + }), + } + defer func() { + for i := range patched { + patched[i].Reset() + } + }() + + tests := []struct { + name string + timestamp string + + exceptedError bool + }{ + { + name: "normal request", + timestamp: strconv.Itoa(int(time.Now().Unix())), + + exceptedError: false, + }, + { + name: "no timestamp", + + exceptedError: true, + }, + } + + for _, test := range tests { + // req, _ := http.NewRequest(http.MethodPost, "http://127.0.0.1:8080", nil) + processCtx := types.CreateInvokeProcessContext() + processCtx.ReqHeader["X-Timestamp-Auth"] = test.timestamp + // processCtx.ReqHeader["Method"] = http.MethodPost + processCtx.ReqHeader["URL"] = "http://127.0.0.1:8080" + // req.Header.Set("X-Timestamp-Auth", test.timestamp) + err := authCheck(processCtx) + if test.exceptedError { + assert.Errorf(t, err, test.name) + } else { + assert.Nil(t, err, test.name) + } + } +} diff --git a/frontend/pkg/frontend/middleware/trafficlimit.go b/frontend/pkg/frontend/middleware/trafficlimit.go new file mode 100644 index 0000000000000000000000000000000000000000..c4537e8ab95e8af16e96aa185d70a03663b47775 --- /dev/null +++ b/frontend/pkg/frontend/middleware/trafficlimit.go @@ -0,0 +1,51 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package middleware - +package middleware + +import ( + "errors" + + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/monitor" + "frontend/pkg/common/faas_common/statuscode" + "frontend/pkg/common/faas_common/urnutils" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/responsehandler" + "frontend/pkg/frontend/tenanttrafficlimit" + "frontend/pkg/frontend/types" +) + +// TrafficLimiter - +func TrafficLimiter(next Handler) Handler { + return func(ctx *types.InvokeProcessContext) error { + if err := tenanttrafficlimit.Limit(urnutils.GetTenantFromFuncKey(ctx.FuncKey)); err != nil { + responsehandler.SetErrorInContext(ctx, statuscode.FrontendStatusTrafficLimitEffective, err.Error()) + log.GetLogger().Errorf("tenant traffic limit err:%s,traceID%s", err.Error(), ctx.TraceID) + return err + } + memoryWant := uint64(float64(len(ctx.ReqBody)) * config.GetConfig(). + MemoryEvaluatorConfig.RequestMemoryEvaluator) + if !monitor.IsAllowByMemory(ctx.FuncKey, memoryWant, ctx.TraceID) { + ErrHeavyLoad := errors.New("http server is under heavy load") + responsehandler.SetErrorInContext(ctx, statuscode.HeavyLoadCode, ErrHeavyLoad.Error()) + return ErrHeavyLoad + } + defer monitor.GetMemInstance().ReleaseFunctionMem(ctx.FuncKey, memoryWant) + return next(ctx) + } +} diff --git a/frontend/pkg/frontend/remoteclientlease/leaseporxy_test.go b/frontend/pkg/frontend/remoteclientlease/leaseporxy_test.go new file mode 100644 index 0000000000000000000000000000000000000000..d87839f88f436a127e47a2bee4c3502b2967e513 --- /dev/null +++ b/frontend/pkg/frontend/remoteclientlease/leaseporxy_test.go @@ -0,0 +1,76 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package remoteclientlease + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/types" +) + +func TestFaasManager(t *testing.T) { + events := []etcd3.Event{ + { + Type: etcd3.PUT, + Value: []byte(`{ + "instanceID": "3f079541-15fc-4009-8c41-50b2b2936772", + "instanceStatus": { + "code": 3, + "msg": "running" + }}`), + }, + { + Type: etcd3.PUT, + Value: []byte(`{ + "instanceID": "3f079541-15fc-4009-8c41-50b2b2936772", + "instanceStatus": { + "code": 5, + "msg": "exiting" + }}`), + }, + { + Type: etcd3.PUT, + Value: []byte("value1"), + }, + } + Convey("Test faas manager handler", t, func() { + instanceInfo := &types.InstanceInfo{ + FunctionName: "faasmanager", + InstanceName: "3f079541-15fc-4009-8c41-50b2b2936772", + } + UpdateFaasManager(&events[0], instanceInfo) + So(info.funcKey, ShouldEqual, "faasmanager") + So(info.instanceID, ShouldEqual, "3f079541-15fc-4009-8c41-50b2b2936772") + DeleteFaasManager(instanceInfo) + So(info, ShouldBeNil) + instanceInfo = &types.InstanceInfo{ + FunctionName: "faasmanager", + InstanceName: "3f079541-15fc-4009-8c41-50b2b2936772", + } + UpdateFaasManager(&events[1], instanceInfo) + So(info, ShouldBeNil) + DeleteFaasManager(instanceInfo) + So(info, ShouldBeNil) + UpdateFaasManager(&events[2], instanceInfo) + So(info, ShouldBeNil) + DeleteFaasManager(instanceInfo) + So(info, ShouldBeNil) + }) +} diff --git a/frontend/pkg/frontend/remoteclientlease/leaseproxy.go b/frontend/pkg/frontend/remoteclientlease/leaseproxy.go new file mode 100644 index 0000000000000000000000000000000000000000..151b986084f2cc05f7e742979442dd348e16ef80 --- /dev/null +++ b/frontend/pkg/frontend/remoteclientlease/leaseproxy.go @@ -0,0 +1,216 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package remoteclientlease - +package remoteclientlease + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "time" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/grpc/pb/commonargs" + "frontend/pkg/common/faas_common/grpc/pb/lease" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/types" + "frontend/pkg/frontend/common/util" + "frontend/pkg/frontend/config" +) + +const invokeManagerTimeout = 10 + +// FaasManagerInfo faasManager Info +type FaasManagerInfo struct { + funcKey string + instanceID string + InstanceStatus types.InstanceStatus +} + +var ( + mtx sync.RWMutex + info *FaasManagerInfo + inUsedInfo = make(map[string]*FaasManagerInfo) +) + +// UpdateFaasManager update faasManager Info +func UpdateFaasManager(event *etcd3.Event, in *types.InstanceInfo) { + instanceInfo := &types.InstanceSpecification{} + err := json.Unmarshal(event.Value, instanceInfo) + if err != nil { + log.GetLogger().Errorf("failed to unmarshal instance event, err: %s", err.Error()) + return + } + mtx.Lock() + log.GetLogger().Infof( + "Success to update faas-manager info in faas-frontend, functionName: %s, instanceName: %s", + in.FunctionName, in.InstanceName, + ) + if info == nil && instanceInfo.InstanceStatus.Code == int32(constant.KernelInstanceStatusRunning) { + info = &FaasManagerInfo{ + funcKey: in.FunctionName, + instanceID: in.InstanceName, + InstanceStatus: instanceInfo.InstanceStatus, + } + mtx.Unlock() + return + } + inUsedInfo[in.InstanceName] = &FaasManagerInfo{ + funcKey: in.FunctionName, + instanceID: in.InstanceName, + InstanceStatus: instanceInfo.InstanceStatus, + } + mtx.Unlock() + +} + +// DeleteFaasManager delete faasManager Info +func DeleteFaasManager(in *types.InstanceInfo) { + mtx.Lock() + defer mtx.Unlock() + log.GetLogger().Infof( + "Success to delete faas-manager info in faas-frontend, functionName: %s, instanceName: %s", + in.FunctionName, in.InstanceName, + ) + if info != nil && info.instanceID == in.InstanceName { + info = nil + } + for k, v := range inUsedInfo { + if k == in.InstanceName { + delete(inUsedInfo, k) + continue + } + if info == nil && v.InstanceStatus.Code == int32(constant.KernelInstanceStatusRunning) { + log.GetLogger().Infof("reset faas-manager info instanceName: %s", v.instanceID) + info = v + delete(inUsedInfo, k) + break + } + } +} + +func invokeFaasManager(traceID, remoteClientID, op string) *lease.LeaseResponse { + args := []*api.Arg{ + { + Type: api.Value, + Data: []byte(op), + }, + { + Type: api.Value, + Data: []byte(remoteClientID), + }, + { + Type: api.Value, + Data: []byte(traceID), + }, + } + resp := &lease.LeaseResponse{ + Code: commonargs.ErrorCode_ERR_NONE, + Message: "success create lease", + } + mtx.RLock() + if info == nil { + err := setEventToEtcd(remoteClientID, op, traceID) + if err != nil { + log.GetLogger().Errorf("failed to invoke faasmanager and failed write to etcd,"+ + " FaasManagerInfo is empty, traceID: %s, err: %v", traceID, err) + resp.Code = commonargs.ErrorCode_ERR_ETCD_OPERATION_ERROR + resp.Message = "failed to invoke faasmanager and etcd save failed, info is empty" + } + mtx.RUnlock() + return resp + } + log.GetLogger().Infof("Start to send request to faas-manager, traceID: %s", traceID) + msg := util.InvokeRequest{ + Function: info.funcKey, + Args: args, + InstanceID: info.instanceID, + TraceID: traceID, + InvokeTimeout: invokeManagerTimeout, + } + mtx.RUnlock() + if config.GetConfig().RetryConfig != nil && config.GetConfig().RetryConfig.InstanceExceptionRetry { + msg.RetryTimes = config.GetConfig().InvokeMaxRetryTimes + } + respData, err := util.NewClient().Invoke(msg) + if err != nil { + log.GetLogger().Errorf("failed to send request, err: %s, traceID: %s", err.Error(), traceID) + err = setEventToEtcd(remoteClientID, op, traceID) + if err != nil { + resp.Code = commonargs.ErrorCode_ERR_ETCD_OPERATION_ERROR + resp.Message = "failed to create new lease, err: " + err.Error() + } + return resp + } + respMsg := &types.CallHandlerResponse{} + if err = json.Unmarshal(respData, respMsg); err != nil { + log.GetLogger().Errorf("failed to unmarshal resp, err: %s, traceID: %s", err.Error(), traceID) + resp.Code = commonargs.ErrorCode_ERR_INNER_SYSTEM_ERROR + resp.Message = "failed to unmarshal resp, err: " + err.Error() + return resp + } + if respMsg.Code != constant.InsReqSuccessCode { + resp.Code = commonargs.ErrorCode_ERR_INNER_SYSTEM_ERROR + resp.Message = fmt.Sprintf("code: %d, message: %s", respMsg.Code, respMsg.Message) + } + return resp +} + +// NewLease the handler of new lease +func NewLease(remoteClientID string, traceID string) *lease.LeaseResponse { + resp := invokeFaasManager(traceID, remoteClientID, constant.NewLease) + return resp +} + +// KeepAlive the handler of KeepAlive +func KeepAlive(remoteClientID string, traceID string) *lease.LeaseResponse { + resp := invokeFaasManager(traceID, remoteClientID, constant.KeepAlive) + return resp +} + +// DelLease the handler of DelLease +func DelLease(remoteClientID string, traceID string) *lease.LeaseResponse { + resp := invokeFaasManager(traceID, remoteClientID, constant.DelLease) + return resp +} + +func setEventToEtcd(remoteClientID string, op string, traceID string) error { + client := etcd3.GetRouterEtcdClient() + key := constant.LeasePrefix + "/" + remoteClientID + event := types.LeaseEvent{ + Type: op, + RemoteClientID: remoteClientID, + Timestamp: time.Now().Unix(), + TraceID: traceID, + } + marshal, err := json.Marshal(event) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), etcd3.DurationContextTimeout) + defer cancel() + _, err = client.Client.Put(ctx, key, string(marshal)) + if err != nil { + return err + } + return nil +} diff --git a/frontend/pkg/frontend/responsehandler/common.go b/frontend/pkg/frontend/responsehandler/common.go new file mode 100644 index 0000000000000000000000000000000000000000..fb40051101eb2e0e121679e2b3df3fbb4c5a45a4 --- /dev/null +++ b/frontend/pkg/frontend/responsehandler/common.go @@ -0,0 +1,91 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package responsehandler + +import ( + "encoding/json" + "fmt" + + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/statuscode" + "frontend/pkg/frontend/types" +) + +func generateRespBodyToUser(innerCode int, message json.RawMessage) ([]byte, error) { + if innerCode == statuscode.InnerResponseSuccessCode { + return message, nil + } + return createErrorResponseBody(innerCode, message) +} + +func createErrorResponseBody(errorCode int, message json.RawMessage) ([]byte, error) { + body, err := marshalJSONResponse(errorCode, message) + if err != nil { + log.GetLogger().Infof("message is not json format") + body, err = marshalStringResponse(errorCode, string(message)) + if err != nil { + return []byte{}, fmt.Errorf("failed to marshal response data: %s", err) + } + } + return body, nil +} + +func marshalJSONResponse(errorCode int, message json.RawMessage) ([]byte, error) { + body, err := json.Marshal(struct { + Code int `json:"code"` + Message json.RawMessage `json:"message"` + }{ + Code: errorCode, + Message: message, + }) + return body, err +} + +func marshalStringResponse(errorCode int, message string) ([]byte, error) { + body, err := json.Marshal(struct { + Code int `json:"code"` + Message string `json:"message"` + }{ + Code: errorCode, + Message: message, + }) + return body, err +} + +func buildResponse(ctx *types.InvokeProcessContext, innerCode int, message interface{}) { + var data []byte + var err error + stringMessage, ok := message.(string) + if ok { + data, err = marshalStringResponse(innerCode, stringMessage) + if err != nil { + log.GetLogger().Errorf("failed to marshal string response data, traceID: %s, err: %s", + ctx.TraceID, err.Error()) + return + } + } + jsonMessage, ok := message.(json.RawMessage) + if ok { + data, err = marshalJSONResponse(innerCode, jsonMessage) + if err != nil { + log.GetLogger().Errorf("failed to marshal json response data, traceID: %s, err: %s", + ctx.TraceID, err.Error()) + return + } + } + ctx.RespBody = data +} diff --git a/frontend/pkg/frontend/responsehandler/common_test.go b/frontend/pkg/frontend/responsehandler/common_test.go new file mode 100644 index 0000000000000000000000000000000000000000..9f5ab5b3e3c05a52143614f37d8d67e48ace8a17 --- /dev/null +++ b/frontend/pkg/frontend/responsehandler/common_test.go @@ -0,0 +1,35 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package responsehandler + +import ( + "testing" + + "encoding/json" + . "github.com/smartystreets/goconvey/convey" +) + +func TestCreateErrorResponseBody(t *testing.T) { + Convey("Test CreateErrorResponseBody", t, func() { + body, err := createErrorResponseBody(404, json.RawMessage(`"Not Found"`)) + want := []byte(`{"code":404,"message":"Not Found"}`) + So(string(body), ShouldEqual, string(want)) + So(err, ShouldBeNil) + body, err = createErrorResponseBody(0, json.RawMessage(`invalid json`)) + So(err, ShouldBeNil) + }) +} diff --git a/frontend/pkg/frontend/responsehandler/fgresponsehandler.go b/frontend/pkg/frontend/responsehandler/fgresponsehandler.go new file mode 100644 index 0000000000000000000000000000000000000000..f9af2801470f8d55fdc9d7e0db71417aa6f0d30c --- /dev/null +++ b/frontend/pkg/frontend/responsehandler/fgresponsehandler.go @@ -0,0 +1,90 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package responsehandler + +import ( + "strconv" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/snerror" + "frontend/pkg/common/faas_common/statuscode" + "frontend/pkg/frontend/common/httpconstant" + "frontend/pkg/frontend/common/util" + "frontend/pkg/frontend/types" +) + +// FGResponseHandler - +type FGResponseHandler struct{} + +// SetResponseFromInvocation - +func (f *FGResponseHandler) SetResponseFromInvocation(ctx *types.InvokeProcessContext, + message []byte) (*types.CallResp, snerror.SNError) { + if len(ctx.RespHeader) > 0 { + log.GetLogger().Errorf("response has been written, traceID: %s", ctx.TraceID) + return &types.CallResp{}, snerror.New(statuscode.FrontendStatusInternalError, + "response has been written") + } + respMsg, err := util.UnmarshalCallResp(message) + if err != nil { + log.GetLogger().Errorf("failed to translate response data, traceID: %s, err: %s", + ctx.TraceID, err.Error()) + return &types.CallResp{}, snerror.New(statuscode.FrontendStatusInternalError, err.Error()) + } + + innerCode, err := strconv.Atoi(respMsg.InnerCode) + if err != nil { + log.GetLogger().Errorf("failed to get the innerCode, traceID: %s, err: %s", + ctx.TraceID, err.Error()) + return respMsg, snerror.New(statuscode.FrontendStatusInternalError, err.Error()) + } + + body, err := generateRespBodyToUser(innerCode, respMsg.Body) + if err != nil { + log.GetLogger().Errorf("failed to generate the returned body, traceID: %s, err: %s", + ctx.TraceID, err.Error()) + return respMsg, snerror.New(statuscode.FrontendStatusInternalError, err.Error()) + } + setHTTPHeader(ctx, respMsg) + ctx.StatusCode = statuscode.Code(innerCode) + ctx.RespBody = body + return respMsg, nil +} + +// SetResponseFromFrontend - +func (f *FGResponseHandler) SetResponseFromFrontend(ctx *types.InvokeProcessContext, + innerCode int, message interface{}) { + if len(ctx.RespHeader) > 0 { + return + } + ctx.StatusCode = statuscode.Code(innerCode) + ctx.RespHeader[constant.HeaderInnerCode] = strconv.Itoa(innerCode) + buildResponse(ctx, innerCode, message) +} + +func setHTTPHeader(ctx *types.InvokeProcessContext, respMsg *types.CallResp) { + ctx.RespHeader[constant.HeaderContentType] = httpconstant.ApplicationJSON + ctx.RespHeader[constant.HeaderInnerCode] = respMsg.InnerCode + ctx.RespHeader[constant.HeaderLogResult] = respMsg.LogResult + ctx.RespHeader[constant.HeaderInvokeSummary] = respMsg.InvokeSummary + ctx.RespHeader[constant.HeaderBillingDuration] = respMsg.BillingDuration + if ctx.NeedReadRespHeader { + for k, v := range respMsg.Headers { + ctx.RespHeader[k] = v + } + } +} diff --git a/frontend/pkg/frontend/responsehandler/responsehandler.go b/frontend/pkg/frontend/responsehandler/responsehandler.go new file mode 100644 index 0000000000000000000000000000000000000000..1a858b640840aaa3c446d41a4c135a1a2345612c --- /dev/null +++ b/frontend/pkg/frontend/responsehandler/responsehandler.go @@ -0,0 +1,62 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package responsehandler - +package responsehandler + +import ( + "frontend/pkg/common/faas_common/snerror" + "frontend/pkg/frontend/types" +) + +// SetErrorInContextWithDefault will set error in process context +func SetErrorInContextWithDefault(ctx *types.InvokeProcessContext, err error, defaultInnerErrorCode int, + defaultMessage interface{}) { + errorCode := defaultInnerErrorCode + errorMessage := defaultMessage + snErr, ok := err.(snerror.SNError) + if ok { + errorCode = snErr.Code() + errorMessage = snErr.Error() + } + if ctx.RequestTraceInfo != nil { + ctx.RequestTraceInfo.InnerCode = errorCode + } + Handler.SetResponseFromFrontend(ctx, errorCode, errorMessage) +} + +// SetErrorInContext will set error in process context +func SetErrorInContext(ctx *types.InvokeProcessContext, innerCode int, message interface{}) { + Handler.SetResponseFromFrontend(ctx, innerCode, message) +} + +// SetResponseInContext will set response body in process context +func SetResponseInContext(ctx *types.InvokeProcessContext, + message []byte) (*types.CallResp, snerror.SNError) { + return Handler.SetResponseFromInvocation(ctx, message) +} + +// HandlerInterface - +type HandlerInterface interface { + SetResponseFromFrontend(ctx *types.InvokeProcessContext, innerCode int, message interface{}) + SetResponseFromInvocation(ctx *types.InvokeProcessContext, + message []byte) (*types.CallResp, snerror.SNError) +} + +var ( + // Handler - + Handler HandlerInterface +) diff --git a/frontend/pkg/frontend/schedulerproxy/proxy.go b/frontend/pkg/frontend/schedulerproxy/proxy.go new file mode 100644 index 0000000000000000000000000000000000000000..53714686a36a05a2de43c5751de36ea40ba31722 --- /dev/null +++ b/frontend/pkg/frontend/schedulerproxy/proxy.go @@ -0,0 +1,243 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package schedulerproxy - +package schedulerproxy + +import ( + "fmt" + "strings" + "sync" + "time" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/loadbalance" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/types" + "frontend/pkg/frontend/instancemanager" +) + +const ( + // hashRingSize the concurrent hash ring length + hashRingSize = 100 + limiterTime = 1 * time.Millisecond +) + +const ( + etcdPathElementsLen = 14 + tenantIndex = 6 + functionNameIndex = 8 + versionIndex = 10 + instanceNameIndex = 13 + funcKeyElementsLen = 3 +) + +const ( + addSchedulerInfoOption = "ADD" + removeSchedulerInfoOption = "REMOVE" +) + +// Proxy is the singleton proxy +var Proxy *ProxyManager + +func init() { + Proxy = newSchedulerProxy( + loadbalance.NewLimiterCHGeneric(limiterTime), + ) +} + +// ProxyManager is used to get instances from FaaSScheduler via a grpc stream +type ProxyManager struct { + faasSchedulers sync.Map + // key is tenantID, value is instanceID + exclusivitySchedulers sync.Map + // used to select a FaaSScheduler by the func info Concurrent Consistent Hash + loadBalance loadbalance.LoadBalance + RTAPI api.LibruntimeAPI +} + +// Add an FaaSScheduler +func (im *ProxyManager) Add(scheduleInfo *types.InstanceInfo, logger api.FormatLogger) { + if im.RTAPI != nil { + switch scheduleInfo.InstanceID != "" { + case true: + im.RTAPI.UpdateSchdulerInfo(scheduleInfo.InstanceName, scheduleInfo.InstanceID, addSchedulerInfoOption) + case false: + im.RTAPI.UpdateSchdulerInfo(scheduleInfo.InstanceName, scheduleInfo.InstanceID, removeSchedulerInfoOption) + default: + + } + } + im.faasSchedulers.Store(scheduleInfo.InstanceName, scheduleInfo) + im.exclusivitySchedulers.Store(scheduleInfo.Exclusivity, scheduleInfo.InstanceName) + if scheduleInfo.Exclusivity != "" { + logger.Infof("no need to add scheduler to load balance for exclusivity %s", scheduleInfo.Exclusivity) + return + } + im.loadBalance.Add(scheduleInfo.InstanceName, 0) + logger.Infof("add scheduler to load balance") +} + +// Exist - +func (im *ProxyManager) Exist(instanceName string, instanceId string) bool { + value, ok := im.faasSchedulers.Load(instanceName) + if !ok { + return false + } + info, _ := value.(*types.InstanceInfo) // no need judge + if info == nil { + return false + } + return info.InstanceID == instanceId +} + +// ExistInstanceName - +func (im *ProxyManager) ExistInstanceName(instanceName string) bool { + _, ok := im.faasSchedulers.Load(instanceName) + return ok +} + +// Remove a FaaSScheduler +func (im *ProxyManager) Remove(schedulerInfo *types.InstanceInfo, logger api.FormatLogger) { + if _, ok := im.faasSchedulers.Load(schedulerInfo.InstanceName); !ok { + logger.Infof("no need delete unexist scheduler") + return + } + if im.RTAPI != nil { + im.RTAPI.UpdateSchdulerInfo(schedulerInfo.InstanceName, schedulerInfo.InstanceID, removeSchedulerInfoOption) + } + im.faasSchedulers.Delete(schedulerInfo.InstanceName) + im.exclusivitySchedulers.Range(func(key, value interface{}) bool { + instanceID, ok := value.(string) + if !ok { + return true + } + if instanceID == schedulerInfo.InstanceName { + im.exclusivitySchedulers.Delete(key) + } + return true + }) + im.loadBalance.Remove(schedulerInfo.InstanceName) + logger.Infof("deleted from load balance") +} + +// Get an instance for this request +func (im *ProxyManager) Get(funcKey string, logger api.FormatLogger) (*types.InstanceInfo, error) { + logger.Debugf("begin to get scheduler for funcKey: %s", funcKey) + next, err := im.getNextScheduler(funcKey, logger) + if err != nil { + return nil, err + } + faasSchedulerName, ok := next.(string) + if !ok { + return nil, fmt.Errorf("failed to parse the result of loadbanlance: %+v", next) + } + if strings.TrimSpace(faasSchedulerName) == "" { + return nil, fmt.Errorf("no avaiable faas scheduler was found") + } + faaSSchedulerData, ok := im.faasSchedulers.Load(faasSchedulerName) + if !ok { + return nil, fmt.Errorf("failed to get the faas scheduler named %s", faasSchedulerName) + } + faaSScheduler, ok := faaSSchedulerData.(*types.InstanceInfo) + if !ok { + return nil, fmt.Errorf("invalid faas scheduler named %s: %#v", faasSchedulerName, faaSSchedulerData) + } + logger.Infof("succeed to get scheduler instanceID: %s for funcKey: %s", faasSchedulerName, funcKey) + return faaSScheduler, nil +} + +// IsEmpty - +func (im *ProxyManager) IsEmpty() bool { + flag := false + im.faasSchedulers.Range(func(k, v any) bool { + instance, ok := v.(*types.InstanceInfo) + if !ok { + return true + } + + ok = instancemanager.GetFaaSSchedulerInstanceManager().IsExist(instance.InstanceID) + if ok { + flag = true + return false + } + return true + }) + return !flag +} + +// GetSchedulerByInstanceName - +func (im *ProxyManager) GetSchedulerByInstanceName(instanceName string, traceID string) (*types.InstanceInfo, error) { + faaSSchedulerData, ok := im.faasSchedulers.Load(instanceName) + if !ok { + return nil, fmt.Errorf("failed to get the faas scheduler named %s,traceID %s", instanceName, traceID) + } + faaSScheduler, ok := faaSSchedulerData.(*types.InstanceInfo) + if !ok { + return nil, fmt.Errorf("invalid faas scheduler named %s: %#v, traceID: %s", + instanceName, faaSSchedulerData, traceID) + } + return faaSScheduler, nil +} + +func (im *ProxyManager) getNextScheduler(funcKey string, logger api.FormatLogger) (any, error) { + var next interface{} + elements := strings.Split(funcKey, constant.KeySeparator) + if len(elements) == funcKeyElementsLen { + var ok bool + tenantID := elements[0] + next, ok = im.exclusivitySchedulers.Load(tenantID) + if ok && next != nil { + return next, nil + } + } else { + logger.Warnf("invalid funcKey: %s", funcKey) + } + + // select one FaaSScheduler by the func key + next = im.loadBalance.Next(funcKey, false) + if next == nil { + log.GetLogger().Errorf("failed to get faaSScheduler instance, function: %s", funcKey) + return nil, fmt.Errorf("failed to get faaSScheduler instance") + } + return next, nil +} + +// DeleteBalancer - +func (im *ProxyManager) DeleteBalancer(funcKey string) { + im.loadBalance.DeleteBalancer(funcKey) +} + +// SetStain - +func (im *ProxyManager) SetStain(funcKey, instanceName string) { + if v, ok := im.loadBalance.(*loadbalance.LimiterCHGeneric); ok { + v.SetStain(funcKey, instanceName) + } +} + +// Reset - reset hash anchor point +func (im *ProxyManager) Reset() { + im.loadBalance.Reset() +} + +// newSchedulerProxy return an instance pool which get the instance from the remote FaaSScheduler +func newSchedulerProxy(lb loadbalance.LoadBalance) *ProxyManager { + return &ProxyManager{ + loadBalance: lb, + } +} diff --git a/frontend/pkg/frontend/schedulerproxy/proxy_test.go b/frontend/pkg/frontend/schedulerproxy/proxy_test.go new file mode 100644 index 0000000000000000000000000000000000000000..9d92468aa5f095c9763d5c390f2dd43b2446a4c4 --- /dev/null +++ b/frontend/pkg/frontend/schedulerproxy/proxy_test.go @@ -0,0 +1,149 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package schedulerproxy + +import ( + "reflect" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/common/faas_common/loadbalance" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/types" +) + +func Test_schedulerProxy_Add(t *testing.T) { + convey.Convey("Add", t, func() { + Proxy.Add(&types.InstanceInfo{InstanceName: "instance1"}, log.GetLogger()) + _, ok := Proxy.faasSchedulers.Load("instance1") + convey.So(ok, convey.ShouldEqual, true) + + convey.So(Proxy.Exist("instance1", ""), convey.ShouldBeTrue) + convey.So(Proxy.ExistInstanceName("instance1"), convey.ShouldBeTrue) + + Proxy.Add(&types.InstanceInfo{InstanceName: "instance1", InstanceID: "1"}, log.GetLogger()) + convey.So(Proxy.Exist("instance1", "1"), convey.ShouldBeTrue) + convey.So(Proxy.ExistInstanceName("instance1"), convey.ShouldBeTrue) + convey.So(Proxy.Exist("instance1", ""), convey.ShouldBeFalse) + }) +} + +func Test_schedulerProxy_Remove(t *testing.T) { + convey.Convey("Remove", t, func() { + Proxy.Add(&types.InstanceInfo{InstanceName: "instance1"}, log.GetLogger()) + _, ok := Proxy.faasSchedulers.Load("instance1") + convey.So(ok, convey.ShouldEqual, true) + + Proxy.Remove(&types.InstanceInfo{InstanceName: "instance1"}, log.GetLogger()) + _, ok = Proxy.faasSchedulers.Load("instance1") + convey.So(ok, convey.ShouldEqual, false) + convey.So(Proxy.ExistInstanceName("instance1"), convey.ShouldBeFalse) + }) +} + +func Test_schedulerProxy_Get(t *testing.T) { + convey.Convey("Get", t, func() { + convey.Convey("assert failed", func() { + patches := []*gomonkey.Patches{ + gomonkey.ApplyMethod(reflect.TypeOf(&loadbalance.ConcurrentCHGeneric{}), "Next", + func(_ *loadbalance.ConcurrentCHGeneric, name string, move bool) interface{} { + return 123 + }), + } + defer func() { + for _, patch := range patches { + time.Sleep(100 * time.Millisecond) + patch.Reset() + } + }() + + _, err := Proxy.Get("functionKey", log.GetLogger()) + convey.So(err, convey.ShouldBeError) + }) + + convey.Convey("no avaiable faas scheduler was found", func() { + patches := []*gomonkey.Patches{ + gomonkey.ApplyMethod(reflect.TypeOf(&loadbalance.ConcurrentCHGeneric{}), "Next", + func(_ *loadbalance.ConcurrentCHGeneric, name string, move bool) interface{} { + return "" + }), + } + defer func() { + for _, patch := range patches { + time.Sleep(100 * time.Millisecond) + patch.Reset() + } + }() + + _, err := Proxy.Get("functionKey", log.GetLogger()) + convey.So(err, convey.ShouldBeError) + }) + + convey.Convey("failed to get the faas scheduler named", func() { + patches := []*gomonkey.Patches{ + gomonkey.ApplyMethod(reflect.TypeOf(&loadbalance.ConcurrentCHGeneric{}), "Next", + func(_ *loadbalance.ConcurrentCHGeneric, name string, move bool) interface{} { + return "faaSScheduler" + }), + } + defer func() { + for _, patch := range patches { + time.Sleep(100 * time.Millisecond) + patch.Reset() + } + }() + + _, err := Proxy.Get("functionKey", log.GetLogger()) + convey.So(err, convey.ShouldBeError) + }) + + convey.Convey("success", func() { + patches := []*gomonkey.Patches{ + gomonkey.ApplyMethod(reflect.TypeOf(&loadbalance.ConcurrentCHGeneric{}), "Next", + func(_ *loadbalance.ConcurrentCHGeneric, name string, move bool) interface{} { + return "instance1" + }), + } + defer func() { + for _, patch := range patches { + time.Sleep(100 * time.Millisecond) + patch.Reset() + } + }() + Proxy.Add(&types.InstanceInfo{InstanceName: "instance1"}, log.GetLogger()) + scheduler, err := Proxy.Get("functionKey", log.GetLogger()) + convey.So(err, convey.ShouldBeNil) + convey.So(scheduler.InstanceName, convey.ShouldEqual, "instance1") + }) + }) +} + +func Test_schedulerProxy_GetSchedulerByInstanceName(t *testing.T) { + convey.Convey("GetSchedulerByInstanceName", t, func() { + scheduler := &types.InstanceInfo{InstanceName: "name1", InstanceID: "id1"} + Proxy.Add(scheduler, log.GetLogger()) + getScheduler, err := Proxy.GetSchedulerByInstanceName("name1", "") + convey.So(err, convey.ShouldBeNil) + convey.So(getScheduler.InstanceID, convey.ShouldEqual, "id1") + + getScheduler, err = Proxy.GetSchedulerByInstanceName("name2", "") + convey.So(err, convey.ShouldNotBeNil) + }) +} diff --git a/frontend/pkg/frontend/schedulerproxy/schedulereventhandler.go b/frontend/pkg/frontend/schedulerproxy/schedulereventhandler.go new file mode 100644 index 0000000000000000000000000000000000000000..bcf27215bfb3c980490319b6c60174e4a2cdf660 --- /dev/null +++ b/frontend/pkg/frontend/schedulerproxy/schedulereventhandler.go @@ -0,0 +1,88 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package schedulerproxy - +package schedulerproxy + +import ( + "encoding/json" + + "go.uber.org/zap" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/types" + "frontend/pkg/common/faas_common/utils" + "frontend/pkg/frontend/config" +) + +// ProcessDelete - +func ProcessDelete(info *types.InstanceInfo, logger api.FormatLogger) { + defer logger.Infof("process delete event over") + if Proxy.ExistInstanceName(info.InstanceName) { + Proxy.Remove(info, logger) + Proxy.Reset() + logger.Infof("deleted from ProxyManager") + } +} + +// ProcessUpdate - +func ProcessUpdate(event *etcd3.Event, info *types.InstanceInfo, logger api.FormatLogger) { + defer logger.Infof("process update event over") + + instanceInfo := &types.InstanceSpecification{} + if len(event.Value) != 0 { + err := json.Unmarshal(event.Value, instanceInfo) + if err != nil { + logger.Errorf("failed to unmarshal ProxyManager to instanceInfo, error %s", err.Error()) + } + } + + logger = logger.With(zap.Any("instanceName", info.InstanceName), zap.Any("instanceId", instanceInfo.InstanceID)) + if instanceInfo.CreateOptions != nil { + info.Exclusivity = instanceInfo.CreateOptions[constant.SchedulerExclusivityKey] + } + info.InstanceID = instanceInfo.InstanceID + info.Address = instanceInfo.RuntimeAddress + + isExist := Proxy.Exist(info.InstanceName, info.InstanceID) + isRunning := instanceInfo.InstanceStatus.Code == int32(constant.KernelInstanceStatusRunning) + logger = logger.With(zap.Any("isExistInProxy", isExist), zap.Any("instanceStatus", instanceInfo.InstanceStatus.Code)) + + // scheduler实例添加到环的逻辑：如果是终端云融合架构场景，则无论scheduler状态如何，亦添加到hash环中,但是如果scheduler id为空，则删除该实例。否则 需要判断其实例状态 + switch config.GetConfig().SchedulerKeyPrefixType { + case constant.SchedulerKeyTypeModule: + Proxy.Add(info, logger) + Proxy.Reset() + logger.Infof("add to ProxyManager") + case constant.SchedulerKeyTypeFunction: + fallthrough + default: + if !isExist && (isRunning || instanceInfo.InstanceStatus.Code == int32(constant.KernelInstanceStatusCreating)) { + Proxy.Add(info, logger) + Proxy.Reset() + logger.Infof("added to ProxyManager") + } else if utils.CheckFaaSSchedulerInstanceFault(instanceInfo.InstanceStatus) && isExist { + Proxy.Remove(info, logger) + Proxy.Reset() + logger.Infof("deleted from ProxyManager") + } else { + logger.Infof("do nothing") + } + } +} diff --git a/frontend/pkg/frontend/schedulerproxy/schedulereventhandler_test.go b/frontend/pkg/frontend/schedulerproxy/schedulereventhandler_test.go new file mode 100644 index 0000000000000000000000000000000000000000..272b15d4ba305e157c02ab2ac59a722ea9705e2b --- /dev/null +++ b/frontend/pkg/frontend/schedulerproxy/schedulereventhandler_test.go @@ -0,0 +1,120 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package schedulerproxy - +package schedulerproxy + +import ( + "encoding/json" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/types" + "frontend/pkg/frontend/config" +) + +func getBytes(info *types.InstanceSpecification) []byte { + bytes, _ := json.Marshal(info) + return bytes +} + +func TestProcessUpdate(t *testing.T) { + foundedCall := 0 + missedCall := 0 + defer gomonkey.ApplyMethod(reflect.TypeOf(Proxy), "Add", func(_ *ProxyManager, scheduler *types.InstanceInfo, _ api.FormatLogger) { + foundedCall++ + }).Reset() + resetCall := 0 + defer gomonkey.ApplyMethod(reflect.TypeOf(Proxy), "Reset", func(_ *ProxyManager) { + resetCall++ + }).Reset() + defer gomonkey.ApplyMethod(reflect.TypeOf(Proxy), "Remove", func(_ *ProxyManager, scheduler *types.InstanceInfo, _ api.FormatLogger) { + missedCall++ + }).Reset() + convey.Convey("Test module scheduler ProcessUpdate", t, func() { + oldType := config.GetConfig().SchedulerKeyPrefixType + config.GetConfig().SchedulerKeyPrefixType = "module" + event := &etcd3.Event{ + Key: "/scheduler1", // no need + } + info := &types.InstanceInfo{ + InstanceName: "instanceName1", + } + + insSpec := &types.InstanceSpecification{ + InstanceID: "", + } + event.Value = getBytes(insSpec) + ProcessUpdate(event, info, log.GetLogger()) + convey.So(foundedCall, convey.ShouldEqual, 1) + convey.So(resetCall, convey.ShouldEqual, 1) + + info = &types.InstanceInfo{InstanceName: "instanceName1"} + insSpec = &types.InstanceSpecification{InstanceID: "instanceId1"} + event.Value = getBytes(insSpec) + ProcessUpdate(event, info, log.GetLogger()) + convey.So(foundedCall, convey.ShouldEqual, 2) + convey.So(resetCall, convey.ShouldEqual, 2) + + event.Value = []byte(`{"createOptions":{}, "instanceStatus":{"code":3, "msg":"ok"}}`) + info = &types.InstanceInfo{InstanceName: "instanceName1"} + insSpec = &types.InstanceSpecification{InstanceID: "instanceId1", CreateOptions: make(map[string]string), InstanceStatus: types.InstanceStatus{ + Code: 3, + Msg: "ok", + }} + event.Value = getBytes(insSpec) + ProcessUpdate(event, info, log.GetLogger()) + convey.So(foundedCall, convey.ShouldEqual, 3) + convey.So(resetCall, convey.ShouldEqual, 3) + config.GetConfig().SchedulerKeyPrefixType = oldType + }) + + convey.Convey("Test function scheduler ProcessUpdate", t, func() { + oldType := config.GetConfig().SchedulerKeyPrefixType + config.GetConfig().SchedulerKeyPrefixType = "function" + + foundedCall = 0 + resetCall = 0 + event := &etcd3.Event{ + Key: "/scheduler1", + Value: []byte(""), + } + info := &types.InstanceInfo{ + InstanceName: "instanceName1", + } + ProcessUpdate(event, info, log.GetLogger()) + convey.So(foundedCall, convey.ShouldEqual, 0) + convey.So(resetCall, convey.ShouldEqual, 0) + + event.Value = []byte("{}") + ProcessUpdate(event, info, log.GetLogger()) + convey.So(foundedCall, convey.ShouldEqual, 0) + convey.So(resetCall, convey.ShouldEqual, 0) + + event.Value = []byte(`{"createOptions":{}, "instanceStatus":{"code":3, "msg":"ok"}}`) + ProcessUpdate(event, info, log.GetLogger()) + convey.So(foundedCall, convey.ShouldEqual, 1) + convey.So(resetCall, convey.ShouldEqual, 1) + config.GetConfig().SchedulerKeyPrefixType = oldType + }) +} diff --git a/frontend/pkg/frontend/selfregister/selfregister.go b/frontend/pkg/frontend/selfregister/selfregister.go new file mode 100644 index 0000000000000000000000000000000000000000..d0846c82cdbce9a5c10cbd4b13a88e462ba73c98 --- /dev/null +++ b/frontend/pkg/frontend/selfregister/selfregister.go @@ -0,0 +1,104 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package selfregister - +package selfregister + +import ( + "fmt" + "os" + + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/frontend/config" +) + +// RegisterFrontendInstanceToEtcd - +func RegisterFrontendInstanceToEtcd(stopCh <-chan struct{}) error { + instanceKey, err := getInstanceKeyWithClusterID() + if err != nil { + return err + } + register := etcd3.EtcdRegister{ + EtcdClient: etcd3.GetMetaEtcdClient(), + InstanceKey: instanceKey, + Value: "active", + StopCh: stopCh, + } + err = register.Register() + if err != nil { + return err + } + return nil +} + +func getInstanceKeyWithClusterID() (string, error) { + clusterID, err := getClusterID() + if err != nil { + return "", err + } + nodeIP := getNodeIP() + podName := os.Getenv("POD_NAME") + err = validateEnvs(nodeIP, podName) + if err != nil { + return "", err + } + key := fmt.Sprintf("/sn/frontend/instances/%s/%s/%s", clusterID, nodeIP, podName) + return key, nil +} + +func getClusterID() (string, error) { + clusterID := config.GetConfig().ClusterID + if clusterID == "" { + clusterID = os.Getenv("CLUSTER_ID") + } + if clusterID == "" { + log.GetLogger().Errorf("get cluster failed, can not register frontend info to etcd") + return "", fmt.Errorf("get cluster failed") + } + return clusterID, nil +} + +func getNodeIP() string { + nodeIP := os.Getenv("HOST_IP") + if nodeIP == "" { + nodeIP = os.Getenv("NODE_IP") + } + return nodeIP +} + +func getInstanceKey() (string, error) { + nodeIP := getNodeIP() + podName := os.Getenv("POD_NAME") + err := validateEnvs(nodeIP, podName) + if err != nil { + return "", err + } + key := fmt.Sprintf("/sn/frontend/instances/%s/%s", nodeIP, podName) + return key, nil +} + +func validateEnvs(nodeIP, podName string) error { + if nodeIP == "" { + log.GetLogger().Errorf("can not find NODE_IP env, can not register frontend info to etcd") + return fmt.Errorf("NODE_IP env not found") + } + if podName == "" { + log.GetLogger().Errorf("can not find POD_NAME env, can not register frontend info to etcd") + return fmt.Errorf("POD_NAME env not found") + } + return nil +} diff --git a/frontend/pkg/frontend/selfregister/selfregister_test.go b/frontend/pkg/frontend/selfregister/selfregister_test.go new file mode 100644 index 0000000000000000000000000000000000000000..ffcc84a225805201d00999b5e2d437627786c31b --- /dev/null +++ b/frontend/pkg/frontend/selfregister/selfregister_test.go @@ -0,0 +1,93 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package register - +package selfregister + +import ( + "os" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/types" +) + +func TestPrepareKey(t *testing.T) { + prepareEnv() + defer cleanEnv() + defer gomonkey.ApplyFunc(config.GetConfig, func() *types.Config { + return &types.Config{ + ClusterID: "cluster1", + } + }).Reset() + convey.Convey("test prepareKey", t, func() { + convey.Convey("getInstanceKey ok", func() { + _ = os.Setenv("HOST_IP", "127.0.0.1") + key, err := getInstanceKey() + _ = os.Setenv("HOST_IP", "") + convey.So(err, convey.ShouldBeNil) + convey.So(key, convey.ShouldEqual, "/sn/frontend/instances/127.0.0.1/frontend_****") + }) + convey.Convey("getInstanceKeyWithClusterID ok", func() { + key, err := getInstanceKeyWithClusterID() + convey.So(err, convey.ShouldBeNil) + convey.So(key, convey.ShouldEqual, "/sn/frontend/instances/cluster1/127.0.0.1/frontend_****") + }) + convey.Convey("validate env", func() { + err := validateEnvs("", "podname") + convey.So(err, convey.ShouldNotBeNil) + convey.So(err.Error(), convey.ShouldEqual, "NODE_IP env not found") + err = validateEnvs("ip", "") + convey.So(err, convey.ShouldNotBeNil) + convey.So(err.Error(), convey.ShouldEqual, "POD_NAME env not found") + }) + convey.Convey("getInstanceKeyWithClusterID cluster in evn", func() { + defer gomonkey.ApplyFunc(config.GetConfig, func() *types.Config { + return &types.Config{ + ClusterID: "", + } + }).Reset() + _ = os.Setenv("CLUSTER_ID", "cluster1") + key, err := getInstanceKeyWithClusterID() + convey.So(err, convey.ShouldBeNil) + convey.So(key, convey.ShouldEqual, "/sn/frontend/instances/cluster1/127.0.0.1/frontend_****") + }) + convey.Convey("getInstanceKeyWithClusterID get cluster failed", func() { + defer gomonkey.ApplyFunc(config.GetConfig, func() *types.Config { + return &types.Config{ + ClusterID: "", + } + }).Reset() + _ = os.Setenv("CLUSTER_ID", "") + _, err := getInstanceKeyWithClusterID() + convey.So(err, convey.ShouldNotBeNil) + convey.So(err.Error(), convey.ShouldEqual, "get cluster failed") + }) + }) +} + +func prepareEnv() { + _ = os.Setenv("NODE_IP", "127.0.0.1") + _ = os.Setenv("POD_NAME", "frontend_****") +} + +func cleanEnv() { + _ = os.Setenv("NODE_IP", "") + _ = os.Setenv("POD_NAME", "") +} diff --git a/frontend/pkg/frontend/server/http.go b/frontend/pkg/frontend/server/http.go new file mode 100644 index 0000000000000000000000000000000000000000..f7c86f12975b1821bd9be1a7806820432ecfdad8 --- /dev/null +++ b/frontend/pkg/frontend/server/http.go @@ -0,0 +1,196 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package server - +package server + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "net/url" + "os" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/gin-gonic/gin" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/healthlog" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/tls" + "frontend/pkg/frontend/serverstatus" + + "frontend/pkg/frontend/api" + "frontend/pkg/frontend/common" + "frontend/pkg/frontend/common/httpconstant" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/functiontask" + "frontend/pkg/frontend/middleware" + "frontend/pkg/frontend/selfregister" + "frontend/pkg/frontend/watcher" +) + +const ( + logFileName = "frontend" + idleTimeOffset = 10 +) + +var ( + server *http.Server + activeConns int64 + wg sync.WaitGroup +) + +// CreateHTTPServer - +func CreateHTTPServer() *http.Server { + listenIP := config.GetConfig().HTTPConfig.ServerListenIP + if len(listenIP) == 0 { + listenIP = os.Getenv(constant.PodIPEnvKey) + log.GetLogger().Warnf("failed to get pod ip from HTTPConfig, try to use %s as listen IP", listenIP) + } + if len(listenIP) == 0 { + log.GetLogger().Errorf("failed to get pod ip from env POD_IP") + return nil + } + gin.SetMode(gin.ReleaseMode) + engine := gin.New() + engine.Use(gin.Recovery()) + api.InitRoute(engine) + server = &http.Server{ + Handler: allowQuerySemicolons(engine), + ReadTimeout: time.Duration(config.GetConfig().HTTPConfig.ServerReadTimeout) * time.Second, + WriteTimeout: time.Duration(config.GetConfig().HTTPConfig.ServerWriteTimeout) * time.Second, + Addr: fmt.Sprintf("%s:%d", listenIP, config.GetConfig().HTTPConfig.ServerListenPort), + ConnState: recordConn, + } + return server +} + +func recordConn(conn net.Conn, state http.ConnState) { + switch state { + case http.StateNew: + atomic.AddInt64(&activeConns, 1) + wg.Add(1) + case http.StateClosed, http.StateHijacked: + atomic.AddInt64(&activeConns, -1) + wg.Done() + default: + return + } +} + +// GetHTTPServer - +func GetHTTPServer() *http.Server { + return server +} + +// Start some watchers +func Start(server *http.Server, stopCh <-chan struct{}) error { + // starts to listen and serve + if server == nil { + return errors.New("http server is nil") + } + log.GetLogger().Infof("FaaS-Frontend HTTP server starting on %s", server.Addr) + if err := watcher.StartWatch(stopCh); err != nil { + return err + } + if err := selfregister.RegisterFrontendInstanceToEtcd(stopCh); err != nil { + return err + } + + if config.GetConfig().HTTPSConfig != nil && config.GetConfig().HTTPSConfig.HTTPSEnable { + err := tls.InitTLSConfig(*config.GetConfig().HTTPSConfig) + if err != nil { + log.GetLogger().Errorf("failed to init the HTTPS config: %s", err.Error()) + return err + } + server.TLSConfig = tls.GetClientTLSConfig() + err = server.ListenAndServeTLS("", "") + if err != nil { + log.GetLogger().Errorf("error when https ListenAndServeTLS: %s", err.Error()) + return err + } + } else { + if err := server.ListenAndServe(); err != nil { + log.GetLogger().Errorf("error when http ListenAndServe: %s", err.Error()) + return err + } + } + go healthlog.PrintHealthLog(stopCh, printInputLog, logFileName) + return nil +} + +// GracefulShutdown Shutdown Gracefully +func GracefulShutdown(httpServer *http.Server) { + if httpServer == nil { + log.GetLogger().Infof("http server is not initialize, no need to shutdown") + return + } + log.GetLogger().Infof("http server start graceful shutdown") + serverstatus.Shutdown() + // wait long-connections closed + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + select { + case <-done: + log.GetLogger().Infof("all connections closed gracefully") + case <-time.After(time.Duration(config.GetConfig().HTTPConfig.ClientIdleTimeout+idleTimeOffset) * time.Second): + log.GetLogger().Infof("timeout reached, forcing shutdown") + } + ctx, cancel := context.WithTimeout(context.Background(), common.GracefulShutdownTimeOut) + defer cancel() + defer func() { + err := httpServer.Shutdown(ctx) + if err != nil { + log.GetLogger().Errorf("http server shutdown error") + } + }() + middleware.GraceExit() + log.GetLogger().Infof("http server shutdown after processing graceful exit") +} + +// getReadBufferSize get the default read buffer size for server and client +func getReadBufferSize() int { + return httpconstant.DefaultGraphReadBufferSize +} + +func allowQuerySemicolons(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.RawQuery, ";") { + r2 := new(http.Request) + *r2 = *r + r2.URL = new(url.URL) + *r2.URL = *r.URL + r2.URL.RawQuery = strings.ReplaceAll(r.URL.RawQuery, ";", httpconstant.SemicolonReplacer) + h.ServeHTTP(w, r2) + } else { + h.ServeHTTP(w, r) + } + }) +} + +func printInputLog() { + busProxyNum := functiontask.GetBusProxies().GetNum() + log.GetLogger().Infof("%s is alive. The number of busProxy is %d", logFileName, busProxyNum) +} diff --git a/frontend/pkg/frontend/server/http_test.go b/frontend/pkg/frontend/server/http_test.go new file mode 100644 index 0000000000000000000000000000000000000000..6c7e745544b4c3cb6458949ca766157ed56a0573 --- /dev/null +++ b/frontend/pkg/frontend/server/http_test.go @@ -0,0 +1,216 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package server + +import ( + "context" + "crypto/tls" + "errors" + "net/http" + "os" + "reflect" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/common/faas_common/etcd3" + commonTls "frontend/pkg/common/faas_common/tls" + mockUtils "frontend/pkg/common/faas_common/utils" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/middleware" + "frontend/pkg/frontend/types" + "frontend/pkg/frontend/watcher" +) + +func TestStart(t *testing.T) { + prepareEnv() + defer cleanEnv() + defer gomonkey.ApplyFunc(config.GetConfig, func() *types.Config { + return &types.Config{ + ClusterID: "cluster1", + AzID: "1", + HTTPConfig: &types.FrontendHTTP{ + ServerListenPort: 8888, + }, + } + }).Reset() + + defer gomonkey.ApplyMethod(reflect.TypeOf(&etcd3.EtcdRegister{}), "Register", + func(_ *etcd3.EtcdRegister) error { + return nil + }).Reset() + convey.Convey("Start", t, func() { + stopCh := make(chan struct{}) + convey.Convey("success", func() { + patches := []*gomonkey.Patches{ + gomonkey.ApplyFunc(watcher.StartWatch, func(stopCh <-chan struct{}) error { + return nil + }), + gomonkey.ApplyMethod(reflect.TypeOf(&http.Server{}), "ListenAndServe", + func(_ *http.Server) error { + return nil + }), + } + defer func() { + for _, patch := range patches { + patch.Reset() + } + }() + err := Start(CreateHTTPServer(), stopCh) + convey.So(err, convey.ShouldBeNil) + }) + + convey.Convey(" start https server success", func() { + patches := []*gomonkey.Patches{ + gomonkey.ApplyFunc(watcher.StartWatch, func(stopCh <-chan struct{}) error { + return nil + }), + gomonkey.ApplyMethod(reflect.TypeOf(&http.Server{}), "ListenAndServeTLS", + func(_ *http.Server, certFile, keyFile string) error { + return nil + }), + // ListenAndServeTLS(certFile, keyFile string) error + gomonkey.ApplyFunc(config.GetConfig, func() *types.Config { + return &types.Config{ + AzID: "1", + ClusterID: "1", + HTTPSConfig: &commonTls.InternalHTTPSConfig{ + HTTPSEnable: true, + }, + HTTPConfig: &types.FrontendHTTP{ + ServerListenPort: 8888, + }, + } + }), + gomonkey.ApplyFunc(commonTls.InitTLSConfig, func(config commonTls.InternalHTTPSConfig) error { + return nil + }), + + gomonkey.ApplyFunc(commonTls.GetClientTLSConfig, func() *tls.Config { + return &tls.Config{} + }), + } + defer func() { + for _, patch := range patches { + patch.Reset() + } + }() + server := CreateHTTPServer() + err := Start(server, stopCh) + convey.So(err, convey.ShouldBeNil) + convey.So(server.TLSConfig, convey.ShouldNotBeNil) + }) + + convey.Convey("server failed", func() { + patches := []*gomonkey.Patches{ + gomonkey.ApplyFunc(watcher.StartWatch, func(stopCh <-chan struct{}) error { + return nil + }), + gomonkey.ApplyMethod(reflect.TypeOf(&http.Server{}), "ListenAndServe", + func(_ *http.Server) error { + return errors.New("server error") + }), + } + defer func() { + for _, patch := range patches { + patch.Reset() + } + }() + err := Start(CreateHTTPServer(), stopCh) + convey.So(err, convey.ShouldNotBeNil) + }) + + convey.Convey("server failed exit", func() { + rt := &mockUtils.FakeLibruntimeSdkClient{} + exit := false + patches := []*gomonkey.Patches{ + gomonkey.ApplyFunc(watcher.StartWatch, func(stopCh <-chan struct{}) error { + return nil + }), + gomonkey.ApplyMethod(reflect.TypeOf(&http.Server{}), "ListenAndServe", + func(_ *http.Server) error { + return errors.New("server error") + }), + gomonkey.ApplyMethod(reflect.TypeOf(&mockUtils.FakeLibruntimeSdkClient{}), "Exit", + func(_ *mockUtils.FakeLibruntimeSdkClient) { + exit = true + }), + } + defer func() { + for _, patch := range patches { + patch.Reset() + } + }() + wait := make(chan int) + go func() { + err := Start(CreateHTTPServer(), stopCh) + if err != nil { + rt.Exit(0, "") + } + wait <- 1 + }() + <-wait + convey.So(exit, convey.ShouldBeTrue) + + }) + }) +} + +func TestGracefulShutdown(t *testing.T) { + config.GetConfig().HTTPConfig = &types.FrontendHTTP{} + config.GetConfig().HTTPConfig.ClientIdleTimeout = 0 + convey.Convey("GracefulShutdown", t, func() { + convey.Convey("success", func() { + patches := []*gomonkey.Patches{ + gomonkey.ApplyFunc(middleware.GraceExit, func() {}), + gomonkey.ApplyFunc(os.Exit, func(code int) {}), + gomonkey.ApplyMethod(reflect.TypeOf(&http.Server{}), "Shutdown", func(_ *http.Server, + ctx context.Context) error { + return nil + }), + } + defer func() { + for _, patch := range patches { + patch.Reset() + } + }() + stopCh := make(chan struct{}) + go func() { + time.Sleep(100 * time.Millisecond) + stopCh <- struct{}{} + }() + GracefulShutdown(&http.Server{}) + }) + convey.Convey("failed", func() { + GracefulShutdown(&http.Server{}) + }) + }) +} + +func prepareEnv() { + _ = os.Setenv("NODE_IP", "127.0.0.1") + _ = os.Setenv("POD_NAME", "frontend_****") + _ = os.Setenv("POD_IP", "127.0.0.1") +} + +func cleanEnv() { + _ = os.Setenv("NODE_IP", "") + _ = os.Setenv("POD_NAME", "") + _ = os.Setenv("POD_IP", "") +} diff --git a/frontend/pkg/frontend/serverstatus/serverstatus.go b/frontend/pkg/frontend/serverstatus/serverstatus.go new file mode 100644 index 0000000000000000000000000000000000000000..32454ccadab31468ca516c577400faff7f681b3c --- /dev/null +++ b/frontend/pkg/frontend/serverstatus/serverstatus.go @@ -0,0 +1,39 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package serverstatus the status of http server +package serverstatus + +import "sync" + +var ( + mu sync.RWMutex + isShutdown = false +) + +// IsShutdown return if server is shutdown +func IsShutdown() bool { + mu.RLock() + defer mu.RUnlock() + return isShutdown +} + +// Shutdown shutdown server +func Shutdown() { + mu.Lock() + isShutdown = true + mu.Unlock() +} diff --git a/frontend/pkg/frontend/state/state.go b/frontend/pkg/frontend/state/state.go new file mode 100644 index 0000000000000000000000000000000000000000..07622130dd70390c1d03eba3acf0fe3fcac0b154 --- /dev/null +++ b/frontend/pkg/frontend/state/state.go @@ -0,0 +1,141 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package state - +package state + +import ( + "encoding/json" + "fmt" + "os" + "sync" + + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/state" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/types" +) + +// FrontendState add the status to be saved here. +type FrontendState struct { + Config types.Config `json:"Config" valid:"optional"` +} + +const defaultHandlerQueueSize = 1000 + +var ( + frontendState = &FrontendState{} + frontendStateLock sync.RWMutex + frontendHandlerQueue *state.Queue + stateKey = "" +) + +func init() { + frontendInstanceIDSelf := os.Getenv("INSTANCE_ID") + stateKey = "/faas/state/recover/faasfrontend/" + frontendInstanceIDSelf +} + +// InitState - +func InitState() { + cfg := config.GetConfig() + if cfg.StateDisable { + log.GetLogger().Warnf("state is disable, skip init state") + return + } + if frontendHandlerQueue != nil { + return + } + frontendHandlerQueue = state.NewStateQueue(defaultHandlerQueueSize) + if frontendHandlerQueue == nil { + return + } + go frontendHandlerQueue.Run(updateState) +} + +// SetState - +func SetState(byte []byte) error { + return json.Unmarshal(byte, frontendState) +} + +// GetState - +func GetState() FrontendState { + frontendStateLock.RLock() + defer frontendStateLock.RUnlock() + return *frontendState +} + +// GetStateByte is used to obtain the local state +func GetStateByte() ([]byte, error) { + if frontendHandlerQueue == nil { + return nil, fmt.Errorf("frontendHandlerQueue is not initialized") + } + frontendStateLock.RLock() + defer frontendStateLock.RUnlock() + stateBytes, err := frontendHandlerQueue.GetState(stateKey) + if err != nil { + return nil, err + } + log.GetLogger().Debugf("get state from etcd frontendState: %v", string(stateBytes)) + return stateBytes, nil +} + +// DeleteStateByte - +func DeleteStateByte() error { + if frontendHandlerQueue == nil { + return fmt.Errorf("frontendHandlerQueue is not initialized") + } + frontendStateLock.RLock() + defer frontendStateLock.RUnlock() + return frontendHandlerQueue.DeleteState(stateKey) +} + +func updateState(value interface{}, tags ...string) { + if frontendHandlerQueue == nil { + log.GetLogger().Errorf("frontend state frontendHandlerQueue is nil") + return + } + frontendStateLock.Lock() + defer frontendStateLock.Unlock() + switch v := value.(type) { + case types.Config: + frontendState.Config = v + log.GetLogger().Infof("update frontend state for config") + default: + log.GetLogger().Warnf("unknown data type for FrontendState") + return + } + + state, err := json.Marshal(frontendState) + if err != nil { + log.GetLogger().Errorf("get frontend state error %s", err.Error()) + return + } + if err = frontendHandlerQueue.SaveState(state, stateKey); err != nil { + log.GetLogger().Errorf("save frontend state error: %s", err.Error()) + return + } + log.GetLogger().Info("update frontend state") +} + +// Update is used to write frontend state to the cache queue +func Update(value interface{}, tags ...string) { + if frontendHandlerQueue == nil { + return + } + if err := frontendHandlerQueue.Push(value, tags...); err != nil { + log.GetLogger().Errorf("failed to push state to state queue: %s", err.Error()) + } +} diff --git a/frontend/pkg/frontend/state/state_test.go b/frontend/pkg/frontend/state/state_test.go new file mode 100644 index 0000000000000000000000000000000000000000..c3bcb22b09d49ec8440f61abfc9042a52a6ad98b --- /dev/null +++ b/frontend/pkg/frontend/state/state_test.go @@ -0,0 +1,157 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package state + +import ( + "errors" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/state" + "frontend/pkg/frontend/types" +) + +func TestUpdateState(t *testing.T) { + convey.Convey("Test updateState", t, func() { + patch := gomonkey.ApplyFunc(etcd3.GetRouterEtcdClient, func() *etcd3.EtcdClient { + return &etcd3.EtcdClient{} + }) + defer patch.Reset() + InitState() + convey.Convey("frontendHandlerQueue is nil", func() { + rawFq := frontendHandlerQueue + frontendHandlerQueue = nil + updateState(nil) + frontendHandlerQueue = rawFq + convey.So(reflect.DeepEqual(frontendState.Config, types.Config{}), convey.ShouldEqual, true) + }) + convey.Convey("value is error type", func() { + updateState("value") + convey.So(reflect.DeepEqual(frontendState.Config, types.Config{}), convey.ShouldEqual, true) + }) + convey.Convey("value is Config type", func() { + q := &state.Queue{} + patch := gomonkey.ApplyMethod(reflect.TypeOf(q), + "SaveState", func(q *state.Queue, state []byte, key string) error { + return nil + }) + defer patch.Reset() + config := &types.Config{} + updateState(config) + convey.So(frontendState.Config, convey.ShouldNotBeNil) + }) + convey.Convey("save state error", func() { + q := &state.Queue{} + patch := gomonkey.ApplyMethod(reflect.TypeOf(q), + "SaveState", func(q *state.Queue, state []byte, key string) error { + return errors.New("save state error") + }) + defer patch.Reset() + config := &types.Config{} + updateState(config) + convey.So(frontendState.Config, convey.ShouldNotBeNil) + }) + }) +} + +func TestGetStateByte(t *testing.T) { + convey.Convey("Test getStateByte", t, func() { + patch := gomonkey.ApplyFunc(etcd3.GetRouterEtcdClient, func() *etcd3.EtcdClient { + return &etcd3.EtcdClient{} + }) + defer patch.Reset() + InitState() + convey.Convey("frontendHandlerQueue is nil", func() { + rawFq := frontendHandlerQueue + frontendHandlerQueue = nil + _, err := GetStateByte() + frontendHandlerQueue = rawFq + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("getStateByte success", func() { + q := &state.Queue{} + patch := gomonkey.ApplyMethod(reflect.TypeOf(q), + "GetState", func(q *state.Queue, key string) ([]byte, error) { + return []byte("state"), nil + }) + defer patch.Reset() + stateBytes, err := GetStateByte() + convey.So(string(stateBytes), convey.ShouldEqual, "state") + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("GetState error", func() { + q := &state.Queue{} + patch := gomonkey.ApplyMethod(reflect.TypeOf(q), + "GetState", func(q *state.Queue, key string) ([]byte, error) { + return []byte{}, errors.New("get state error") + }) + defer patch.Reset() + _, err := GetStateByte() + convey.So(err, convey.ShouldNotBeNil) + }) + }) +} + +func TestDeleteStateByte(t *testing.T) { + convey.Convey("Test deleteState", t, func() { + patch := gomonkey.ApplyFunc(etcd3.GetRouterEtcdClient, func() *etcd3.EtcdClient { + return &etcd3.EtcdClient{} + }) + defer patch.Reset() + InitState() + convey.Convey("frontendHandlerQueue is nil", func() { + rawFq := frontendHandlerQueue + frontendHandlerQueue = nil + err := DeleteStateByte() + frontendHandlerQueue = rawFq + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("DeleteState success", func() { + q := &state.Queue{} + patch := gomonkey.ApplyMethod(reflect.TypeOf(q), + "DeleteState", func(q *state.Queue, key string) error { + return nil + }) + defer patch.Reset() + err := DeleteStateByte() + convey.So(err, convey.ShouldBeNil) + }) + }) +} + +func TestUpdate(t *testing.T) { + convey.Convey("Test Update", t, func() { + patch := gomonkey.ApplyFunc(etcd3.GetRouterEtcdClient, func() *etcd3.EtcdClient { + return &etcd3.EtcdClient{} + }) + defer patch.Reset() + InitState() + convey.Convey("frontendHandlerQueue is nil", func() { + rawFq := frontendHandlerQueue + frontendHandlerQueue = nil + Update("value") + frontendHandlerQueue = rawFq + }) + convey.Convey("update success", func() { + Update("value") + }) + }) +} diff --git a/frontend/pkg/frontend/stream/requeststream.go b/frontend/pkg/frontend/stream/requeststream.go new file mode 100644 index 0000000000000000000000000000000000000000..c5c3da5e1f7339d2a273438744212762a13ad8e6 --- /dev/null +++ b/frontend/pkg/frontend/stream/requeststream.go @@ -0,0 +1,60 @@ +//go:build module + +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stream + +import ( + "net/http" + "strings" + + "github.com/gin-gonic/gin" + + "frontend/pkg/frontend/common/httpconstant" + "frontend/pkg/frontend/types" +) + +const ( + readOneMB = 1024 * 1024 +) + +var ( + upStreamContentTypes = []string{httpconstant.FormContentType, httpconstant.MultipartFormContentType} +) + +// BuildStreamContext - +func BuildStreamContext(ctx *gin.Context, processCtx *types.InvokeProcessContext) { + processCtx.StreamInfo = &types.StreamInvokeInfo{ + ReqStream: ctx.Request.Body, + // 下载流请求是一个普通的http请求，无法区分，因此所有场景rspStream都需要透传 + RspStream: ctx.Writer, + } +} + +// IsHTTPUploadStream - +func IsHTTPUploadStream(r *http.Request) bool { + contentType := r.Header.Get(httpconstant.ContentTypeHeaderKey) + if len(contentType) < 1 { + return false + } + for _, k := range upStreamContentTypes { + if strings.Contains(contentType, k) { + return true + } + } + return false +} diff --git a/frontend/pkg/frontend/stream/requeststream_function.go b/frontend/pkg/frontend/stream/requeststream_function.go new file mode 100644 index 0000000000000000000000000000000000000000..6b174567b8bffd06eea117977cfdbe991cb0759e --- /dev/null +++ b/frontend/pkg/frontend/stream/requeststream_function.go @@ -0,0 +1,36 @@ +//go:build function + +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stream + +import "frontend/pkg/frontend/types" + +// BuildStreamContext - +func BuildStreamContext(ctx interface{}, processCtx *types.InvokeProcessContext) { + processCtx.StreamInfo = &types.StreamInvokeInfo{} +} + +// HTTPStreamInvokeHandler - +func HTTPStreamInvokeHandler(ctx interface{}, timeout interface{}) error { + return nil +} + +// IsHTTPUploadStream - +func IsHTTPUploadStream(r interface{}) bool { + return false +} diff --git a/frontend/pkg/frontend/stream/requeststream_test.go b/frontend/pkg/frontend/stream/requeststream_test.go new file mode 100644 index 0000000000000000000000000000000000000000..ec41dbf83d75618e80e119bc578a4bc6bfab8c0f --- /dev/null +++ b/frontend/pkg/frontend/stream/requeststream_test.go @@ -0,0 +1,127 @@ +//go:build module + +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stream + +import ( + "bufio" + "bytes" + "frontend/pkg/frontend/types" + "github.com/gin-gonic/gin" + "github.com/smartystreets/goconvey/convey" + "net" + "net/http" + "net/http/httptest" + "testing" +) + +func TestIsHTTPUploadStream(t *testing.T) { + convey.Convey("Test IsHTTPUploadStream", t, func() { + req, err := http.NewRequest("POST", "http://example.com", nil) + convey.So(err, convey.ShouldBeNil) + // when no content-type + convey.So(IsHTTPUploadStream(req), convey.ShouldBeFalse) + + // when is stream req + req.Header.Set("Content-Type", "multipart/form-data") + convey.So(IsHTTPUploadStream(req), convey.ShouldBeTrue) + + // when not stream req + req.Header.Set("Content-Type", "json") + convey.So(IsHTTPUploadStream(req), convey.ShouldBeFalse) + }) +} + +func TestBuildStreamContext(t *testing.T) { + convey.Convey("Test BuildStreamContext", t, func() { + ctx, _ := gin.CreateTestContext(httptest.NewRecorder()) + reqBody := bytes.NewBufferString("test body") + ctx.Request, _ = http.NewRequest("POST", "/", reqBody) + + processCtx := &types.InvokeProcessContext{} + BuildStreamContext(ctx, processCtx) + + convey.So(processCtx.StreamInfo.ReqStream, convey.ShouldResemble, ctx.Request.Body) + convey.So(processCtx.StreamInfo.RspStream, convey.ShouldResemble, ctx.Writer) + }) +} + +type MockReadCloser struct{} + +func (m MockReadCloser) Close() error { + return nil +} + +func (m MockReadCloser) Read([]byte) (int, error) { + return 0, nil +} + +type fakeResponseWriter struct { + close bool + header http.Header + body []byte + statusCode int +} + +func (f fakeResponseWriter) Header() http.Header { + return f.header +} + +func (f fakeResponseWriter) Write(i []byte) (int, error) { + panic("implement me") +} + +func (f *fakeResponseWriter) WriteHeader(statusCode int) { + f.statusCode = statusCode +} + +func (f fakeResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + panic("implement me") +} + +func (f fakeResponseWriter) Flush() { + panic("implement me") +} + +func (f fakeResponseWriter) CloseNotify() <-chan bool { + panic("implement me") +} + +func (f fakeResponseWriter) Status() int { + panic("implement me") +} + +func (f fakeResponseWriter) Size() int { + panic("implement me") +} + +func (f fakeResponseWriter) WriteString(s string) (int, error) { + panic("implement me") +} + +func (f fakeResponseWriter) Written() bool { + panic("implement me") +} + +func (f fakeResponseWriter) WriteHeaderNow() { + panic("implement me") +} + +func (f fakeResponseWriter) Pusher() http.Pusher { + panic("implement me") +} diff --git a/frontend/pkg/frontend/stream/responsestream.go b/frontend/pkg/frontend/stream/responsestream.go new file mode 100644 index 0000000000000000000000000000000000000000..6d27e92eadcf490e90c02314a2fb145b179f9b5c --- /dev/null +++ b/frontend/pkg/frontend/stream/responsestream.go @@ -0,0 +1,105 @@ +//go:build module + +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stream + +import ( + "errors" + "github.com/google/uuid" + "os" + "sync" + "time" + + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/types" +) + +const ( + failedRetrySubscribeSleepTime = 1 * time.Second + zeroElementSleepTimeLimit = 15 * time.Second +) + +var ( + responseStreamMap sync.Map + frontendResponseStreamName = os.Getenv("HOSTNAME") + errRecNodata = errors.New("receive no data") +) + +// ResponseStream - +type ResponseStream struct { + processContext *types.InvokeProcessContext + stopChan *types.StreamStopChan + isStream bool + processTimes int32 +} + +// NewStopChan - +func NewStopChan() *types.StreamStopChan { + return &types.StreamStopChan{C: make(chan struct{})} +} + +// GetFrontendResponseStreamName - +func GetFrontendResponseStreamName() string { + return frontendResponseStreamName +} + +// responseInfo - +type responseInfo struct { + StatusCode int `json:"statusCode"` + RequestID string `json:"requestID"` + ResponseHeaders map[string][]string `json:"responseHeaders"` + ResponseStreamName string `json:"responseStreamName"` +} + +// RegisterResponse - +func RegisterResponse(ctx *types.InvokeProcessContext) bool { + if !config.GetConfig().StreamEnable { + return false + } + // 流下载请求是普通http请求无法区分，等监听流收到数据才能确定是下载流请求。所有http请求都要先注册响应流 + ctx.StreamInfo.ResponseStreamName = uuid.New().String() + responseStopChan := NewStopChan() + r := &ResponseStream{processContext: ctx, stopChan: responseStopChan} + responseStreamMap.Store(ctx.StreamInfo.ResponseStreamName, r) + ctx.StreamInfo.ResponseStopChan = responseStopChan + return true +} + +// ReleaseResponse - +func ReleaseResponse(responseStreamName string) { + if !config.GetConfig().StreamEnable { + return + } + responseStreamMap.Delete(responseStreamName) +} + +// CheckIsResponseStream - +func CheckIsResponseStream(responseStreamName string) bool { + if !config.GetConfig().StreamEnable { + return false + } + v, ok := responseStreamMap.Load(responseStreamName) + if !ok { + return false + } + res, ok := v.(*ResponseStream) + if !ok { + return false + } + return res.isStream +} diff --git a/frontend/pkg/frontend/stream/responsestream_function.go b/frontend/pkg/frontend/stream/responsestream_function.go new file mode 100644 index 0000000000000000000000000000000000000000..6bd0a932918fbb192d0b08ad87ead07eefefb881 --- /dev/null +++ b/frontend/pkg/frontend/stream/responsestream_function.go @@ -0,0 +1,38 @@ +//go:build function + +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stream + +// GetFrontendResponseStreamName - +func GetFrontendResponseStreamName() string { + return "" +} + +// RegisterResponse - +func RegisterResponse(ctx interface{}) bool { + return false +} + +// ReleaseResponse - +func ReleaseResponse(responseStreamName string) { +} + +// CheckIsResponseStream - +func CheckIsResponseStream(responseStreamName string) bool { + return false +} diff --git a/frontend/pkg/frontend/stream/responsestream_test.go b/frontend/pkg/frontend/stream/responsestream_test.go new file mode 100644 index 0000000000000000000000000000000000000000..82c9baad6d94410b913ece9518a50f0b20672a06 --- /dev/null +++ b/frontend/pkg/frontend/stream/responsestream_test.go @@ -0,0 +1,116 @@ +//go:build module + +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stream + +import ( + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/types" +) + +type MockResponseWriter struct{} + +func (m MockResponseWriter) Header() http.Header { + return http.Header{} +} + +func (m MockResponseWriter) Write([]byte) (int, error) { + return 0, nil +} + +func (m MockResponseWriter) WriteHeader(int) { + return +} + +func TestRegisterResponse(t *testing.T) { + t.Run("not enable dataSystem", func(t *testing.T) { + config.GetConfig().StreamEnable = false + reqInfo := &ResponseStream{ + processContext: &types.InvokeProcessContext{ + RequestTraceInfo: &types.RequestTraceInfo{ + Deadline: time.Now().Add(-1 * time.Second), + }, + StreamInfo: &types.StreamInvokeInfo{ + RspStream: MockResponseWriter{}}, + }, + } + RegisterResponse(reqInfo.processContext) + assert.Nil(t, reqInfo.processContext.StreamInfo.ResponseStopChan) + _, ok := responseStreamMap.Load(reqInfo.processContext.StreamInfo.ResponseStreamName) + assert.False(t, ok) + }) + t.Run("enable dataSystem", func(t *testing.T) { + config.GetConfig().StreamEnable = true + reqInfo := &ResponseStream{ + processContext: &types.InvokeProcessContext{ + RequestTraceInfo: &types.RequestTraceInfo{ + Deadline: time.Now().Add(-1 * time.Second), + }, + StreamInfo: &types.StreamInvokeInfo{ + RspStream: MockResponseWriter{}}, + }, + } + RegisterResponse(reqInfo.processContext) + assert.NotNil(t, reqInfo.processContext.StreamInfo.ResponseStopChan) + _, ok := responseStreamMap.Load(reqInfo.processContext.StreamInfo.ResponseStreamName) + assert.True(t, ok) + }) +} + +func TestCheckIsResponseStream(t *testing.T) { + t.Run("not enable datasystem", func(t *testing.T) { + config.GetConfig().StreamEnable = false + result := CheckIsResponseStream("xxx") + assert.False(t, result) + }) + t.Run("not exist", func(t *testing.T) { + config.GetConfig().StreamEnable = true + traceID := "not exist" + result := CheckIsResponseStream(traceID) + assert.False(t, result) + }) + t.Run("not_response_stream_type", func(t *testing.T) { + config.GetConfig().StreamEnable = true + traceID := "test_trace_id" + responseStreamMap.Store(traceID, "not_response_stream_type") + result := CheckIsResponseStream(traceID) + assert.False(t, result) + }) + t.Run("not stream", func(t *testing.T) { + config.GetConfig().StreamEnable = true + traceID := "test_trace_id" + responseStreamMap.Store(traceID, &ResponseStream{isStream: false}) + defer responseStreamMap.Delete(traceID) + result := CheckIsResponseStream(traceID) + assert.False(t, result) + }) + t.Run("is stream", func(t *testing.T) { + config.GetConfig().StreamEnable = true + traceID := "test_trace_id" + responseStreamMap.Store(traceID, &ResponseStream{isStream: true}) + defer responseStreamMap.Delete(traceID) + result := CheckIsResponseStream(traceID) + assert.True(t, result) + }) +} diff --git a/frontend/pkg/frontend/stream/subscriber.go b/frontend/pkg/frontend/stream/subscriber.go new file mode 100644 index 0000000000000000000000000000000000000000..7900dde3e5d7b6dd170ba6ca8914ba1435f7ef40 --- /dev/null +++ b/frontend/pkg/frontend/stream/subscriber.go @@ -0,0 +1,112 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package stream - +package stream + +import ( + "errors" + "strconv" + "strings" + + "github.com/gin-gonic/gin" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/datasystemclient" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/statuscode" + "frontend/pkg/common/faas_common/utils" + "frontend/pkg/frontend/common/httpconstant" + "frontend/pkg/frontend/responsehandler" + "frontend/pkg/frontend/types" +) + +// SubscribeHandler - +func SubscribeHandler(ctx *types.InvokeProcessContext) error { + streamName := ctx.ReqHeader[httpconstant.HeaderStreamName] + if isEmptyString(streamName) { + responsehandler.SetErrorInContext(ctx, statuscode.FrontendStatusBadRequest, "stream name is invalid") + log.GetLogger().Errorf("stream name with is missing") + return errors.New("stream name missing") + } + + if isEmptyString(ctx.ReqHeader[httpconstant.HeaderTimeoutMs]) { + responsehandler.SetErrorInContext(ctx, statuscode.FrontendStatusBadRequest, + "parameter: "+httpconstant.HeaderTimeoutMs+" is missing in header") + log.GetLogger().Errorf("parameter: '%s' is missing in header", httpconstant.HeaderTimeoutMs) + return errors.New("parameter missing in header") + } + + statusTransTimeout, timeoutMs := transformToNumber(ctx.ReqHeader[httpconstant.HeaderTimeoutMs]) + if !statusTransTimeout { + responsehandler.SetErrorInContext(ctx, statuscode.FrontendStatusBadRequest, + "parameter: "+httpconstant.HeaderTimeoutMs+" is invalid") + log.GetLogger().Errorf("parameter: '%s' is invalid", httpconstant.HeaderTimeoutMs) + return errors.New("parameter invalid") + } + + statusExpectNum, expectReceiveNum := transformToNumber(ctx.ReqHeader[httpconstant.HeaderExpectNum]) + if !isEmptyString(ctx.ReqHeader[httpconstant.HeaderExpectNum]) && !statusExpectNum { + responsehandler.SetErrorInContext(ctx, statuscode.FrontendStatusBadRequest, + "parameter: "+httpconstant.HeaderExpectNum+" is invalid") + log.GetLogger().Errorf("parameter: '%s' is invalid", httpconstant.HeaderExpectNum) + return errors.New("parameter invalid") + } + + streamCtx := &types.StreamContext{ + StreamName: streamName, + ExpectNum: int32(expectReceiveNum), + TimeoutMs: uint32(timeoutMs), + } + ctx.StreamCtx = streamCtx + return nil +} + +// StartSubscribeStream - +func StartSubscribeStream(ctx *types.InvokeProcessContext, httpCtx *gin.Context) error { + log.GetLogger().Infof("start subscribe stream with name: %s, timeout_ms: %d, expect_num: %d", + ctx.StreamCtx.StreamName, ctx.StreamCtx.TimeoutMs, ctx.StreamCtx.ExpectNum) + resultError := datasystemclient.SubscribeStream(datasystemclient.SubscribeParam{ + StreamName: ctx.StreamCtx.StreamName, + TimeoutMs: ctx.StreamCtx.TimeoutMs, + ExpectReceiveNum: ctx.StreamCtx.ExpectNum, + }, &datasystemclient.GinCtxAdapter{Context: httpCtx}) + if resultError != nil { + errInfo, ok := resultError.(api.ErrorInfo) + if !ok { + responsehandler.SetErrorInContext(ctx, statuscode.InternalErrorCode, + "subscribeStream return invalid error type") + } + responsehandler.SetErrorInContext(ctx, errInfo.Code, utils.MessageTruncation(errInfo.Error())) + } + return resultError +} + +func isEmptyString(s string) bool { + return len(strings.TrimSpace(s)) == 0 +} + +func transformToNumber(s string) (bool, int) { + n, err := strconv.Atoi(s) + if err != nil { + return false, -1 + } + if n <= 0 { + return false, n + } + return true, n +} diff --git a/frontend/pkg/frontend/stream/subscriber_test.go b/frontend/pkg/frontend/stream/subscriber_test.go new file mode 100644 index 0000000000000000000000000000000000000000..50b8b0f99f33c71f0570885afc61b965e476b54f --- /dev/null +++ b/frontend/pkg/frontend/stream/subscriber_test.go @@ -0,0 +1,65 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stream + +import ( + "testing" + + "github.com/smartystreets/goconvey/convey" +) + +func TestIsEmptyString(t *testing.T) { + convey.Convey("test something is Empty String", t, func() { + x := "" + convey.So(isEmptyString(x), convey.ShouldEqual, true) + }) + + convey.Convey("test something is not Empty String", t, func() { + x := "some" + convey.So(isEmptyString(x), convey.ShouldEqual, false) + }) +} + +func TestTransformToNumber(t *testing.T) { + convey.Convey("test something is positive num string", t, func() { + x := "1" + isNum, res := transformToNumber(x) + convey.So(isNum, convey.ShouldEqual, true) + convey.So(res, convey.ShouldEqual, 1) + }) + + convey.Convey("test something is empty string", t, func() { + x := "" + isNum, res := transformToNumber(x) + convey.So(isNum, convey.ShouldEqual, false) + convey.So(res, convey.ShouldEqual, -1) + }) + + convey.Convey("test something is not num string", t, func() { + x := "a" + isNum, res := transformToNumber(x) + convey.So(isNum, convey.ShouldEqual, false) + convey.So(res, convey.ShouldEqual, -1) + }) + + convey.Convey("test something is negative num string", t, func() { + x := "-2" + isNum, res := transformToNumber(x) + convey.So(isNum, convey.ShouldEqual, false) + convey.So(res, convey.ShouldEqual, -2) + }) +} diff --git a/frontend/pkg/frontend/subscriber/subscriber.go b/frontend/pkg/frontend/subscriber/subscriber.go new file mode 100644 index 0000000000000000000000000000000000000000..02ac6345fd2942c79c71ecbed502df78c34c2232 --- /dev/null +++ b/frontend/pkg/frontend/subscriber/subscriber.go @@ -0,0 +1,131 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package subscriber - +package subscriber + +import ( + "sync" + "sync/atomic" +) + +const ( + // Delete - + Delete = "delete" + // Update - + Update = "update" +) + +var ( + enableEventTypes = map[string]struct{}{ + Update: {}, + Delete: {}, + } +) + +// Observer - +type Observer struct { + Update func(data interface{}) + Delete func(data interface{}) +} + +type event struct { + eventType string + eventValue interface{} +} + +// Subject - +type Subject struct { + enable atomic.Bool + observers []*Observer + eventChan chan *event + sync.RWMutex +} + +// NewSubject - +func NewSubject() *Subject { + return &Subject{ + observers: make([]*Observer, 0), + eventChan: make(chan *event, 1000), + RWMutex: sync.RWMutex{}, + } +} + +// PublishEvent - +func (s *Subject) PublishEvent(eventType string, data interface{}) { + if !s.enable.Load() { + return + } + + if _, ok := enableEventTypes[eventType]; !ok { + return + } + + s.eventChan <- &event{ + eventType: eventType, + eventValue: data, + } +} + +// Subscribe - +func (s *Subject) Subscribe(o *Observer) { + s.Lock() + defer s.Unlock() + s.observers = append(s.observers, o) +} + +func (s *Subject) notifyUpdateEvent(data interface{}) { + s.RLock() + defer s.RUnlock() + for _, observer := range s.observers { + observer.Update(data) + } +} + +func (s *Subject) notifyDeleteEvent(data interface{}) { + s.RLock() + defer s.RUnlock() + for _, observer := range s.observers { + observer.Delete(data) + } +} + +// StartLoop - +func (s *Subject) StartLoop(stopCh <-chan struct{}) { + if s.enable.Load() { + return // 防重放 + } + s.enable.Store(true) + go func() { + for { + select { + case e, ok := <-s.eventChan: + if !ok { + return + } + switch e.eventType { + case Update: + s.notifyUpdateEvent(e.eventValue) + case Delete: + s.notifyDeleteEvent(e.eventValue) + default: // do nothing + } + case <-stopCh: + return + } + } + }() +} diff --git a/frontend/pkg/frontend/subscriber/subscriber_test.go b/frontend/pkg/frontend/subscriber/subscriber_test.go new file mode 100644 index 0000000000000000000000000000000000000000..d74ccc5ca4e20d9e21ae1c63b640f9f5c2e7191c --- /dev/null +++ b/frontend/pkg/frontend/subscriber/subscriber_test.go @@ -0,0 +1,251 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package subscriber + +import ( + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/stretchr/testify/assert" +) + +func TestNewSubject(t *testing.T) { + sub := NewSubject() + + assert.False(t, sub.enable.Load()) + assert.Equal(t, 0, len(sub.observers)) + assert.NotNil(t, sub.eventChan) +} + +func TestPublishEvent(t *testing.T) { + t.Run("禁用状态下不发布事件", func(t *testing.T) { + sub := NewSubject() + data := "test data" + + sub.PublishEvent(Update, data) + sub.PublishEvent(Delete, data) + + // 验证事件没有被放入channel + assert.Equal(t, 0, len(sub.eventChan)) + }) + + t.Run("启用状态下发布Update事件", func(t *testing.T) { + sub := NewSubject() + sub.enable.Store(true) + data := "test data" + + sub.PublishEvent(Update, data) + + assert.Equal(t, 1, len(sub.eventChan)) + e := <-sub.eventChan + assert.Equal(t, data, e.eventValue) + assert.Equal(t, Update, e.eventType) + }) + + t.Run("启用状态下发布Delete事件", func(t *testing.T) { + sub := NewSubject() + sub.enable.Store(true) + data := "test data" + + sub.PublishEvent(Delete, data) + + assert.Equal(t, 1, len(sub.eventChan)) + e := <-sub.eventChan + assert.Equal(t, data, e.eventValue) + assert.Equal(t, Delete, e.eventType) + }) + + t.Run("不支持的eventType", func(t *testing.T) { + sub := NewSubject() + sub.enable.Store(true) + data := "test data" + + sub.PublishEvent("invalid", data) + + assert.Equal(t, 0, len(sub.eventChan)) + }) +} + +func TestSubscribe(t *testing.T) { + sub := NewSubject() + observer1 := &Observer{ + Update: func(data interface{}) {}, + Delete: func(data interface{}) {}, + } + observer2 := &Observer{ + Update: func(data interface{}) {}, + Delete: func(data interface{}) {}, + } + + sub.Subscribe(observer1) + sub.Subscribe(observer2) + + assert.Equal(t, 2, len(sub.observers)) + assert.Equal(t, observer1, sub.observers[0]) + assert.Equal(t, observer2, sub.observers[1]) +} + +func TestNotifyEvents(t *testing.T) { + t.Run("通知Update事件", func(t *testing.T) { + sub := NewSubject() + data := "test data" + + var updateCalled0 bool + var updateCalled1 bool + observer0 := &Observer{ + Update: func(d interface{}) { + updateCalled0 = true + assert.Equal(t, data, d) + }, + Delete: func(d interface{}) {}, + } + observer1 := &Observer{ + Update: func(d interface{}) { + updateCalled1 = true + assert.Equal(t, data, d) + }, + Delete: func(d interface{}) {}, + } + sub.Subscribe(observer0) + sub.Subscribe(observer1) + + sub.notifyUpdateEvent(data) + + assert.True(t, updateCalled0) + assert.True(t, updateCalled1) + }) + + t.Run("通知Delete事件", func(t *testing.T) { + sub := NewSubject() + data := "test data" + + var deleteCalled0 bool + var deleteCalled1 bool + + observer0 := &Observer{ + Update: func(d interface{}) {}, + Delete: func(d interface{}) { + deleteCalled0 = true + assert.Equal(t, data, d) + }, + } + observer1 := &Observer{ + Update: func(d interface{}) {}, + Delete: func(d interface{}) { + deleteCalled1 = true + assert.Equal(t, data, d) + }, + } + sub.Subscribe(observer0) + sub.Subscribe(observer1) + + sub.notifyDeleteEvent(data) + + assert.True(t, deleteCalled0) + assert.True(t, deleteCalled1) + }) +} + +func TestStartLoop(t *testing.T) { + t.Run("防重放启动", func(t *testing.T) { + sub := NewSubject() + + patches := gomonkey.NewPatches() + defer patches.Reset() + + count := 0 + observer := &Observer{ + Update: func(d interface{}) { + count++ + }, + Delete: func(d interface{}) {}, + } + sub.Subscribe(observer) + + stopCh := make(chan struct{}) + sub.StartLoop(stopCh) + sub.StartLoop(stopCh) + sub.eventChan <- &event{ + eventType: Update, + eventValue: "???", + } + time.Sleep(1 * time.Second) + close(stopCh) + assert.Equal(t, count, 1) + sub.eventChan <- &event{ + eventType: Update, + eventValue: "???", + } + time.Sleep(1 * time.Second) + assert.Equal(t, count, 1) + }) + + t.Run("正常处理Update事件", func(t *testing.T) { + sub := NewSubject() + stopCh := make(chan struct{}) + defer close(stopCh) + data := "test data" + + var updateCalled bool + observer := &Observer{ + Update: func(d interface{}) { + updateCalled = true + assert.Equal(t, data, d) + }, + Delete: func(d interface{}) {}, + } + sub.Subscribe(observer) + + sub.eventChan <- &event{ + eventType: Update, + eventValue: data, + } + + sub.StartLoop(stopCh) + time.Sleep(100 * time.Millisecond) // 等待goroutine处理 + + assert.True(t, updateCalled) + }) + + t.Run("正常处理Delete事件", func(t *testing.T) { + sub := NewSubject() + stopCh := make(chan struct{}) + defer close(stopCh) + data := "test data" + + var deleteCalled bool + observer := &Observer{ + Update: func(d interface{}) {}, + Delete: func(d interface{}) { + deleteCalled = true + assert.Equal(t, data, d) + }, + } + sub.Subscribe(observer) + + sub.eventChan <- &event{ + eventType: Delete, + eventValue: data, + } + + sub.StartLoop(stopCh) + time.Sleep(100 * time.Millisecond) // 等待goroutine处理 + + assert.True(t, deleteCalled) + }) +} diff --git a/frontend/pkg/frontend/tenanttrafficlimit/trafficlimit.go b/frontend/pkg/frontend/tenanttrafficlimit/trafficlimit.go new file mode 100644 index 0000000000000000000000000000000000000000..06237f93e4417322a1a3b6b453672b519113a8b5 --- /dev/null +++ b/frontend/pkg/frontend/tenanttrafficlimit/trafficlimit.go @@ -0,0 +1,101 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package tenanttrafficlimit is for trigger traffic limitation +package tenanttrafficlimit + +import ( + "errors" + "fmt" + "sync" + + "golang.org/x/time/rate" + + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/urnutils" + "frontend/pkg/frontend/config" +) + +// TenantContainer tenant information and tenant's Limiter +type TenantContainer struct { + tenantInfo *sync.Map +} + +// TenantInfo - +type TenantInfo struct { + Quota int + Limiter *rate.Limiter +} + +var ( + // tenantBuf - + tenantBuf = &TenantContainer{ + tenantInfo: &sync.Map{}, + } +) + +// Limit is the main function of tenant traffic limitation +func Limit(tenantID string) error { + allow, quota := tenantBuf.tenantTakeOneToken(tenantID) + if !allow { + errMsg := fmt.Sprintf("Requests reached the max quota %d of the tenant %s", + quota, urnutils.Anonymize(tenantID)) + return errors.New(errMsg) + } + return nil +} + +func (t *TenantContainer) tenantTakeOneToken(tenantID string) (bool, int) { + tenantInfo := t.getTenantInfo(tenantID) + if tenantInfo.Limiter == nil { + log.GetLogger().Infof("traffic limiter is invalid") + tenantInfo.Limiter = getDefaultLimiter() + t.tenantInfo.Store(tenantID, tenantInfo) + return tenantInfo.Limiter.Allow(), tenantInfo.Quota + } + return tenantInfo.Limiter.Allow(), tenantInfo.Quota +} + +// getTenantInfo to generator the tenant limiter +func (t *TenantContainer) getTenantInfo(tenantID string) TenantInfo { + getInfo, ok := t.tenantInfo.Load(tenantID) + if !ok { + log.GetLogger().Warnf("failed to get tenant info for tenant %s.", urnutils.Anonymize(tenantID)) + return TenantInfo{Limiter: nil, Quota: config.GetConfig().DefaultTenantLimitQuota} + } + tenantInfo, ok := getInfo.(TenantInfo) + if !ok { + log.GetLogger().Warnf("invalid tenant info type for tenant %s.", urnutils.Anonymize(tenantID)) + return TenantInfo{Limiter: nil, Quota: config.GetConfig().DefaultTenantLimitQuota} + } + return tenantInfo +} + +func (t *TenantContainer) syncLimiter(tenantID string, tenantInfo TenantInfo) { + tenantInfo.Limiter = t.getTenantLimiter(tenantInfo) + t.tenantInfo.Store(tenantID, tenantInfo) +} + +// ProcessUpdate - +func ProcessUpdate(event *etcd3.Event) error { + return tenantBuf.ProcessUpdate(event) +} + +// ProcessDelete - +func ProcessDelete(event *etcd3.Event) { + tenantBuf.ProcessDelete(event) +} diff --git a/frontend/pkg/frontend/tenanttrafficlimit/trafficlimit_test.go b/frontend/pkg/frontend/tenanttrafficlimit/trafficlimit_test.go new file mode 100644 index 0000000000000000000000000000000000000000..5d1cc13930f09f3ab744016d754376e60fe08565 --- /dev/null +++ b/frontend/pkg/frontend/tenanttrafficlimit/trafficlimit_test.go @@ -0,0 +1,125 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package tenanttrafficlimit + +import ( + "fmt" + "sync" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/stretchr/testify/assert" + "golang.org/x/time/rate" + + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/types" +) + +// TestTenantTraffic test for tenant traffic limiter +func TestTenantTraffic(t *testing.T) { + defer gomonkey.ApplyFunc(config.GetConfig, func() *types.Config { + return &types.Config{ + InstanceNum: 3, + DefaultTenantLimitQuota: 1800, + } + }).Reset() + tenantID01 := "tenantID1" + tenantID02 := "tenantID2" + tenantID03 := "tenantID3" + tenanting1 := TenantInfo{Quota: 17, Limiter: &rate.Limiter{}} + tenanting2 := TenantInfo{Quota: 13, Limiter: &rate.Limiter{}} + tenanting3 := TenantInfo{Quota: 20, Limiter: &rate.Limiter{}} + tenantBuf.tenantInfo.Store(tenantID01, tenanting1) + tenantBuf.tenantInfo.Store(tenantID02, tenanting2) + tenantBuf.tenantInfo.Store(tenantID03, tenanting3) + tenantBuf.syncLimiter(tenantID01, tenanting1) + tenantBuf.syncLimiter(tenantID02, tenanting2) + tenantBuf.syncLimiter(tenantID03, tenanting3) + for i := 0; i < 6; i++ { + assert.Equal(t, nil, Limit(tenantID01)) + } + for i := 7; i < 20; i++ { + assert.Error(t, Limit(tenantID01)) + } + for i := 0; i < 4; i++ { + assert.Equal(t, nil, Limit(tenantID02)) + } + for i := 5; i < 20; i++ { + assert.Error(t, Limit(tenantID02)) + } + for i := 0; i < 7; i++ { + assert.Equal(t, nil, Limit(tenantID03)) + } + for i := 8; i < 20; i++ { + assert.Error(t, Limit(tenantID03)) + } + fmt.Println("test update") + time.Sleep(5 * time.Second) + //update tenant quota + event1 := &etcd3.Event{ + Type: etcd3.PUT, + Key: "/sn/qos/business/yrk/tenant/tenantID1", + Value: []byte(`{"quota":10}`), + } + tenantBuf.ProcessUpdate(event1) + for i := 0; i < 3; i++ { + assert.Equal(t, nil, Limit(tenantID01)) + } + for i := 4; i < 20; i++ { + assert.Error(t, Limit(tenantID01)) + } + time.Sleep(5 * time.Second) + //delete tenant quota + fmt.Println("test delete") + event2 := &etcd3.Event{ + Type: etcd3.DELETE, + Key: "/sn/qos/business/yrk/tenant/tenantID2", + } + tenantBuf.ProcessDelete(event2) + for i := 0; i < 40; i++ { + assert.Equal(t, nil, Limit(tenantID02)) + } + fmt.Println("test large scale") + event3 := &etcd3.Event{ + Type: etcd3.PUT, + Key: "/sn/qos/business/yrk/tenant/tenantID3", + Value: []byte(`{"quota":200000}`), + } + tenantBuf.ProcessUpdate(event3) + for i := 0; i < 40000; i++ { + assert.Equal(t, nil, Limit(tenantID03)) + } +} + +func TestTenantTakeOneToken(t *testing.T) { + defer gomonkey.ApplyFunc(config.GetConfig, func() *types.Config { + return &types.Config{ + InstanceNum: 3, + DefaultTenantLimitQuota: 1800, + } + }).Reset() + tenantBuf.tenantInfo = &sync.Map{} + allow, quota := tenantBuf.tenantTakeOneToken("test") + assert.Equal(t, allow, true) + assert.Equal(t, quota, 1800) + tenantBuf.tenantInfo.Store("test", 18) + allow, quota = tenantBuf.tenantTakeOneToken("test") + assert.Equal(t, allow, true) + assert.Equal(t, quota, 1800) +} diff --git a/frontend/pkg/frontend/tenanttrafficlimit/variable.go b/frontend/pkg/frontend/tenanttrafficlimit/variable.go new file mode 100644 index 0000000000000000000000000000000000000000..722378c418b08f1da1dbc8cb8c3518a50037ae08 --- /dev/null +++ b/frontend/pkg/frontend/tenanttrafficlimit/variable.go @@ -0,0 +1,112 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package tenanttrafficlimit + +import ( + "encoding/json" + "math" + "strings" + + "golang.org/x/time/rate" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/urnutils" + "frontend/pkg/frontend/config" +) + +const ( + tenantIDIndex = 6 + zero = 0 + defaultFrontendNum = 1 +) + +func (t *TenantContainer) getTenantLimiter(info TenantInfo) *rate.Limiter { + frontendNum := getFrontendNum() + limitRate := float64(info.Quota) / float64(frontendNum) + limitBucketSize := int(math.Ceil(float64(info.Quota)) / + float64(frontendNum) * constant.TrafficRedundantRate) + tenantLimiter := rate.NewLimiter(rate.Limit(limitRate), limitBucketSize) + return tenantLimiter +} + +func getFrontendNum() int { + frontendNum := config.GetConfig().InstanceNum + if frontendNum <= zero { + frontendNum = defaultFrontendNum + } + return frontendNum +} + +// ProcessUpdate - +func (t *TenantContainer) ProcessUpdate(event *etcd3.Event) error { + var data map[string]int + s := strings.Split(event.Key, constant.KeySeparator) + if len(s) <= tenantIDIndex { + log.GetLogger().Errorf("failed to get the tenantID") + return nil + } + tenantID := s[tenantIDIndex] + if err := json.Unmarshal(event.Value, &data); err != nil { + log.GetLogger().Errorf("failed to unmarshal the etcd event value") + return err + } + if data == nil { + log.GetLogger().Errorf("failed to update the quota value") + return nil + } + quota, ok := data["quota"] + if !ok { + log.GetLogger().Errorf("failed to get the quota value") + return nil + } + tenantMsg := t.getTenantInfo(tenantID) + tenantMsg.Quota = quota + t.syncLimiter(tenantID, tenantMsg) + log.GetLogger().Infof("update tenant %s quota update to %d.", urnutils.Anonymize(tenantID), quota) + return nil +} + +// ProcessDelete - +func (t *TenantContainer) ProcessDelete(event *etcd3.Event) { + s := strings.Split(event.Key, constant.KeySeparator) + if len(s) <= tenantIDIndex { + log.GetLogger().Errorf("failed to get the tenantID") + return + } + tenantID := s[tenantIDIndex] + quota := config.GetConfig().DefaultTenantLimitQuota + tenantMsg := t.getTenantInfo(tenantID) + if tenantMsg.Limiter == nil { + log.GetLogger().Errorf("tenantID limiter is not exit,no need to delete") + return + } + tenantMsg.Quota = quota + t.syncLimiter(tenantID, tenantMsg) + log.GetLogger().Infof("delete tenant %s quota, and set it to default %d.", + urnutils.Anonymize(tenantID), quota) +} + +func getDefaultLimiter() *rate.Limiter { + frontendNum := getFrontendNum() + limitRate := float64(config.GetConfig().DefaultTenantLimitQuota) / float64(frontendNum) + limitBucketSize := int(math.Ceil(float64(config.GetConfig().DefaultTenantLimitQuota)) / + float64(frontendNum) * constant.TrafficRedundantRate) + tenantLimiter := rate.NewLimiter(rate.Limit(limitRate), limitBucketSize) + return tenantLimiter +} diff --git a/frontend/pkg/frontend/types/streamtypes.go b/frontend/pkg/frontend/types/streamtypes.go new file mode 100644 index 0000000000000000000000000000000000000000..624fd955bf468e60e669071ae290080de02f371e --- /dev/null +++ b/frontend/pkg/frontend/types/streamtypes.go @@ -0,0 +1,57 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package types + +import ( + "io" + "net/http" + "sync" + "sync/atomic" +) + +// StreamInvokeInfo - +type StreamInvokeInfo struct { + RequestStreamName string + ResponseStreamName string + ReqStream io.ReadCloser + RspStream http.ResponseWriter + RequestStreamErrorCode int32 + ResponseStopChan *StreamStopChan +} + +// SetRequestStreamErrorCode - +func (r *StreamInvokeInfo) SetRequestStreamErrorCode(errorCode int32) { + atomic.StoreInt32(&r.RequestStreamErrorCode, errorCode) +} + +// GetRequestStreamErrorCode - +func (r *StreamInvokeInfo) GetRequestStreamErrorCode() int32 { + return atomic.LoadInt32(&r.RequestStreamErrorCode) +} + +// StreamStopChan - +type StreamStopChan struct { + C chan struct{} + once sync.Once +} + +// SafeClose - +func (mc *StreamStopChan) SafeClose() { + mc.once.Do(func() { + close(mc.C) + }) +} diff --git a/frontend/pkg/frontend/types/type.go b/frontend/pkg/frontend/types/type.go new file mode 100644 index 0000000000000000000000000000000000000000..f0873dde3fac314b414e84cd5794b7c5ca40e14f --- /dev/null +++ b/frontend/pkg/frontend/types/type.go @@ -0,0 +1,349 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package types - +package types + +import ( + "encoding/json" + "time" + + "frontend/pkg/common/faas_common/alarm" + "frontend/pkg/common/faas_common/crypto" + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/localauth" + "frontend/pkg/common/faas_common/logger/config" + "frontend/pkg/common/faas_common/redisclient" + "frontend/pkg/common/faas_common/sts/raw" + "frontend/pkg/common/faas_common/tls" + "frontend/pkg/common/faas_common/types" + wisecloudTypes "frontend/pkg/common/faas_common/wisecloudtool/types" +) + +// FunctionRequestInfo function response info +type FunctionRequestInfo struct { + URN string `json:"Frn"` + BusinessID string `json:"BusinessId"` + TenantID string `json:"TenantId"` + Name string `json:"FuncName"` + Version string `json:"FuncVersion"` + TraceID string `json:"TraceId"` + Alias string `json:"Alias"` + AppID string `json:"AppID"` + StateKey string `json:"StateKey"` + NodeLabel string `json:"NodeLabel"` + FutureID string `json:"-"` +} + +// InvokeErrorResponse invoke error response +type InvokeErrorResponse struct { + Code int `json:"code"` + Message string `json:"message"` +} + +// ResourceSpecification contains resource specification of a requested instance +type ResourceSpecification struct { + CPU int64 `json:"cpu"` + Memory int64 `json:"memory"` + CustomResource map[string]int64 `json:"customResource"` +} + +// CallReq is the msg structure sent from the frontend to the executor +type CallReq struct { + Header map[string]string `json:"header"` + Path string `json:"path"` + Method string `json:"method"` + Query string `json:"query"` + Body json.RawMessage `json:"body"` +} + +// CallResp is the msg structure returned by the executor to the frontend +type CallResp struct { + Headers map[string]string `json:"headers"` + BillingDuration string `json:"billingDuration"` + InnerCode string `json:"innerCode"` + InvokeSummary string `json:"invokeSummary"` + LogResult string `json:"logResult"` + UserFuncTime float64 `json:"userFuncTime"` + ExecutorTime float64 `json:"executorTime"` + Body json.RawMessage `json:"body"` +} + +// InitResp - +type InitResp struct { + ErrorCode string `json:"errorCode"` + Message json.RawMessage `json:"message"` +} + +// Config is the config used by faas frontend function +type Config struct { + InstanceNum int `json:"instanceNum"` + CPU float64 `json:"cpu" valid:"optional"` + Memory float64 `json:"memory" valid:"optional"` + SLAQuota int `json:"slaQuota" valid:"optional"` + Runtime RuntimeConfig `json:"runtime" valid:"optional"` + LocalAuth *localauth.AuthConfig `json:"localAuth"` + MetaEtcd etcd3.EtcdConfig `json:"metaEtcd" valid:"required"` + DataSystemEtcd etcd3.EtcdConfig `json:"dataSystemEtcd" valid:"optional"` + CAEMetaEtcd etcd3.EtcdConfig `json:"caeMetaEtcd" valid:"optional"` + RouterEtcd etcd3.EtcdConfig `json:"routerEtcd" valid:"required"` + RedisConfig RedisConfig `json:"redisConfig" valid:"optional"` + HTTPConfig *FrontendHTTP `json:"http" valid:"optional"` + HTTPSConfig *tls.InternalHTTPSConfig `json:"httpsConfig" valid:"optional"` + DataSystemConfig *types.DataSystemConfig `json:"dataSystemConfig" valid:"optional"` + StreamEnable bool `json:"streamEnable" valid:"optional"` + StateDisable bool `json:"stateDisable" valid:"optional"` + BusinessType int `json:"businessType"` + FunctionInvokeBackend int `json:"functionInvokeBackend" valid:"optional"` + SccConfig crypto.SccConfig `json:"sccConfig" valid:"optional"` + Image string `json:"image" valid:"optional"` + SchedulerKeyPrefixType string `json:"schedulerKeyPrefixType" valid:"optional"` + MemoryControlConfig *types.MemoryControlConfig `json:"memoryControlConfig" valid:"optional"` + MemoryEvaluatorConfig *MemoryEvaluatorConfig `json:"memoryEvaluatorConfig" valid:"optional"` + DefaultTenantLimitQuota int `json:"defaultTenantLimitQuota" valid:"optional"` + // frontend pool + DynamicPoolEnable bool `json:"dynamicPoolEnable" valid:"optional"` + // CaaS config + AuthenticationEnable bool `json:"authenticationEnable" valid:"optional"` + RawStsConfig raw.StsConfig `json:"rawStsConfig,omitempty"` + TrafficLimitParams *TrafficLimitParams `json:"trafficLimitParams" valid:"optional"` + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + AzID string `json:"azID" valid:"optional"` + ClusterID string `json:"clusterID" valid:"optional"` + ClusterName string `json:"clusterName" valid:"optional"` + AlarmConfig alarm.Config `json:"alarmConfig" valid:"optional"` + Version string `json:"version" valid:"optional"` + // FunctionGraph config + FunctionNameSeparator string `json:"functionNameSeparator" valid:"optional"` + AlarmServerAddress string `json:"alarmServerAddress" valid:"optional"` + InvokeMaxRetryTimes int `json:"invokeMaxRetryTimes" valid:"optional"` + EtcdLeaseConfig *EtcdLeaseConfig `json:"etcdLeaseConfig" valid:"optional"` + HeartbeatConfig *HeartbeatConfig `json:"heartbeatConfig" valid:"optional"` + E2EMaxDelayTime int64 `json:"e2eMaxDelayTime" valid:"optional"` + RetryConfig *RetryConfig `json:"retry" valid:"optional"` + ShareKeys ShareKeys `json:"shareKeys" valid:"optional"` + Affinity string `json:"affinity"` + RPCClientConcurrentNum int `json:"rpcClientConcurrentNum" valid:"optional"` + NodeAffinity string `json:"nodeAffinity" valid:"optional"` + NodeAffinityPolicy string `json:"nodeAffinityPolicy" valid:"optional"` + AuthConfig AuthConfig `json:"authConfig" valid:"optional"` + WiseCloudConfig WiseCloudConfig `json:"wiseCloudConfig" valid:"optional"` +} + +// WiseCloudConfig - +type WiseCloudConfig struct { + ServiceAccountJwt wisecloudTypes.ServiceAccountJwt `json:"serviceAccountJwt" valid:"optional"` +} + +// RetryConfig define retry config +type RetryConfig struct { + InstanceExceptionRetry bool `json:"instanceExceptionRetry" valid:"optional"` +} + +// RedisConfig redis config +type RedisConfig struct { + ClusterID string `json:"clusterID,omitempty" valid:",optional"` + ServerAddr string `json:"serverAddr,omitempty" valid:",optional"` + ServerMode string `json:"serverMode,omitempty" valid:",optional"` + Password string `json:"password,omitempty" valid:",optional"` + EnableTLS bool `json:"enableTLS,omitempty" valid:",optional"` + TimeoutConf redisclient.TimeoutConf `json:"timeoutConf,omitempty" valid:",optional"` +} + +// MemoryEvaluatorConfig memory evaluator config +type MemoryEvaluatorConfig struct { + RequestMemoryEvaluator float64 `json:"requestMemoryEvaluator" valid:",optional"` +} + +// ShareKeys - +type ShareKeys struct { + AccessKey string `json:"accessKey" valid:"optional"` +} + +// RuntimeConfig config info +type RuntimeConfig struct { + Port string `json:"port" valid:",optional"` + AvailableZoneKey string `json:"azkey,omitempty" valid:",optional"` + + // SDK + LogConfig config.CoreInfo `json:"logConfig" valid:"optional"` + SystemAuthConfig SystemAuthConfig `json:"systemAuthConfig" valid:"optional"` + EnableSigaction bool `json:"enableSigaction" valid:"optional"` +} + +// FrontendHTTP Used to configure the ResponseTimeout +type FrontendHTTP struct { + RespTimeOut int64 `json:"resptimeout" valid:",optional"` + WorkerInstanceReadTimeOut int64 `json:"workerInstanceReadTimeOut" valid:",optional"` + // MaxRequestBodySize unit is M + MaxRequestBodySize int `json:"maxRequestBodySize" valid:"required"` + // MaxStreamRequestBodySize unit is M + MaxStreamRequestBodySize int `json:"maxStreamRequestBodySize" valid:"optional"` + // ServerReadTimeout unit is S + ServerReadTimeout int `json:"serverReadTimeout" valid:"optional"` + // ServerWriteTimeout unit is S + ServerWriteTimeout int `json:"serverWriteTimeout" valid:"optional"` + // ClientIdleTimeout unit is S + ClientIdleTimeout int `json:"clientIdleTimeout" valid:"optional"` + // MaxDataSystemMultiDataBodySize unit is M + MaxDataSystemMultiDataBodySize int `json:"maxDataSystemMultiDataBodySize" valid:"optional"` + ServerListenPort int `json:"serverListenPort" valid:"optional"` + ServerListenIP string `json:"serverListenIP" valid:"optional"` +} + +// TrafficLimitParams parameters of traffic limitation +type TrafficLimitParams struct { + InstanceLimitRate float64 `json:"instanceLimitRate" valid:",optional"` + InstanceBucketSize int `json:"instanceBucketSize" valid:",optional"` + FuncLimitRate float64 `json:"funcLimitRate" valid:",optional"` + FuncBucketSize int `json:"funcBucketSize" valid:",optional"` +} + +// StreamContext - +type StreamContext struct { + StreamName string + TimeoutMs uint32 + ExpectNum int32 +} + +// InvokeProcessContext - +type InvokeProcessContext struct { + // func basic info + TraceID string + RequestID string + FuncKey string + ShouldRetry bool + TrafficLimited bool + StartTime time.Time + RequestTraceInfo *RequestTraceInfo + IsHTTPUploadStream bool + StreamInfo *StreamInvokeInfo + AcquireTimeout int64 + InvokeTimeout int64 + InvokeWithoutScheduler bool + + // request info + ReqHeader map[string]string + ReqPath string + ReqMethod string + ReqQuery string + ReqBody []byte + // response info + StatusCode int + RespHeader map[string]string + RespBody []byte + + // 响应透传 + NeedReadRespHeader bool + + // stream + StreamCtx *StreamContext +} + +// CreateInvokeProcessContext - +func CreateInvokeProcessContext() *InvokeProcessContext { + return &InvokeProcessContext{ + ReqHeader: make(map[string]string), + RespHeader: make(map[string]string), + StartTime: time.Now(), + } +} + +// AuthConfig - +type AuthConfig struct { + LocalAuthConfig LocalAuthConfig `json:"localAuthConfig"` +} + +// PolicyConfig - +type PolicyConfig struct { + Allow string `json:"allow"` + Deny string `json:"deny"` +} + +// LocalAuthConfig - +type LocalAuthConfig struct { + LocalAuthCryptoPath string `json:"localAuthCryptoPath"` +} + +// SystemAuthConfig - +type SystemAuthConfig struct { + Enable bool `json:"enable" validate:"optional"` + AccessKey string `json:"accessKey" validate:"optional"` + SecretKey string `json:"secretKey" validate:"optional"` +} + +// APIGTriggerResponse extern interface of web response +type APIGTriggerResponse struct { + Body string `json:"body"` + Headers map[string][]string `json:"headers"` + StatusCode int `json:"statusCode"` + IsBase64Encoded bool `json:"isBase64Encoded"` +} + +// APIGTriggerEvent extern interface of web request +type APIGTriggerEvent struct { + IsBase64Encoded bool `json:"isBase64Encoded"` + HTTPMethod string `json:"httpMethod"` + Path string `json:"path"` + Body string `json:"body"` + PathParameters map[string]string `json:"pathParameters"` + RequestContext APIGRequestContext `json:"requestContext"` + Headers map[string]interface{} `json:"headers"` + QueryStringParameters map[string]interface{} `json:"queryStringParameters"` + UserData string `json:"user_data"` +} + +// APIGRequestContext - +type APIGRequestContext struct { + APIID string `json:"apiId"` + RequestID string `json:"requestId"` + Stage string `json:"stage"` + SourceIP string `json:"sourceIp"` +} + +// EtcdLeaseConfig etcd lease config +type EtcdLeaseConfig struct { + LeaseTTL int64 `yaml:"leaseTTL" valid:"optional"` + RenewTTL int64 `yaml:"renewTTL" valid:"optional"` +} + +// HeartbeatConfig heartbeat config +type HeartbeatConfig struct { + HeartbeatTimeout int `json:"heartbeatTimeout" valid:",optional"` + HeartbeatInterval int `json:"heartbeatInterval" valid:"optional"` + HeartbeatTimeoutThreshold int `json:"heartbeatTimeoutThreshold" valid:"optional"` +} + +// RequestTraceInfo - +type RequestTraceInfo struct { + URN string + BusinessID string + TenantID string + FuncName string + Version string + AnonymizeURN string + TryCount int + InnerCode int + AllBusCost time.Duration + LastBusCost time.Duration + Deadline time.Time + CallInstance string + CallNode string + TotalCost time.Duration + FrontendCost time.Duration + BusCost time.Duration + WorkerCost time.Duration +} diff --git a/frontend/pkg/frontend/watcher/alias.go b/frontend/pkg/frontend/watcher/alias.go new file mode 100644 index 0000000000000000000000000000000000000000..4f2457de8b9e65074f7d60e436b38d8e11d2c8ae --- /dev/null +++ b/frontend/pkg/frontend/watcher/alias.go @@ -0,0 +1,73 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package watcher + +import ( + "strings" + + "frontend/pkg/common/faas_common/aliasroute" + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/logger/log" +) + +const ( + aliasEtcdKeyLen = 10 + aliasedIndex = 2 + defaultAliasSign = "aliases" + defaultTenantSign = "tenant" + defaultFuncSign = "function" +) + +func startWatchAlias(stopCh <-chan struct{}) { + etcdClient := etcd3.GetMetaEtcdClient() + watcher := etcd3.NewEtcdWatcher(constant.AliasPrefix, aliasFilter, + aliasHandler, stopCh, etcdClient) + watcher.StartWatch() +} + +// key: /sn/aliases/business//tenant//function// +func aliasFilter(event *etcd3.Event) bool { + etcdKey := event.Key + keyParts := strings.Split(etcdKey, constant.ETCDEventKeySeparator) + if len(keyParts) != aliasEtcdKeyLen { + return true + } + if keyParts[aliasedIndex] != defaultAliasSign || keyParts[tenantsIndex] != defaultTenantSign || + keyParts[functionIndex] != defaultFuncSign { + return true + } + + return false +} + +func aliasHandler(event *etcd3.Event) { + log.GetLogger().Infof("handling alias event type %d, key:%s", event.Type, event.Key) + switch event.Type { + case etcd3.PUT: + _, err := aliasroute.ProcessUpdate(event) + if err != nil { + return + } + case etcd3.DELETE: + aliasroute.ProcessDelete(event) + case etcd3.ERROR: + log.GetLogger().Warnf("etcd error event: %s", event.Value) + default: + log.GetLogger().Warnf("unsupported event, key: %s", event.Key) + } +} diff --git a/frontend/pkg/frontend/watcher/alias_test.go b/frontend/pkg/frontend/watcher/alias_test.go new file mode 100644 index 0000000000000000000000000000000000000000..c383dce0aeeeea528c07dbd4bcbe6f778553bd2e --- /dev/null +++ b/frontend/pkg/frontend/watcher/alias_test.go @@ -0,0 +1,95 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package watcher + +import ( + "encoding/json" + "reflect" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/common/faas_common/aliasroute" + "frontend/pkg/common/faas_common/etcd3" +) + +func TestStartWatchAlias(t *testing.T) { + convey.Convey("StartWatch", t, func() { + patches := []*gomonkey.Patches{ + gomonkey.ApplyMethod(reflect.TypeOf(&etcd3.EtcdWatcher{}), "StartWatch", func(ew *etcd3.EtcdWatcher) { + }), + gomonkey.ApplyFunc(etcd3.GetMetaEtcdClient, func() *etcd3.EtcdClient { + return &etcd3.EtcdClient{} + }), + } + defer func() { + for _, patch := range patches { + time.Sleep(100 * time.Millisecond) + patch.Reset() + } + }() + startWatchAlias(make(chan struct{})) + }) +} + +func Test_AliasHandler(t *testing.T) { + aliasByte, _ := json.Marshal(&aliasroute.AliasElement{}) + convey.Convey("handler", t, func() { + convey.Convey("PUT", func() { + aliasHandler(&etcd3.Event{ + Type: etcd3.PUT, + Value: aliasByte, + }) + }) + + convey.Convey("DELETE", func() { + aliasHandler(&etcd3.Event{Type: etcd3.DELETE}) + }) + + convey.Convey("SYNCED", func() { + aliasHandler(&etcd3.Event{Type: etcd3.SYNCED}) + }) + + convey.Convey("DEFAULT", func() { + aliasHandler(&etcd3.Event{}) + }) + }) +} + +func Test_AliasFilter(t *testing.T) { + convey.Convey("filter", t, func() { + convey.Convey("len true", func() { + filter := aliasFilter(&etcd3.Event{ + Key: "sn/aliases/business/yrk/tenant/{tenantId}/function/{function-name}/{alias-name}"}) + convey.So(filter, convey.ShouldEqual, true) + }) + + convey.Convey("true", func() { + filter := aliasFilter(&etcd3.Event{ + Key: "sn/functions/business/yrk/tenant/{tenantId}/function/{function-name}/{alias-name}"}) + convey.So(filter, convey.ShouldEqual, true) + }) + + convey.Convey("success", func() { + filter := aliasFilter(&etcd3.Event{ + Key: "/sn/aliases/business/yrk/tenant/{tenantId}/function/{function-name}/{alias-name}"}) + convey.So(filter, convey.ShouldEqual, false) + }) + }) +} diff --git a/frontend/pkg/frontend/watcher/functionmeta.go b/frontend/pkg/frontend/watcher/functionmeta.go new file mode 100644 index 0000000000000000000000000000000000000000..c78367f1046136ccf2c49fd32fd3c29ffaa24327 --- /dev/null +++ b/frontend/pkg/frontend/watcher/functionmeta.go @@ -0,0 +1,88 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package watcher + +import ( + "strings" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/frontend/functionmeta" +) + +const ( + functionEtcdKeyLen = 11 + functionsIndex = 2 + tenantsIndex = 5 + functionIndex = 7 +) + +func startWatchFunctionMeta(stopCh <-chan struct{}) { + etcdClient := etcd3.GetMetaEtcdClient() + watcher := etcd3.NewEtcdWatcher(constant.FunctionPrefix, functionMetaFilter, + functionMetaHandler, stopCh, etcdClient) + watcher.StartWatch() +} + +// key: /sn/functions/business//tenant//function//version/ +func functionMetaFilter(event *etcd3.Event) bool { + etcdKey := event.Key + keyParts := strings.Split(etcdKey, constant.ETCDEventKeySeparator) + + if len(keyParts) != functionEtcdKeyLen { + return true + } + if keyParts[functionsIndex] != "functions" || keyParts[tenantsIndex] != "tenant" || + keyParts[functionIndex] != "function" { + return true + } + + return false +} + +func functionMetaHandler(event *etcd3.Event) { + log.GetLogger().Infof("handling function meta event type %d, key:%s", event.Type, event.Key) + switch event.Type { + case etcd3.PUT: + if err := functionmeta.ProcessUpdate(event.Key, event.Value, event.ETCDType); err != nil { + return + } + return + case etcd3.DELETE: + if err := functionmeta.ProcessDelete(event.Key, event.ETCDType); err != nil { + log.GetLogger().Errorf("failed to process delete event, err:%s", err) + return + } + return + case etcd3.SYNCED: + log.GetLogger().Infof("frontend function ready to receive etcd kv") + default: + log.GetLogger().Errorf("undefined etcd event") + return + } +} + +func startWatchCAEFunctionMeta(stopCh <-chan struct{}) { + etcdClient := etcd3.GetCAEMetaEtcdClient() + if etcdClient == nil { + return + } + watcher := etcd3.NewEtcdWatcher(constant.FunctionPrefix, functionMetaFilter, + functionMetaHandler, stopCh, etcdClient) + watcher.StartWatch() +} diff --git a/frontend/pkg/frontend/watcher/functionmeta_test.go b/frontend/pkg/frontend/watcher/functionmeta_test.go new file mode 100644 index 0000000000000000000000000000000000000000..0ae8ede3a377ac9738569edc7d33519a70ee78dd --- /dev/null +++ b/frontend/pkg/frontend/watcher/functionmeta_test.go @@ -0,0 +1,121 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package watcher + +import ( + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + "reflect" + "testing" + "time" + + "frontend/pkg/common/faas_common/etcd3" +) + +func TestStartWatchFunctionMeta(t *testing.T) { + convey.Convey("StartWatch", t, func() { + patches := []*gomonkey.Patches{ + gomonkey.ApplyMethod(reflect.TypeOf(&etcd3.EtcdWatcher{}), "StartWatch", func(ew *etcd3.EtcdWatcher) { + }), + gomonkey.ApplyFunc(etcd3.GetMetaEtcdClient, func() *etcd3.EtcdClient { + return &etcd3.EtcdClient{} + }), + } + defer func() { + for _, patch := range patches { + time.Sleep(100 * time.Millisecond) + patch.Reset() + } + }() + startWatchFunctionMeta(make(chan struct{})) + }) +} + +func TestStartWatchCAEFunctionMeta(t *testing.T) { + convey.Convey("StartWatch 01", t, func() { + patches := []*gomonkey.Patches{ + gomonkey.ApplyMethod(reflect.TypeOf(&etcd3.EtcdWatcher{}), "StartWatch", func(ew *etcd3.EtcdWatcher) { + }), + gomonkey.ApplyFunc(etcd3.GetCAEMetaEtcdClient, func() *etcd3.EtcdClient { + return &etcd3.EtcdClient{} + }), + } + defer func() { + for _, patch := range patches { + time.Sleep(100 * time.Millisecond) + patch.Reset() + } + }() + startWatchCAEFunctionMeta(make(chan struct{})) + }) + + convey.Convey("StartWatch 02", t, func() { + patches := []*gomonkey.Patches{ + gomonkey.ApplyMethod(reflect.TypeOf(&etcd3.EtcdWatcher{}), "StartWatch", func(ew *etcd3.EtcdWatcher) { + }), + } + defer func() { + for _, patch := range patches { + time.Sleep(100 * time.Millisecond) + patch.Reset() + } + }() + startWatchCAEFunctionMeta(make(chan struct{})) + }) +} + +func Test_FunctionMetaFilter(t *testing.T) { + convey.Convey("filter", t, func() { + convey.Convey("len false", func() { + filter := functionMetaFilter(&etcd3.Event{ + Key: "sn/functions/business//tenant//function//version/"}) + convey.So(filter, convey.ShouldEqual, true) + }) + + convey.Convey("false", func() { + filter := functionMetaFilter(&etcd3.Event{ + Key: "/sn/functions/business//tenant//function//version/"}) + convey.So(filter, convey.ShouldEqual, false) + }) + + convey.Convey("true", func() { + filter := functionMetaFilter(&etcd3.Event{ + Key: "/sn/instance/business//tenant//function//version/"}) + convey.So(filter, convey.ShouldEqual, true) + }) + }) +} + +func Test_FunctionMetaHandler(t *testing.T) { + convey.Convey("handler", t, func() { + convey.Convey("PUT", func() { + functionMetaHandler(&etcd3.Event{Type: etcd3.PUT}) + }) + + convey.Convey("DELETE", func() { + functionMetaHandler(&etcd3.Event{Type: etcd3.DELETE}) + }) + + convey.Convey("SYNCED", func() { + functionMetaHandler(&etcd3.Event{Type: etcd3.SYNCED}) + }) + + convey.Convey("DEFAULT", func() { + functionMetaHandler(&etcd3.Event{}) + }) + }) +} diff --git a/frontend/pkg/frontend/watcher/functiontask.go b/frontend/pkg/frontend/watcher/functiontask.go new file mode 100644 index 0000000000000000000000000000000000000000..f4be7b095472b30c19794ef707e268ef26591e05 --- /dev/null +++ b/frontend/pkg/frontend/watcher/functiontask.go @@ -0,0 +1,81 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package watcher + +import ( + "encoding/json" + "strings" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/functiontask" +) + +const ( + // NodeEtcdPrefix is the etcd prefix for live node + NodeEtcdPrefix = "/sn/workers/business/yrk/tenant/0/function/function-task" + // ValidNodePathCount is the seprator count of Node Path + ValidNodePathCount = 13 +) + +// StartWatchFunctionProxy - +func startWatchFunctionProxy(stopCh <-chan struct{}) { + if config.GetConfig().FunctionInvokeBackend != constant.BackendTypeFG { + return + } + etcdClient := etcd3.GetRouterEtcdClient() + watcher := etcd3.NewEtcdWatcher(NodeEtcdPrefix, IsTaskNode, processFunctionTaskEvent, stopCh, etcdClient) + watcher.StartWatch() +} + +func processFunctionTaskEvent(event *etcd3.Event) { + switch event.Type { + case etcd3.PUT: + ft := &struct { + NodeIP string // 目前只对nodeIP感兴趣，其他字段不重要，基本没用，如果有用到，再加上 + }{} + err := json.Unmarshal(event.Value, ft) + if err != nil { + log.GetLogger().Errorf("error is %s", err) + log.GetLogger().Warnf("unmarshal functiontask etcd event failed: %s", event.Value) + return + } + functiontask.GetBusProxies().Add(event.Key, ft.NodeIP) + + case etcd3.DELETE: + functiontask.GetBusProxies().Delete(event.Key) + case etcd3.ERROR: + log.GetLogger().Warnf("etcd error event: %s", event.Value) + default: + log.GetLogger().Warnf("unsupported event: %s", event.Value) + } +} + +// IsTaskNode /sn/workers/business/yrk/tenant/0/function/function-task/version/$latest/defaultaz/node01 +func IsTaskNode(event *etcd3.Event) bool { + + strs := strings.Split(event.Key, constant.KeySeparator) + if len(strs) != ValidNodePathCount { + return true + } + if strs[6] != "0" || strs[8] != "function-task" { + return true + } + return false +} diff --git a/frontend/pkg/frontend/watcher/functiontask_test.go b/frontend/pkg/frontend/watcher/functiontask_test.go new file mode 100644 index 0000000000000000000000000000000000000000..14ab93d3ec8c9f4c51d93c8aab14c7d5e811d7fc --- /dev/null +++ b/frontend/pkg/frontend/watcher/functiontask_test.go @@ -0,0 +1,121 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package watcher + +import ( + "encoding/json" + "errors" + "frontend/pkg/frontend/functiontask" + "reflect" + "testing" + "time" + + "frontend/pkg/common/faas_common/etcd3" + "github.com/agiledragon/gomonkey/v2" + + "github.com/smartystreets/goconvey/convey" +) + +func TestProcessNodeEvent(t *testing.T) { + convey.Convey("test process node event", t, func() { + convey.Convey("process err", func() { + defer gomonkey.ApplyFunc(json.Unmarshal, func(data []byte, v interface{}) error { + return errors.New("") + }).Reset() + convey.Convey("add event", func() { + defer gomonkey.ApplyMethod(reflect.TypeOf(functiontask.GetBusProxies()), "Add", func(_ *functiontask.BusProxies, + nodeID, nodeIp string) { + }).Reset() + event := &etcd3.Event{Type: etcd3.PUT} + processFunctionTaskEvent(event) + }) + convey.Convey("remove event", func() { + defer gomonkey.ApplyMethod(reflect.TypeOf(functiontask.GetBusProxies()), "Delete", func(_ *functiontask.BusProxies, + nodeID string) { + }).Reset() + event := &etcd3.Event{Type: etcd3.DELETE} + processFunctionTaskEvent(event) + }) + }) + convey.Convey("process ok", func() { + defer gomonkey.ApplyFunc(json.Unmarshal, func(data []byte, v interface{}) error { + return nil + }).Reset() + convey.Convey("add event", func() { + defer gomonkey.ApplyMethod(reflect.TypeOf(functiontask.GetBusProxies()), "Add", func(_ *functiontask.BusProxies, + nodeID, nodeIp string) { + }).Reset() + event := &etcd3.Event{Type: etcd3.PUT} + processFunctionTaskEvent(event) + }) + convey.Convey("remove event", func() { + defer gomonkey.ApplyMethod(reflect.TypeOf(functiontask.GetBusProxies()), "Delete", func(_ *functiontask.BusProxies, + nodeID string) { + }).Reset() + event := &etcd3.Event{Type: etcd3.DELETE} + processFunctionTaskEvent(event) + }) + }) + convey.Convey("test other event", func() { + convey.Convey("test etcd err event", func() { + event := &etcd3.Event{Type: etcd3.ERROR} + processFunctionTaskEvent(event) + }) + convey.Convey("test etcd undefined event", func() { + event := &etcd3.Event{Type: -1} + processFunctionTaskEvent(event) + }) + }) + }) +} + +func TestIsTaskNode(t *testing.T) { + convey.Convey("Test IsTaskNode", t, func() { + convey.Convey("is not TaskNode", func() { + key1 := &etcd3.Event{ + Key: "", + } + convey.So(IsTaskNode(key1), convey.ShouldBeTrue) + key2 := &etcd3.Event{ + Key: " /sn/workers/business/yrk/tenant/0/function/function/version/$latest/defaultaz/node01", + } + convey.So(IsTaskNode(key2), convey.ShouldBeTrue) + }) + convey.Convey("is node", func() { + key := &etcd3.Event{ + Key: " /sn/workers/business/yrk/tenant/0/function/function-task/version/$latest/defaultaz/node01", + } + convey.So(IsTaskNode(key), convey.ShouldBeFalse) + }) + }) +} + +func TestStartWatchFunctionProxy(t *testing.T) { + convey.Convey("StartWatch", t, func() { + patches := []*gomonkey.Patches{ + gomonkey.ApplyMethod(reflect.TypeOf(&etcd3.EtcdWatcher{}), "StartWatch", func(ew *etcd3.EtcdWatcher) { + }), + } + defer func() { + for _, patch := range patches { + time.Sleep(100 * time.Millisecond) + patch.Reset() + } + }() + startWatchFunctionProxy(make(chan struct{})) + }) +} diff --git a/frontend/pkg/frontend/watcher/instanceconfig.go b/frontend/pkg/frontend/watcher/instanceconfig.go new file mode 100644 index 0000000000000000000000000000000000000000..14d11513547d206dd19dc257e08785f85d641b51 --- /dev/null +++ b/frontend/pkg/frontend/watcher/instanceconfig.go @@ -0,0 +1,50 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package watcher + +import ( + "go.uber.org/zap" + + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/instanceconfig" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/instanceconfigmanager" +) + +func startWatchInstanceConfig(stopCh <-chan struct{}) { + etcdClient := etcd3.GetRouterEtcdClient() + watcher := etcd3.NewEtcdWatcher(instanceconfig.InsConfigEtcdPrefix, + instanceconfig.GetWatcherFilter(config.GetConfig().ClusterID), + instanceConfigHandler, stopCh, etcdClient) + watcher.StartWatch() +} + +func instanceConfigHandler(event *etcd3.Event) { + logger := log.GetLogger().With(zap.Any("eventType", event.Type), zap.Any("rev", event.Rev)) + log.GetLogger().Infof("handling instance config, key: %s", event.Key) + switch event.Type { + case etcd3.PUT: + instanceconfigmanager.ProcessUpdate(event, logger) + case etcd3.DELETE: + instanceconfigmanager.ProcessDelete(event, logger) + case etcd3.ERROR: + logger.Warnf("etcd error event: %s", event.Value) + default: + logger.Warnf("unsupported event") + } +} diff --git a/frontend/pkg/frontend/watcher/instanceinfo.go b/frontend/pkg/frontend/watcher/instanceinfo.go new file mode 100644 index 0000000000000000000000000000000000000000..e386f344515ee7bb72037cf58f4cf9180709a622 --- /dev/null +++ b/frontend/pkg/frontend/watcher/instanceinfo.go @@ -0,0 +1,63 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package watcher + +import ( + "strings" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/frontend/instancemanager" +) + +const ( + instanceEtcdKeyLen = 14 +) + +func startWatchInstanceInfo(stopCh <-chan struct{}) { + etcdClient := etcd3.GetRouterEtcdClient() + watcher := etcd3.NewEtcdWatcher(constant.InstancePathPrefix, instanceInfoFilter, + instanceInfoHandler, stopCh, etcdClient) + watcher.StartWatch() +} + +func instanceInfoHandler(event *etcd3.Event) { + log.GetLogger().Infof("handling instance info event type %d, key:%s", event.Type, event.Key) + switch event.Type { + case etcd3.PUT: + instancemanager.ProcessInstanceUpdate(event) + case etcd3.DELETE: + instancemanager.ProcessInstanceDelete(event) + case etcd3.SYNCED: + instancemanager.ProcessInstanceSync(event) + case etcd3.ERROR: + log.GetLogger().Warnf("etcd error event: %s", event.Value) + default: + log.GetLogger().Warnf("unsupported event, key: %s", event.Key) + } +} + +// key: /sn/aliases/business//tenant//function// // todo 有问题 +func instanceInfoFilter(event *etcd3.Event) bool { + etcdKey := event.Key + keyParts := strings.Split(etcdKey, constant.ETCDEventKeySeparator) + if len(keyParts) != instanceEtcdKeyLen { + return true + } + return false +} diff --git a/frontend/pkg/frontend/watcher/instanceinfo_test.go b/frontend/pkg/frontend/watcher/instanceinfo_test.go new file mode 100644 index 0000000000000000000000000000000000000000..b1d2a7eb2ba4627c31ad24011f553b9b6ace8754 --- /dev/null +++ b/frontend/pkg/frontend/watcher/instanceinfo_test.go @@ -0,0 +1,79 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package watcher + +import ( + "encoding/json" + "testing" + + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/types" +) + +func Test_handler(t *testing.T) { + instanceEtcdInfoBytes, _ := json.Marshal(&types.InstanceSpecification{InstanceStatus: types.InstanceStatus{}}) + type args struct { + event *etcd3.Event + } + tests := []struct { + name string + args args + }{ + {"case1 event put", args{event: &etcd3.Event{ + Type: etcd3.PUT, + Value: instanceEtcdInfoBytes, + }}}, + {"case2 event delete", args{event: &etcd3.Event{ + Type: etcd3.DELETE, + Key: "/sn/instance/business/yrk/tenant/12/function/0-system-faasscheduler/version/$latest/defaultaz/requestID/123", + }}}, + {"case3 event error", args{event: &etcd3.Event{ + Type: etcd3.ERROR, + }}}, + {"case4 event default", args{event: &etcd3.Event{ + Type: etcd3.SYNCED, + }}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + instanceInfoHandler(tt.args.event) + }) + } +} + +func Test_InstanceInfoFilter(t *testing.T) { + type args struct { + event *etcd3.Event + } + tests := []struct { + name string + args args + want bool + }{ + {"case1", args{event: &etcd3.Event{ + Type: etcd3.PUT, + Key: "/sn/instance/business/yrk/tenant/12/function/0-system-faasscheduler/version/$latest/defaultaz/requestID/123", + }}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := instanceInfoFilter(tt.args.event); got != tt.want { + t.Errorf("filter() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/frontend/pkg/frontend/watcher/remoteclientlease.go b/frontend/pkg/frontend/watcher/remoteclientlease.go new file mode 100644 index 0000000000000000000000000000000000000000..8f8e1c0a9bc8691ae54c466fa58b08b7ec210fca --- /dev/null +++ b/frontend/pkg/frontend/watcher/remoteclientlease.go @@ -0,0 +1,67 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package watcher + +import ( + "strings" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/utils" + "frontend/pkg/frontend/remoteclientlease" +) + +func startWatchRemoteClientLease(stopCh <-chan struct{}) { + etcdClient := etcd3.GetRouterEtcdClient() + watcher := etcd3.NewEtcdWatcher(constant.InstancePathPrefix, remoteClientLeaseFilter, + remoteClientLeaseHandler, stopCh, etcdClient) + watcher.StartWatch() +} + +func isFaaSManager(etcdPath string) bool { + info, err := utils.GetFunctionInstanceInfoFromEtcdKey(etcdPath) + if err != nil { + return false + } + return strings.Contains(info.FunctionName, "faasmanager") +} + +func remoteClientLeaseFilter(event *etcd3.Event) bool { + return !isFaaSManager(event.Key) +} + +func remoteClientLeaseHandler(event *etcd3.Event) { + log.GetLogger().Infof("handling faas manager event type %d, key:%s", event.Type, event.Key) + if event.Type == etcd3.SYNCED { + log.GetLogger().Infof("faaSManager ready to receive etcd kv") + return + } + info, err := utils.GetFunctionInstanceInfoFromEtcdKey(event.Key) + if err != nil { + log.GetLogger().Errorf("failed to parse event key of %+v: %s", event, err) + return + } + switch event.Type { + case etcd3.PUT: + remoteclientlease.UpdateFaasManager(event, info) + case etcd3.DELETE: + remoteclientlease.DeleteFaasManager(info) + default: + log.GetLogger().Warnf("unsupported event: %#v", event) + } +} diff --git a/frontend/pkg/frontend/watcher/remoteclientlease_test.go b/frontend/pkg/frontend/watcher/remoteclientlease_test.go new file mode 100644 index 0000000000000000000000000000000000000000..b695ec399c4ca6ed01009dd2c6ef586d6cdbf469 --- /dev/null +++ b/frontend/pkg/frontend/watcher/remoteclientlease_test.go @@ -0,0 +1,99 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package watcher + +import ( + "reflect" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + . "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/common/faas_common/etcd3" +) + +func TestStartWatchRemoteClientLease(t *testing.T) { + Convey("StartWatchRemoteClientLease", t, func() { + patches := []*gomonkey.Patches{ + gomonkey.ApplyMethod(reflect.TypeOf(&etcd3.EtcdWatcher{}), "StartWatch", func(ew *etcd3.EtcdWatcher) { + }), + gomonkey.ApplyFunc(etcd3.GetRouterEtcdClient, func() *etcd3.EtcdClient { + return &etcd3.EtcdClient{} + }), + } + defer func() { + for _, patch := range patches { + time.Sleep(100 * time.Millisecond) + patch.Reset() + } + }() + startWatchRemoteClientLease(make(chan struct{})) + }) +} + +func TestHandler(t *testing.T) { + events := []etcd3.Event{ + { + Type: etcd3.PUT, + Key: "/sn/instance/business/yrk/tenant/1/function/faasmanager/version/latest/defaultaz/requestID/3f079541-15fc-4009-8c41-50b2b2936772", + Value: []byte(`{ + "instanceID": "3f079541-15fc-4009-8c41-50b2b2936772", + "instanceStatus": { + "code": 3, + "msg": "running" + }}`), + }, + { + Type: etcd3.DELETE, + Key: "/sn/instance/business/yrk/tenant/1/function/faasmanager/version/latest/defaultaz/requestID/3f079541-15fc-4009-8c41-50b2b2936772", + }, + { + Type: etcd3.PUT, + Key: "/business/yrk/tenant/1/function/xxxxmanager/version/latest/defaultaz/3f079541-15fc-4009-8c41-50b2b2936772", + Value: []byte(`{ + "instanceID": "3f079541-15fc-4009-8c41-50b2b2936772", + "instanceStatus": { + "code": 5, + "msg": "exiting" + }}`), + }, + { + Type: etcd3.DELETE, + Key: "/business/yrk/tenant/1/function/xxxxmanager/version/latest/defaultaz/3f079541-15fc-4009-8c41-50b2b2936772", + }, + } + Convey("Test faas manager handler", t, func() { + remoteClientLeaseHandler(&events[0]) + So(events[0].Type, ShouldEqual, etcd3.PUT) + remoteClientLeaseHandler(&events[1]) + So(events[1].Type, ShouldEqual, etcd3.DELETE) + remoteClientLeaseHandler(&events[2]) + So(events[2].Type, ShouldEqual, etcd3.PUT) + remoteClientLeaseHandler(&events[3]) + So(events[3].Type, ShouldEqual, etcd3.DELETE) + }) +} + +func Test_isFaaSManager(t *testing.T) { + Convey("test isFaaSManager", t, func() { + manager := isFaaSManager("/sn/instance/business/yrk/tenant/0/function/0-system-faasmanager/version/$latest/defaultaz/d9300b9aec177ed300/0826cb0b-40bc-4e90-ab19-eb2e16b223eb") + So(manager, ShouldBeTrue) + manager = isFaaSManager("aaa") + So(manager, ShouldBeFalse) + }) +} diff --git a/frontend/pkg/frontend/watcher/scheduler.go b/frontend/pkg/frontend/watcher/scheduler.go new file mode 100644 index 0000000000000000000000000000000000000000..4b297021f1a022e26de664d9cf3a33ec5cb43f1f --- /dev/null +++ b/frontend/pkg/frontend/watcher/scheduler.go @@ -0,0 +1,120 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package watcher + +import ( + "strings" + + "go.uber.org/zap" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/types" + "frontend/pkg/common/faas_common/utils" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/schedulerproxy" +) + +// start to watch the schedulers by the etcd +func startWatchScheduler(stopCh <-chan struct{}) { + switch config.GetConfig().SchedulerKeyPrefixType { + case constant.SchedulerKeyTypeFunction: + startWatchInstanceScheduler(stopCh) + case constant.SchedulerKeyTypeModule: + startWatchModuleScheduler(stopCh) + default: + startWatchInstanceScheduler(stopCh) + } +} + +// start to watch the instance faas schedulers by the etcd +func startWatchInstanceScheduler(stopCh <-chan struct{}) { + etcdClient := etcd3.GetRouterEtcdClient() + watcher := etcd3.NewEtcdWatcher(constant.InstancePathPrefix, instanceSchedulerFilter, instanceSchedulerHandler, + stopCh, etcdClient) + watcher.StartWatch() +} + +func isFaaSScheduler(etcdPath string) bool { + info, err := utils.GetFunctionInstanceInfoFromEtcdKey(etcdPath) + if err != nil { + return false + } + return strings.Contains(info.FunctionName, "faasscheduler") +} + +func instanceSchedulerFilter(event *etcd3.Event) bool { + return !isFaaSScheduler(event.Key) +} + +func instanceSchedulerHandler(event *etcd3.Event) { + logger := log.GetLogger().With(zap.Any("eventType", event.Type), zap.Any("eventKey", event.Key), + zap.Any("revisionId", event.Rev), zap.Any("schedulerType", "function")) + logger.Infof("recv scheduler event type") + if event.Type == etcd3.SYNCED { + logger.Infof("faaSFrontend scheduler ready to receive etcd kv") + return + } + info, err := utils.GetFunctionInstanceInfoFromEtcdKey(event.Key) + if err != nil { + logger.Errorf("failed to parse event key: %s", err.Error()) + return + } + handleEvent(event, info, logger) +} + +// start to watch the module schedulers by the etcd +func startWatchModuleScheduler(stopCh <-chan struct{}) { + etcdClient := etcd3.GetRouterEtcdClient() + watcher := etcd3.NewEtcdWatcher(constant.ModuleSchedulerPrefix, moduleSchedulerFilter, moduleSchedulerHandler, + stopCh, etcdClient) + watcher.StartWatch() +} + +func moduleSchedulerFilter(event *etcd3.Event) bool { + return !strings.Contains(event.Key, constant.ModuleSchedulerPrefix) +} + +func moduleSchedulerHandler(event *etcd3.Event) { + logger := log.GetLogger().With(zap.Any("eventType", event.Type), zap.Any("eventKey", event.Key), + zap.Any("revisionId", event.Rev), zap.Any("schedulerType", "module")) + logger.Infof("recv module scheduler event type") + if event.Type == etcd3.SYNCED { + logger.Infof("faaSFrontend scheduler ready to receive etcd kv") + return + } + info, err := utils.GetModuleSchedulerInfoFromEtcdKey(event.Key) + if err != nil { + logger.Errorf("failed to parse event key: %s", err.Error()) + return + } + handleEvent(event, info, logger) +} + +func handleEvent(event *etcd3.Event, info *types.InstanceInfo, logger api.FormatLogger) { + switch event.Type { + case etcd3.PUT: + schedulerproxy.ProcessUpdate(event, info, logger) + case etcd3.DELETE: + schedulerproxy.ProcessDelete(info, logger) + default: + logger.Warnf("unsupported event, type is %d", event.Type) + } +} diff --git a/frontend/pkg/frontend/watcher/scheduler_test.go b/frontend/pkg/frontend/watcher/scheduler_test.go new file mode 100644 index 0000000000000000000000000000000000000000..4b3265f69257009492ad7f1e2d6628f97237fca8 --- /dev/null +++ b/frontend/pkg/frontend/watcher/scheduler_test.go @@ -0,0 +1,209 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package watcher + +import ( + "reflect" + "sync/atomic" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + . "github.com/smartystreets/goconvey/convey" + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/types" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/schedulerproxy" +) + +func TestIsFaaSScheduler(t *testing.T) { + Convey("TestIsFaaSScheduler", t, func() { + key := "/sn/instance/business/yrk/tenant/0/function/faasscheduler/version/latest/defaultaz/requestID/3f079541-15fc-4009-8c41-50b2b2936772" + So(isFaaSScheduler(key), ShouldBeTrue) + key = "/sn/instance/business/yrk/tenant/1/function/faasscheduler/version/latest/defaultaz/requestID/3f079541-15fc-4009-8c41-50b2b2936772" + So(isFaaSScheduler(key), ShouldBeTrue) + key = "/sn/instance/business/yrk/tenant/0/function/scheduler/version/latest/defaultaz/requestID/3f079541-15fc-4009-8c41-50b2b2936772" + So(isFaaSScheduler(key), ShouldBeFalse) + key = "/instance/business/yrk/tenant/0/function/faasscheduler/version/latest/defaultaz/requestID/3f079541-15fc-4009-8c41-50b2b2936772" + So(isFaaSScheduler(key), ShouldBeFalse) + }) +} + +func TestInstanceSchedulerHandler(t *testing.T) { + var ( + founded int32 + missed int32 + ) + + store := make(map[string]string) + defer gomonkey.ApplyMethod(reflect.TypeOf(schedulerproxy.Proxy), "Add", func(_ *schedulerproxy.ProxyManager, i *types.InstanceInfo, _ api.FormatLogger) { + store[i.InstanceName] = i.InstanceID + atomic.AddInt32(&founded, 1) + }).Reset() + defer gomonkey.ApplyMethod(reflect.TypeOf(schedulerproxy.Proxy), "Remove", func(_ *schedulerproxy.ProxyManager, i *types.InstanceInfo, _ api.FormatLogger) { + delete(store, i.InstanceName) + atomic.AddInt32(&missed, 1) + }).Reset() + defer gomonkey.ApplyMethod(reflect.TypeOf(schedulerproxy.Proxy), "Exist", func(_ *schedulerproxy.ProxyManager, instanceName string, instanceId string) bool { + id, ok := store[instanceName] + if !ok { + return false + } + if id != instanceId { + return false + } + return true + }).Reset() + defer gomonkey.ApplyMethod(reflect.TypeOf(schedulerproxy.Proxy), "ExistInstanceName", func(_ *schedulerproxy.ProxyManager, instanceName string) bool { + _, ok := store[instanceName] + return ok + }).Reset() + + events := []etcd3.Event{ + { + Type: etcd3.PUT, + Key: "/sn/instance/business/yrk/tenant/1/function/faasscheduler/version/latest/defaultaz/requestID/3f079541-15fc-4009-8c41-50b2b2936772", + Value: []byte(`{ + "instanceID": "1f060613-68af-4a02-8000-000000e077ce", + "instanceStatus": { + "code": 3, + "msg": "running" + }}`), + }, + { + Type: etcd3.DELETE, + Key: "/sn/instance/business/yrk/tenant/1/function/faasscheduler/version/latest/defaultaz/requestID/3f079541-15fc-4009-8c41-50b2b2936772", + }, + { + Type: etcd3.PUT, + Key: "/business/yrk/tenant/1/function/xxxxscheduler/version/latest/defaultaz/3f079541-15fc-4009-8c41-50b2b2936772", + }, + { + Type: etcd3.DELETE, + Key: "/business/yrk/tenant/1/function/xxxxscheduler/version/latest/defaultaz/3f079541-15fc-4009-8c41-50b2b2936772", + }, + } + + Convey("Test instance scheduler Handler", t, func() { + for _, event := range events { + instanceSchedulerHandler(&event) + } + So(atomic.LoadInt32(&founded), ShouldEqual, 1) + So(atomic.LoadInt32(&missed), ShouldEqual, 1) + }) +} + +func TestStartWatchScheduler(t *testing.T) { + Convey("StartWatchScheduler", t, func() { + patches := []*gomonkey.Patches{ + gomonkey.ApplyMethod(reflect.TypeOf(&etcd3.EtcdWatcher{}), "StartWatch", func(ew *etcd3.EtcdWatcher) { + }), + gomonkey.ApplyFunc(etcd3.GetRouterEtcdClient, func() *etcd3.EtcdClient { + return &etcd3.EtcdClient{} + }), + } + defer func() { + for _, patch := range patches { + time.Sleep(100 * time.Millisecond) + patch.Reset() + } + }() + startWatchScheduler(make(chan struct{})) + }) +} + +func TestModuleSchedulerFilter(t *testing.T) { + Convey("TestModuleSchedulerFilter", t, func() { + event := &etcd3.Event{ + Key: "/sn/faas-scheduler/instances/cluster001/7.218.100.25/faas-scheduler-59ddbc4b75-8xdjf", + } + So(moduleSchedulerFilter(event), ShouldBeFalse) + event = &etcd3.Event{ + Key: "/sn/instance/business/yrk/tenant/0/function/scheduler/version/latest/defaultaz/requestID/3f079541-15fc-4009-8c41-50b2b2936772", + } + So(moduleSchedulerFilter(event), ShouldBeTrue) + }) +} + +func TestModuleSchedulerHandler(t *testing.T) { + var ( + founded int32 + missed int32 + ) + + store := make(map[string]string) + defer gomonkey.ApplyMethod(reflect.TypeOf(schedulerproxy.Proxy), "Add", func(_ *schedulerproxy.ProxyManager, i *types.InstanceInfo, _ api.FormatLogger) { + store[i.InstanceName] = i.InstanceID + atomic.AddInt32(&founded, 1) + }).Reset() + defer gomonkey.ApplyMethod(reflect.TypeOf(schedulerproxy.Proxy), "Remove", func(_ *schedulerproxy.ProxyManager, i *types.InstanceInfo, _ api.FormatLogger) { + delete(store, i.InstanceName) + atomic.AddInt32(&missed, 1) + }).Reset() + defer gomonkey.ApplyMethod(reflect.TypeOf(schedulerproxy.Proxy), "Exist", func(_ *schedulerproxy.ProxyManager, instanceName string, instanceId string) bool { + id, ok := store[instanceName] + if !ok { + return false + } + if id != instanceId { + return false + } + return true + }).Reset() + defer gomonkey.ApplyMethod(reflect.TypeOf(schedulerproxy.Proxy), "ExistInstanceName", func(_ *schedulerproxy.ProxyManager, instanceName string) bool { + _, ok := store[instanceName] + return ok + }).Reset() + + events := []etcd3.Event{ + { + Type: etcd3.PUT, + Key: "/sn/faas-scheduler/instances/cluster001/7.218.100.25/faas-scheduler-59ddbc4b75-8xdjf", + Value: []byte(`{ + "instanceID": "faas-scheduler-59ddbc4b75-8xdjf", + "instanceStatus": { + "code": 3, + "msg": "running" + }}`), + }, + { + Type: etcd3.DELETE, + Key: "/sn/faas-scheduler/instances/cluster001/7.218.100.25/faas-scheduler-59ddbc4b75-8xdjf", + }, + { + Type: etcd3.PUT, + Key: "/business/yrk/tenant/1/function/xxxxscheduler/version/latest/defaultaz/3f079541-15fc-4009-8c41-50b2b2936772", + }, + { + Type: etcd3.DELETE, + Key: "/business/yrk/tenant/1/function/xxxxscheduler/version/latest/defaultaz/3f079541-15fc-4009-8c41-50b2b2936772", + }, + } + + oldType := config.GetConfig().SchedulerKeyPrefixType + config.GetConfig().SchedulerKeyPrefixType = "module" + Convey("Test module scheduler Handler", t, func() { + for _, event := range events { + moduleSchedulerHandler(&event) + } + So(atomic.LoadInt32(&founded), ShouldEqual, 1) + So(atomic.LoadInt32(&missed), ShouldEqual, 1) + }) + config.GetConfig().SchedulerKeyPrefixType = oldType +} diff --git a/frontend/pkg/frontend/watcher/tenantqos.go b/frontend/pkg/frontend/watcher/tenantqos.go new file mode 100644 index 0000000000000000000000000000000000000000..437fae6fe2320f872175cd8be56ec63390bac60c --- /dev/null +++ b/frontend/pkg/frontend/watcher/tenantqos.go @@ -0,0 +1,73 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package watcher + +import ( + "strings" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/etcd3" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/frontend/tenanttrafficlimit" +) + +const ( + // QOSEtcdPrefix is the etcd prefix for tenant qos limit + QOSEtcdPrefix = "/sn/qos/business/yrk" + // ValidTenantPathCount is the separator count of tenant Path + ValidTenantPathCount = 7 + // tenantMark is mark of etcd of tenant + tenantMark = "tenant" + tenantIndex = 5 +) + +// StartWatch - +func startWatchTenantQOS(stopCh <-chan struct{}) { + etcdClient := etcd3.GetMetaEtcdClient() + watcher := etcd3.NewEtcdWatcher(QOSEtcdPrefix, tenantQOSFilter, processTenantEvent, + stopCh, etcdClient) + watcher.StartWatch() +} + +func processTenantEvent(event *etcd3.Event) { + log.GetLogger().Infof("handling tenant qos event type %d, key:%s", event.Type, event.Key) + var err error + switch event.Type { + case etcd3.PUT: + if err = tenanttrafficlimit.ProcessUpdate(event); err != nil { + return + } + case etcd3.DELETE: + tenanttrafficlimit.ProcessDelete(event) + case etcd3.ERROR: + log.GetLogger().Warnf("etcd error event: %s", event.Value) + default: + log.GetLogger().Warnf("unsupported event: %s", event.Value) + } +} + +func tenantQOSFilter(event *etcd3.Event) bool { + etcdKey := event.Key + s := strings.Split(etcdKey, constant.KeySeparator) + if len(s) != ValidTenantPathCount { + return true + } + if s[tenantIndex] != tenantMark { + return true + } + return false +} diff --git a/frontend/pkg/frontend/watcher/tenantqos_test.go b/frontend/pkg/frontend/watcher/tenantqos_test.go new file mode 100644 index 0000000000000000000000000000000000000000..f7ba97274635cc174a1aab44a942a2787ec798cb --- /dev/null +++ b/frontend/pkg/frontend/watcher/tenantqos_test.go @@ -0,0 +1,97 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package watcher + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "frontend/pkg/common/faas_common/etcd3" +) + +func TestProcessTenantEvent(t *testing.T) { + // put error + event := &etcd3.Event{ + Type: etcd3.PUT, + Key: "/sn/qos/business/yrk/tenant", + Value: []byte{}, + } + processTenantEvent(event) + // put error + event = &etcd3.Event{ + Type: etcd3.PUT, + Key: "/sn/qos/business/yrk/tenant/tenantID", + Value: []byte{}, + } + processTenantEvent(event) + event = &etcd3.Event{ + Type: etcd3.PUT, + Key: "/sn/qos/business/yrk/tenant/tenantID", + Value: []byte(`{"quo":10}`), + } + processTenantEvent(event) + // delete error + event = &etcd3.Event{ + Type: etcd3.DELETE, + Key: "/sn/qos/business/yrk/tenant", + Value: []byte{}, + } + processTenantEvent(event) + + // error + event = &etcd3.Event{ + Type: etcd3.ERROR, + Key: "/sn/qos/business/yrk/tenant", + Value: []byte{}, + } + processTenantEvent(event) + + // error + event = &etcd3.Event{ + Type: etcd3.SYNCED, + Key: "/sn/qos/business/yrk/tenant", + Value: []byte{}, + } + processTenantEvent(event) +} + +func TestTenantQOSFilter(t *testing.T) { + event := &etcd3.Event{ + Type: etcd3.PUT, + Key: "/sn/qos/business/yrk/tenant", + Value: []byte{}, + } + ok := tenantQOSFilter(event) + assert.Equal(t, true, ok) + + event = &etcd3.Event{ + Type: etcd3.PUT, + Key: "/sn/qos/business/tenant/yrk/TenantID", + Value: []byte{}, + } + ok = tenantQOSFilter(event) + assert.Equal(t, true, ok) + + event = &etcd3.Event{ + Type: etcd3.PUT, + Key: "/sn/qos/business/yrk/tenant/TenantID", + Value: []byte{}, + } + ok = tenantQOSFilter(event) + assert.Equal(t, false, ok) +} diff --git a/frontend/pkg/frontend/watcher/watch.go b/frontend/pkg/frontend/watcher/watch.go new file mode 100644 index 0000000000000000000000000000000000000000..03f2a7f5eba0f139a8e69f7bd03cd264d8e1e50e --- /dev/null +++ b/frontend/pkg/frontend/watcher/watch.go @@ -0,0 +1,43 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package watcher - +package watcher + +import ( + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/frontend/config" +) + +// StartWatch start watching etcd with scheduler, alias and function +func StartWatch(stopCh <-chan struct{}) error { + log.GetLogger().Infof("FaaS-Frontend etcd watcher starting...") + go startWatchScheduler(stopCh) + go startWatchRemoteClientLease(stopCh) + go startWatchFunctionMeta(stopCh) + if config.GetConfig().BusinessType == constant.BusinessTypeFG { + go startWatchCAEFunctionMeta(stopCh) + go startWatchFunctionProxy(stopCh) + } + go startWatchCAEFunctionMeta(stopCh) + go startWatchFunctionProxy(stopCh) + go startWatchAlias(stopCh) + go startWatchTenantQOS(stopCh) + go startWatchInstanceInfo(stopCh) + go startWatchInstanceConfig(stopCh) + return nil +} diff --git a/frontend/pkg/frontend/watcher/watcher_test.go b/frontend/pkg/frontend/watcher/watcher_test.go new file mode 100644 index 0000000000000000000000000000000000000000..2296afa676af0a60f1becde1f0362e231f0d82b0 --- /dev/null +++ b/frontend/pkg/frontend/watcher/watcher_test.go @@ -0,0 +1,52 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package watcher + +import ( + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" +) + +func TestStartWatch(t *testing.T) { + + convey.Convey("StartWatch", t, func() { + patches := []*gomonkey.Patches{ + gomonkey.ApplyFunc(startWatchScheduler, func(stopCh <-chan struct{}) {}), + gomonkey.ApplyFunc(startWatchRemoteClientLease, func(stopCh <-chan struct{}) {}), + gomonkey.ApplyFunc(startWatchFunctionMeta, func(stopCh <-chan struct{}) {}), + gomonkey.ApplyFunc(startWatchCAEFunctionMeta, func(stopCh <-chan struct{}) {}), + gomonkey.ApplyFunc(startWatchAlias, func(stopCh <-chan struct{}) {}), + gomonkey.ApplyFunc(startWatchTenantQOS, func(stopCh <-chan struct{}) {}), + gomonkey.ApplyFunc(startWatchInstanceInfo, func(stopCh <-chan struct{}) {}), + gomonkey.ApplyFunc(startWatchInstanceConfig, func(stopCh <-chan struct{}) {}), + } + defer func() { + for _, patch := range patches { + time.Sleep(100 * time.Millisecond) + patch.Reset() + } + }() + convey.Convey("success ", func() { + err := StartWatch(make(chan struct{})) + convey.So(err, convey.ShouldBeNil) + }) + + }) +} diff --git a/frontend/pkg/frontend/wisecloud/auth.go b/frontend/pkg/frontend/wisecloud/auth.go new file mode 100644 index 0000000000000000000000000000000000000000..62db476b5bfadea8463edcc32df1258a27aefa5a --- /dev/null +++ b/frontend/pkg/frontend/wisecloud/auth.go @@ -0,0 +1,166 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package wisecloud - +package wisecloud + +import ( + "crypto/hmac" + "crypto/sha256" + "strings" + "time" + + "github.com/valyala/fasthttp" +) + +// Auth - +func Auth(ctx *fasthttp.RequestCtx, ak string, sk []byte) bool { + + headers := make(map[string]string) + ctx.Request.Header.VisitAll(func(key, value []byte) { + headers[strings.ToLower(string(key))] = string(value) + }) + + return AuthDownGradeFunctionCall(string(ctx.Request.URI().Path()), headers, ctx.Request.Body(), ak, sk) +} + +// AuthDownGradeFunctionCall - +func AuthDownGradeFunctionCall(url string, headers map[string]string, body []byte, ak string, sk []byte) bool { + headerAuthorization := headers["authorization"] + layout := "20060102T150405Z" + info, ok := parseAuthorization(headerAuthorization) + if !ok { + return false + } + t, err := time.Parse(layout, info.timeStamp) + if err != nil { + return false + } + if time.Now().Sub(t) > 5*time.Minute { // 时间戳有效期5分钟 + return false + } + signature := generateSignature(url, info.timeStamp, body, ak, sk) + if encodeHex(signature) != info.signature { + return false + } + return true +} + +type authorizationStruct struct { + timeStamp string + ak string + signature string +} + +func parseAuthorization(authorization string) (*authorizationStruct, bool) { + suffix, flag := strings.CutPrefix(authorization, "HMAC-SHA256 ") + if !flag { + return nil, false + } + splits := strings.Split(suffix, ",") + if len(splits) != 3 { // 固定格式 + return nil, false + } + timeStamp, flag0 := strings.CutPrefix(splits[0], "timestamp=") + ak, flag1 := strings.CutPrefix(splits[1], "access_key=") + signature, flag2 := strings.CutPrefix(splits[2], "signature=") + if flag0 && flag1 && flag2 { + return &authorizationStruct{ + timeStamp: timeStamp, + ak: ak, + signature: signature, + }, true + } + return nil, false +} + +func generateSignature(url string, timeStamp string, body []byte, ak string, sk []byte) []byte { + digestBytes := buildDigest(url, timeStamp, body, ak) + digestHex := sha256AndHex(digestBytes) + return sign(sk, []byte(digestHex)) +} + +func buildDigest(url string, timeStamp string, body []byte, ak string) []byte { + var builder strings.Builder + builder.WriteString(url) + builder.WriteString("\n") + builder.WriteString("X-Timestamp: ") + builder.WriteString(timeStamp) + builder.WriteString("\n") + builder.WriteString("X-Access-Key: ") + builder.WriteString(ak) + builder.WriteString("\n") + builder.Write(body) + return []byte(builder.String()) +} + +func sign(sk, content []byte) []byte { + h := hmac.New(sha256.New, sk) + _, err := h.Write(content) + if err != nil { + return nil + } + return h.Sum(nil) +} + +const ( + firstFourBitShift = 4 +) + +// EncodeHex encode hex +func encodeHex(data []byte) string { + if data == nil || len(data) == 0 { + return "" + } + l := len(data) + out := make([]byte, l<<1) + j := 0 + for i := 0; i < l; i++ { + if j >= l<<1 { + return "" + } + out[j] = hexDigits[(data[i]>>4)&0xF] // magic number + j++ + if j >= l<<1 { + return "" + } + out[j] = hexDigits[(data[i] & 0xF)] // magic number + j++ + } + return string(out) +} + +func sha256AndHex(input []byte) string { + // 计算SHA256哈希 + hash := sha256.Sum256(input) + + // 将哈希值转换为16进制字符串 + var builder strings.Builder + for _, b := range hash { + // 获取高4位和低4位，并转换为对应的16进制字符 + builder.WriteByte(hexDigits[b>>firstFourBitShift]) + builder.WriteByte(hexDigits[b&0x0f]) // 取后四位 + } + builder.WriteByte('\n') // 与C++版本一致添加换行符 + + return builder.String() +} + +// 16进制字符集，使用小写字母与C++版本保持一致 +var hexDigits = []byte{ + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', +} diff --git a/frontend/pkg/frontend/wisecloud/auth_test.go b/frontend/pkg/frontend/wisecloud/auth_test.go new file mode 100644 index 0000000000000000000000000000000000000000..2371485e1c160f70289165f257b8486e2f0e7044 --- /dev/null +++ b/frontend/pkg/frontend/wisecloud/auth_test.go @@ -0,0 +1,115 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wisecloud + +import ( + "testing" + "time" + + "github.com/smartystreets/goconvey/convey" + "github.com/valyala/fasthttp" +) + +func TestSign(t *testing.T) { + convey.Convey("test authorization format", t, func() { + tests := []struct { + authorizaton string + timestamp string + ak string + signature string + parseOk bool + }{ + { + authorizaton: "HMAC-SHA256 timestamp=20250829T091626Z,access_key=access_key,signature=d334c14fd0d493c1f8d4490ff62d409ac8e9c4ffd35d4d96f5656aa02922c1a8", + timestamp: "20250829T091626Z", + ak: "access_key", + signature: "d334c14fd0d493c1f8d4490ff62d409ac8e9c4ffd35d4d96f5656aa02922c1a8", + parseOk: true, + }, + { + authorizaton: "HMAC-SHA25", + parseOk: false, + }, + { + authorizaton: "HMAC-SHA257 timestamp=20250829T091626Z,access_key=access_key,signature=d334c14fd0d493c1f8d4490ff62d409ac8e9c4ffd35d4d96f5656aa02922c1a8", + parseOk: false, + }, + { + authorizaton: "HMAC-SHA256 timestamp20250829T091626Z,access_keyaccess_key,signatured334c14fd0d493c1f8d4490ff62d409ac8e9c4ffd35d4d96f5656aa02922c1a8", + parseOk: false, + }, + } + + for _, tt := range tests { + info, ok := parseAuthorization(tt.authorizaton) + convey.So(ok, convey.ShouldEqual, tt.parseOk) + if !ok { + convey.So(info, convey.ShouldBeNil) + } else { + convey.So(info.timeStamp, convey.ShouldEqual, tt.timestamp) + convey.So(info.ak, convey.ShouldEqual, tt.ak) + convey.So(info.signature, convey.ShouldEqual, tt.signature) + } + } + }) + + ctx := &fasthttp.RequestCtx{ + Request: fasthttp.Request{}, + Response: fasthttp.Response{}, + } + ctx.Request.SetRequestURI("http://127.0.0.1:8080/serverless/v2/functions/wisefunction:cn:iot:8d86c63b22e24d9ab650878b75408ea6:function:0@faas@python:latest/invocations") + ctx.Request.Header.Set("Authorization", "HMAC-SHA256 timestamp=20250829T091626Z,access_key=access_key,signature=edc450dcccdc7f46e701dcbc03409d4c465915ef42cb8da51f8afbfc25818cd6") + ctx.Request.SetBody([]byte("123")) + ak := "access_key" + sk := []byte("secret_key") + url := "/serverless/v2/functions/wisefunction:cn:iot:8d86c63b22e24d9ab650878b75408ea6:function:0@faas@python:latest/invocations" + convey.Convey("test buildDigest", t, func() { + expectDigest := "/serverless/v2/functions/wisefunction:cn:iot:8d86c63b22e24d9ab650878b75408ea6:function:0@faas@python:latest/invocations\n" + + "X-Timestamp: 20250830T034742Z\n" + + "X-Access-Key: access_key\n" + + "123" + + convey.So(expectDigest, convey.ShouldEqual, string(buildDigest(url, "20250830T034742Z", []byte("123"), "access_key"))) + }) + + convey.Convey("Test signature", t, func() { + signature := encodeHex(generateSignature(url, "20250830T034742Z", ctx.Request.Body(), ak, sk)) + convey.So(signature, convey.ShouldEqual, "050e8457b82a6c366ff800827df1a75c58fafa9f9319c99b01c4b23a6d9cbd09") + }) + + convey.Convey("test timeout", t, func() { + tests := []struct { + timeStamp string + ok bool + }{ + { + timeStamp: time.Now().Add(-6 * time.Minute).UTC().Format("20060102T150405Z"), + ok: false, + }, + { + timeStamp: time.Now().Add(-4*time.Minute - 55*time.Second).UTC().Format("20060102T150405Z"), + ok: true, + }, + } + for _, tt := range tests { + signature := generateSignature(string(ctx.Request.URI().Path()), tt.timeStamp, ctx.Request.Body(), ak, sk) + acutalAuthorization := "HMAC-SHA256 timestamp=" + tt.timeStamp + ",access_key=" + ak + ",signature=" + encodeHex(signature) + ctx.Request.Header.Set("Authorization", acutalAuthorization) + convey.So(Auth(ctx, ak, sk), convey.ShouldEqual, tt.ok) + } + }) +} diff --git a/frontend/pkg/frontend/wisecloud/coldstart.go b/frontend/pkg/frontend/wisecloud/coldstart.go new file mode 100644 index 0000000000000000000000000000000000000000..2602a2f15c6fbbc0c6ff9ce7ca97dea10b4db815 --- /dev/null +++ b/frontend/pkg/frontend/wisecloud/coldstart.go @@ -0,0 +1,36 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package wisecloud - +package wisecloud + +import ( + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/wisecloudtool" + "frontend/pkg/common/faas_common/wisecloudtool/types" +) + +var coldStartProvider *wisecloudtool.PodOperator + +// NewColdStartProvider - +func NewColdStartProvider(config *types.ServiceAccountJwt) *wisecloudtool.PodOperator { + if config == nil || config.ServiceAccount == nil { + log.GetLogger().Infof("NewColdStartProvider failed, config empty") + return nil + } + coldStartProvider = wisecloudtool.NewColdStarter(config, log.GetLogger()) + return coldStartProvider +} diff --git a/frontend/pkg/frontend/wisecloud/metrics.go b/frontend/pkg/frontend/wisecloud/metrics.go new file mode 100644 index 0000000000000000000000000000000000000000..3282f2eb398e75b823c51e6ac90f668befceec51 --- /dev/null +++ b/frontend/pkg/frontend/wisecloud/metrics.go @@ -0,0 +1,153 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package wisecloud - +package wisecloud + +import ( + "sync" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/instanceconfig" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/resspeckey" + "frontend/pkg/common/faas_common/types" + "frontend/pkg/common/faas_common/wisecloudtool" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/functionmeta" + "frontend/pkg/frontend/instancemanager" +) + +var metricsManager = &MetricsManager{ + RWMutex: sync.RWMutex{}, + metricsProvider: wisecloudtool.NewMetricProvider(), + logger: log.GetLogger(), +} + +// GetMetricsManager - +func GetMetricsManager() *MetricsManager { + return metricsManager +} + +// MetricsManager - +type MetricsManager struct { + sync.RWMutex + metricsProvider *wisecloudtool.MetricProvider + logger api.FormatLogger // key: {funcKey}#{invokeLabel}, value: {namespace, podName} +} + +// ProcessFunctionDelete - +func (m *MetricsManager) ProcessFunctionDelete(funcSpec *types.FuncSpec) { + if funcSpec == nil { + return + } + m.Lock() + defer m.Unlock() + m.metricsProvider.ClearMetricsForFunction(&funcSpec.FuncMetaData) + m.logger.Infof("delete function: %s wisecloud metrics", funcSpec.FunctionKey) +} + +// ProcessInsConfigDelete - +func (m *MetricsManager) ProcessInsConfigDelete(insConfig *instanceconfig.Configuration) { + if insConfig == nil { + return + } + m.Lock() + defer m.Unlock() + funcSpec, ok := functionmeta.LoadFuncSpec(insConfig.FuncKey) + if !ok { + m.logger.Warnf("funcKey: %s's functionMetaData not found", insConfig.FuncKey) + return + } + m.metricsProvider.ClearMetricsForInsConfig(&funcSpec.FuncMetaData, insConfig.InstanceLabel) + m.logger.Infof("delete function: %s, invokeLabel: %s wisecloud metrics", + funcSpec.FunctionKey, insConfig.InstanceLabel) +} + +// ProcessInstanceDelete - +func (m *MetricsManager) ProcessInstanceDelete(instance *types.InstanceSpecification) { + if instance == nil { + return + } + m.Lock() + defer m.Unlock() + funcKey, ok := instance.CreateOptions[constant.FunctionKeyNote] + if !ok { + m.logger.Warnf("delete instance: %s wisecloud metrics failed, no functionMeta", instance.InstanceID) + return + } + funcSpec, ok := functionmeta.LoadFuncSpec(funcKey) + if !ok { + return + } + resSpecKey, err := resspeckey.GetResKeyFromStr(instance.CreateOptions[constant.ResourceSpecNote]) + if err != nil { + return + } + labels := wisecloudtool.GetMetricLabels(&funcSpec.FuncMetaData, resSpecKey.InvokeLabel, + instance.Extensions.PodNamespace, instance.Extensions.PodDeploymentName, instance.Extensions.PodName) + m.metricsProvider.ClearLeaseRequestTotalWithLabel(labels) + m.metricsProvider.ClearConcurrencyGaugeWithLabel(labels) + m.logger.Infof("delete instance: %s wisecloud metrics, function: %s, invokeLabel: %s", + instance.InstanceID, funcSpec.FunctionKey, resSpecKey.InvokeLabel) +} + +// InvokeStart - +func (m *MetricsManager) InvokeStart(funcKey string, resSpecKeyStr string, instanceId string) { + if config.GetConfig().BusinessType != constant.BusinessTypeWiseCloud { + return + } + funcSpec, ok := functionmeta.LoadFuncSpec(funcKey) + if !ok { + return + } + instance := instancemanager.GetGlobalInstanceScheduler().GetInstance(funcKey, resSpecKeyStr, instanceId) + if instance == nil { + return + } + resSpecKey, err := resspeckey.GetResKeyFromStr(instance.CreateOptions[constant.ResourceSpecNote]) + if err != nil { + return + } + labels := wisecloudtool.GetMetricLabels(&funcSpec.FuncMetaData, resSpecKey.InvokeLabel, + instance.Extensions.PodNamespace, instance.Extensions.PodDeploymentName, instance.Extensions.PodName) + m.metricsProvider.IncConcurrencyGaugeWithLabel(labels) + m.metricsProvider.IncLeaseRequestTotalWithLabel(labels) +} + +// InvokeEnd - +func (m *MetricsManager) InvokeEnd(funcKey, resSpecKeyStr string, instanceId string) { + if config.GetConfig().BusinessType != constant.BusinessTypeWiseCloud { + return + } + funcSpec, ok := functionmeta.LoadFuncSpec(funcKey) + if !ok { + return + } + instance := instancemanager.GetGlobalInstanceScheduler().GetInstance(funcKey, resSpecKeyStr, instanceId) + if instance == nil { + return + } + resSpecKey, err := resspeckey.GetResKeyFromStr(instance.CreateOptions[constant.ResourceSpecNote]) + if err != nil { + return + } + labels := wisecloudtool.GetMetricLabels(&funcSpec.FuncMetaData, resSpecKey.InvokeLabel, + instance.Extensions.PodNamespace, instance.Extensions.PodDeploymentName, instance.Extensions.PodName) + m.metricsProvider.DecConcurrencyGaugeWithLabel(labels) +} diff --git a/frontend/pkg/frontend/wisecloud/metrics_test.go b/frontend/pkg/frontend/wisecloud/metrics_test.go new file mode 100644 index 0000000000000000000000000000000000000000..13031f7ebf712569c49f189d8d438df728a8b695 --- /dev/null +++ b/frontend/pkg/frontend/wisecloud/metrics_test.go @@ -0,0 +1,356 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wisecloud + +import ( + "fmt" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/instanceconfig" + "frontend/pkg/common/faas_common/resspeckey" + commontypes "frontend/pkg/common/faas_common/types" + "frontend/pkg/frontend/config" + "frontend/pkg/frontend/functionmeta" + "frontend/pkg/frontend/instancemanager" + "frontend/pkg/frontend/types" +) + +// 测试数据准备 +var ( + mockFuncSpec = &commontypes.FuncSpec{ + FunctionKey: "test-function", + FuncMetaData: commontypes.FuncMetaData{ + // 填充元数据字段 + }, + } + mockInsConfig = &instanceconfig.Configuration{ + FuncKey: "test-function", + InstanceLabel: "test-label", + } + mockInstance = &commontypes.InstanceSpecification{ + CreateOptions: map[string]string{ + constant.FunctionKeyNote: "test-function", + constant.ResourceSpecNote: "test-res-spec", + }, + Extensions: commontypes.Extensions{ + PodNamespace: "test-ns", + PodDeploymentName: "test-deploy", + PodName: "test-pod", + }, + InstanceID: "test-instance-id", + } +) + +func TestWiseCloudMetricsManager_ProcessFunctionDelete(t *testing.T) { + convey.Convey("测试 ProcessFunctionDelete", t, func() { + // 使用 gomonkey mock MetricProvider + patches := gomonkey.NewPatches() + defer patches.Reset() + + var called bool + patches.ApplyMethodFunc(metricsManager.metricsProvider, "ClearMetricsForFunction", + func(*commontypes.FuncMetaData) { + called = true + }) + + // 执行测试 + metricsManager.ProcessFunctionDelete(mockFuncSpec) + + // 验证结果 + convey.So(called, convey.ShouldBeTrue) + }) +} + +func TestWiseCloudMetricsManager_ProcessInsConfigDelete(t *testing.T) { + convey.Convey("测试 ProcessInsConfigDelete", t, func() { + // 使用 gomonkey mock 依赖 + patches := gomonkey.NewPatches() + defer patches.Reset() + + // mock 函数元数据加载 + patches.ApplyFunc(functionmeta.LoadFuncSpec, func(string) (*commontypes.FuncSpec, bool) { + return mockFuncSpec, true + }) + + var called bool + patches.ApplyMethodFunc(metricsManager.metricsProvider, "ClearMetricsForInsConfig", + func(*commontypes.FuncMetaData, string) { + called = true + }) + + // 执行测试 + metricsManager.ProcessInsConfigDelete(mockInsConfig) + + // 验证结果 + convey.So(called, convey.ShouldBeTrue) + }) + + convey.Convey("测试 ProcessInsConfigDelete 函数元数据不存在", t, func() { + // 使用 gomonkey mock 依赖 + patches := gomonkey.NewPatches() + defer patches.Reset() + + // mock 函数元数据加载返回不存在 + patches.ApplyFunc(functionmeta.LoadFuncSpec, func(string) (*commontypes.FuncSpec, bool) { + return nil, false + }) + var called bool + patches.ApplyMethodFunc(metricsManager.metricsProvider, "ClearMetricsForInsConfig", + func(*commontypes.FuncMetaData, string) { + called = true + }) + // 执行测试并验证日志输出 + metricsManager.ProcessInsConfigDelete(mockInsConfig) + convey.So(called, convey.ShouldBeFalse) + }) +} + +func TestWiseCloudMetricsManager_ProcessInstanceDelete(t *testing.T) { + // 使用 gomonkey mock 依赖 + patches := gomonkey.NewPatches() + defer patches.Reset() + var leaseCalled, concurrencyCalled bool + // mock 函数元数据加载 + patches.ApplyFunc(functionmeta.LoadFuncSpec, func(string) (*commontypes.FuncSpec, bool) { + return mockFuncSpec, true + }) + patches.ApplyMethodFunc(metricsManager.metricsProvider, "ClearLeaseRequestTotalWithLabel", + func([]string) error { + leaseCalled = true + return nil + }) + patches.ApplyMethodFunc(metricsManager.metricsProvider, "ClearConcurrencyGaugeWithLabel", + func([]string) error { + concurrencyCalled = true + return nil + }) + // mock 资源规格解析 + patches.ApplyFunc(resspeckey.GetResKeyFromStr, func(string) (resspeckey.ResSpecKey, error) { + return resspeckey.ResSpecKey{InvokeLabel: "test-label"}, nil + }) + + convey.Convey("测试 ProcessInstanceDelete", t, func() { + leaseCalled = false + concurrencyCalled = false + + // 执行测试 + metricsManager.ProcessInstanceDelete(mockInstance) + + // 验证结果 + convey.So(leaseCalled, convey.ShouldBeTrue) + convey.So(concurrencyCalled, convey.ShouldBeTrue) + }) + + convey.Convey("测试 ProcessInstanceDelete 异常情况", t, func() { + convey.Convey("函数元数据不存在", func() { + localPatches := gomonkey.NewPatches() + defer localPatches.Reset() + leaseCalled = false + concurrencyCalled = false + localPatches.ApplyFunc(functionmeta.LoadFuncSpec, func(string) (*commontypes.FuncSpec, bool) { + return nil, false + }) + metricsManager.ProcessInstanceDelete(mockInstance) + + convey.So(leaseCalled, convey.ShouldBeFalse) + convey.So(concurrencyCalled, convey.ShouldBeFalse) + }) + + convey.Convey("资源规格解析失败", func() { + localPatches := gomonkey.NewPatches() + defer localPatches.Reset() + leaseCalled = false + concurrencyCalled = false + localPatches.ApplyFunc(resspeckey.GetResKeyFromStr, func(string) (*resspeckey.ResSpecKey, error) { + return nil, fmt.Errorf("parse error") + }) + + metricsManager.ProcessInstanceDelete(mockInstance) + convey.So(leaseCalled, convey.ShouldBeFalse) + convey.So(concurrencyCalled, convey.ShouldBeFalse) + }) + }) +} + +func TestWiseCloudMetricsManager_InvokeStart(t *testing.T) { + convey.Convey("测试 InvokeStart", t, func() { + // 使用 gomonkey mock 依赖 + patches := gomonkey.NewPatches() + defer patches.Reset() + + // mock 配置检查 + patches.ApplyFunc(config.GetConfig, func() *types.Config { + return &types.Config{BusinessType: constant.BusinessTypeWiseCloud} + }) + + // mock 函数元数据加载 + patches.ApplyFunc(functionmeta.LoadFuncSpec, func(string) (*commontypes.FuncSpec, bool) { + return mockFuncSpec, true + }) + + // mock 资源规格解析 + patches.ApplyFunc(resspeckey.GetResKeyFromStr, func(string) (resspeckey.ResSpecKey, error) { + return resspeckey.ResSpecKey{InvokeLabel: "test-label"}, nil + }) + + var leaseCalled, concurrencyCalled bool + patches.ApplyMethodFunc(metricsManager.metricsProvider, "IncLeaseRequestTotalWithLabel", + func([]string) error { + leaseCalled = true + return nil + }) + patches.ApplyMethodFunc(metricsManager.metricsProvider, "IncConcurrencyGaugeWithLabel", + func([]string) error { + concurrencyCalled = true + return nil + }) + patches.ApplyMethodFunc(instancemanager.GetGlobalInstanceScheduler(), "GetInstance", + func(string, string, string) *commontypes.InstanceSpecification { + return &commontypes.InstanceSpecification{ + Extensions: commontypes.Extensions{ + PodName: "1", + PodNamespace: "2", + PodDeploymentName: "3", + }, + } + }) + + // 执行测试 + metricsManager.InvokeStart("test-function", "test-res-spec", "test-function") + + // 验证结果 + convey.So(leaseCalled, convey.ShouldBeTrue) + convey.So(concurrencyCalled, convey.ShouldBeTrue) + }) + + convey.Convey("测试 InvokeStart 非CaaS业务类型", t, func() { + patches := gomonkey.NewPatches() + defer patches.Reset() + + patches.ApplyFunc(config.GetConfig, func() *types.Config { + return &types.Config{BusinessType: 2} + }) + + var called bool + patches.ApplyMethodFunc(metricsManager.metricsProvider, "IncConcurrencyGaugeWithLabel", + func([]string) error { + called = true + return nil + }) + + metricsManager.InvokeStart("test-function", "test-res-spec", "test-function") + convey.So(called, convey.ShouldBeFalse) + }) +} + +func TestWiseCloudMetricsManager_InvokeEnd(t *testing.T) { + convey.Convey("测试 InvokeEnd", t, func() { + // 使用 gomonkey mock 依赖 + patches := gomonkey.NewPatches() + defer patches.Reset() + + // mock 配置检查 + patches.ApplyFunc(config.GetConfig, func() *types.Config { + return &types.Config{BusinessType: constant.BusinessTypeWiseCloud} + }) + + // mock 函数元数据加载 + patches.ApplyFunc(functionmeta.LoadFuncSpec, func(string) (*commontypes.FuncSpec, bool) { + return mockFuncSpec, true + }) + + // mock 实例获取 + patches.ApplyMethodFunc(instancemanager.GetGlobalInstanceScheduler(), "GetInstance", + func(string, string, string) *commontypes.InstanceSpecification { + return mockInstance + }) + + // mock 资源规格解析 + patches.ApplyFunc(resspeckey.GetResKeyFromStr, func(string) (resspeckey.ResSpecKey, error) { + return resspeckey.ResSpecKey{InvokeLabel: "test-label"}, nil + }) + + patches.ApplyMethodFunc(instancemanager.GetGlobalInstanceScheduler(), "GetInstance", + func(string, string, string) *commontypes.InstanceSpecification { + return &commontypes.InstanceSpecification{ + Extensions: commontypes.Extensions{ + PodName: "1", + PodNamespace: "2", + PodDeploymentName: "3", + }, + } + }) + + var called bool + patches.ApplyMethodFunc(metricsManager.metricsProvider, "DecConcurrencyGaugeWithLabel", + func([]string) error { + called = true + return nil + }) + + // 执行测试 + metricsManager.InvokeEnd("test-function", "test-res-spec", "test-instance-id") + + // 验证结果 + convey.So(called, convey.ShouldBeTrue) + }) + + convey.Convey("测试 InvokeEnd 异常情况", t, func() { + convey.Convey("非CaaS业务类型", func() { + patches := gomonkey.NewPatches() + defer patches.Reset() + + patches.ApplyFunc(config.GetConfig, func() *types.Config { + return &types.Config{BusinessType: 2} + }) + + var called bool + patches.ApplyMethodFunc(metricsManager.metricsProvider, "DecConcurrencyGaugeWithLabel", + func([]string) error { + called = true + return nil + }) + + metricsManager.InvokeEnd("test-function", "test-res-spec", "test-instance-id") + convey.So(called, convey.ShouldBeFalse) + }) + + convey.Convey("实例不存在", func() { + patches := gomonkey.NewPatches() + defer patches.Reset() + + patches.ApplyMethodFunc(instancemanager.GetGlobalInstanceScheduler(), "GetInstance", + func(string, string, string) *commontypes.InstanceSpecification { + return nil + }) + + var called bool + patches.ApplyMethodFunc(metricsManager.metricsProvider, "DecConcurrencyGaugeWithLabel", + func([]string) error { + called = true + return nil + }) + + metricsManager.InvokeEnd("test-function", "test-res-spec", "test-instance-id") + convey.So(called, convey.ShouldBeFalse) + }) + }) +} diff --git a/frontend/pkg/frontend/wisecloud/queue.go b/frontend/pkg/frontend/wisecloud/queue.go new file mode 100644 index 0000000000000000000000000000000000000000..b3fc628473edd84dc8fe21b4b444ff9f5970dadf --- /dev/null +++ b/frontend/pkg/frontend/wisecloud/queue.go @@ -0,0 +1,391 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package wisecloud - +package wisecloud + +import ( + "sync" + "sync/atomic" + "time" + + "go.uber.org/zap" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/instanceconfig" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/queue" + "frontend/pkg/common/faas_common/resspeckey" + "frontend/pkg/common/faas_common/snerror" + "frontend/pkg/common/faas_common/statuscode" + "frontend/pkg/common/faas_common/types" + "frontend/pkg/common/faas_common/utils" + "frontend/pkg/frontend/functionmeta" + "frontend/pkg/frontend/instanceconfigmanager" + "frontend/pkg/frontend/instancemanager" +) + +// queueManager - +var queueManager = &QueueManager{ + RWMutex: sync.RWMutex{}, + queuesMap: make(map[string]map[string]*reqQueue), + logger: log.GetLogger(), +} + +// GetQueueManager - +func GetQueueManager() *QueueManager { + return queueManager +} + +// QueueManager - +type QueueManager struct { + sync.RWMutex + queuesMap map[string]map[string]*reqQueue + logger api.FormatLogger +} + +// ProcessFunctionDelete - +func (m *QueueManager) ProcessFunctionDelete(funcMeta *types.FuncSpec) { + m.Lock() + queues, ok := m.queuesMap[funcMeta.FunctionKey] + if !ok { + m.Unlock() + return + } + delete(m.queuesMap, funcMeta.FunctionKey) + m.Unlock() + + resKeySum := "" + for resKey, q := range queues { + q.destroy() + resKeySum += resKey + "," + } + m.logger.Infof("recv function delete event, delete queues success, funcKey: %s, resKeysum: %s", + funcMeta.FunctionKey, resKeySum) +} + +// ProcessInsConfigDelete - +func (m *QueueManager) ProcessInsConfigDelete(insConfig *instanceconfig.Configuration) { + m.Lock() + queues, ok := m.queuesMap[insConfig.FuncKey] + if !ok { + m.Unlock() + return + } + deleteQueue := make(map[string]*reqQueue) + + for resSpecKey, q := range queues { + if q.resSpec.InvokeLabel == insConfig.InstanceLabel { + deleteQueue[resSpecKey] = q + } + } + for k, _ := range deleteQueue { + delete(queues, k) + } + + if len(queues) == 0 { + delete(m.queuesMap, insConfig.FuncKey) + } + m.Unlock() + + resKeySum := "" + for _, q := range deleteQueue { + q.destroy() + resKeySum += q.resSpec.String() + } + m.logger.Infof("recv insConfig delete event, delete queue success, funcKey: %s, reskey: %s", + insConfig.FuncKey, resKeySum) +} + +// AddPendingRequest - +func (m *QueueManager) AddPendingRequest(funcKey string, resSpec *resspeckey.ResSpecKey, pendingReq *PendingRequest) { + _, ok := functionmeta.LoadFuncSpec(funcKey) + if !ok { + pendingReq.ResultChan <- &PendingResponse{ + Error: snerror.New(statuscode.FuncMetaNotFoundErrCode, statuscode.FuncMetaNotFoundErrMsg), + } + return + } + + insConfig, ok := instanceconfigmanager.Load(funcKey, resSpec.InvokeLabel) + if !ok { + pendingReq.ResultChan <- &PendingResponse{ + Error: snerror.New(statuscode.FuncMetaNotFoundErrCode, "instance label not found"), + } + return + } + m.RLock() + queues, ok := m.queuesMap[funcKey] + if !ok { + m.RUnlock() + m.Lock() + queues, ok = m.queuesMap[funcKey] + if !ok { + queues = make(map[string]*reqQueue) + m.queuesMap[funcKey] = queues + } + m.Unlock() + m.RLock() + } + + queue, ok := queues[resSpec.String()] + if !ok { + m.RUnlock() + m.Lock() + queue, ok = queues[resSpec.String()] + if !ok { + queue = newQueue(funcKey, resSpec, insConfig) + queues[resSpec.String()] = queue + } + m.Unlock() + m.RLock() + } + + queue.addPendingRequest(pendingReq) + m.RUnlock() +} + +// ProcessInstanceUpdate - +func (m *QueueManager) ProcessInstanceUpdate(instance *types.InstanceSpecification) { + funcKey, ok := instance.CreateOptions[constant.FunctionKeyNote] + if !ok { + return + } + resSpec, err := resspeckey.GetResKeyFromStr(instance.CreateOptions[constant.ResourceSpecNote]) + if err != nil { + return + } + + m.RLock() + queues, ok := m.queuesMap[funcKey] + if !ok { + m.RUnlock() + return + } + + queue, ok := queues[resSpec.String()] + if !ok { + m.RUnlock() + return + } + queue.handleInstanceUpdate(instance) + if queue.Len() == 0 { + m.RUnlock() + m.Lock() + if queue.Len() == 0 { + queue.destroy() + delete(queues, resSpec.String()) + if len(queues) == 0 { + delete(m.queuesMap, funcKey) + } + } + m.Unlock() + m.RLock() + } + + m.RUnlock() +} + +// ProcessQueueEmpty - +func (m *QueueManager) ProcessQueueEmpty(funcKey string, resSpec *resspeckey.ResSpecKey) { + m.Lock() + defer m.Unlock() + queues, ok := m.queuesMap[funcKey] + if !ok { + return + } + + queue, ok := queues[resSpec.String()] + if !ok { + return + } + queue.Lock() + defer queue.Unlock() + if queue.Len() != 0 { + return + } + go queue.destroy() + delete(queues, resSpec.String()) + if len(queues) == 0 { + delete(m.queuesMap, funcKey) + } +} + +type reqQueue struct { + funcKey string + resSpec *resspeckey.ResSpecKey + insConfig *instanceconfig.Configuration + sync.RWMutex + *queue.FifoQueue + logger api.FormatLogger + stopCh chan struct{} + running atomic.Bool +} + +func newQueue(funcKey string, resSpec *resspeckey.ResSpecKey, + insConfig *instanceconfig.Configuration) *reqQueue { + q := &reqQueue{ + funcKey: funcKey, + resSpec: resSpec, + RWMutex: sync.RWMutex{}, + FifoQueue: queue.NewFifoQueue(nil), + insConfig: insConfig, + logger: log.GetLogger().With(zap.Any("funcKey", funcKey), zap.Any("resSpecKey", resSpec.String())), + stopCh: make(chan struct{}), + } + + q.running.Store(true) + go q.timeoutLoop() + return q +} + +// PendingRequest - +type PendingRequest struct { + CreatedTime time.Time + ScheduleTimeout time.Duration + ResultChan chan *PendingResponse +} + +// PendingResponse - +type PendingResponse struct { + Instance *types.InstanceSpecification + Error error +} + +func (q *reqQueue) destroy() { + q.Lock() + if !q.running.Load() { + q.Unlock() + return + } + q.running.Store(false) + utils.SafeCloseChannel(q.stopCh) + q.Unlock() +} + +func (q *reqQueue) addPendingRequest(req *PendingRequest) { + q.Lock() + defer q.Unlock() + if q.Len() >= 100 { // magic number + req.ResultChan <- &PendingResponse{ + Instance: nil, + Error: snerror.New(statuscode.FrontendStatusTooManyRequests, "queue has too many requests"), + } + } + err := q.PushBack(req) + if err != nil { + return + } + if q.Len() == 1 && coldStartProvider != nil { + go func() { + err := coldStartProvider.ColdStart(q.funcKey, *q.resSpec, &q.insConfig.NuwaRuntimeInfo) + if err != nil { + q.clearQueueWithError(err) + } + }() + } +} + +func (q *reqQueue) handleInstanceUpdate(_ *types.InstanceSpecification) { + q.Lock() + defer q.Unlock() + for { + if q.Len() == 0 { + break + } + pendingReq, ok := q.Front().(*PendingRequest) + if !ok { + q.PopFront() + continue + } + instance := instancemanager.GetGlobalInstanceScheduler().GetRandomInstanceWithoutUnexpectedInstance(q.funcKey, + q.resSpec.String(), nil, q.logger) + if instance == nil { + break + } + pendingReq.ResultChan <- &PendingResponse{ + Instance: instance, + Error: nil, + } + q.PopFront() + } + + if q.Len() == 0 { + go queueManager.ProcessQueueEmpty(q.funcKey, q.resSpec) + } +} + +func (q *reqQueue) clearQueueWithError(err error) { + q.Lock() + defer q.Unlock() + for q.Len() != 0 { + pendingReq, ok := q.PopFront().(*PendingRequest) + if !ok { + continue + } + pendingReq.ResultChan <- &PendingResponse{ + Instance: nil, + Error: err, + } + } + go queueManager.ProcessQueueEmpty(q.funcKey, q.resSpec) +} + +// timeoutLoop - +func (q *reqQueue) timeoutLoop() { + timeoutTicker := time.NewTicker(1 * time.Second) + defer func() { + timeoutTicker.Stop() + q.logger.Infof("exit queue") + }() + + for { + select { + case <-q.stopCh: + q.logger.Infof("recv stop event") + err := snerror.New(statuscode.FuncMetaNotFoundErrCode, statuscode.FuncMetaNotFoundErrMsg) + q.clearQueueWithError(err) + return + case <-timeoutTicker.C: + } + q.checkQueueTimeout() + } +} + +func (q *reqQueue) checkQueueTimeout() { + q.Lock() + defer q.Unlock() + for q.Len() != 0 { + pendingReq, ok := q.Front().(*PendingRequest) + if !ok { + q.PopFront() + continue + } + waitTime := time.Now().Sub(pendingReq.CreatedTime) + if waitTime.Milliseconds() > pendingReq.ScheduleTimeout.Milliseconds() { + err := snerror.New(statuscode.ErrAcquireTimeoutCode, statuscode.InsThdReqTimeoutErrMsg) + pendingReq.ResultChan <- &PendingResponse{ + Instance: nil, + Error: err, + } + q.PopFront() + continue + } + break + } +} diff --git a/frontend/pkg/frontend/wisecloud/queue_test.go b/frontend/pkg/frontend/wisecloud/queue_test.go new file mode 100644 index 0000000000000000000000000000000000000000..68fa4a4f14002f1fec2bf217f3428918986c1706 --- /dev/null +++ b/frontend/pkg/frontend/wisecloud/queue_test.go @@ -0,0 +1,539 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wisecloud + +import ( + "encoding/json" + "fmt" + "reflect" + "sync" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + "go.uber.org/zap" + + "yuanrong.org/kernel/runtime/libruntime/api" + + "frontend/pkg/common/faas_common/constant" + "frontend/pkg/common/faas_common/instanceconfig" + "frontend/pkg/common/faas_common/logger/log" + "frontend/pkg/common/faas_common/queue" + "frontend/pkg/common/faas_common/resspeckey" + "frontend/pkg/common/faas_common/types" + "frontend/pkg/common/faas_common/wisecloudtool" + "frontend/pkg/frontend/functionmeta" + "frontend/pkg/frontend/instanceconfigmanager" + "frontend/pkg/frontend/instancemanager" +) + +func TestQueueManager_ProcessFunctionDelete(t *testing.T) { + convey.Convey("测试 ProcessFunctionDelete", t, func() { + // 初始化数据 + funcKey1 := "test-func1" + resSpec1 := &resspeckey.ResSpecKey{InvokeLabel: "test-label1"} + + funcKey2 := "test-func2" + resSpec2 := &resspeckey.ResSpecKey{InvokeLabel: "test-label2"} + // 创建 QueueManager 并添加测试队列 + manager := &QueueManager{ + queuesMap: map[string]map[string]*reqQueue{ + funcKey1: { + resSpec1.String(): newQueue(funcKey1, resSpec1, &instanceconfig.Configuration{}), + }, + funcKey2: { + resSpec2.String(): newQueue(funcKey2, resSpec2, &instanceconfig.Configuration{}), + }, + }, + logger: log.GetLogger(), + RWMutex: sync.RWMutex{}, + } + + // 执行测试 + manager.ProcessFunctionDelete(&types.FuncSpec{FunctionKey: funcKey1}) + manager.ProcessFunctionDelete(&types.FuncSpec{FunctionKey: funcKey2}) + + // 验证结果 + convey.So(manager.queuesMap, convey.ShouldNotContainKey, funcKey1) + convey.So(manager.queuesMap, convey.ShouldNotContainKey, funcKey2) + }) +} + +func TestQueueManager_ProcessInsConfigDelete(t *testing.T) { + convey.Convey("测试 ProcessInsConfigDelete", t, func() { + // 初始化数据 + funcKey := "test-func" + invokeLabel1 := "test-label1" + resSpec1 := &resspeckey.ResSpecKey{InvokeLabel: invokeLabel1} + + invokeLabel2 := "test-label2" + resSpec2 := &resspeckey.ResSpecKey{InvokeLabel: invokeLabel2} + + // 创建 QueueManager 并添加测试队列 + queue1 := newQueue(funcKey, resSpec1, &instanceconfig.Configuration{}) + queue2 := newQueue(funcKey, resSpec2, &instanceconfig.Configuration{}) + + manager := &QueueManager{ + queuesMap: map[string]map[string]*reqQueue{ + funcKey: { + resSpec1.String(): queue1, + resSpec2.String(): queue2, + }, + }, + logger: log.GetLogger(), + RWMutex: sync.RWMutex{}, + } + + // 执行测试1 + manager.ProcessInsConfigDelete(&instanceconfig.Configuration{ + FuncKey: funcKey, + InstanceLabel: invokeLabel1, + }) + + // 验证结果1 + convey.So(len(manager.queuesMap[funcKey]), convey.ShouldEqual, 1) + + // 执行测试2 + manager.ProcessInsConfigDelete(&instanceconfig.Configuration{ + FuncKey: funcKey, + InstanceLabel: invokeLabel2, + }) + + // 验证结果2 + convey.So(len(manager.queuesMap[funcKey]), convey.ShouldEqual, 0) + }) +} + +func TestQueueManager_AddPendingRequest(t *testing.T) { + convey.Convey("测试 addPendingRequest", t, func() { + // 使用 gomonkey mock 依赖 + patches := gomonkey.NewPatches() + defer patches.Reset() + + funcKey := "test-func" + invokeLabel := "test-label" + resSpec := &resspeckey.ResSpecKey{InvokeLabel: invokeLabel} + pendingReq0 := &PendingRequest{ + CreatedTime: time.Now(), + ScheduleTimeout: 10 * time.Second, + ResultChan: make(chan *PendingResponse, 1), + } + pendingReq1 := &PendingRequest{ + CreatedTime: time.Now(), + ScheduleTimeout: 10 * time.Second, + ResultChan: make(chan *PendingResponse, 1), + } + + // mock 函数元数据加载 + patches.ApplyFunc(functionmeta.LoadFuncSpec, func(string) (*types.FuncSpec, bool) { + return &types.FuncSpec{FunctionKey: funcKey}, true + }) + + // mock 实例配置加载 + patches.ApplyFunc(instanceconfigmanager.Load, func(string, string) (*instanceconfig.Configuration, bool) { + return &instanceconfig.Configuration{ + FuncKey: funcKey, + InstanceLabel: invokeLabel, + }, true + }) + + coldStartCount := 0 + wg := sync.WaitGroup{} + wg.Add(1) + coldStartProvider = &wisecloudtool.PodOperator{} + patches.ApplyMethod(reflect.TypeOf(coldStartProvider), "ColdStart", func(_ *wisecloudtool.PodOperator, _ string, _ resspeckey.ResSpecKey) error { + coldStartCount++ + if coldStartCount == 1 { + wg.Done() + } + return nil + }) + + // 创建 QueueManager + manager := &QueueManager{ + queuesMap: make(map[string]map[string]*reqQueue), + logger: log.GetLogger(), + } + + // 执行测试 + manager.AddPendingRequest(funcKey, resSpec, pendingReq0) + manager.AddPendingRequest(funcKey, resSpec, pendingReq1) + + // 验证结果 + convey.So(manager.queuesMap[funcKey], convey.ShouldNotBeNil) + convey.So(manager.queuesMap[funcKey][resSpec.String()], convey.ShouldNotBeNil) + + wg.Wait() + convey.So(coldStartCount, convey.ShouldEqual, 1) + + // 收尾 + manager.ProcessFunctionDelete(&types.FuncSpec{FunctionKey: funcKey}) + + // 测试冷启动失败场景 + wgFailed := sync.WaitGroup{} + wgFailed.Add(1) + coldStartCount = 0 + pendingReq0 = &PendingRequest{ + CreatedTime: time.Now(), + ScheduleTimeout: 10 * time.Second, + ResultChan: make(chan *PendingResponse, 1), + } + pendingReq1 = &PendingRequest{ + CreatedTime: time.Now(), + ScheduleTimeout: 10 * time.Second, + ResultChan: make(chan *PendingResponse, 1), + } + patches.ApplyMethod(reflect.TypeOf(coldStartProvider), "ColdStart", func(_ *wisecloudtool.PodOperator, _ string, _ resspeckey.ResSpecKey) error { + coldStartCount++ + if coldStartCount == 1 { + wgFailed.Done() + } + return fmt.Errorf("error") + }) + + processEmptyChan := sync.WaitGroup{} + processEmptyChan.Add(1) + processEmptyCount := 0 + patches.ApplyMethod(reflect.TypeOf(queueManager), "ProcessQueueEmpty", func(_ *QueueManager, _ string, _ *resspeckey.ResSpecKey) { + processEmptyCount++ + if processEmptyCount == 1 { + processEmptyChan.Done() + } + }) + + manager.AddPendingRequest(funcKey, resSpec, pendingReq0) + manager.AddPendingRequest(funcKey, resSpec, pendingReq1) + wgFailed.Wait() + + convey.So(coldStartCount, convey.ShouldEqual, 1) + result0 := <-pendingReq0.ResultChan + convey.So(result0.Error, convey.ShouldNotBeNil) + result1 := <-pendingReq1.ResultChan + convey.So(result1.Error, convey.ShouldNotBeNil) + processEmptyChan.Wait() + }) +} + +func TestQueueManager_AddPendingRequest_FuncMetaNotFound(t *testing.T) { + convey.Convey("测试 addPendingRequest 函数元数据不存在", t, func() { + // 使用 gomonkey mock 依赖 + patches := gomonkey.NewPatches() + defer patches.Reset() + + // mock 函数元数据加载返回不存在 + patches.ApplyFunc(functionmeta.LoadFuncSpec, func(string) (*types.FuncSpec, bool) { + return nil, false + }) + + manager := &QueueManager{ + queuesMap: make(map[string]map[string]*reqQueue), + } + + // 创建测试用的 pending 请求 + resultChan := make(chan *PendingResponse, 1) + pendingReq := &PendingRequest{ + ResultChan: resultChan, + } + + // 执行测试 + manager.AddPendingRequest("not-exist", &resspeckey.ResSpecKey{}, pendingReq) + + // 验证结果 + convey.So(len(resultChan), convey.ShouldEqual, 1) + resp := <-resultChan + convey.So(resp.Error, convey.ShouldNotBeNil) + convey.So(resp.Error.Error(), convey.ShouldContainSubstring, "function metadata not found") + }) +} + +func TestQueueManager_ProcessInstanceUpdate(t *testing.T) { + convey.Convey("测试 ProcessInstanceUpdate", t, func() { + // 使用 gomonkey mock 依赖 + patches := gomonkey.NewPatches() + defer patches.Reset() + + funcKey := "test-func" + invokeLabel := "test-label" + invokeLabelFake := "test-label-fake" + resSpec := &resspeckey.ResSpecKey{InvokeLabel: "test-label"} + resJson := &resspeckey.ResourceSpecification{ + InvokeLabel: invokeLabel, + } + + resSpecFake := &resspeckey.ResSpecKey{InvokeLabel: invokeLabelFake} + resJsonBytes, _ := json.Marshal(resJson) + resJsonFakeBytes, _ := json.Marshal(resSpecFake) + + instance := &types.InstanceSpecification{ + CreateOptions: map[string]string{ + constant.FunctionKeyNote: funcKey, + constant.ResourceSpecNote: string(resJsonBytes), + }, + } + + // 创建 QueueManager 并添加测试队列 + queue := &reqQueue{ + funcKey: funcKey, + resSpec: resSpec, + FifoQueue: queue.NewFifoQueue(nil), + } + + // 添加一个待处理请求 + queue.Lock() + queue.PushBack(&PendingRequest{ + ResultChan: make(chan *PendingResponse, 1), + }) + queue.Unlock() + + manager := &QueueManager{ + queuesMap: map[string]map[string]*reqQueue{ + funcKey: { + resSpec.String(): queue, + }, + }, + } + + // mock 实例管理器 + patches.ApplyMethodFunc(instancemanager.GetGlobalInstanceScheduler(), + "GetRandomInstanceWithoutUnexpectedInstance", + func(string, string, []string, api.FormatLogger) *types.InstanceSpecification { + return instance + }) + + // 执行测试,传入非对应函数的key + manager.ProcessInstanceUpdate(&types.InstanceSpecification{CreateOptions: map[string]string{ + constant.FunctionKeyNote: "mock-test", + constant.ResourceSpecNote: string(resJsonBytes), + }}) + // 验证结果 + convey.So(queue.Len(), convey.ShouldEqual, 1) + + // 执行测试,传入对应函数非对应label的key + manager.ProcessInstanceUpdate(&types.InstanceSpecification{CreateOptions: map[string]string{ + constant.FunctionKeyNote: funcKey, + constant.ResourceSpecNote: string(resJsonFakeBytes), + }}) + // 验证结果 + convey.So(queue.Len(), convey.ShouldEqual, 1) + + // 执行测试，传入对应的函数的key + manager.ProcessInstanceUpdate(instance) + + // 验证结果 + convey.So(queue.Len(), convey.ShouldEqual, 0) + }) +} + +func TestQueue_AddPendingRequest_ExceedLimit(t *testing.T) { + convey.Convey("测试队列请求超过限制", t, func() { + queue := &reqQueue{ + FifoQueue: queue.NewFifoQueue(nil), + logger: log.GetLogger().With(zap.String("funcKey", "test"), zap.String("resSpecKey", "test")), + } + + // 填充队列到上限 + for i := 0; i < 100; i++ { + queue.PushBack(&PendingRequest{ + ResultChan: make(chan *PendingResponse, 1), + }) + } + + // 创建一个会触发超限的请求 + resultChan := make(chan *PendingResponse, 1) + pendingReq := &PendingRequest{ + ResultChan: resultChan, + } + + // 执行测试 + queue.addPendingRequest(pendingReq) + + // 验证结果 + convey.So(len(resultChan), convey.ShouldEqual, 1) + resp := <-resultChan + convey.So(resp.Error, convey.ShouldNotBeNil) + convey.So(resp.Error.Error(), convey.ShouldContainSubstring, "too many request") + }) +} + +func TestQueueManager_ProcessQueueEmpty(t *testing.T) { + convey.Convey("测试 ProcessQueueEmpty 方法", t, func() { + // 准备测试数据 + funcKey := "test-func" + resSpec := &resspeckey.ResSpecKey{InvokeLabel: "test-label"} + resSpecStr := resSpec.String() + + // 测试用例组 + convey.Convey("当队列存在且为空时", func() { + // 创建 mock 队列 + mockQueue := &reqQueue{ + RWMutex: sync.RWMutex{}, + FifoQueue: queue.NewFifoQueue(nil), + } + + // 准备 QueueManager + manager := &QueueManager{ + queuesMap: map[string]map[string]*reqQueue{ + funcKey: { + resSpecStr: mockQueue, + }, + }, + } + + // mock reqQueue 方法 + patches := gomonkey.NewPatches() + defer patches.Reset() + + var destroyCalled bool + wg := sync.WaitGroup{} + wg.Add(1) + patches.ApplyPrivateMethod(mockQueue, "destroy", func() { + destroyCalled = true + wg.Done() + }) + + // 执行测试 + manager.ProcessQueueEmpty(funcKey, resSpec) + + // 验证结果 + wg.Wait() + convey.So(destroyCalled, convey.ShouldBeTrue) + convey.So(manager.queuesMap[funcKey], convey.ShouldNotContainKey, resSpecStr) + }) + + convey.Convey("当队列不存在时", func() { + manager := &QueueManager{ + queuesMap: make(map[string]map[string]*reqQueue), + } + + // 执行测试 + manager.ProcessQueueEmpty(funcKey, resSpec) + + // 验证 queuesMap 未被修改 + convey.So(manager.queuesMap, convey.ShouldNotContainKey, funcKey) + }) + + convey.Convey("当队列不为空时", func() { + mockQueue := &reqQueue{ + RWMutex: sync.RWMutex{}, + FifoQueue: queue.NewFifoQueue(nil), + } + mockQueue.PushBack(&PendingRequest{ + CreatedTime: time.Time{}, + ScheduleTimeout: 0, + ResultChan: make(chan *PendingResponse), + }) + + manager := &QueueManager{ + queuesMap: map[string]map[string]*reqQueue{ + funcKey: { + resSpecStr: mockQueue, + }, + }, + } + + // mock 队列不为空 + patches := gomonkey.NewPatches() + defer patches.Reset() + + var destroyCalled bool + patches.ApplyPrivateMethod(mockQueue, "destroy", func() { + destroyCalled = true + }) + + // 执行测试 + manager.ProcessQueueEmpty(funcKey, resSpec) + + // 验证队列未被销毁 + time.Sleep(500 * time.Millisecond) + convey.So(destroyCalled, convey.ShouldBeFalse) + convey.So(manager.queuesMap[funcKey], convey.ShouldContainKey, resSpecStr) + }) + + convey.Convey("当删除最后一个队列时清理父map", func() { + mockQueue := &reqQueue{ + RWMutex: sync.RWMutex{}, + FifoQueue: queue.NewFifoQueue(nil), + } + + manager := &QueueManager{ + queuesMap: map[string]map[string]*reqQueue{ + funcKey: { + resSpecStr: mockQueue, + }, + }, + } + + // 执行测试 + manager.ProcessQueueEmpty(funcKey, resSpec) + + // 验证整个 funcKey 映射被清除 + convey.So(manager.queuesMap, convey.ShouldNotContainKey, funcKey) + }) + }) +} + +func TestQueue_TimeoutLoop(t *testing.T) { + convey.Convey("测试队列超时处理循环", t, func() { + // 使用 gomonkey mock 时间 + patches := gomonkey.NewPatches() + defer patches.Reset() + + now := time.Now() + coldStartProvider = &wisecloudtool.PodOperator{} + patches.ApplyMethodFunc(reflect.TypeOf(coldStartProvider), "ColdStart", func(funcKeyWithRes string, resSpec resspeckey.ResSpecKey, nuwaRuntimeInfo *types.NuwaRuntimeInfo) error { + return nil + }) + queue := newQueue("test-func", &resspeckey.ResSpecKey{}, &instanceconfig.Configuration{}) + + // 添加一个已经超时的请求 + timeoutReq := &PendingRequest{ + CreatedTime: now.Add(-11 * time.Second), + ScheduleTimeout: 10 * time.Second, + ResultChan: make(chan *PendingResponse, 1), + } + timeoutReq0 := &PendingRequest{ + CreatedTime: now.Add(-11 * time.Second), + ScheduleTimeout: 10 * time.Second, + ResultChan: make(chan *PendingResponse, 1), + } + timeoutReq1 := &PendingRequest{ + CreatedTime: now.Add(-8 * time.Second), + ScheduleTimeout: 10 * time.Second, + ResultChan: make(chan *PendingResponse, 1), + } + + queue.Lock() + queue.PushBack(timeoutReq) + queue.PushBack(timeoutReq0) + queue.PushBack(timeoutReq1) + queue.Unlock() + + // 等待超时处理 + time.Sleep(1*time.Second + 100*time.Millisecond) + + // 验证结果 + convey.So(len(timeoutReq.ResultChan), convey.ShouldEqual, 1) + convey.So(queue.Len(), convey.ShouldEqual, 1) + + time.Sleep(2 * time.Second) + convey.So(queue.Len(), convey.ShouldEqual, 0) + + // 停止队列 + queue.destroy() + }) +} diff --git a/frontend/posix/README.md b/frontend/posix/README.md new file mode 100644 index 0000000000000000000000000000000000000000..530cda7d8ac0f871d577734a729b6df863be751c --- /dev/null +++ b/frontend/posix/README.md @@ -0,0 +1,3 @@ +# posix + +posix文件存放 \ No newline at end of file diff --git a/frontend/posix/proto/affinity.proto b/frontend/posix/proto/affinity.proto new file mode 100644 index 0000000000000000000000000000000000000000..3dd493108d40779a62d93b97bbf5467d1dfea369 --- /dev/null +++ b/frontend/posix/proto/affinity.proto @@ -0,0 +1,98 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package affinity; + +option go_package = "frontend/pkg/common/faas_common/grpc/pb/affinity;affinity"; + + +// IN: True if the candinate has label with desired label key and its label value is in the specified value set +message LabelIn { + repeated string values = 1; +} + +// NOT_IN: True if the candinate has label with desired label key and its label value is NOT in the specified value set +message LabelNotIn { + repeated string values = 1; +} + +// EXISTS: True if the candinate has label with desired label key +message LabelExists {} + +// DOES_NOT_EXIST: True if the candinate does't have any label with desired label key +message LabelDoesNotExist {} + +message LabelOperator { + oneof LabelOperator { + LabelIn in = 1; + LabelNotIn notIn = 2; + LabelExists exists = 3; + LabelDoesNotExist notExist = 4; + } +} + +message LabelExpression { + string key = 1; // label key + LabelOperator op = 2; +} + +message SubCondition { + repeated LabelExpression expressions = 1; // AND between expressions + int64 weight = 2; // weight of this sub condition for ranking +} + +message Condition { + repeated SubCondition subConditions = 1; // OR between sub conditions + bool orderPriority = 2; // in order of priority instead of weights rank +} + +message Selector { + Condition condition = 1; +} + +enum AffinityType { + PreferredAffinity = 0; + PreferredAntiAffinity = 1; + RequiredAffinity = 2; + RequiredAntiAffinity = 3; +} + +enum AffinityScope { + POD = 0; + NODE = 1; +} + +message ResourceAffinity { + Selector preferredAffinity = 1; + Selector preferredAntiAffinity = 2; + Selector requiredAffinity = 3; + Selector requiredAntiAffinity = 4; +} + +message InstanceAffinity { + Selector preferredAffinity = 1; + Selector preferredAntiAffinity = 2; + Selector requiredAffinity = 3; + Selector requiredAntiAffinity = 4; + AffinityScope scope = 5; +} + +message Affinity { + ResourceAffinity resource = 1; + InstanceAffinity instance = 2; +} diff --git a/frontend/posix/proto/common.proto b/frontend/posix/proto/common.proto new file mode 100644 index 0000000000000000000000000000000000000000..1d0fc0d5879efbf45a641e154e96684bb801e452 --- /dev/null +++ b/frontend/posix/proto/common.proto @@ -0,0 +1,240 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package common; + +option go_package = "frontend/pkg/common/faas_common/grpc/pb/common;common"; + +message Arg { + enum ArgType { + VALUE = 0; + OBJECT_REF = 1; + } + ArgType type = 1; + bytes value = 2; + repeated string nested_refs = 3; +} + +enum ErrorCode { + ERR_NONE = 0; + ERR_PARAM_INVALID = 1001; + ERR_RESOURCE_NOT_ENOUGH = 1002; + ERR_INSTANCE_NOT_FOUND = 1003; + ERR_INSTANCE_DUPLICATED = 1004; + ERR_INVOKE_RATE_LIMITED = 1005; + ERR_RESOURCE_CONFIG_ERROR = 1006; + ERR_INSTANCE_EXITED = 1007; + ERR_EXTENSION_META_ERROR = 1008; + ERR_INSTANCE_SUB_HEALTH = 1009; + ERR_GROUP_SCHEDULE_FAILED = 1010; + ERR_GROUP_EXIT_TOGETHER = 1011; + ERR_CREATE_RATE_LIMITED = 1012; + ERR_INSTANCE_EVICTED = 1013; + ERR_AUTHORIZE_FAILED = 1014; + ERR_FUNCTION_META_NOT_FOUND = 1015; + ERR_INSTANCE_INFO_INVALID = 1016; + ERR_SCHEDULE_CANCELED = 1017; + ERR_SCHEDULE_PLUGIN_CONFIG = 1018; + ERR_SUB_STATE_INVALID = 1019; + ERR_USER_CODE_LOAD = 2001; + ERR_USER_FUNCTION_EXCEPTION = 2002; + ERR_REQUEST_BETWEEN_RUNTIME_BUS = 3001; + ERR_INNER_COMMUNICATION = 3002; + ERR_INNER_SYSTEM_ERROR = 3003; + ERR_DISCONNECT_FRONTEND_BUS = 3004; + ERR_ETCD_OPERATION_ERROR = 3005; + ERR_BUS_DISCONNECTION = 3006; + ERR_REDIS_OPERATION_ERROR = 3007; + ERR_K8S_UNAVAILABLE = 3008; + ERR_FUNCTION_AGENT_OPERATION_ERROR = 3009; + ERR_STATE_MACHINE_ERROR = 3010; + ERR_LOCAL_SCHEDULER_OPERATION_ERROR = 3011; + ERR_RUNTIME_MANAGER_OPERATION_ERROR = 3012; + ERR_INSTANCE_MANAGER_OPERATION_ERROR= 3013; + ERR_LOCAL_SCHEDULER_ABNORMAL = 3014; + ERR_DS_UNAVAILABLE = 3015; + ERR_NPU_FAULT_ERROR = 3016; +} + +enum HealthCheckCode { + HEALTHY = 0; + HEALTH_CHECK_FAILED = 1; + SUB_HEALTH = 2; +} + +message SmallObject { + string id = 1; + bytes value = 2; // sbuffer +} + +message StackTraceInfo { + string type = 1; // type of exception thrown by user code + string message = 2; // message in user code thrown exception + repeated StackTraceElement stackTraceElements = 3; // stack trace elements in user code thrown exception + string language = 4; // language of user code +} + +message StackTraceElement { + string className = 1; // class name of user code exception + string methodName = 2; // method name of user code exception + string fileName = 3; // file name of user code exception + int64 lineNumber = 4; // line number of user code exception + map extensions = 5; // extensions for different language +} + +message TLSConfig { + bool dsAuthEnable = 1; + bool dsEncryptEnable = 2; + bytes dsClientPublicKey = 3; + bytes dsClientPrivateKey = 4; + bytes dsServerPublicKey = 5; + bool serverAuthEnable = 6; + bytes rootCertData = 7; + bytes moduleCertData = 8; + bytes moduleKeyData = 9; + string token = 10; + bool enableServerMode = 11; + string serverNameOverride = 12; + string posixPort = 13; + string salt = 14; + string accessKey = 15; // component-level access key + string securityKey = 16; // component-level security key +} + +message HeteroDeviceInfo +{ + int64 deviceId = 1; + string deviceIp = 2; + int64 rankId = 3; +} + +message ServerInfo +{ + repeated HeteroDeviceInfo devices = 1; + string serverId = 2; +} + +message FunctionGroupRunningInfo +{ + repeated ServerInfo serverList = 1; + int64 instanceRankId = 2; + int64 worldSize = 3; + string deviceName = 4; +} + +// message used in unix domain socket +message SocketMessage { + string magicNumber = 1; // header info(magicNumber/version/packetType/packetID) used to check + string version = 2; + string packetType = 3; + string packetID = 4; + BusinessMessage businessMsg = 5; +} + +message BusinessMessage { + MessageType type = 1; + oneof payload { + FunctionLog functionLog = 2; + } +} + +// Used in domain socket between runtime and runtime manager +enum MessageType { + LogProcess = 0; +} + +// user function log, one kind of businessMessage payload +message FunctionLog { + string level = 1; // log level + string timestamp = 2; + string content = 3; // log content + string invokeID = 4; + string traceID = 5; + string stage = 6; // log stage + bool isStart = 7; // first log sign + bool isFinish = 8; // last log sign + string logType = 9; // "tail": return log to user when invoke finishes, "": do not return log + int32 errorCode = 10; + string functionInfo = 11; // user function version urn + string instanceId = 12; + string logSource = 13; // std or logger + string logGroupId = 14; // used in FG + string logStreamId = 15; // used in FG +} + +message RuntimeInfo { + string serverIpAddr = 1; + int32 serverPort = 2; + string route = 3; // for low-reliability instance, format is "ip:port" +} + +message Bundle { + map resources = 1; + // custom label for reserved unit + repeated string labels = 2; // "key:value" or "key2" +} + +message ResourceGroupSpec { + string name = 1; + // indicated which rg is the resource group was created from, default is primary + string owner = 2; + // indicated which app submitted(job/app) + string appID = 3; + string tenantID = 4; + // multiple units which is reserved defined in a resource group + repeated Bundle bundles = 5; + message Option { + // resource group schedule priority + int64 priority = 1; + // etc: + // "lifetime" : "detached" + map extension = 100; + } + Option opt = 6; +} + +message InstanceTermination { + string instanceID = 1; +} + +message FunctionMasterObserve {} + +message FunctionMasterEvent { + string address = 1; +} + +message SubscriptionPayload { + oneof Content { + InstanceTermination instanceTermination = 1; // Subscribe to instance termination event + FunctionMasterObserve functionMaster = 2; // Subscribe to function-master election changed + } +} + +message UnsubscriptionPayload { + oneof Content { + InstanceTermination instanceTermination = 1; // Unsubscribe specified instance's termination event + FunctionMasterObserve functionMaster = 2; // UnSubscribe to function-master election changed + } +} + +message NotificationPayload { + oneof Content { + InstanceTermination instanceTermination = 1; // Instance termination event notification + FunctionMasterEvent functionMasterEvent = 2; // function-master election changed event + } +} \ No newline at end of file diff --git a/frontend/posix/proto/core_service.proto b/frontend/posix/proto/core_service.proto new file mode 100644 index 0000000000000000000000000000000000000000..7caab2b09e04eb7934b616791d9c07760d3ae2fd --- /dev/null +++ b/frontend/posix/proto/core_service.proto @@ -0,0 +1,206 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package core_service; + +import "common.proto"; +import "affinity.proto"; + +option go_package = "frontend/pkg/common/faas_common/grpc/pb/core;core"; + +// Core service provides APIs to runtime, +service CoreService { + // Create an instance for specify function + rpc Create (CreateRequest) returns (CreateResponse) {} + // invoke the created instance + rpc Invoke (InvokeRequest) returns (InvokeResponse) {} + // exit the created instance + rpc Exit (ExitRequest) returns (ExitResponse) {} + // save state of the created instance + rpc SaveState (StateSaveRequest) returns (StateSaveResponse) {} + // load state of the created instance + rpc LoadState (StateLoadRequest) returns (StateLoadResponse) {} + // Kill the signal to instance + rpc Kill (KillRequest) returns (KillResponse) {} +} + +enum AffinityType { + PreferredAffinity = 0; + PreferredAntiAffinity = 1; + RequiredAffinity = 2; + RequiredAntiAffinity = 3; +} + +message SchedulingOptions { + int32 priority = 1; + map resources = 2; + map extension = 3; + // will deprecate in future + map affinity = 4; + affinity.Affinity scheduleAffinity = 5; + InstanceRange range = 6; + int64 scheduleTimeoutMs = 7; + bool preemptedAllowed = 8; + // indicated which rgroup submit to + string rGroupName = 9; +} + +message InstanceRange { + int32 min = 1; + int32 max = 2; + int32 step = 3; +} + +message CreateRequest { + string function = 1; + repeated common.Arg args = 2; + SchedulingOptions schedulingOps = 3; + string requestID = 4; + string traceID = 5; + repeated string labels = 6; // "key:value" or "key2" + // optional. if designated instanceID is not empty, the created instance id will be assigned designatedInstanceID + string designatedInstanceID = 7; + map createOptions = 8; +} + +message CreateResourceGroupRequest { + common.ResourceGroupSpec rGroupSpec = 1; + string requestID = 2; + string traceID = 3; +} + +message CreateResourceGroupResponse { + common.ErrorCode code = 1; + string message = 2; + string requestID = 3; +} + +message CreateResponse { + common.ErrorCode code = 1; + string message = 2; + string instanceID = 3; +} + +message GroupOptions { + // group schedule timeout (sec) + int64 timeout = 1; + // group alias name, this field cannot be used for life cycle management. + string groupName = 2; + bool sameRunningLifecycle = 3; + // indicated which rgroup submit to + string rGroupName = 4; +} + +// gang scheduling +message CreateRequests { + repeated CreateRequest requests = 1; + string tenantID = 2; + string requestID = 3; + string traceID = 4; + GroupOptions groupOpt = 5; +} + +// gang scheduling +message CreateResponses { + common.ErrorCode code = 1; + string message = 2; + repeated string instanceIDs = 3; + // used for life cycle management and the unique ID of the corresponding group. + // when you want to recycle a group, use signal 4 to send a kill request for the ID. + string groupID = 4; +} + +message InvokeOptions { + map customTag = 1; +} + +message InvokeRequest { + string function = 1; + repeated common.Arg args = 2; + string instanceID = 3; + string requestID = 4; + string traceID = 5; + repeated string returnObjectIDs = 6; + string spanID = 7; + InvokeOptions invokeOptions = 8; +} + +message InvokeResponse { + common.ErrorCode code = 1; + string message = 2; + string returnObjectID = 3; +} + +message CallResult { + common.ErrorCode code = 1; + string message = 2; + string instanceID = 3; + string requestID = 4; + repeated common.SmallObject smallObjects = 5; + repeated common.StackTraceInfo stackTraceInfos = 6; + common.RuntimeInfo runtimeInfo = 7; +} + +message CallResultAck { + common.ErrorCode code = 1; + string message = 2; +} + +message ExitRequest { + common.ErrorCode code = 1; + string message = 2; +} + +message ExitResponse { + common.ErrorCode code = 1; + string message = 2; +} + +message StateSaveRequest { + bytes state = 1; + string requestID = 2; +} + +message StateSaveResponse { + common.ErrorCode code = 1; + string message = 2; + string checkpointID = 3; +} + +message StateLoadRequest { + string checkpointID = 1; + string requestID = 2; +} + +message StateLoadResponse { + common.ErrorCode code = 1; + string message = 2; + bytes state = 3; +} + +message KillRequest { + string instanceID = 1; + int32 signal = 2; + bytes payload = 3; + string requestID = 4; +} + +message KillResponse { + common.ErrorCode code = 1; + string message = 2; +} diff --git a/frontend/posix/proto/runtime_rpc.proto b/frontend/posix/proto/runtime_rpc.proto new file mode 100644 index 0000000000000000000000000000000000000000..6384ac1ff72eafb82bac076ef2b90db2abfe2bde --- /dev/null +++ b/frontend/posix/proto/runtime_rpc.proto @@ -0,0 +1,125 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package runtime_rpc; + +import "core_service.proto"; +import "runtime_service.proto"; + +option go_package = "frontend/pkg/common/faas_common/grpc/pb;api"; + +// RuntimeRPC provide bidirectional streaming RPC interface +service RuntimeRPC { + // build bidirection grpc communication channel, different message body type specify different api handler + rpc MessageStream (stream StreamingMessage) returns (stream StreamingMessage) {} +} + +// InvocationRPC provide bidirectional streaming RPC for function instance invocation +service InvocationRPC { + rpc MessageStream (stream StreamingMessage) returns (stream StreamingMessage) {} +} + + +message StreamingMessage { + string messageID = 1; + oneof body { + + // Create an instance for specify function + // handle by core + core_service.CreateRequest createReq = 2; + core_service.CreateResponse createRsp = 3; + + // invoke the created instance + // handle by core + core_service.InvokeRequest invokeReq = 4; + core_service.InvokeResponse invokeRsp = 5; + + // exit the created instance + // only support to be called by instance itself + // handle by core + core_service.ExitRequest exitReq = 6; + core_service.ExitResponse exitRsp = 7; + + // save state of the created instance + // handle by core + core_service.StateSaveRequest saveReq = 8; + core_service.StateSaveResponse saveRsp = 9; + + // load state of the created instance + // handle by core + core_service.StateLoadRequest loadReq = 10; + core_service.StateLoadResponse loadRsp = 11; + + // send the signal to instance or core + // 1 ~ 63: core defined signal + // 64 ~ 1024: custom runtime defined signal + // handle by core + core_service.KillRequest killReq = 12; + core_service.KillResponse killRsp = 13; + + // send call request result to sender + // handle by core + core_service.CallResult callResultReq = 14; + core_service.CallResultAck callResultAck = 15; + + // Call a method or init state of instance + // handle by runtime + runtime_service.CallRequest callReq = 16; + runtime_service.CallResponse callRsp = 17; + + // NotifyResult is applied to async notify result of create or invoke request invoked by runtime + // handle by runtime + runtime_service.NotifyRequest notifyReq = 18; + runtime_service.NotifyResponse notifyRsp = 19; + + // Checkpoint request a state to save for failure recovery and state migration + // handle by runtime + runtime_service.CheckpointRequest checkpointReq = 20; + runtime_service.CheckpointResponse checkpointRsp = 21; + + // Recover state + // handle by runtime + runtime_service.RecoverRequest recoverReq = 22; + runtime_service.RecoverResponse recoverRsp = 23; + + // request an instance to shutdown + // handle by runtime + runtime_service.ShutdownRequest shutdownReq = 24; + runtime_service.ShutdownResponse shutdownRsp = 25; + + // receive the signal send by other runtime or driver + // handle by runtime + runtime_service.SignalRequest signalReq = 26; + runtime_service.SignalResponse signalRsp = 27; + + // check whether the runtime is alive + // handle by runtime + runtime_service.HeartbeatRequest heartbeatReq = 28; + runtime_service.HeartbeatResponse heartbeatRsp = 29; + + // Create group instance for specify function + // handle by core + core_service.CreateRequests createReqs = 30; + core_service.CreateResponses createRsps = 31; + + // Create resource group to reserve multiple bundle of resource + core_service.CreateResourceGroupRequest rGroupReq = 32; + core_service.CreateResourceGroupResponse rGroupRsp = 33; + } + map metaData = 100; +} \ No newline at end of file diff --git a/frontend/posix/proto/runtime_service.proto b/frontend/posix/proto/runtime_service.proto new file mode 100644 index 0000000000000000000000000000000000000000..03706cccb6c5c3dfe115b67a43c8afc9888f1d93 --- /dev/null +++ b/frontend/posix/proto/runtime_service.proto @@ -0,0 +1,131 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package runtime_service; + +import "common.proto"; + +option go_package = "frontend/pkg/common/faas_common/grpc/pb/runtime;runtime"; + +// Runtime service provides APIs to core, +service RuntimeService { + // Call a method or init state of instance + rpc Call (CallRequest) returns (CallResponse) {} + // NotifyResult is applied to async notify result of create or invoke request invoked by runtime + rpc NotifyResult (NotifyRequest) returns (NotifyResponse) {} + // Checkpoint request a state to save for failure recovery and state migration + rpc Checkpoint (CheckpointRequest) returns (CheckpointResponse) {} + // Recover state + rpc Recover (RecoverRequest) returns (RecoverResponse) {} + // GracefulExit request an instance graceful exit + rpc GracefulExit (GracefulExitRequest) returns (GracefulExitResponse) {} + // Shutdown request an instance shutdown + rpc Shutdown (ShutdownRequest) returns (ShutdownResponse) {} + // check whether the runtime is alive + rpc Heartbeat (HeartbeatRequest) returns (HeartbeatResponse) {} + // Signal the signal to instance + rpc Signal (SignalRequest) returns (SignalResponse) {} +} + +message CallRequest { + string function = 1; + repeated common.Arg args = 2; + string traceID = 3; + string returnObjectID = 4; + // isCreate specify the request whether initialization or runtime invoke + bool isCreate = 5; + // senderID specify the caller identity + // while process done, it should be send back to core by CallResult.instanceID + string senderID = 6; + // while process done, it should be send back to core by CallResult.requestID + string requestID = 7; + repeated string returnObjectIDs = 8; + map createOptions = 9; + string spanID = 10; +} + +message CallResponse { + common.ErrorCode code = 1; + string message = 2; + +} + +message CheckpointRequest { + string checkpointID = 1; +} + +message CheckpointResponse { + common.ErrorCode code = 1; + string message = 2; + bytes state = 3; +} + +message RecoverRequest { + bytes state = 1; + map createOptions = 2; +} + +message RecoverResponse { + common.ErrorCode code = 1; + string message = 2; +} + +message GracefulExitRequest { + uint64 gracePeriodSecond = 1; +} + +message GracefulExitResponse { + common.ErrorCode code = 1; + string message = 2; +} + +message ShutdownRequest { + uint64 gracePeriodSecond = 1; +} + +message ShutdownResponse { + common.ErrorCode code = 1; + string message = 2; +} + +message NotifyRequest { + string requestID = 1; + common.ErrorCode code = 2; + string message = 3; + repeated common.SmallObject smallObjects = 4; + repeated common.StackTraceInfo stackTraceInfos = 5; + common.RuntimeInfo runtimeInfo = 7; +} + +message NotifyResponse {} + +message HeartbeatRequest {} + +message HeartbeatResponse { + common.HealthCheckCode code = 1; +} + +message SignalRequest { + int32 signal = 1; + bytes payload = 2; +} + +message SignalResponse { + common.ErrorCode code = 1; + string message = 2; +} \ No newline at end of file diff --git a/frontend/test/common/test.sh b/frontend/test/common/test.sh new file mode 100644 index 0000000000000000000000000000000000000000..641e51388a03d2970ec6a06f64824854c6b4fd6a --- /dev/null +++ b/frontend/test/common/test.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# global environment +CUR_DIR=$(dirname "$(readlink -f "$0")") +ROOT_PATH=${CUR_DIR}/../.. +SRC_PATH=${ROOT_PATH}/pkg/common/faas_common +OUTPUT_PATH=${CUR_DIR}/output +echo LD_LIBRARY_PATH=$LD_LIBRARY_PATH + +# run go test and report +run_gocover_report() +{ + rm -rf "${OUTPUT_PATH}" + mkdir -p "${OUTPUT_PATH}" + + cd ${SRC_PATH} + go test -v -gcflags=all=-l -covermode="${GOCOVER_MODE}" -coverprofile="$OUTPUT_PATH/common.cover" -coverpkg="./..." "./..." + + if [ $? -ne 0 ]; then + log_error "failed to go test common" + exit 1 + fi + + # export llt coverage result + cd "$OUTPUT_PATH" + echo "mode: ${GOCOVER_MODE}" > coverage.out && cat ./*.cover | grep -v mode: | grep -v pb.go | sort -r | \ + awk '{if($1 != last) {print $0;last=$1}}' >> coverage.out + + gocov convert coverage.out > coverage.json + gocov report coverage.json > CoverResult.txt + gocov-html coverage.json > coverage.html +} + +run_gocover_report +exit 0 \ No newline at end of file diff --git a/frontend/test/faasfrontend/test.sh b/frontend/test/faasfrontend/test.sh new file mode 100644 index 0000000000000000000000000000000000000000..e47173643f7d51d1782db79f606b4fe9904e4267 --- /dev/null +++ b/frontend/test/faasfrontend/test.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# global environment +CUR_DIR=$(dirname "$(readlink -f "$0")") +ROOT_PATH=${CUR_DIR}/../.. +SRC_PATH=${ROOT_PATH}/pkg/frontend +OUTPUT_PATH=${CUR_DIR}/output +echo LD_LIBRARY_PATH=$LD_LIBRARY_PATH + +go mod tidy +ARCH="x86_64" +# run go test and report +run_gocover_report() +{ + rm -rf "${OUTPUT_PATH}" + mkdir -p "${OUTPUT_PATH}" + + cd ${SRC_PATH} + find frontendsdk/ -type f -print0 | xargs -0 sed -i 's#"yuanrong.org/kernel/runtime/posixsdk"#"functionsystem/pkg/frontend/frontendsdk/posixsdk"#g' + go test -tags function -v -gcflags=all=-l -covermode="${GOCOVER_MODE}" -coverprofile="$OUTPUT_PATH/faasfrontend.cover" -coverpkg="./..." "./..." + + if [ $? -ne 0 ]; then + log_error "failed to go test faasfrontend" + exit 1 + fi + + # export llt coverage result + cd "$OUTPUT_PATH" + echo "mode: ${GOCOVER_MODE}" > coverage.out && cat ./*.cover | grep -v mode: | grep -v pb.go | grep -v fb.go | sort -r | \ + awk '{if($1 != last) {print $0;last=$1}}' >> coverage.out + + gocov convert coverage.out > coverage.json + gocov report coverage.json > CoverResult.txt + gocov-html coverage.json > coverage.html +} + +run_gocover_report +exit 0 \ No newline at end of file diff --git a/frontend/test/test.sh b/frontend/test/test.sh new file mode 100644 index 0000000000000000000000000000000000000000..f13531f0684c2576d86e2bbd9e56dc210e98bb32 --- /dev/null +++ b/frontend/test/test.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CUR_DIR=$(dirname "$(readlink -f "$0")") +PROJECT_DIR=$(cd "${CUR_DIR}/.."; pwd) +ROOT_PATH=$PROJECT_DIR + +# resolve missing go.sum entry +go mod tidy +go env -w "GOFLAGS"="-mod=mod" + +# coverage mode +# set: 每个语句是否执行？ +# count: 每个语句执行了几次？ +# atomic: 类似于count, 但表示的是并行程序中的精确计数 +export GOCOVER_MODE="set" + +# test module name +MODULE_LIST=(\ +"common" \ +"faasfrontend" +) + +. "${ROOT_PATH}"/build/compile_functions.sh + +# $1: source file name, In the format of xxx.go +# $2: target file name, In the format of xxx_mock.go +function generate_mock() +{ + if ! mockgen -destination "$2" -source "$1" -package mock; then + log_error "Failed to generate mock file." + return 1; + fi +} +export -f generate_mock + +# create source code link, go cover report dependent on GOPATH src +link_source_code() +{ + rm -rf "${GOPATH}/pkg" + rm -rf "${GOPATH}/src/frontend" + + mkdir -p "${GOPATH}"/src/ + ln -s "${ROOT_PATH}" "${GOPATH}"/src/frontend +} + +link_source_code + + +if ! sh -x "${CUR_DIR}/${MODULE_LIST[$i]}/test.sh"; then + echo "Failed to test ${MODULE_LIST[$i]}" + exit 1 +fi +echo "Succeed to test ${MODULE_LIST[$i]}" + +exit 0