From 957de5a34b604a8ffd592047d33bba60207eb3f6 Mon Sep 17 00:00:00 2001 From: zhoupan39 Date: Wed, 30 Apr 2025 10:40:15 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E3=80=90clusterd=E3=80=91=E3=80=90?= =?UTF-8?q?=E4=BF=AE=E6=94=B9=E8=AF=B4=E6=98=8E=E3=80=91=E6=8E=A8=E9=80=81?= =?UTF-8?q?=E6=95=85=E9=9A=9C=E4=BF=A1=E6=81=AF=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../application/fault/publish_fault_plugin.go | 30 +++++++++++++---- .../fault/publish_fault_plugin_test.go | 13 ++++++++ .../faultrank/job_fault_rank_processor.go | 33 ++++++++++++------- .../clusterd/pkg/common/constant/const.go | 6 ++++ 4 files changed, 64 insertions(+), 18 deletions(-) diff --git a/component/clusterd/pkg/application/fault/publish_fault_plugin.go b/component/clusterd/pkg/application/fault/publish_fault_plugin.go index f5120a6b..a0af2bf4 100644 --- a/component/clusterd/pkg/application/fault/publish_fault_plugin.go +++ b/component/clusterd/pkg/application/fault/publish_fault_plugin.go @@ -40,11 +40,14 @@ func (s *FaultServer) checkPublishFault(allJobFaultInfo map[string]constant.JobF hwlog.RunLog.Errorf("jobId=%s jobInfo is not found", jobId) continue } - jobFaultMap[jobId] = append(jobFaultMap[jobId], jobFaultInfo.FaultDevice...) + filterList := filterFault(jobFaultInfo.FaultDevice) + if len(filterList) == 0 { + continue + } + jobFaultMap[jobId] = append(jobFaultMap[jobId], filterList...) if len(jobInfo.MultiInstanceJobId) > 0 { - jobFaultMap[jobInfo.MultiInstanceJobId] = append(jobFaultMap[jobInfo.MultiInstanceJobId], - jobFaultInfo.FaultDevice...) + jobFaultMap[jobInfo.MultiInstanceJobId] = append(jobFaultMap[jobInfo.MultiInstanceJobId], filterList...) } } s.dealWithFaultInfoForMergedJob(jobFaultMap) @@ -55,7 +58,7 @@ func (s *FaultServer) dealWithFaultInfoForMergedJob(jobFaultDeviceMap map[string s.lock.Lock() for targetJobId, faultPublisher := range s.faultPublisher { if faultPublisher == nil || !faultPublisher.IsSubscribed() { - hwlog.RunLog.Errorf("jobId=%s not registered or subscribe fault service", targetJobId) + hwlog.RunLog.Debugf("jobId=%s not registered or subscribe fault service", targetJobId) continue } jobPublisherMap[targetJobId] = faultPublisher @@ -207,9 +210,9 @@ func getStateLevelByFaultLevel(faultLevel string) (string, int) { return constant.HealthyState, constant.HealthyLevel } switch faultLevel { - case constant.NotHandleFault: + case constant.NotHandleFault, constant.NotHandleFaultLevelStr: return constant.HealthyState, constant.HealthyLevel - case constant.SubHealthFault: + case constant.SubHealthFault, constant.PreSeparateFault, constant.PreSeparateFaultLevelStr: return constant.SubHealthyState, constant.SubHealthyLevel default: return constant.UnHealthyState, constant.UnHealthyLevel @@ -245,3 +248,18 @@ func compareFaultMsg(this, other *fault.FaultMsgSignal) bool { } return proto.Equal(this, other) } + +func filterFault(faultDeviceList []constant.FaultDevice) []constant.FaultDevice { + if len(faultDeviceList) == 0 { + return nil + } + filteredList := make([]constant.FaultDevice, 0, len(faultDeviceList)) + for _, faultDevice := range faultDeviceList { + if _, level := getStateLevelByFaultLevel(faultDevice.FaultLevel); level == constant.HealthyLevel { + hwlog.RunLog.Debugf("fileter fault device %v", faultDevice) + continue + } + filteredList = append(filteredList, faultDevice) + } + return filteredList +} diff --git a/component/clusterd/pkg/application/fault/publish_fault_plugin_test.go b/component/clusterd/pkg/application/fault/publish_fault_plugin_test.go index ed0378ce..408c96a0 100644 --- a/component/clusterd/pkg/application/fault/publish_fault_plugin_test.go +++ b/component/clusterd/pkg/application/fault/publish_fault_plugin_test.go @@ -177,3 +177,16 @@ func TestCheckFaultFromFaultCenter(t *testing.T) { convey.So(success, convey.ShouldBeTrue) }) } + +func TestFilterFault(t *testing.T) { + convey.Convey("filter NotHandle and NotHandFault error", t, func() { + faultDeviceList := []constant.FaultDevice{ + {ServerName: "node1", DeviceId: "0", FaultLevel: constant.SeparateNPU}, + {ServerName: "node1", DeviceId: "1", FaultLevel: constant.NotHandleFault}, + {ServerName: "node1", DeviceId: "1", FaultLevel: constant.NotHandleFaultLevelStr}} + filteredList := filterFault(faultDeviceList) + convey.So(len(filteredList), convey.ShouldEqual, 1) + convey.So(filteredList, convey.ShouldResemble, []constant.FaultDevice{ + {ServerName: "node1", DeviceId: "0", FaultLevel: constant.SeparateNPU}}) + }) +} diff --git a/component/clusterd/pkg/application/faultmanager/jobprocess/faultrank/job_fault_rank_processor.go b/component/clusterd/pkg/application/faultmanager/jobprocess/faultrank/job_fault_rank_processor.go index b344c13c..01d91fe8 100644 --- a/component/clusterd/pkg/application/faultmanager/jobprocess/faultrank/job_fault_rank_processor.go +++ b/component/clusterd/pkg/application/faultmanager/jobprocess/faultrank/job_fault_rank_processor.go @@ -4,6 +4,7 @@ package faultrank import ( + "encoding/json" "strconv" "strings" "sync" @@ -198,7 +199,10 @@ func (processor *jobRankFaultInfoProcessor) findNodeDeviceAndSwitchFault( if ok { nodeStatusList = append(nodeStatusList, switchInfo.NodeStatus) } - faultDeviceList = append(faultDeviceList, getFaultDeviceInfoBySwitchInfo(&server, switchInfo)...) + switchFaultDev := getFaultDeviceInfoBySwitchInfo(&server, switchInfo) + if switchFaultDev.DeviceId == constant.EmptyDeviceId { + faultDeviceList = append(faultDeviceList, switchFaultDev) + } if ok && switchInfo.NodeStatus == constant.UnHealthyState { hwlog.RunLog.Debugf("node %s switch is unhealthy", nodeName) faultCode := strings.Join(switchInfo.FaultCode, constant.Comma) @@ -245,26 +249,31 @@ func getFaultDeviceInfoByNodeInfo(server *constant.ServerHccl, nodeInfo *constan } faultList := make([]constant.FaultDevice, 0) for _, faultDev := range nodeInfo.FaultDevList { - for _, faultCode := range faultDev.FaultCode { - deviceId := strconv.FormatInt(faultDev.DeviceId, constant.FormatBase) - faultList = append(faultList, convertToFaultDevice(server, faultCode, faultDev.FaultLevel, - deviceId, faultDev.DeviceType)) - } + deviceId := strconv.FormatInt(faultDev.DeviceId, constant.FormatBase) + assembledFaultCode := strings.Join(faultDev.FaultCode, constant.Comma) + faultList = append(faultList, convertToFaultDevice(server, assembledFaultCode, faultDev.FaultLevel, + deviceId, faultDev.DeviceType)) } return faultList } func getFaultDeviceInfoBySwitchInfo(server *constant.ServerHccl, - switchInfo *constant.SwitchInfo) []constant.FaultDevice { + switchInfo *constant.SwitchInfo) constant.FaultDevice { if switchInfo == nil { - return nil + return constant.FaultDevice{} } - faultList := make([]constant.FaultDevice, 0) + faultCodeList := make([]string, 0, len(switchInfo.SwitchFaultInfo.FaultCode)) for _, faultCode := range switchInfo.SwitchFaultInfo.FaultCode { - faultList = append(faultList, convertToFaultDevice(server, faultCode, switchInfo.SwitchFaultInfo.FaultLevel, - constant.EmptyDeviceId, constant.FaultTypeSwitch)) + switchFaultInfo := constant.SimpleSwitchFaultInfo{} + if err := json.Unmarshal([]byte(faultCode), &switchFaultInfo); err != nil { + hwlog.RunLog.Warnf("unmarshal switch SimpleSwitchFaultInfo error, faultCode=%s err=%v", faultCode, err) + faultCodeList = append(faultCodeList, faultCode) + continue + } + faultCodeList = append(faultCodeList, switchFaultInfo.AssembledFaultCode) } - return faultList + return convertToFaultDevice(server, strings.Join(faultCodeList, constant.Comma), + switchInfo.SwitchFaultInfo.FaultLevel, constant.EmptyDeviceId, constant.FaultTypeSwitch) } func convertToFaultDevice(server *constant.ServerHccl, faultCode, diff --git a/component/clusterd/pkg/common/constant/const.go b/component/clusterd/pkg/common/constant/const.go index 8789d84a..0ae36b92 100644 --- a/component/clusterd/pkg/common/constant/const.go +++ b/component/clusterd/pkg/common/constant/const.go @@ -175,6 +175,12 @@ const ( CardNetworkUnhealthy = "CardNetworkUnhealthy" // SubHealthFault sub healthy fault SubHealthFault = "SubHealthFault" + // NotHandleFaultLevelStr NotHandle Fault Level Str + NotHandleFaultLevelStr = "NotHandle" + // PreSeparateFaultLevelStr PreSeparate Fault Level Str + PreSeparateFaultLevelStr = "PreSeparate" + // PreSeparateFault pre-separate fault + PreSeparateFault = "PreSeparateFault" ) // About cm keys -- Gitee From e3ee85838e75b009a6f0c9676f9e3dddfd338700 Mon Sep 17 00:00:00 2001 From: zhoupan39 Date: Wed, 30 Apr 2025 16:08:09 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E3=80=90clusterd=E3=80=91=E3=80=90?= =?UTF-8?q?=E4=BF=AE=E6=94=B9=E8=AF=B4=E6=98=8E=E3=80=91=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?DT?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../job_fault_rank_processor_test.go | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/component/clusterd/pkg/application/faultmanager/jobprocess/faultrank/job_fault_rank_processor_test.go b/component/clusterd/pkg/application/faultmanager/jobprocess/faultrank/job_fault_rank_processor_test.go index ddbefdd6..89cd2092 100644 --- a/component/clusterd/pkg/application/faultmanager/jobprocess/faultrank/job_fault_rank_processor_test.go +++ b/component/clusterd/pkg/application/faultmanager/jobprocess/faultrank/job_fault_rank_processor_test.go @@ -4,6 +4,7 @@ package faultrank import ( + "encoding/json" "fmt" "testing" @@ -308,3 +309,80 @@ func testFilterFaultLevel(processor *jobRankFaultInfoProcessor) { convey.So(result["job1"].FaultList[0].FaultLevel, convey.ShouldEqual, "NoRestart") }) } + +func TestGetFaultDeviceInfo(t *testing.T) { + const nodeSN = "fakeNodeSN" + convey.Convey("Test GetFaultDeviceInfo by faultRank,nodeInfo and switchInfo", t, func() { + server := &constant.ServerHccl{ + ServerName: nodeName, + ServerSN: nodeSN, + ServerID: "1", + } + testGetFautDeviceInfoByFaultRank(server, nodeSN) + testGetFaultDeviceInfoByNodeInfo(server, nodeSN) + testGetFaultDeviceInfoBySwitchInfo(server, nodeSN) + }) +} + +func testGetFautDeviceInfoByFaultRank(server *constant.ServerHccl, nodeSN string) { + convey.Convey("Test getFautDeviceInfoByFaultRank", func() { + faultRankList := []constant.FaultRank{ + {FaultCode: "code1", FaultLevel: constant.RestartBusiness, DeviceId: "0"}, + } + faultDeviceList := getFautDeviceInfoByFaultRank(server, faultRankList) + convey.So(faultDeviceList, convey.ShouldHaveLength, 1) + convey.So(faultDeviceList, convey.ShouldResemble, []constant.FaultDevice{ + {ServerName: nodeName, ServerSN: nodeSN, ServerId: "1", DeviceId: "0", FaultCode: "code1", + FaultLevel: constant.RestartBusiness, DeviceType: constant.FaultTypeNPU}, + }) + }) +} + +func testGetFaultDeviceInfoByNodeInfo(server *constant.ServerHccl, nodeSN string) { + convey.Convey("Test getFaultDeviceInfoByNodeInfo", func() { + nodeInfo := &constant.NodeInfo{ + NodeInfoNoName: constant.NodeInfoNoName{FaultDevList: []*constant.FaultDev{ + {DeviceType: "CPU", DeviceId: 0, FaultCode: []string{"code1", "code2"}, + FaultLevel: constant.PreSeparateFault}, + }}} + faultDeviceList := getFaultDeviceInfoByNodeInfo(server, nodeInfo) + convey.So(faultDeviceList, convey.ShouldHaveLength, 1) + convey.So(faultDeviceList, convey.ShouldResemble, []constant.FaultDevice{ + {ServerName: nodeName, ServerSN: nodeSN, ServerId: "1", DeviceId: "0", FaultCode: "code1,code2", + FaultLevel: constant.PreSeparateFault, DeviceType: "CPU"}, + }) + }) +} + +func testGetFaultDeviceInfoBySwitchInfo(server *constant.ServerHccl, nodeSN string) { + convey.Convey("Test getFaultDeviceInfoBySwitchInfo", func() { + switchFaultInfo := constant.SimpleSwitchFaultInfo{ + AssembledFaultCode: "[0x00f103b0,155907,na,na]", + } + switchFaultStr, _ := json.Marshal(switchFaultInfo) + switchInfo := &constant.SwitchInfo{ + SwitchFaultInfo: constant.SwitchFaultInfo{ + FaultCode: []string{"code1", string(switchFaultStr)}, + FaultLevel: constant.PreSeparateFaultLevelStr, + }, + } + faultDevice := getFaultDeviceInfoBySwitchInfo(server, switchInfo) + convey.So(faultDevice, convey.ShouldResemble, constant.FaultDevice{ + ServerName: nodeName, ServerSN: nodeSN, ServerId: "1", DeviceId: constant.EmptyDeviceId, + FaultCode: "code1,[0x00f103b0,155907,na,na]", + FaultLevel: constant.PreSeparateFaultLevelStr, DeviceType: constant.FaultTypeSwitch}) + }) +} + +func TestGetFaultCodeByNodeInfo(t *testing.T) { + convey.Convey("Test getFaultCodeByNodeInfo", t, func() { + nodeInfo := &constant.NodeInfo{ + NodeInfoNoName: constant.NodeInfoNoName{FaultDevList: []*constant.FaultDev{ + {FaultCode: []string{"code1", "code2"}}, + {FaultCode: []string{"code2", "code3"}}, + }}, + } + codeList := getFaultCodeByNodeInfo(nodeInfo) + convey.So(codeList, convey.ShouldResemble, []string{"code1", "code2", "code3"}) + }) +} -- Gitee From be5470b995a1f3d7c25cae414dbd8ba3de9d3225 Mon Sep 17 00:00:00 2001 From: zhoupan39 Date: Wed, 30 Apr 2025 16:25:19 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E3=80=90clusterd=E3=80=91=E3=80=90?= =?UTF-8?q?=E4=BF=AE=E6=94=B9=E8=AF=B4=E6=98=8E=E3=80=91=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?DT?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../faultrank/job_fault_rank_processor.go | 33 +++++++------------ .../job_fault_rank_processor_test.go | 21 ++++++------ 2 files changed, 22 insertions(+), 32 deletions(-) diff --git a/component/clusterd/pkg/application/faultmanager/jobprocess/faultrank/job_fault_rank_processor.go b/component/clusterd/pkg/application/faultmanager/jobprocess/faultrank/job_fault_rank_processor.go index 01d91fe8..b344c13c 100644 --- a/component/clusterd/pkg/application/faultmanager/jobprocess/faultrank/job_fault_rank_processor.go +++ b/component/clusterd/pkg/application/faultmanager/jobprocess/faultrank/job_fault_rank_processor.go @@ -4,7 +4,6 @@ package faultrank import ( - "encoding/json" "strconv" "strings" "sync" @@ -199,10 +198,7 @@ func (processor *jobRankFaultInfoProcessor) findNodeDeviceAndSwitchFault( if ok { nodeStatusList = append(nodeStatusList, switchInfo.NodeStatus) } - switchFaultDev := getFaultDeviceInfoBySwitchInfo(&server, switchInfo) - if switchFaultDev.DeviceId == constant.EmptyDeviceId { - faultDeviceList = append(faultDeviceList, switchFaultDev) - } + faultDeviceList = append(faultDeviceList, getFaultDeviceInfoBySwitchInfo(&server, switchInfo)...) if ok && switchInfo.NodeStatus == constant.UnHealthyState { hwlog.RunLog.Debugf("node %s switch is unhealthy", nodeName) faultCode := strings.Join(switchInfo.FaultCode, constant.Comma) @@ -249,31 +245,26 @@ func getFaultDeviceInfoByNodeInfo(server *constant.ServerHccl, nodeInfo *constan } faultList := make([]constant.FaultDevice, 0) for _, faultDev := range nodeInfo.FaultDevList { - deviceId := strconv.FormatInt(faultDev.DeviceId, constant.FormatBase) - assembledFaultCode := strings.Join(faultDev.FaultCode, constant.Comma) - faultList = append(faultList, convertToFaultDevice(server, assembledFaultCode, faultDev.FaultLevel, - deviceId, faultDev.DeviceType)) + for _, faultCode := range faultDev.FaultCode { + deviceId := strconv.FormatInt(faultDev.DeviceId, constant.FormatBase) + faultList = append(faultList, convertToFaultDevice(server, faultCode, faultDev.FaultLevel, + deviceId, faultDev.DeviceType)) + } } return faultList } func getFaultDeviceInfoBySwitchInfo(server *constant.ServerHccl, - switchInfo *constant.SwitchInfo) constant.FaultDevice { + switchInfo *constant.SwitchInfo) []constant.FaultDevice { if switchInfo == nil { - return constant.FaultDevice{} + return nil } - faultCodeList := make([]string, 0, len(switchInfo.SwitchFaultInfo.FaultCode)) + faultList := make([]constant.FaultDevice, 0) for _, faultCode := range switchInfo.SwitchFaultInfo.FaultCode { - switchFaultInfo := constant.SimpleSwitchFaultInfo{} - if err := json.Unmarshal([]byte(faultCode), &switchFaultInfo); err != nil { - hwlog.RunLog.Warnf("unmarshal switch SimpleSwitchFaultInfo error, faultCode=%s err=%v", faultCode, err) - faultCodeList = append(faultCodeList, faultCode) - continue - } - faultCodeList = append(faultCodeList, switchFaultInfo.AssembledFaultCode) + faultList = append(faultList, convertToFaultDevice(server, faultCode, switchInfo.SwitchFaultInfo.FaultLevel, + constant.EmptyDeviceId, constant.FaultTypeSwitch)) } - return convertToFaultDevice(server, strings.Join(faultCodeList, constant.Comma), - switchInfo.SwitchFaultInfo.FaultLevel, constant.EmptyDeviceId, constant.FaultTypeSwitch) + return faultList } func convertToFaultDevice(server *constant.ServerHccl, faultCode, diff --git a/component/clusterd/pkg/application/faultmanager/jobprocess/faultrank/job_fault_rank_processor_test.go b/component/clusterd/pkg/application/faultmanager/jobprocess/faultrank/job_fault_rank_processor_test.go index 89cd2092..fe7a042b 100644 --- a/component/clusterd/pkg/application/faultmanager/jobprocess/faultrank/job_fault_rank_processor_test.go +++ b/component/clusterd/pkg/application/faultmanager/jobprocess/faultrank/job_fault_rank_processor_test.go @@ -4,7 +4,6 @@ package faultrank import ( - "encoding/json" "fmt" "testing" @@ -356,21 +355,21 @@ func testGetFaultDeviceInfoByNodeInfo(server *constant.ServerHccl, nodeSN string func testGetFaultDeviceInfoBySwitchInfo(server *constant.ServerHccl, nodeSN string) { convey.Convey("Test getFaultDeviceInfoBySwitchInfo", func() { - switchFaultInfo := constant.SimpleSwitchFaultInfo{ - AssembledFaultCode: "[0x00f103b0,155907,na,na]", - } - switchFaultStr, _ := json.Marshal(switchFaultInfo) switchInfo := &constant.SwitchInfo{ SwitchFaultInfo: constant.SwitchFaultInfo{ - FaultCode: []string{"code1", string(switchFaultStr)}, + FaultCode: []string{"code1", "code2"}, FaultLevel: constant.PreSeparateFaultLevelStr, }, } - faultDevice := getFaultDeviceInfoBySwitchInfo(server, switchInfo) - convey.So(faultDevice, convey.ShouldResemble, constant.FaultDevice{ - ServerName: nodeName, ServerSN: nodeSN, ServerId: "1", DeviceId: constant.EmptyDeviceId, - FaultCode: "code1,[0x00f103b0,155907,na,na]", - FaultLevel: constant.PreSeparateFaultLevelStr, DeviceType: constant.FaultTypeSwitch}) + faultDeviceList := getFaultDeviceInfoBySwitchInfo(server, switchInfo) + convey.So(faultDeviceList, convey.ShouldResemble, []constant.FaultDevice{ + {ServerName: nodeName, ServerSN: nodeSN, ServerId: "1", DeviceId: constant.EmptyDeviceId, + FaultCode: "code1", + FaultLevel: constant.PreSeparateFaultLevelStr, DeviceType: constant.FaultTypeSwitch}, + {ServerName: nodeName, ServerSN: nodeSN, ServerId: "1", DeviceId: constant.EmptyDeviceId, + FaultCode: "code2", + FaultLevel: constant.PreSeparateFaultLevelStr, DeviceType: constant.FaultTypeSwitch}, + }) }) } -- Gitee From be7964c7e238eb315ad2b25e079daea684d3b86d Mon Sep 17 00:00:00 2001 From: zhoupan39 Date: Wed, 30 Apr 2025 17:24:46 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E3=80=90clusterd=E3=80=91=E3=80=90?= =?UTF-8?q?=E4=BF=AE=E6=94=B9=E8=AF=B4=E6=98=8E=E3=80=91=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?DT?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../jobprocess/faultrank/job_fault_rank_processor_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/component/clusterd/pkg/application/faultmanager/jobprocess/faultrank/job_fault_rank_processor_test.go b/component/clusterd/pkg/application/faultmanager/jobprocess/faultrank/job_fault_rank_processor_test.go index fe7a042b..10178f14 100644 --- a/component/clusterd/pkg/application/faultmanager/jobprocess/faultrank/job_fault_rank_processor_test.go +++ b/component/clusterd/pkg/application/faultmanager/jobprocess/faultrank/job_fault_rank_processor_test.go @@ -338,6 +338,7 @@ func testGetFautDeviceInfoByFaultRank(server *constant.ServerHccl, nodeSN string } func testGetFaultDeviceInfoByNodeInfo(server *constant.ServerHccl, nodeSN string) { + fauleDeviceLen := 2 convey.Convey("Test getFaultDeviceInfoByNodeInfo", func() { nodeInfo := &constant.NodeInfo{ NodeInfoNoName: constant.NodeInfoNoName{FaultDevList: []*constant.FaultDev{ @@ -345,9 +346,11 @@ func testGetFaultDeviceInfoByNodeInfo(server *constant.ServerHccl, nodeSN string FaultLevel: constant.PreSeparateFault}, }}} faultDeviceList := getFaultDeviceInfoByNodeInfo(server, nodeInfo) - convey.So(faultDeviceList, convey.ShouldHaveLength, 1) + convey.So(faultDeviceList, convey.ShouldHaveLength, fauleDeviceLen) convey.So(faultDeviceList, convey.ShouldResemble, []constant.FaultDevice{ - {ServerName: nodeName, ServerSN: nodeSN, ServerId: "1", DeviceId: "0", FaultCode: "code1,code2", + {ServerName: nodeName, ServerSN: nodeSN, ServerId: "1", DeviceId: "0", FaultCode: "code1", + FaultLevel: constant.PreSeparateFault, DeviceType: "CPU"}, + {ServerName: nodeName, ServerSN: nodeSN, ServerId: "1", DeviceId: "0", FaultCode: "code2", FaultLevel: constant.PreSeparateFault, DeviceType: "CPU"}, }) }) -- Gitee