diff --git a/debug/accuracy_tools/msprobe/nan_analyze/analyzer.py b/debug/accuracy_tools/msprobe/nan_analyze/analyzer.py index e147f23b7c7bd514a13251830e0365928876bc75..a32ad7c351cfcaf33eb2b3e6fb78d30994b1015a 100644 --- a/debug/accuracy_tools/msprobe/nan_analyze/analyzer.py +++ b/debug/accuracy_tools/msprobe/nan_analyze/analyzer.py @@ -221,7 +221,7 @@ class NanAnalyzer: node = get_next_node(nodes) if not node: continue - if not groups or node.node_id in all_ids_in_groups: + if not groups or node.node_id not in all_ids_in_groups: new_group = find_all_members(node) groups.append(new_group) all_ids_in_groups.update(new_group) diff --git a/debug/accuracy_tools/msprobe/nan_analyze/graph.py b/debug/accuracy_tools/msprobe/nan_analyze/graph.py index 5a4f8fb87296a39796b5124854ba7060be71d53a..3e6e7d71e3a7b7ecf99eb47e7e7163ae72a55648 100644 --- a/debug/accuracy_tools/msprobe/nan_analyze/graph.py +++ b/debug/accuracy_tools/msprobe/nan_analyze/graph.py @@ -16,8 +16,8 @@ from dataclasses import dataclass from msprobe.core.common.const import Const from msprobe.core.common.log import logger -from msprobe.core.common.exceptions import MsprobeException from msprobe.nan_analyze.utils import FileCache, RankPath, is_ignore_op, check_item_anomaly, NanAnalyseConst +from msprobe.core.common.exceptions import MsprobeException @dataclass @@ -99,13 +99,13 @@ class CommunicationNode: self.link_nodes = kwargs.get('link_nodes', {}) self.dst_nodes = kwargs.get('dst_nodes', {}) self.src_nodes = kwargs.get('src_nodes', {}) - self.next_nodes = kwargs.get('next_nodes', {}) + self.next_node = kwargs.get('next_node') self.compute_ops = kwargs.get('compute_ops', []) self.type = self._resolve_type() self.connected = False def add_next(self, node): - self.next_nodes[node.node_id] = node + self.next_node = node node.pre_node = self node.layer = self.layer + 1 node.data.layer = node.layer @@ -113,7 +113,9 @@ class CommunicationNode: def add_link(self, node): self.link_nodes[node.node_id] = node node.link_nodes[self.node_id] = self - node.layer = self.layer + layer = max(node.layer, self.layer) + self.update_layer(layer) + node.update_layer(layer) node.data.layer = node.layer self.connected = True node.connected = True @@ -121,14 +123,16 @@ class CommunicationNode: def add_dst(self, node): self.dst_nodes[node.node_id] = node node.src_nodes[self.node_id] = self - node.layer = self.layer + layer = max(node.layer, self.layer) + self.update_layer(layer) + node.update_layer(layer) node.data.layer = node.layer self.connected = True node.connected = True def delete(self): - for node in self.next_nodes.values(): - node.pre_node = None + if self.next_node: + self.next_node.pre_node = None for node in self.dst_nodes.values(): node.src_nodes.pop(self.node_id) for node in self.src_nodes.values(): @@ -138,9 +142,19 @@ class CommunicationNode: if self.pre_node: self.pre_node.next_nodes.pop(self.node_id) + def update_layer(self, layer): + if layer == self.layer: + return + self.layer = layer + next_node = self.next_node + while next_node: + layer += 1 + next_node.layer = layer + next_node = next_node.next_node + def has_nan_inf(self): return self.input_has_nan_inf() or check_item_anomaly(self.data.outputs) - + def input_has_nan_inf(self): return check_item_anomaly(self.data.input_args) or check_item_anomaly(self.data.input_kwargs) diff --git a/debug/accuracy_tools/msprobe/visualization/graph_service.py b/debug/accuracy_tools/msprobe/visualization/graph_service.py index 69718d9c26fa4e3d8b787a936d01f624e7411915..bb219b423afbf3fae19aa78f6725ef96ca9b5bff 100644 --- a/debug/accuracy_tools/msprobe/visualization/graph_service.py +++ b/debug/accuracy_tools/msprobe/visualization/graph_service.py @@ -69,7 +69,7 @@ def _compare_graph_result(input_param, args): # 对两个数据进行构图 graph_n = _build_graph_info(input_param.get('npu_path'), args) graph_b = _build_graph_info(input_param.get('bench_path'), args) - logger.info('Model graphs built successfully, start Comparing graphs...') + logger.info('Model graphs built successfully, start comparing graphs...') # 基于graph、stack和data进行比较 graph_comparator = _compare_graph(graph_n, graph_b, input_param, args) # 增加micro step标记