From a2b10fd4d63ca062f78ece81a8a761b6ca0380e3 Mon Sep 17 00:00:00 2001 From: wuyulong11 Date: Mon, 26 Jun 2023 09:14:24 +0800 Subject: [PATCH] =?UTF-8?q?=E3=80=90=E4=BF=AE=E6=94=B9=E8=AF=B4=E6=98=8E?= =?UTF-8?q?=E3=80=91=20Memory=20View=E7=95=8C=E9=9D=A2=E6=8A=98=E7=BA=BF?= =?UTF-8?q?=E5=9B=BE=E7=BB=84=E4=BB=B6=E6=9B=BF=E6=8D=A2=E4=B8=BAecharts?= =?UTF-8?q?=20=E3=80=90=E4=BF=AE=E6=94=B9=E4=BA=BA=E3=80=91=20wuyulong=203?= =?UTF-8?q?0031080?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../profiling/tb_plugin/fe/package.json | 4 +- .../fe/src/components/MemoryView.tsx | 64 ++- .../fe/src/components/charts/NewLineChart.tsx | 410 ++++++++++++++++++ .../tb_plugin/torch_tb_profiler/plugin.py | 10 +- .../profiler/run_generator.py | 122 +++--- 5 files changed, 528 insertions(+), 82 deletions(-) create mode 100644 tb_plugins/profiling/tb_plugin/fe/src/components/charts/NewLineChart.tsx diff --git a/tb_plugins/profiling/tb_plugin/fe/package.json b/tb_plugins/profiling/tb_plugin/fe/package.json index 7beeb696c42..caaa61a9b1d 100644 --- a/tb_plugins/profiling/tb_plugin/fe/package.json +++ b/tb_plugins/profiling/tb_plugin/fe/package.json @@ -16,10 +16,12 @@ "@material-ui/icons": "^4.11.2", "antd": "^4.17.0", "clsx": "^1.1.1", + "echarts": "^5.4.2", "portable-fetch": "^3.0.0", "react": "^16.13.1", "react-dom": "^16.13.1", - "react-flame-graph": "^1.4.0" + "react-flame-graph": "^1.4.0", + "url": "^0.11.1" }, "devDependencies": { "@types/react": "^16.9.51", diff --git a/tb_plugins/profiling/tb_plugin/fe/src/components/MemoryView.tsx b/tb_plugins/profiling/tb_plugin/fe/src/components/MemoryView.tsx index 060a18d1679..e9ec99fa3f0 100644 --- a/tb_plugins/profiling/tb_plugin/fe/src/components/MemoryView.tsx +++ b/tb_plugins/profiling/tb_plugin/fe/src/components/MemoryView.tsx @@ -26,7 +26,7 @@ import { } from '../api' import { useSearchDirectly } from '../utils/search' import { AntTableChart } from './charts/AntTableChart' -import { LineChart } from './charts/LineChart' +import { LineChart } from './charts/NewLineChart' import { DataLoading } from './DataLoading' import { MemoryStatsTable } from './tables/MemoryStatsTable' @@ -337,19 +337,62 @@ export const MemoryView: React.FC = React.memo((props) => { } const onTagChanged: SelectProps['onChange'] = (event) => { + event.target.value === 'Operator' && setSelectedRange(undefined) setTag(event.target.value as string) - setSelectedRange(undefined) } const onSelectedRangeChanged = (start: number, end: number) => { - let bias = memoryCurveData?.metadata.first_ts ?? 0 - let scale = 1 / (memoryCurveData?.metadata.time_factor ?? 1) - let startTs = Math.round(start * scale + bias) - let endTs = Math.round(end * scale + bias) - if (startTs == endTs) { + if (start > end) { + setSelectedRange(undefined) + return + } + + let allDatas = deviceTarget === 'Ascend' ? + memoryCurveData?.rows[device]?.Allocated : memoryCurveData?.rows[device] + if (allDatas.length <= 1) { setSelectedRange(undefined) return } + + let startTs = 0 + let endTs = 0 + let realStart = 0 + let realEnd = 0 + let startId = 1 + let endId = 0 + let needLoopStart = true + for (let i = 1; i < allDatas.length; i++) { + if (startId > start && needLoopStart) { + needLoopStart = false + realStart = i - 1 + } + if (allDatas[i] !== allDatas[i - 1]) { + if (startId <= start) { + startId += 1 + } + endId += 1 + } + if (endId > end) { + realEnd = i - 1 + break + } else { + realEnd = i + if (needLoopStart) { + realStart = i + } + } + } + + if (deviceTarget === 'Ascend') { + startTs = allDatas[realStart][0] + endTs = allDatas[realEnd][0] + } else { + let bias = memoryCurveData?.metadata.first_ts ?? 0 + let scale = 1 / (memoryCurveData?.metadata.time_factor ?? 1) + startTs = Math.round(allDatas[realStart][0] * scale + bias) + endTs = Math.round(allDatas[realEnd][0] * scale + bias) + } + setSelectedRange({ start, end, startTs, endTs }) } @@ -401,13 +444,6 @@ export const MemoryView: React.FC = React.memo((props) => { deviceTarget={deviceTarget} tag={tag} onSelectionChanged={tag !== 'Component' ? onSelectedRangeChanged : undefined} - explorerOptions={{ - actions: ['dragToZoom', 'rightClickToReset'], - axis: 'horizontal', - keepInBounds: true, - maxZoomIn: 0.000001, - maxZoomOut: 10 - }} record={selectedRecord} /> diff --git a/tb_plugins/profiling/tb_plugin/fe/src/components/charts/NewLineChart.tsx b/tb_plugins/profiling/tb_plugin/fe/src/components/charts/NewLineChart.tsx new file mode 100644 index 00000000000..526efe0bb08 --- /dev/null +++ b/tb_plugins/profiling/tb_plugin/fe/src/components/charts/NewLineChart.tsx @@ -0,0 +1,410 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2023 Huawei Technologies Co., Ltd + * All rights reserved. + * + * Licensed under the BSD 3-Clause License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/license/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *--------------------------------------------------------------------------------------------*/ + +import { makeStyles } from '@material-ui/core/styles' +import * as React from 'react' +import { Graph, GraphAscend } from '../../api' +import { useResizeEventDependency } from '../../utils/resize' +import { binarySearch } from '../../utils/binarysearch' +import * as echarts from 'echarts' + +interface IProps { + graph: Graph | GraphAscend + height?: number + deviceTarget: string + tag: string + hAxisTitle?: string + vAxisTitle?: string + explorerOptions?: object + onSelectionChanged?: (start: number, end: number) => void + record?: any +} + +const useStyles = makeStyles(() => ({ + root: { + height: (props: Pick) => props.height + } +})) + +export const LineChart: React.FC = (props) => { + const { + graph, + height = 400, + deviceTarget, + tag, + hAxisTitle, + vAxisTitle, + onSelectionChanged, + record + } = props + const classes = useStyles({ height }) + const graphRef = React.useRef(null) + const [resizeEventDependency] = useResizeEventDependency() + const [chartObj, setChartObj] = React.useState() + const selectedPoints = React.useRef>([]) + + React.useLayoutEffect(() => { + const element = graphRef.current + if (!element) return + element.oncontextmenu = () => { return false } + + let myChart = echarts.init(element) + myChart.clear() + + let option: echarts.EChartsOption = { + title: { + text: graph.title, + textStyle: { + fontSize: 16 + } + }, + tooltip: { trigger: 'axis' }, + legend: { + type: 'scroll', + bottom: 0 + }, + xAxis: { + type: 'category', + boundaryGap: false, + name: hAxisTitle + }, + yAxis: { + type: 'value', + name: vAxisTitle, + scale: true + }, + toolbox: { + feature: { + dataZoom: { + yAxisIndex: 'none' + }, + restore: {} + } + } + } + + if (deviceTarget === 'Ascend') { + if (tag === 'Component') { + if (graph.columns.length === 3) { + option = { + ...option, + dataset: { + source: [ + graph.columns.map(column => column.name), + ...(graph.rows['PTA'] ?? graph.rows['GE']) + ] + }, + series: Array(2).fill( + { + type: 'line', + select: { + itemStyle: { + borderWidth: 5, + shadowBlur: 5 + } + }, + emphasis: { + itemStyle: { + borderWidth: 5, + shadowBlur: 5 + } + }, + selectedMode: 'single', + } + ) + } + } else if (graph.columns.length === 5) { + const datasetTitle1: Array = [] + const datasetTitle2: Array = [] + graph.columns.forEach((column, index) => { + if (index === 0 || index < 3) { + datasetTitle1.push(column.name) + } + if (index === 0 || index >= 3) { + datasetTitle2.push(column.name) + } + }) + option = { + ...option, + dataset: [ + { + source: [ + datasetTitle1, + ...graph.rows['PTA'] + ] + }, + { + source: [ + datasetTitle2, + ...graph.rows['GE'] + ] + } + ], + series: Array(2).fill( + { + type: 'line', + select: { + itemStyle: { + borderWidth: 5, + shadowBlur: 5 + } + }, + emphasis: { + itemStyle: { + borderWidth: 5, + shadowBlur: 5 + } + }, + selectedMode: 'single', + datasetIndex: 0 + }).concat(Array(2).fill( + { + type: 'line', + select: { + itemStyle: { + borderWidth: 5, + shadowBlur: 5 + } + }, + emphasis: { + itemStyle: { + borderWidth: 5, + shadowBlur: 5 + } + }, + selectedMode: 'single', + datasetIndex: 1 + })) + } + } + } else { + if (graph.columns.length === 3) { + const datasetTitle1: Array = [] + const datasetTitle2: Array = [] + graph.columns.forEach((column, index) => { + if (index === 0 || index < 2) { + datasetTitle1.push(column.name) + } + if (index === 0 || index >= 2) { + datasetTitle2.push(column.name) + } + }) + option = { + ...option, + dataset: [ + { + source: [ + datasetTitle1, + ...graph.rows['Allocated'] + ] + }, + { + source: [ + datasetTitle2, + ...graph.rows['Reserved'] + ] + } + ], + series: [ + { + type: 'line', + name: 'Allocated', + emphasis: { + label: { + show: true + }, + itemStyle: { + borderWidth: 5, + shadowBlur: 5 + } + }, + select: { + itemStyle: { + borderWidth: 5, + shadowBlur: 5 + } + }, + datasetIndex: 0 + }, + { + type: 'line', + name: 'Reserved', + select: { + itemStyle: { + borderWidth: 5, + shadowBlur: 5 + } + }, + emphasis: { + itemStyle: { + borderWidth: 5, + shadowBlur: 5 + } + }, + selectedMode: 'single', + datasetIndex: 1 + } + ] + } + } + } + } else { + option = { + ...option, + dataset: { + source: [ + graph.columns.map(column => column.name), + ...graph.rows + ] + }, + series: [ + { + type: 'line', + name: 'Allocated', + select: { + itemStyle: { + borderWidth: 5, + shadowBlur: 5 + } + }, + emphasis: { + itemStyle: { + borderWidth: 5, + shadowBlur: 5 + } + }, + selectedMode: 'single' + }, + { + type: 'line', + name: 'Reserved', + select: { + itemStyle: { + borderWidth: 5, + shadowBlur: 5 + } + }, + emphasis: { + itemStyle: { + borderWidth: 5, + shadowBlur: 5 + } + }, + selectedMode: 'single' + } + ] + } + } + + option && myChart.setOption(option, true) + myChart.dispatchAction({ + type: 'takeGlobalCursor', + key: 'dataZoomSelect', + dataZoomSelectActive: true + }) + + myChart.off('dataZoom') + myChart.on('dataZoom', (param: any) => { + if (onSelectionChanged) { + onSelectionChanged(param.batch[0].startValue, param.batch[0].endValue) + } + }) + + myChart.off('restore') + myChart.on('restore', () => { + if (onSelectionChanged) { + // Set startId greater than endId to query all memory events. + onSelectionChanged(0, -1) + } + }) + + myChart.off('click') + myChart.on('click', (param) => { + myChart.dispatchAction({ + type: 'unselect', + seriesId: param.seriesId, + dataIndex: selectedPoints.current + }) + myChart.dispatchAction({ + type: 'select', + seriesId: param.seriesId, + dataIndex: param.dataIndex + }) + + selectedPoints.current = [param.dataIndex] + }) + + myChart.off('contextmenu') + myChart.getZr().on('contextmenu', () => { + myChart.dispatchAction({ + type: 'restore' + }) + myChart.dispatchAction({ + type: 'takeGlobalCursor', + key: 'dataZoomSelect', + dataZoomSelectActive: true + }) + }) + + setChartObj(myChart) + }, [graph, height, resizeEventDependency]) + + React.useEffect(() => { + const compare_fn = (key: number, mid: Array) => + key - parseFloat(mid[0].toFixed(2)) + if (chartObj && tag === 'Operator') { + if (record) { + let startId = -1 + let endId = -1 + if (deviceTarget === 'Ascend') { + startId = binarySearch(graph.rows['Allocated'], record.col2, compare_fn) + endId = binarySearch(graph.rows['Allocated'], record.col3, compare_fn) + } else { + startId = binarySearch(graph.rows, record.col2, compare_fn) + endId = binarySearch(graph.rows, record.col3, compare_fn) + } + let selection = [] + startId >= 0 && selection.push(startId) + endId >= 0 && selection.push(endId) + chartObj.dispatchAction({ + type: 'downplay', + seriesName: 'Allocated', + dataIndex: selectedPoints.current + }) + chartObj.dispatchAction({ + type: 'highlight', + seriesName: 'Allocated', + dataIndex: selection + }) + selectedPoints.current = selection + } else { + chartObj.dispatchAction({ + type: 'downplay', + seriesName: 'Allocated', + dataIndex: selectedPoints.current + }) + selectedPoints.current = [] + } + } + }, [graph, record, chartObj]) + + return ( +
+
+
+ ) +} diff --git a/tb_plugins/profiling/tb_plugin/torch_tb_profiler/plugin.py b/tb_plugins/profiling/tb_plugin/torch_tb_profiler/plugin.py index 0150fcdcf1e..777d778ca8a 100644 --- a/tb_plugins/profiling/tb_plugin/torch_tb_profiler/plugin.py +++ b/tb_plugins/profiling/tb_plugin/torch_tb_profiler/plugin.py @@ -2,6 +2,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # -------------------------------------------------------------------------- import atexit +import copy import gzip import json import os @@ -314,11 +315,12 @@ class TorchProfilerPlugin(base_plugin.TBPlugin): time_metric = request.args.get('time_metric', 'ms') memory_metric = request.args.get('memory_metric', 'KB') if profile.device_target == 'Ascend': - operator_memory_events = profile.memory_events['operator']['rows'] + temp_memory_events = copy.deepcopy(profile.memory_events) + operator_memory_events = temp_memory_events['operator']['rows'] if start_ts is not None: - start_ts = int(start_ts) + start_ts = float(start_ts) if end_ts is not None: - end_ts = int(end_ts) + end_ts = float(end_ts) for key in operator_memory_events: if start_ts is not None and end_ts is not None: operator_memory_events[key] = [i for i in operator_memory_events[key] if @@ -329,7 +331,7 @@ class TorchProfilerPlugin(base_plugin.TBPlugin): elif end_ts is not None: operator_memory_events[key] = [i for i in operator_memory_events[key] if i[2] and end_ts >= i[2]] - return self.respond_as_json(profile.memory_events, True) + return self.respond_as_json(temp_memory_events, True) else: if start_ts is not None: start_ts = int(start_ts) diff --git a/tb_plugins/profiling/tb_plugin/torch_tb_profiler/profiler/run_generator.py b/tb_plugins/profiling/tb_plugin/torch_tb_profiler/profiler/run_generator.py index 333af12b040..09e38c03e59 100644 --- a/tb_plugins/profiling/tb_plugin/torch_tb_profiler/profiler/run_generator.py +++ b/tb_plugins/profiling/tb_plugin/torch_tb_profiler/profiler/run_generator.py @@ -347,13 +347,13 @@ class RunGenerator(object): 'rows': self.process_data } for device in process_devices_type: - if self.process_data.get(device).get('Allocated') is not None: + if self.process_data.get(device).get('Allocated') is not None and self.process_data.get(device).get( + 'Reserved') is not None: total_result['columns'][device].append( {'name': f'Allocated ({cano.memory_metric})', 'type': 'number', 'tooltip': 'PTA+GE memory in use.'}) - if self.process_data.get(device).get('Reserved') is not None: total_result['columns'][device].append( {'name': f'Reserved ({cano.memory_metric})', 'type': 'number', - 'tooltip': 'APP reserved memory by allocator, both used and unused.'}) + 'tooltip': 'PTA+GE reserved memory by allocator, both used and unused.'}) if len(total_result['columns'][device]) > 0: total_result['columns'][device].insert(0, {'name': f'Time ({cano.time_metric})', 'type': 'number', 'tooltip': 'Time since profiler starts.'}) @@ -422,19 +422,12 @@ class RunGenerator(object): max_reserved = 0 for array_value in process_data.get(device).get(component): max_reserved = max(array_value[2], max_reserved) - peaks[device] += f'{component} Peak Memory Usage: {max_reserved:.1f}{memory_metric}\n' + peaks[device] += f'{component} Reserved Peak Memory Usage: {max_reserved:.1f}{memory_metric}\n' return devices_type, peaks @staticmethod - def _check_csv_columns(columns: list): + def _check_csv_columns(columns: list, column_idxs: dict): column_exist_count = 0 - column_idxs = { - 'Component': -1, - 'Device Type': -1, - 'Timestamp(us)': -1, - 'Total Reserved(MB)': -1, - 'Total Allocated(MB)': -1 - } for idx, column in enumerate(columns): if column in column_idxs: column_idxs[column] = idx @@ -456,21 +449,29 @@ class RunGenerator(object): {'name': 'Time(ms)', 'type': 'number'}] } peak_memory_rows = defaultdict(list) + required_column_idxs = { + 'Component': -1, + 'Device Type': -1, + 'Timestamp(us)': -1, + 'Total Reserved(MB)': -1, + 'Total Allocated(MB)': -1 + } (tag_type_idx, device_type_idx, time_idx, reserved_idx, allocated_idx), column_exist_count = \ - RunGenerator._check_csv_columns(datas[0]) + RunGenerator._check_csv_columns(datas[0], required_column_idxs) if column_exist_count < 5: logger.error('Required column is missing in file "memory_record.csv"') else: for ls in datas[1:]: - time_column = round((float(ls[time_idx]) - self.profile_data.start_ts) / 1000, 3) + time_column = round((float(ls[time_idx]) - self.profile_data.start_ts) / 1000, 2) device_type = ls[device_type_idx] if ls[tag_type_idx] == 'PTA+GE': process_data.setdefault(device_type, {}).setdefault('Allocated', []).append( - [time_column, float(ls[allocated_idx])]) + [time_column, round(float(ls[allocated_idx]), 3)]) process_data.setdefault(device_type, {}).setdefault('Reserved', []).append( - [time_column, float(ls[reserved_idx])]) + [time_column, round(float(ls[reserved_idx]), 3)]) elif ls[tag_type_idx] in ('PTA', 'GE'): - line_chart_data = [time_column, float(ls[allocated_idx]), float(ls[reserved_idx])] + line_chart_data = [time_column, round(float(ls[allocated_idx]), 3), + round(float(ls[reserved_idx]), 3)] pta_or_ge_data.setdefault(device_type, {}).setdefault(ls[tag_type_idx], []).append(line_chart_data) else: self._handle_peak_memory_rows(device_type_idx, ls, peak_memory_rows, reserved_idx, tag_type_idx, @@ -853,35 +854,33 @@ class RunGenerator(object): return datas def _generate_kernel_table_npu(self): - display_columns = ('Step Id', 'Name', 'Type', 'Accelerator Core', 'Start Time(us)', 'Duration(us)', - 'Wait Time(us)', 'Block Dim', 'Input Shapes', 'Input Data Types', 'Input Formats', - 'Output Shapes', 'Output Data Types', 'Output Formats') - display_idxs = [] table = {'columns': [], 'rows': []} result = { 'metadata': { - 'sort': 'Total Duration (us)' + 'sort': 'Duration (us)' }, 'data': table } path = self.profile_data.kernel_file_path datas = RunGenerator._get_csv_data(path) - for idx, column in enumerate(datas[0]): - if column == 'Name': - self.name_idx = idx - elif column == 'Duration(us)': - self.duration_idx = idx - elif column == 'Accelerator Core': - self.core_type_idx = idx - - if column in display_columns: - display_idxs.append(idx) - if column in ('Duration(us)', 'Start Time', 'Wait Time(us)', 'Block Dim'): + required_column_idxs = { + 'Name': -1, + 'Duration(us)': -1, + 'Accelerator Core': -1 + } + (name_idx, duration_idx, core_type_idx), column_exist_count = \ + RunGenerator._check_csv_columns(datas[0], required_column_idxs) + if column_exist_count < 3: + logger.error('Required column is missing in file "kernel_details.csv"') + else: + for column in datas[0]: + if column in ('Duration(us)', 'Start Time(us)', 'Wait Time(us)', 'Block Dim'): table['columns'].append({'type': 'number', 'name': column}) else: table['columns'].append({'type': 'string', 'name': column}) - table['rows'] = [self._handle_kernel_table_rows(display_idxs, ls) for idx, ls in - enumerate(datas) if idx != 0] + + self._handle_kernel_table_rows(name_idx, duration_idx, core_type_idx, datas[1:]) + table['rows'] = datas[1:] return result @staticmethod @@ -924,34 +923,31 @@ class RunGenerator(object): return gpu_info - def _handle_kernel_table_rows(self, ids, row): - display_row = [] - for idx in ids: - display_row.append(row[idx]) - call_name = row[self.name_idx] - call_duration = float(row[self.duration_idx]) - call_type = row[self.core_type_idx] - if self.accelerator_data.get(call_type) is not None: - self.accelerator_data[call_type] += call_duration - else: - self.accelerator_data[call_type] = call_duration - - if self.statistic_data.get(call_name) is not None: - temp = self.statistic_data[call_name] - temp['Max'] = max(temp['Max'], call_duration) - temp['Min'] = min(temp['Min'], call_duration) - temp['Total'] = round(temp['Total'] + call_duration, 2) - temp['Calls'] += 1 - temp['Average'] = round(temp['Total'] / temp['Calls'], 2) - else: - self.statistic_data[call_name] = { - 'Calls': 1, - 'Total': call_duration, - 'Min': call_duration, - 'Average': call_duration, - 'Max': call_duration - } - return display_row + def _handle_kernel_table_rows(self, name_idx, duration_idx, core_type_idx, rows): + for row in rows: + call_name = row[name_idx] + call_duration = float(row[duration_idx]) + call_type = row[core_type_idx] + if self.accelerator_data.get(call_type) is not None: + self.accelerator_data[call_type] += call_duration + else: + self.accelerator_data[call_type] = call_duration + + if self.statistic_data.get(call_name) is not None: + temp = self.statistic_data[call_name] + temp['Max'] = max(temp['Max'], call_duration) + temp['Min'] = min(temp['Min'], call_duration) + temp['Total'] = round(temp['Total'] + call_duration, 2) + temp['Calls'] += 1 + temp['Average'] = round(temp['Total'] / temp['Calls'], 2) + else: + self.statistic_data[call_name] = { + 'Calls': 1, + 'Total': call_duration, + 'Min': call_duration, + 'Average': call_duration, + 'Max': call_duration + } class DistributedRunGenerator(object): -- Gitee