From 4cb714c59fdfb8675075a7746a995b160214c974 Mon Sep 17 00:00:00 2001 From: hummel mao Date: Wed, 13 Aug 2025 17:34:15 +0800 Subject: [PATCH 01/11] =?UTF-8?q?feat(csv=5Fparser)=20=E5=AE=8C=E6=88=90?= =?UTF-8?q?=E8=A7=A3=E6=9E=90=20csv=20=E7=AC=AC=E4=B8=80=E9=83=A8=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ModelVis/rust/csv_parser/src/lib.rs | 41 ++++++++- .../ModelVis/rust/csv_parser/src/operator.rs | 22 ++++- .../ModelVis/rust/csv_parser/src/parser.rs | 87 ++++++++++++++++++- 3 files changed, 141 insertions(+), 9 deletions(-) diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs index c47534cacb..b419045870 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs @@ -11,15 +11,50 @@ pub mod operator; pub mod parser; pub fn is_valid_csv(name: &str) -> bool { - todo!() + // 情况1：精确匹配 + if name == "kernel_detail.csv" { + return true; + } + + // 情况2：匹配 op_summary_YYYYMMDDHHMMSS.csv + if !name.starts_with("op_summary_") || !name.ends_with(".csv") { + return false; + } + + // 去掉前缀和后缀 + let body = &name["op_summary_".len()..name.len() - 4]; // 去掉 .csv + + // 必须是 14 位：YYYYMMDDHHMMSS + if body.len() != 14 { + return false; + } + + // 检查是否全为数字 + let all_digits = |s: &str| s.chars().all(|c| c.is_ascii_digit()); + + all_digits(body) } pub fn get_valid_fields(name: &str) -> Vec { - todo!() + if name == "kernel_detail.csv" { + return vec![ + ValidHeaderField { h_field: "Name".to_string(), h_key: NAME_KEY.to_string() }, + ValidHeaderField { h_field: "Type".to_string(), h_key: TYPE_KEY.to_string() }, + ValidHeaderField { h_field: "Duration(us)".to_string(), h_key: DURATION_KEY.to_string() }, + ValidHeaderField { h_field: "aiv_mte2_time(us)".to_string(), h_key: MTE2_TIME_KEY.to_string() }, + ]; + } + vec![ + ValidHeaderField { h_field: "Op Name".to_string(), h_key: NAME_KEY.to_string() }, + ValidHeaderField { h_field: "Op Type".to_string(), h_key: TYPE_KEY.to_string() }, + ValidHeaderField { h_field: "Task Duration(us)".to_string(), h_key: DURATION_KEY.to_string() }, + ValidHeaderField { h_field: "aiv_mte2_time(us)".to_string(), h_key: MTE2_TIME_KEY.to_string() }, + ] } pub fn parse_operator_csv(path: &str) -> Result> { - todo!() + let map = HashMap::new(); + Ok(map) } pub fn get_filename(path: &str) -> Result { diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs index c1ff226c54..9c1458228f 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs @@ -66,11 +66,29 @@ pub fn group_times_by_op_type(operators: &Vec) -> Vec { impl Creator for Operator { fn create_by_row(row: &HashMap<&str, &str>) -> Result { - todo!() + // 解析 duration + let duration_us = transform_f64(row.get(DURATION_KEY).unwrap())?; + // 解析 mte2_time + let mte2_time_us = transform_f64(row.get(MTE2_TIME_KEY).unwrap())?; + if duration_us < 0.0 || mte2_time_us < 0.0 { + return Err(CreateError::InvalidNumber); + } + Ok(Operator { + name: row.get(NAME_KEY).unwrap().to_string(), + op_type: row.get(TYPE_KEY).unwrap().to_string(), + duration_us, + mte2_time_us, + }) } } /// 解析 string 转为 f64 fn transform_f64(field: &str) -> Result { - todo!() + let num = match field.parse::() { + Ok(d) => d, + Err(_) => { + return Err(CreateError::FailTransformNumber(field.to_string())) + } + }; + Ok(num) } diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs index 87902ac976..22620997d9 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs @@ -40,7 +40,13 @@ pub struct ValidHeaderField { } pub fn parse_operator(path: &str, parse_fields: Vec) -> Result, ParseError> { - todo!() + let file = std::fs::File::open(path)?; + let reader = BufReader::new(file); + + // 调用 parse_csv 函数解析 CSV 文件 + let result = parse_csv(reader, parse_fields, Operator::create_by_row)?; + + Ok(result.rows) // 返回解析后的 Operator 列表 } /// 从字节流中解析 CSV，返回算子列表与结构汇总 @@ -49,16 +55,89 @@ pub fn parse_csv(reader: R, fields: Vec) -> Result { - todo!() + let mut csv_reader = Reader::from_reader(reader); + let mut rows = Vec::new(); + let mut warnings = Vec::new(); + let mut row_count = 0; + + let header_index_map = get_header_index_map(&mut csv_reader, fields)?; + let col_len = header_index_map.len(); + + for result in csv_reader.records() { + row_count += 1; + let record = match result { + Ok(rec) => rec, + Err(e) => { + eprintln!("{}, {:?} 跳过非法 CSV 行", row_count, e); + warnings.push(format!("第 {} 行解析失败: {}", row_count, e)); + continue; + } + }; + + let mut value_map = HashMap::new(); + + // 解析字段 + for (k, v) in header_index_map.iter() { + let field = record.get(*v).unwrap_or("").trim(); + // === 安全过滤：防止 CSV 注入（公式注入）=== + if check_csv_injection(field, row_count, &mut warnings) { + break; + } + value_map.insert(k.as_str(), field); + } + + if value_map.len() < col_len { + continue; + } + + match creator(&value_map) { + Ok(value) => rows.push(value), + Err(e) => { + warnings.push(format!("第 {} 行创建失败: {}", row_count, e)); + continue; + }, + }; + } + + if rows.len() == 0 { + return Err(ParseError::EmptyData); + } + + Ok(ParseResult { + rows, + warnings, + }) } fn get_header_index_map(csv_reader: &mut Reader, fields: Vec) -> Result, ParseError> { - todo!() + if !csv_reader.has_headers() { + return Err(ParseError::EmptyHeader); + } + let headers = csv_reader.headers()?; + // 初始化配置结构体 + let mut header_index = HashMap::new(); + + // 尝试找到每个有效头部字段的索引 + for f in fields.iter() { + let index = headers.iter().position(|h| h == f.h_field) + .ok_or(ParseError::InvalidHeader(f.h_field.to_string()))?; + header_index.insert(f.h_key.to_string(), index); + } + + Ok(header_index) } /// 安全过滤：防止 CSV 注入（公式注入） /// 常见恶意前缀：= + @ - fn check_csv_injection(field: &str, row_count: usize, warnings: &mut Vec) -> bool { - todo!() + if let Some(first_char) = field.chars().next() { + if "=+@-".contains(first_char) { + eprintln!("{} 检测到潜在 CSV 公式注入，已过滤", field); + warnings.push(format!("第 {} 行解析失败: 公式 {}", row_count, field)); + // 可选择跳过或转义 + return true; + } + } + false } -- Gitee From 896039e7d476deba5d2dc1876805e76588ef62a8 Mon Sep 17 00:00:00 2001 From: hummel mao Date: Wed, 13 Aug 2025 20:29:02 +0800 Subject: [PATCH 02/11] =?UTF-8?q?feat(csv=5Fparser)=20=E5=AE=9E=E7=8E=B0?= =?UTF-8?q?=20parse=5Foperator=5Fcsv=20=E5=87=BD=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ModelVis/rust/csv_parser/src/lib.rs | 14 +++-- .../ModelVis/rust/csv_parser/src/operator.rs | 52 +++++++++---------- .../ModelVis/rust/csv_parser/src/parser.rs | 1 + .../ModelVis/rust/csv_parser/tests/test.rs | 10 ++-- .../csv_parser/tests/valid/kernel_detail.csv | 2 +- 5 files changed, 45 insertions(+), 34 deletions(-) diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs index b419045870..723c944cf0 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs @@ -4,8 +4,8 @@ use std::collections::HashMap; use std::path::Path; use anyhow::{anyhow, Result}; -use crate::operator::{OperatorGroup, DURATION_KEY, MTE2_TIME_KEY, NAME_KEY, TYPE_KEY}; -use crate::parser::ValidHeaderField; +use crate::operator::{group_times_by_op_type, OperatorGroup, DURATION_KEY, MTE2_TIME_KEY, NAME_KEY, TYPE_KEY}; +use crate::parser::{parse_operator, ValidHeaderField}; pub mod operator; pub mod parser; @@ -53,8 +53,14 @@ pub fn get_valid_fields(name: &str) -> Vec { } pub fn parse_operator_csv(path: &str) -> Result> { - let map = HashMap::new(); - Ok(map) + let name = get_filename(path)?; + if !is_valid_csv(&name) { + return Err(anyhow!("非法的 csv 文件名: {}", &name)); + } + let fields = get_valid_fields(&name); + let operators = parse_operator(path, fields)?; + + Ok(group_times_by_op_type(&operators)) } pub fn get_filename(path: &str) -> Result { diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs index 9c1458228f..ef8fbcde2e 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs @@ -1,7 +1,7 @@ /* * Copyright (c), Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. */ -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::string::ToString; use serde::Serialize; use anyhow::Result; @@ -32,36 +32,36 @@ pub struct Operator { pub mte2_time_us: f64, // 算子内存搬运时间 } -impl Default for Operator { - fn default() -> Self { - Operator { - name: "".to_string(), - op_type: "".to_string(), - duration_us: 0.0, - mte2_time_us: 0.0, - } - } -} - #[derive(Debug, Serialize)] pub struct OperatorGroup { - op_type: String, - duration_us: f64, - mte2_time_us: f64, + pub op_type: String, + pub duration_us: f64, + pub mte2_time_us: f64, } -impl From for OperatorGroup { - fn from(value: Operator) -> Self { - OperatorGroup { - op_type: value.op_type, - duration_us: value.duration_us, - mte2_time_us: value.mte2_time_us, - } - } -} +pub fn group_times_by_op_type(operators: &Vec) -> HashMap { + // 创建一个临时 HashMap 用于存储每种类型的操作符信息 + let mut type_map: BTreeMap = BTreeMap::new(); -pub fn group_times_by_op_type(operators: &Vec) -> Vec { - vec![] + // 使用 fold 方法来累积数据 + operators.iter().fold(&mut type_map, |map, operator| { + let duration = operator.duration_us; + let mte2_time = operator.mte2_time_us; + let entry = map.entry(operator.op_type.to_string()).or_insert((0.0, 0.0, 0)); + entry.0 += duration; + entry.1 += mte2_time; + entry.2 += 1; + map + }); + + // 计算平均值并转换为 OperatorGroup + type_map.into_iter().map(|(op_type, (total_duration, total_mte2, count))| { + (op_type.clone(), OperatorGroup { + op_type, + duration_us: total_duration / count as f64, + mte2_time_us: total_mte2 / count as f64, + }) + }).collect() } impl Creator for Operator { diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs index 22620997d9..1f2b059957 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs @@ -87,6 +87,7 @@ where } if value_map.len() < col_len { + warnings.push(format!("第 {} 行取值失败: {:?}", row_count, value_map)); continue; } diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/test.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/test.rs index fafdbb5445..f442cd4bc2 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/test.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/test.rs @@ -37,13 +37,17 @@ mod tests { fn test_parse_valid_csv() { // 获取当前工作目录 let current_dir = env::current_dir().expect("Failed to get current directory"); - let filename = current_dir.join(r#"tests\invalid\kernel_detail.csv"#); + let filename = current_dir.join(r#"tests\valid\kernel_detail.csv"#); let path = filename.to_str().expect("Failed to convert path to string"); match parse_operator_csv(path) { Ok(ops) => { - assert_eq!(14, ops.len()); + assert_eq!(ops.len(), 2); + assert_eq!(ops.get("MatMul").unwrap().duration_us, 160.0); + assert_eq!(ops.get("MatMul").unwrap().mte2_time_us, 20.0); + assert_eq!(ops.get("Active").unwrap().duration_us, 30.0); + assert_eq!(ops.get("Active").unwrap().mte2_time_us, 3.0); }, - Err(err) => panic!("Error occurred while parsing ONNX model: {}", err), + Err(err) => panic!("Error occurred while parsing csv: {}", err), } } diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/valid/kernel_detail.csv b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/valid/kernel_detail.csv index 93bff0ced6..48a8cc2198 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/valid/kernel_detail.csv +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/valid/kernel_detail.csv @@ -1,4 +1,4 @@ Name,Type,Duration(us),aiv_mte2_time(us), conv2d,MatMul,120.0,20, relu,Active,30,3, -matmul,MatMul,200,20 \ No newline at end of file +matmul,MatMul,200,20, \ No newline at end of file -- Gitee From e2873e2c718dcecdd22638db5e39dc5d55a77085 Mon Sep 17 00:00:00 2001 From: hummel mao Date: Fri, 15 Aug 2025 10:05:36 +0800 Subject: [PATCH 03/11] =?UTF-8?q?fix(csv)=20=E4=BF=AE=E6=94=B9=E4=B8=AD?= =?UTF-8?q?=E6=96=87=E4=BF=9D=E5=AD=98=EF=BC=8C=E8=A7=A3=E5=86=B3=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E6=B3=A8=E5=85=A5=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ModelVis/rust/csv_parser/src/lib.rs | 4 +- .../ModelVis/rust/csv_parser/src/operator.rs | 8 ++- .../ModelVis/rust/csv_parser/src/parser.rs | 60 ++++++++++++++----- 3 files changed, 52 insertions(+), 20 deletions(-) diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs index 723c944cf0..7306ad76ca 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs @@ -55,7 +55,7 @@ pub fn get_valid_fields(name: &str) -> Vec { pub fn parse_operator_csv(path: &str) -> Result> { let name = get_filename(path)?; if !is_valid_csv(&name) { - return Err(anyhow!("非法的 csv 文件名: {}", &name)); + return Err(anyhow!("Invalid csv filename: {}", &name)); } let fields = get_valid_fields(&name); let operators = parse_operator(path, fields)?; @@ -67,7 +67,7 @@ pub fn get_filename(path: &str) -> Result { let path = Path::new(path); let file_name = match path.file_name() { Some(name) => name.to_string_lossy(), - None => return Err(anyhow!("没有文件名: {}", path.display())), + None => return Err(anyhow!("No filename: {}", path.display())), }; Ok(file_name.as_ref().to_string()) diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs index ef8fbcde2e..f0363cf56e 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs @@ -6,6 +6,8 @@ use std::string::ToString; use serde::Serialize; use anyhow::Result; use thiserror::Error; +use crate::parser::escape_special_chars; + pub const NAME_KEY: &str = "name"; pub const TYPE_KEY: &str = "op_type"; pub const DURATION_KEY: &str = "duration_us"; @@ -13,10 +15,10 @@ pub const MTE2_TIME_KEY: &str = "mte2_time_us"; #[derive(Error, Debug)] pub enum CreateError { - #[error("转换数字失败: {0}")] + #[error("Transform number fail: {0}")] FailTransformNumber(String), - #[error("无效的数字")] + #[error("Invalid number")] InvalidNumber, } @@ -87,7 +89,7 @@ fn transform_f64(field: &str) -> Result { let num = match field.parse::() { Ok(d) => d, Err(_) => { - return Err(CreateError::FailTransformNumber(field.to_string())) + return Err(CreateError::FailTransformNumber(escape_special_chars(field))) } }; Ok(num) diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs index 1f2b059957..a75d3c086b 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs @@ -15,23 +15,20 @@ pub struct ParseResult { #[derive(Error, Debug)] pub enum ParseError { - #[error("CSV 格式错误: {0}")] + #[error("CSV format error: {0}")] CsvError(#[from] csv::Error), - #[error("IO 错误: {0}")] + #[error("IO error: {0}")] IoError(#[from] std::io::Error), - #[error("文件为空或无有效数据")] + #[error("File is empty or has invalid data")] EmptyData, - #[error("文件头为空")] + #[error("Header is empty")] EmptyHeader, - #[error("文件头格式错误，缺少需要的列: {0}")] + #[error("Header format error，missing required column: {0}")] InvalidHeader(String), - - #[error("创建失败: {0}")] - CreateFail(String), } pub struct ValidHeaderField { @@ -68,8 +65,8 @@ where let record = match result { Ok(rec) => rec, Err(e) => { - eprintln!("{}, {:?} 跳过非法 CSV 行", row_count, e); - warnings.push(format!("第 {} 行解析失败: {}", row_count, e)); + eprintln!("Line {}, Skip illegal CSV lines: {}", row_count, e.to_string()); + warnings.push(format!("Line {} parse fail", row_count)); continue; } }; @@ -87,14 +84,14 @@ where } if value_map.len() < col_len { - warnings.push(format!("第 {} 行取值失败: {:?}", row_count, value_map)); + warnings.push(format!("Line {} get value fail", row_count)); continue; } match creator(&value_map) { Ok(value) => rows.push(value), Err(e) => { - warnings.push(format!("第 {} 行创建失败: {}", row_count, e)); + warnings.push(format!("Line {} create error: {}", row_count, e.to_string())); continue; }, }; @@ -122,7 +119,7 @@ fn get_header_index_map(csv_reader: &mut Reader, fields: Ve // 尝试找到每个有效头部字段的索引 for f in fields.iter() { let index = headers.iter().position(|h| h == f.h_field) - .ok_or(ParseError::InvalidHeader(f.h_field.to_string()))?; + .ok_or(ParseError::InvalidHeader(escape_special_chars(&f.h_field)))?; header_index.insert(f.h_key.to_string(), index); } @@ -134,11 +131,44 @@ fn get_header_index_map(csv_reader: &mut Reader, fields: Ve fn check_csv_injection(field: &str, row_count: usize, warnings: &mut Vec) -> bool { if let Some(first_char) = field.chars().next() { if "=+@-".contains(first_char) { - eprintln!("{} 检测到潜在 CSV 公式注入，已过滤", field); - warnings.push(format!("第 {} 行解析失败: 公式 {}", row_count, field)); + let e_field = escape_special_chars(field); + eprintln!("Potential CSV formula injection detected, filtered: {}", &e_field); + warnings.push(format!("Line {} parse fail: {}", row_count, &e_field)); // 可选择跳过或转义 return true; } } false } + +/// 处理字符串中的特殊符号，例如将 \n 替换为 \\n, \t 替换为 \\t 等 +/// +/// # 参数 +/// * `input` - 需要处理的原始字符串 +/// +/// # 返回值 +/// * 处理后的字符串 +pub fn escape_special_chars(input: &str) -> String { + let mut result = input.to_string(); + + // 特殊字符及其对应的转义序列 + let special_chars = [ + ("\n", "\\n"), + ("\t", "\\t"), + ("\r", "\\r"), + ("\"", "\\\""), + ("'", "\\'"), + ("\x08", "\\b"), // \b + ("\x0c", "\\f"), // \f + ("\x0b", "\\v"), // \v + ("\\", "\\\\"), + ]; + + // 使用 for 循环遍历并替换所有特殊字符 + for (char, escaped_char) in special_chars.iter() { + result = result.replace(*char, *escaped_char); + } + + result +} + -- Gitee From d87f6e4e127ffe8ba262cc358661dabc63fbd599 Mon Sep 17 00:00:00 2001 From: hummel mao Date: Fri, 15 Aug 2025 10:50:27 +0800 Subject: [PATCH 04/11] =?UTF-8?q?feat(csv)=20=E6=B7=BB=E5=8A=A0=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E5=AE=89=E5=85=A8=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ModelVis/rust/csv_parser/src/lib.rs | 14 ++- .../ModelVis/rust/csv_parser/src/operator.rs | 2 +- .../ModelVis/rust/csv_parser/src/parser.rs | 46 +------ .../ModelVis/rust/csv_parser/src/utils.rs | 117 ++++++++++++++++++ 4 files changed, 128 insertions(+), 51 deletions(-) create mode 100644 plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/utils.rs diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs index 7306ad76ca..8e895eb8f4 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs @@ -6,9 +6,11 @@ use std::path::Path; use anyhow::{anyhow, Result}; use crate::operator::{group_times_by_op_type, OperatorGroup, DURATION_KEY, MTE2_TIME_KEY, NAME_KEY, TYPE_KEY}; use crate::parser::{parse_operator, ValidHeaderField}; +use crate::utils::check_file_safety; pub mod operator; pub mod parser; +mod utils; pub fn is_valid_csv(name: &str) -> bool { // 情况1：精确匹配 @@ -53,9 +55,10 @@ pub fn get_valid_fields(name: &str) -> Vec { } pub fn parse_operator_csv(path: &str) -> Result> { + check_file_safety(path)?; let name = get_filename(path)?; if !is_valid_csv(&name) { - return Err(anyhow!("Invalid csv filename: {}", &name)); + return Err(anyhow!("Invalid CSV filename: {}", &name)); } let fields = get_valid_fields(&name); let operators = parse_operator(path, fields)?; @@ -65,10 +68,9 @@ pub fn parse_operator_csv(path: &str) -> Result> pub fn get_filename(path: &str) -> Result { let path = Path::new(path); - let file_name = match path.file_name() { - Some(name) => name.to_string_lossy(), - None => return Err(anyhow!("No filename: {}", path.display())), - }; + let file_name = path.file_name() + .ok_or_else(|| anyhow::anyhow!("No filename component in path: {}", path.display()))?; - Ok(file_name.as_ref().to_string()) + // to_string_lossy() 返回 Cow，直接调用 to_string() 即可 + Ok(file_name.to_string_lossy().to_string()) } \ No newline at end of file diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs index f0363cf56e..bce4a17beb 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs @@ -6,7 +6,7 @@ use std::string::ToString; use serde::Serialize; use anyhow::Result; use thiserror::Error; -use crate::parser::escape_special_chars; +use crate::utils::escape_special_chars; pub const NAME_KEY: &str = "name"; pub const TYPE_KEY: &str = "op_type"; diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs index a75d3c086b..0434bccf3c 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs @@ -7,6 +7,8 @@ use csv::Reader; use serde::Serialize; use thiserror::Error; use crate::operator::{CreateError, Creator, Operator}; +use crate::utils::{check_csv_injection, escape_special_chars}; + #[derive(Debug, Serialize)] pub struct ParseResult { pub rows: Vec, @@ -126,49 +128,5 @@ fn get_header_index_map(csv_reader: &mut Reader, fields: Ve Ok(header_index) } -/// 安全过滤：防止 CSV 注入（公式注入） -/// 常见恶意前缀：= + @ - -fn check_csv_injection(field: &str, row_count: usize, warnings: &mut Vec) -> bool { - if let Some(first_char) = field.chars().next() { - if "=+@-".contains(first_char) { - let e_field = escape_special_chars(field); - eprintln!("Potential CSV formula injection detected, filtered: {}", &e_field); - warnings.push(format!("Line {} parse fail: {}", row_count, &e_field)); - // 可选择跳过或转义 - return true; - } - } - false -} -/// 处理字符串中的特殊符号，例如将 \n 替换为 \\n, \t 替换为 \\t 等 -/// -/// # 参数 -/// * `input` - 需要处理的原始字符串 -/// -/// # 返回值 -/// * 处理后的字符串 -pub fn escape_special_chars(input: &str) -> String { - let mut result = input.to_string(); - - // 特殊字符及其对应的转义序列 - let special_chars = [ - ("\n", "\\n"), - ("\t", "\\t"), - ("\r", "\\r"), - ("\"", "\\\""), - ("'", "\\'"), - ("\x08", "\\b"), // \b - ("\x0c", "\\f"), // \f - ("\x0b", "\\v"), // \v - ("\\", "\\\\"), - ]; - - // 使用 for 循环遍历并替换所有特殊字符 - for (char, escaped_char) in special_chars.iter() { - result = result.replace(*char, *escaped_char); - } - - result -} diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/utils.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/utils.rs new file mode 100644 index 0000000000..a8a338e59f --- /dev/null +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/utils.rs @@ -0,0 +1,117 @@ +use std::{fs, io}; +use std::path::Path; + +/// 打开文件前进行安全校验 +/// +/// # 参数 +/// * `file_path` - 文件路径 +/// +/// # 返回值 +/// * `Result<(), io::Error>` - 如果校验通过返回 `Ok(())`，否则返回 `Err(io::Error)` +pub fn check_file_safety(file_path: &str) -> io::Result<()> { + let path = Path::new(file_path); + + // 1. 检查路径是否存在（fs::metadata 和 fs::symlink_metadata 都会检查） + // 如果不存在，会返回错误，直接传播即可。 + + // 2. 检查是否为符号链接（使用 symlink_metadata 避免解析） + let symlink_metadata = fs::symlink_metadata(path)?; + if symlink_metadata.file_type().is_symlink() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "File path points to a symbolic link", + )); + } + + // 3. 获取目标文件的实际元数据（因为不是符号链接，所以就是文件本身） + let metadata = fs::metadata(path)?; + + // 4. 检查文件大小是否超过 1GB + const MAX_FILE_SIZE: u64 = 1024 * 1024 * 1024; // 1 GiB + if metadata.len() > MAX_FILE_SIZE { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "File size exceeds 1GB", + )); + } + + // 5. 平台特定的权限检查 + // Unix: 检查其他用户（others）和组（group）是否有写权限是常见的安全关注点。 + // Windows: 权限模型更复杂（ACL），标准库不直接暴露。通常，如果用户能获取 metadata， + // 我们不打算在此处做复杂的 ACL 检查，可以跳过或进行简单检查。 + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = metadata.permissions().mode(); + + // 在 Unix 中，权限位：user(3位) group(3位) others(3位) + // 我们通常关心 group 和 others 是否有写权限 (w)。 + // 掩码 0o022 表示 group-write 和 others-write 位。 + if mode & 0o022 != 0 { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "File is writable by group or others", + )); + } + } + + // 注意： + // - **不检查“读权限”**：最准确、最安全的“可读”检查是尝试用 `File::open()` 打开文件。 + // 在此函数中检查读权限既复杂（跨平台）又可能不准确（例如，受父目录权限、ACL、capabilities 影响）。 + // 让调用者在 `File::open()` 时处理 `PermissionDenied` 错误是标准做法。 + // - **Windows 权限**：标准库没有提供简单的方法来检查复杂的 Windows ACL。 + // 如果你的应用有严格的 Windows 安全需求，可能需要调用 Win32 API（例如通过 `winapi` crate）， + // 但这超出了标准库的范围，且增加了复杂性。对于大多数用途，依赖 `File::open()` 的结果是合理的。 + + Ok(()) +} + + + +/// 安全过滤：防止 CSV 注入（公式注入） +/// 常见恶意前缀：= + @ - +pub fn check_csv_injection(field: &str, row_count: usize, warnings: &mut Vec) -> bool { + if let Some(first_char) = field.chars().next() { + if "=+@-".contains(first_char) { + let e_field = escape_special_chars(field); + eprintln!("Potential CSV formula injection detected, filtered: {}", &e_field); + warnings.push(format!("Line {} parse fail: {}", row_count, &e_field)); + // 可选择跳过或转义 + return true; + } + } + false +} + +/// 处理字符串中的特殊符号，例如将 \n 替换为 \\n, \t 替换为 \\t 等 +/// +/// # 参数 +/// * `input` - 需要处理的原始字符串 +/// +/// # 返回值 +/// * 处理后的字符串 +pub fn escape_special_chars(input: &str) -> String { + let mut result = input.to_string(); + + // 特殊字符及其对应的转义序列 + let special_chars = [ + ("\n", "\\n"), + ("\t", "\\t"), + ("\r", "\\r"), + ("\"", "\\\""), + ("'", "\\'"), + ("\x08", "\\b"), // \b + ("\x0c", "\\f"), // \f + ("\x0b", "\\v"), // \v + ("\\", "\\\\"), + ]; + + // 使用 for 循环遍历并替换所有特殊字符 + for (char, escaped_char) in special_chars.iter() { + result = result.replace(*char, *escaped_char); + } + + result +} + + -- Gitee From b3c961d7b52e436b94eba9c3f3e73da31ada2e23 Mon Sep 17 00:00:00 2001 From: hummel mao Date: Fri, 15 Aug 2025 10:59:31 +0800 Subject: [PATCH 05/11] =?UTF-8?q?feat(csv)=20=E4=BF=AE=E6=94=B9=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E4=BC=A0=E9=80=92=E5=8F=82=E6=95=B0=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ModelVis/rust/csv_parser/src/operator.rs | 8 ++++---- .../ModelVis/rust/csv_parser/tests/test.rs | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs index bce4a17beb..200d6a122c 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs @@ -37,8 +37,8 @@ pub struct Operator { #[derive(Debug, Serialize)] pub struct OperatorGroup { pub op_type: String, - pub duration_us: f64, - pub mte2_time_us: f64, + pub avg_duration_us: f64, + pub avg_mte2_time_us: f64, } pub fn group_times_by_op_type(operators: &Vec) -> HashMap { @@ -60,8 +60,8 @@ pub fn group_times_by_op_type(operators: &Vec) -> HashMap { assert_eq!(ops.len(), 2); - assert_eq!(ops.get("MatMul").unwrap().duration_us, 160.0); - assert_eq!(ops.get("MatMul").unwrap().mte2_time_us, 20.0); - assert_eq!(ops.get("Active").unwrap().duration_us, 30.0); - assert_eq!(ops.get("Active").unwrap().mte2_time_us, 3.0); + assert_eq!(ops.get("MatMul").unwrap().avg_duration_us, 160.0); + assert_eq!(ops.get("MatMul").unwrap().avg_mte2_time_us, 20.0); + assert_eq!(ops.get("Active").unwrap().avg_duration_us, 30.0); + assert_eq!(ops.get("Active").unwrap().avg_mte2_time_us, 3.0); }, Err(err) => panic!("Error occurred while parsing csv: {}", err), } -- Gitee From 3cfc3774ccaa0301a03efed34504b55075fa6dda Mon Sep 17 00:00:00 2001 From: hummel mao Date: Mon, 18 Aug 2025 17:11:34 +0800 Subject: [PATCH 06/11] =?UTF-8?q?feat(csv=5Fparser):=20=E6=9B=B4=E6=96=B0?= =?UTF-8?q?=E8=A7=A3=E6=9E=90=20csv=20=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ModelVis/rust/csv_parser/src/lib.rs | 24 +-- .../ModelVis/rust/csv_parser/src/operator.rs | 59 ++---- .../ModelVis/rust/csv_parser/src/parser.rs | 188 +++++++++--------- .../ModelVis/rust/csv_parser/src/utils.rs | 5 +- .../rust/csv_parser/tests/parser_test.rs | 64 +++--- 5 files changed, 153 insertions(+), 187 deletions(-) diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs index 8e895eb8f4..874d8d54b0 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs @@ -4,8 +4,8 @@ use std::collections::HashMap; use std::path::Path; use anyhow::{anyhow, Result}; -use crate::operator::{group_times_by_op_type, OperatorGroup, DURATION_KEY, MTE2_TIME_KEY, NAME_KEY, TYPE_KEY}; -use crate::parser::{parse_operator, ValidHeaderField}; +use crate::operator::{group_times_by_op_type, OperatorGroup}; +use crate::parser::{parse_operator, ParseFile}; use crate::utils::check_file_safety; pub mod operator; @@ -37,21 +37,11 @@ pub fn is_valid_csv(name: &str) -> bool { all_digits(body) } -pub fn get_valid_fields(name: &str) -> Vec { +pub fn get_parse_file(name: &str) -> ParseFile { if name == "kernel_detail.csv" { - return vec![ - ValidHeaderField { h_field: "Name".to_string(), h_key: NAME_KEY.to_string() }, - ValidHeaderField { h_field: "Type".to_string(), h_key: TYPE_KEY.to_string() }, - ValidHeaderField { h_field: "Duration(us)".to_string(), h_key: DURATION_KEY.to_string() }, - ValidHeaderField { h_field: "aiv_mte2_time(us)".to_string(), h_key: MTE2_TIME_KEY.to_string() }, - ]; + return ParseFile::KernelDetail; } - vec![ - ValidHeaderField { h_field: "Op Name".to_string(), h_key: NAME_KEY.to_string() }, - ValidHeaderField { h_field: "Op Type".to_string(), h_key: TYPE_KEY.to_string() }, - ValidHeaderField { h_field: "Task Duration(us)".to_string(), h_key: DURATION_KEY.to_string() }, - ValidHeaderField { h_field: "aiv_mte2_time(us)".to_string(), h_key: MTE2_TIME_KEY.to_string() }, - ] + ParseFile::OpSummary } pub fn parse_operator_csv(path: &str) -> Result> { @@ -60,8 +50,8 @@ pub fn parse_operator_csv(path: &str) -> Result> if !is_valid_csv(&name) { return Err(anyhow!("Invalid CSV filename: {}", &name)); } - let fields = get_valid_fields(&name); - let operators = parse_operator(path, fields)?; + let file_type = get_parse_file(&name); + let operators = parse_operator(path, file_type)?; Ok(group_times_by_op_type(&operators)) } diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs index 200d6a122c..ed3e8f58d3 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs @@ -4,27 +4,7 @@ use std::collections::{BTreeMap, HashMap}; use std::string::ToString; use serde::Serialize; -use anyhow::Result; -use thiserror::Error; -use crate::utils::escape_special_chars; - -pub const NAME_KEY: &str = "name"; -pub const TYPE_KEY: &str = "op_type"; -pub const DURATION_KEY: &str = "duration_us"; -pub const MTE2_TIME_KEY: &str = "mte2_time_us"; - -#[derive(Error, Debug)] -pub enum CreateError { - #[error("Transform number fail: {0}")] - FailTransformNumber(String), - - #[error("Invalid number")] - InvalidNumber, -} - -pub trait Creator { - fn create_by_row(row: &HashMap<&str, &str>) -> Result where Self: Sized; -} +use crate::parser::{OperatorFromKernelDetail, OperatorFromOpSummary}; #[derive(Debug, Serialize)] pub struct Operator { @@ -66,31 +46,24 @@ pub fn group_times_by_op_type(operators: &Vec) -> HashMap) -> Result { - // 解析 duration - let duration_us = transform_f64(row.get(DURATION_KEY).unwrap())?; - // 解析 mte2_time - let mte2_time_us = transform_f64(row.get(MTE2_TIME_KEY).unwrap())?; - if duration_us < 0.0 || mte2_time_us < 0.0 { - return Err(CreateError::InvalidNumber); +impl From for Operator { + fn from(value: OperatorFromKernelDetail) -> Self { + Self { + name: value.name, + op_type: value.type_field, + duration_us: value.duration_us, + mte2_time_us: value.aiv_mte2_time_us, } - Ok(Operator { - name: row.get(NAME_KEY).unwrap().to_string(), - op_type: row.get(TYPE_KEY).unwrap().to_string(), - duration_us, - mte2_time_us, - }) } } -/// 解析 string 转为 f64 -fn transform_f64(field: &str) -> Result { - let num = match field.parse::() { - Ok(d) => d, - Err(_) => { - return Err(CreateError::FailTransformNumber(escape_special_chars(field))) +impl From for Operator { + fn from(value: OperatorFromOpSummary) -> Self { + Self { + name: value.name, + op_type: value.type_field, + duration_us: value.duration_us, + mte2_time_us: value.aiv_mte2_time_us, } - }; - Ok(num) + } } diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs index 0434bccf3c..678b3b6c47 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs @@ -1,18 +1,64 @@ /* * Copyright (c), Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. */ -use std::collections::HashMap; +use std::fs::File; use std::io::BufReader; use csv::Reader; -use serde::Serialize; +use serde::Deserialize; use thiserror::Error; -use crate::operator::{CreateError, Creator, Operator}; -use crate::utils::{check_csv_injection, escape_special_chars}; +use crate::operator::Operator; +use crate::utils::check_csv_injection; -#[derive(Debug, Serialize)] -pub struct ParseResult { - pub rows: Vec, - pub warnings: Vec, +// 自定义解析函数：将字符串解析为 f64（自动提取数字，忽略单位） +fn parse_micros_f64<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let s = String::deserialize(deserializer)?; + // 只保留数字、小数点、负号 + let cleaned: String = s + .chars() + .filter(|c| c.is_ascii_digit() || *c == '.' || *c == '-') + .collect(); + + if cleaned.is_empty() { + return Err(serde::de::Error::custom("empty or invalid number")); + } + + cleaned + .parse::() + .map_err(|_| serde::de::Error::custom("failed to parse as f64")) +} + +// 设置来自 kernel_detail.csv 文件的解析 +#[derive(Debug, Deserialize)] +pub struct OperatorFromKernelDetail { + #[serde(rename = "Name")] + pub name: String, + + #[serde(rename = "Type")] + pub type_field: String, + + #[serde(rename = "Duration(us)", deserialize_with = "parse_micros_f64")] + pub duration_us: f64, + + #[serde(rename = "aiv_mte2_time(us)", deserialize_with = "parse_micros_f64")] + pub aiv_mte2_time_us: f64, +} +// 设置来自 op_summary_{时间戳}.csv 文件的解析 +#[derive(Debug, Deserialize)] +pub struct OperatorFromOpSummary { + #[serde(rename = "Op Name")] + pub name: String, + + #[serde(rename = "Op Type")] + pub type_field: String, + + #[serde(rename = "Task Duration(us)", deserialize_with = "parse_micros_f64")] + pub duration_us: f64, + + #[serde(rename = "aiv_mte2_time(us)", deserialize_with = "parse_micros_f64")] + pub aiv_mte2_time_us: f64, } #[derive(Error, Debug)] @@ -25,107 +71,69 @@ pub enum ParseError { #[error("File is empty or has invalid data")] EmptyData, - - #[error("Header is empty")] - EmptyHeader, - - #[error("Header format error，missing required column: {0}")] - InvalidHeader(String), } -pub struct ValidHeaderField { - pub h_field: String, - pub h_key: String, +#[derive(Debug)] +pub enum ParseFile { + KernelDetail, + OpSummary, } -pub fn parse_operator(path: &str, parse_fields: Vec) -> Result, ParseError> { - let file = std::fs::File::open(path)?; +pub fn parse_operator(path: &str, parse_file: ParseFile) -> Result, ParseError> { + let file = File::open(path)?; let reader = BufReader::new(file); - // 调用 parse_csv 函数解析 CSV 文件 - let result = parse_csv(reader, parse_fields, Operator::create_by_row)?; - - Ok(result.rows) // 返回解析后的 Operator 列表 + // 解析 CSV 文件 + let result = match parse_file { + ParseFile::KernelDetail => { + parse_kernel_detail(reader)? + }, + ParseFile::OpSummary => { + parse_op_summary(reader)? + }, + }; + + Ok(result) // 返回解析后的 Operator 列表 } -/// 从字节流中解析 CSV，返回算子列表与结构汇总 -pub fn parse_csv(reader: R, fields: Vec, creator: F) - -> Result, ParseError> -where - T: Creator, - F: Fn(&HashMap<&str, &str>) -> Result { - let mut csv_reader = Reader::from_reader(reader); - let mut rows = Vec::new(); - let mut warnings = Vec::new(); - let mut row_count = 0; - - let header_index_map = get_header_index_map(&mut csv_reader, fields)?; - let col_len = header_index_map.len(); - - for result in csv_reader.records() { - row_count += 1; - let record = match result { - Ok(rec) => rec, - Err(e) => { - eprintln!("Line {}, Skip illegal CSV lines: {}", row_count, e.to_string()); - warnings.push(format!("Line {} parse fail", row_count)); - continue; - } - }; - - let mut value_map = HashMap::new(); - - // 解析字段 - for (k, v) in header_index_map.iter() { - let field = record.get(*v).unwrap_or("").trim(); - // === 安全过滤：防止 CSV 注入（公式注入）=== - if check_csv_injection(field, row_count, &mut warnings) { - break; - } - value_map.insert(k.as_str(), field); - } +pub fn parse_kernel_detail(reader: R) -> Result, ParseError> { + let mut reader = Reader::from_reader(reader); + let mut operators = Vec::new(); - if value_map.len() < col_len { - warnings.push(format!("Line {} get value fail", row_count)); + for (row_count, result) in reader.deserialize().enumerate() { + let op_detail: OperatorFromKernelDetail = result?; + if check_csv_injection(&op_detail.name, row_count) || + check_csv_injection(&op_detail.type_field, row_count) { continue; } - - match creator(&value_map) { - Ok(value) => rows.push(value), - Err(e) => { - warnings.push(format!("Line {} create error: {}", row_count, e.to_string())); - continue; - }, - }; + let operator: Operator = op_detail.into(); // 转换 + operators.push(operator); } - if rows.len() == 0 { + if operators.len() < 1 { return Err(ParseError::EmptyData); } - - Ok(ParseResult { - rows, - warnings, - }) + Ok(operators) } -fn get_header_index_map(csv_reader: &mut Reader, fields: Vec) - -> Result, ParseError> { - if !csv_reader.has_headers() { - return Err(ParseError::EmptyHeader); - } - let headers = csv_reader.headers()?; - // 初始化配置结构体 - let mut header_index = HashMap::new(); - - // 尝试找到每个有效头部字段的索引 - for f in fields.iter() { - let index = headers.iter().position(|h| h == f.h_field) - .ok_or(ParseError::InvalidHeader(escape_special_chars(&f.h_field)))?; - header_index.insert(f.h_key.to_string(), index); +pub fn parse_op_summary(reader: R) -> Result, ParseError> { + let mut reader = Reader::from_reader(reader); + let mut operators = Vec::new(); + + for (row_count, result) in reader.deserialize().enumerate() { + let op_summary: OperatorFromOpSummary = result?; + if check_csv_injection(&op_summary.name, row_count) || + check_csv_injection(&op_summary.type_field, row_count) { + continue; + } + let operator: Operator = op_summary.into(); // 转换 + operators.push(operator); } - Ok(header_index) + if operators.len() < 1 { + return Err(ParseError::EmptyData); + } + Ok(operators) } diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/utils.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/utils.rs index a8a338e59f..8d89d16462 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/utils.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/utils.rs @@ -70,12 +70,11 @@ pub fn check_file_safety(file_path: &str) -> io::Result<()> { /// 安全过滤：防止 CSV 注入（公式注入） /// 常见恶意前缀：= + @ - -pub fn check_csv_injection(field: &str, row_count: usize, warnings: &mut Vec) -> bool { +pub fn check_csv_injection(field: &str, row_count: usize) -> bool { if let Some(first_char) = field.chars().next() { if "=+@-".contains(first_char) { let e_field = escape_special_chars(field); - eprintln!("Potential CSV formula injection detected, filtered: {}", &e_field); - warnings.push(format!("Line {} parse fail: {}", row_count, &e_field)); + eprintln!("Line {}, Potential CSV formula injection detected, filtered: {}", row_count, &e_field); // 可选择跳过或转义 return true; } diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/parser_test.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/parser_test.rs index 1684770544..f9f5c5290a 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/parser_test.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/parser_test.rs @@ -4,65 +4,61 @@ #[cfg(test)] mod parser_tests { use std::io::Cursor; - use csv_parser::parser::{parse_csv, ValidHeaderField}; - use csv_parser::operator::{Creator, Operator, DURATION_KEY, MTE2_TIME_KEY, NAME_KEY, TYPE_KEY}; + use csv_parser::parser::{parse_kernel_detail, parse_op_summary}; - fn get_fields() -> Vec { - vec![ - ValidHeaderField { h_field: "op_name".to_string(), h_key: NAME_KEY.to_string() }, - ValidHeaderField { h_field: "op_type".to_string(), h_key: TYPE_KEY.to_string() }, - ValidHeaderField { h_field: "duration".to_string(), h_key: DURATION_KEY.to_string() }, - ValidHeaderField { h_field: "mte2_time".to_string(), h_key: MTE2_TIME_KEY.to_string() }, - ] + #[test] + fn test_parse_valid_kernel_detail_csv() { + let csv_data = "Name,Type,Duration(us),aiv_mte2_time(us)\nconv2d,MatMul,120.0,20\nrelu,Active,30,3\nmatmul,MatMul,200,20"; + let cursor = Cursor::new(csv_data); + let result = parse_kernel_detail(cursor) + .unwrap(); + + assert_eq!(result.len(), 3); + assert_eq!(result[0].name, "conv2d"); + assert_eq!(result[0].duration_us, 120.0); + assert_eq!(result[0].mte2_time_us, 20.0); } #[test] - fn test_parse_valid_csv() { - let csv_data = "op_name,op_type,duration,mte2_time\nconv2d,MatMul,120.0,20\nrelu,Active,30,3\nmatmul,MatMul,200,20"; + fn test_parse_valid_op_summary_csv() { + let csv_data = "Op Name,Op Type,Task Duration(us),aiv_mte2_time(us)\nconv2d,MatMul,120.0,20\nrelu,Active,30,3\nmatmul,MatMul,200,20"; let cursor = Cursor::new(csv_data); - let fields = get_fields(); - let result = parse_csv(cursor, fields, Operator::create_by_row) + let result = parse_op_summary(cursor) .unwrap(); - assert_eq!(result.rows.len(), 3); - assert_eq!(result.rows[0].name, "conv2d"); - assert_eq!(result.rows[0].duration_us, 120.0); - assert_eq!(result.rows[0].mte2_time_us, 20.0); + assert_eq!(result.len(), 3); + assert_eq!(result[0].name, "conv2d"); + assert_eq!(result[0].duration_us, 120.0); + assert_eq!(result[0].mte2_time_us, 20.0); } #[test] fn test_parse_with_invalid_duration() { - let csv_data = "op_name,op_type,duration,mte2_time\nconv2d,MatMul,abc,1\nrelu,Active,30,3"; + let csv_data = "Name,Type,Duration(us),aiv_mte2_time(us)\nconv2d,MatMul,abc,1\nrelu,Active,30,3"; let cursor = Cursor::new(csv_data); - let fields = get_fields(); - let result = parse_csv(cursor, fields, Operator::create_by_row) - .unwrap(); - - assert_eq!(result.rows.len(), 1); // 只有 relu 被解析 - assert_eq!(result.warnings.len(), 1); - assert!(result.warnings[0].contains("abc")); + let result = parse_kernel_detail(cursor); + assert!(result.is_err()); + assert_eq!(result.unwrap_err().to_string(), "CSV format error: CSV deserialize error: record 1 (line: 2, byte: 41): empty or invalid number"); } #[test] fn test_csv_injection_filter() { - let csv_data = "op_name,op_type,duration,mte2_time\n=CMD|' /C calc'!A0,A,100,10\nsafe_op,Op,50,5"; + let csv_data = "Op Name,Op Type,Task Duration(us),aiv_mte2_time(us)\n=CMD|' /C calc'!A0,A,100,10\nsafe_op,Op,50,5"; let cursor = Cursor::new(csv_data); - let fields = get_fields(); - let result = parse_csv(cursor, fields, Operator::create_by_row) + let result = parse_op_summary(cursor) .unwrap(); - assert_eq!(result.rows.len(), 1); - assert_eq!(result.rows[0].name, "safe_op"); + assert_eq!(result.len(), 1); + assert_eq!(result[0].name, "safe_op"); // 第一行因以 '=' 开头被过滤 } #[test] fn test_empty_csv() { - let csv_data = "op_name,op_type,duration,mte2_time"; // 无数据行 + let csv_data = "Op Name,Op Type,Task Duration(us),aiv_mte2_time(us)"; // 无数据行 let cursor = Cursor::new(csv_data); - let fields = get_fields(); - let result = parse_csv(cursor, fields, Operator::create_by_row); + let result = parse_op_summary(cursor); assert!(result.is_err()); - assert_eq!(result.unwrap_err().to_string(), "文件为空或无有效数据"); + assert_eq!(result.unwrap_err().to_string(), "File is empty or has invalid data"); } } \ No newline at end of file -- Gitee From dac59c83d08cde437c0cff1b84001d3ca4cf86cb Mon Sep 17 00:00:00 2001 From: hummel mao Date: Mon, 18 Aug 2025 17:27:30 +0800 Subject: [PATCH 07/11] =?UTF-8?q?feat(csv=5Fparser):=20=E5=88=A0=E9=99=A4?= =?UTF-8?q?=E4=B8=8D=E9=9C=80=E8=A6=81=E7=9A=84=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ModelVis/rust/csv_parser/src/lib.rs | 2 - .../ModelVis/rust/csv_parser/src/utils.rs | 70 ------------------- 2 files changed, 72 deletions(-) diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs index 874d8d54b0..2babdfb362 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs @@ -6,7 +6,6 @@ use std::path::Path; use anyhow::{anyhow, Result}; use crate::operator::{group_times_by_op_type, OperatorGroup}; use crate::parser::{parse_operator, ParseFile}; -use crate::utils::check_file_safety; pub mod operator; pub mod parser; @@ -45,7 +44,6 @@ pub fn get_parse_file(name: &str) -> ParseFile { } pub fn parse_operator_csv(path: &str) -> Result> { - check_file_safety(path)?; let name = get_filename(path)?; if !is_valid_csv(&name) { return Err(anyhow!("Invalid CSV filename: {}", &name)); diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/utils.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/utils.rs index 8d89d16462..a96f00dc42 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/utils.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/utils.rs @@ -1,73 +1,3 @@ -use std::{fs, io}; -use std::path::Path; - -/// 打开文件前进行安全校验 -/// -/// # 参数 -/// * `file_path` - 文件路径 -/// -/// # 返回值 -/// * `Result<(), io::Error>` - 如果校验通过返回 `Ok(())`，否则返回 `Err(io::Error)` -pub fn check_file_safety(file_path: &str) -> io::Result<()> { - let path = Path::new(file_path); - - // 1. 检查路径是否存在（fs::metadata 和 fs::symlink_metadata 都会检查） - // 如果不存在，会返回错误，直接传播即可。 - - // 2. 检查是否为符号链接（使用 symlink_metadata 避免解析） - let symlink_metadata = fs::symlink_metadata(path)?; - if symlink_metadata.file_type().is_symlink() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "File path points to a symbolic link", - )); - } - - // 3. 获取目标文件的实际元数据（因为不是符号链接，所以就是文件本身） - let metadata = fs::metadata(path)?; - - // 4. 检查文件大小是否超过 1GB - const MAX_FILE_SIZE: u64 = 1024 * 1024 * 1024; // 1 GiB - if metadata.len() > MAX_FILE_SIZE { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "File size exceeds 1GB", - )); - } - - // 5. 平台特定的权限检查 - // Unix: 检查其他用户（others）和组（group）是否有写权限是常见的安全关注点。 - // Windows: 权限模型更复杂（ACL），标准库不直接暴露。通常，如果用户能获取 metadata， - // 我们不打算在此处做复杂的 ACL 检查，可以跳过或进行简单检查。 - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let mode = metadata.permissions().mode(); - - // 在 Unix 中，权限位：user(3位) group(3位) others(3位) - // 我们通常关心 group 和 others 是否有写权限 (w)。 - // 掩码 0o022 表示 group-write 和 others-write 位。 - if mode & 0o022 != 0 { - return Err(io::Error::new( - io::ErrorKind::PermissionDenied, - "File is writable by group or others", - )); - } - } - - // 注意： - // - **不检查“读权限”**：最准确、最安全的“可读”检查是尝试用 `File::open()` 打开文件。 - // 在此函数中检查读权限既复杂（跨平台）又可能不准确（例如，受父目录权限、ACL、capabilities 影响）。 - // 让调用者在 `File::open()` 时处理 `PermissionDenied` 错误是标准做法。 - // - **Windows 权限**：标准库没有提供简单的方法来检查复杂的 Windows ACL。 - // 如果你的应用有严格的 Windows 安全需求，可能需要调用 Win32 API（例如通过 `winapi` crate）， - // 但这超出了标准库的范围，且增加了复杂性。对于大多数用途，依赖 `File::open()` 的结果是合理的。 - - Ok(()) -} - - - /// 安全过滤：防止 CSV 注入（公式注入） /// 常见恶意前缀：= + @ - pub fn check_csv_injection(field: &str, row_count: usize) -> bool { -- Gitee From 325ce8a264c9f12dbb76995123d97bfddd0c1263 Mon Sep 17 00:00:00 2001 From: hummel mao Date: Thu, 14 Aug 2025 17:58:57 +0800 Subject: [PATCH 08/11] =?UTF-8?q?feat(csv=5Fparser):=20=E5=AE=9E=E7=8E=B0?= =?UTF-8?q?=20analyze=5Fduration=20=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plugins/mindstudio-insight-plugins/ModelVis/Cargo.lock | 2 ++ .../ModelVis/app/src-tauri/Cargo.toml | 1 + .../ModelVis/app/src-tauri/src/commands.rs | 9 ++++++++- .../ModelVis/app/src-tauri/src/lib.rs | 4 ++-- .../ModelVis/rust/csv_parser/Cargo.toml | 1 + .../ModelVis/rust/csv_parser/src/lib.rs | 2 +- .../ModelVis/rust/csv_parser/src/operator.rs | 3 ++- 7 files changed, 17 insertions(+), 5 deletions(-) diff --git a/plugins/mindstudio-insight-plugins/ModelVis/Cargo.lock b/plugins/mindstudio-insight-plugins/ModelVis/Cargo.lock index c038a7e206..32daa20846 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/Cargo.lock +++ b/plugins/mindstudio-insight-plugins/ModelVis/Cargo.lock @@ -572,6 +572,7 @@ dependencies = [ name = "csv_parser" version = "0.1.0" dependencies = [ + "ahash", "anyhow", "csv", "serde", @@ -4394,6 +4395,7 @@ version = "0.1.0" dependencies = [ "ahash", "anyhow", + "csv_parser", "fsg", "layout", "parser", diff --git a/plugins/mindstudio-insight-plugins/ModelVis/app/src-tauri/Cargo.toml b/plugins/mindstudio-insight-plugins/ModelVis/app/src-tauri/Cargo.toml index c81a357748..2f20e61470 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/app/src-tauri/Cargo.toml +++ b/plugins/mindstudio-insight-plugins/ModelVis/app/src-tauri/Cargo.toml @@ -18,6 +18,7 @@ smartstring = { workspace = true } ahash = { workspace = true, features = ["serde"] } serde = { workspace = true, features = ["derive"] } serde_json.workspace = true +csv_parser = { path = "../../rust/csv_parser" } parser = { path = "../../rust/parser" } layout = { path = "../../rust/layout" } fsg = { path = "../../rust/fsg" } diff --git a/plugins/mindstudio-insight-plugins/ModelVis/app/src-tauri/src/commands.rs b/plugins/mindstudio-insight-plugins/ModelVis/app/src-tauri/src/commands.rs index ab5f7458b8..9f5bfbc2b1 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/app/src-tauri/src/commands.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/app/src-tauri/src/commands.rs @@ -10,6 +10,8 @@ use fsg::{result::JSONResult, fsgs_model}; use tauri::{path::BaseDirectory, AppHandle, Emitter, Manager, Result as InvokeResult}; use tauri::async_runtime::spawn_blocking; use tauri_plugin_shell::ShellExt; +use csv_parser::operator::OperatorGroup; +use csv_parser::parse_operator_csv; #[allow(non_snake_case)] #[derive(Debug, Clone, Serialize, Deserialize)] @@ -388,7 +390,7 @@ fn read_from<'a, T: DeserializeOwned, P: AsRef>(path: P) -> T { } #[tauri::command] -pub fn mine_fsg(path: &str, name: &str, min_sup: usize, min: usize, max: usize) -> InvokeResult> { +pub async fn mine_fsg(path: &str, name: &str, min_sup: usize, min: usize, max: usize) -> InvokeResult> { let mut model = parse_bin(&path)?; let single_graph = if model.name == name { Model { @@ -425,6 +427,11 @@ fn recursive_get_single_graph(model: &mut Model, name: &str, depth: usize) -> Re Err(anyhow!("Subgraph with name '{}' not found", name)) } +#[tauri::command] +pub async fn analyze_duration(path: &str) -> InvokeResult> { + wrap(parse_operator_csv(path)) +} + fn wrap(src: Result) -> InvokeResult { src.map_err(|e| e.into()) } diff --git a/plugins/mindstudio-insight-plugins/ModelVis/app/src-tauri/src/lib.rs b/plugins/mindstudio-insight-plugins/ModelVis/app/src-tauri/src/lib.rs index 83c6237d48..e37f2d4a16 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/app/src-tauri/src/lib.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/app/src-tauri/src/lib.rs @@ -1,13 +1,13 @@ mod commands; -use commands::{layout_bin, mine_fsg}; +use commands::{layout_bin, mine_fsg, analyze_duration}; #[cfg_attr(mobile, tauri::mobile_entry_point)] pub fn run() { tauri::Builder::default() .plugin(tauri_plugin_dialog::init()) .plugin(tauri_plugin_shell::init()) - .invoke_handler(tauri::generate_handler![layout_bin, mine_fsg]) + .invoke_handler(tauri::generate_handler![layout_bin, mine_fsg, analyze_duration]) .run(tauri::generate_context!()) .expect("error while running tauri application"); } diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/Cargo.toml b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/Cargo.toml index 5dd1fa0c1e..3634b7f90d 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/Cargo.toml +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/Cargo.toml @@ -6,5 +6,6 @@ edition = "2024" [dependencies] anyhow = { workspace = true } thiserror = { workspace = true } +ahash = { workspace = true, features = ["serde"] } csv = "1.3.1" serde = { version = "1.0", features = ["derive"] } diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs index 2babdfb362..ee60aefd75 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs @@ -1,7 +1,7 @@ /* * Copyright (c), Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. */ -use std::collections::HashMap; +use ahash::HashMap; use std::path::Path; use anyhow::{anyhow, Result}; use crate::operator::{group_times_by_op_type, OperatorGroup}; diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs index ed3e8f58d3..ec4af27946 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs @@ -1,7 +1,8 @@ /* * Copyright (c), Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. */ -use std::collections::{BTreeMap, HashMap}; +use std::collections::BTreeMap; +use ahash::HashMap; use std::string::ToString; use serde::Serialize; use crate::parser::{OperatorFromKernelDetail, OperatorFromOpSummary}; -- Gitee From f9347a3c85af9ef167c84291b3d474ccefcde943 Mon Sep 17 00:00:00 2001 From: hummel mao Date: Mon, 18 Aug 2025 20:19:24 +0800 Subject: [PATCH 09/11] =?UTF-8?q?fix(csv=5Fparser):=20=E5=A4=84=E7=90=86?= =?UTF-8?q?=E6=A3=80=E8=A7=86=E6=84=8F=E8=A7=81=EF=BC=8C=E7=AE=80=E5=8C=96?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E5=8C=B9=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ModelVis/rust/csv_parser/src/lib.rs | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs index ee60aefd75..b37e085fb5 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs @@ -17,23 +17,12 @@ pub fn is_valid_csv(name: &str) -> bool { return true; } - // 情况2：匹配 op_summary_YYYYMMDDHHMMSS.csv - if !name.starts_with("op_summary_") || !name.ends_with(".csv") { + // 情况2：匹配 op_summary{*}.csv + if !name.starts_with("op_summary") || !name.ends_with(".csv") { return false; } - // 去掉前缀和后缀 - let body = &name["op_summary_".len()..name.len() - 4]; // 去掉 .csv - - // 必须是 14 位：YYYYMMDDHHMMSS - if body.len() != 14 { - return false; - } - - // 检查是否全为数字 - let all_digits = |s: &str| s.chars().all(|c| c.is_ascii_digit()); - - all_digits(body) + true } pub fn get_parse_file(name: &str) -> ParseFile { -- Gitee From 41c92a84876bf1332b7078e6efeb7a71e66815fe Mon Sep 17 00:00:00 2001 From: hummel mao Date: Tue, 19 Aug 2025 21:00:21 +0800 Subject: [PATCH 10/11] =?UTF-8?q?fix(csv=5Fparser):=20=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E6=9C=AA=E8=80=83=E8=99=91=E5=88=B0=E7=9A=84=E5=9C=BA=E6=99=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ModelVis/rust/csv_parser/src/operator.rs | 12 +-- .../ModelVis/rust/csv_parser/src/parser.rs | 96 +++++++++++++------ .../ModelVis/rust/csv_parser/src/utils.rs | 1 - .../rust/csv_parser/tests/parser_test.rs | 94 ++++++++++++++---- .../ModelVis/rust/csv_parser/tests/test.rs | 6 +- .../csv_parser/tests/valid/kernel_detail.csv | 8 +- .../tests/valid/op_summary_20250812203130.csv | 8 +- 7 files changed, 159 insertions(+), 66 deletions(-) diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs index ec4af27946..fe8e66e1b9 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/operator.rs @@ -12,14 +12,14 @@ pub struct Operator { pub name: String, pub op_type: String, pub duration_us: f64, // 算子耗时 - pub mte2_time_us: f64, // 算子内存搬运时间 + pub memory_move_time_us: f64, // 算子内存搬运时间 } #[derive(Debug, Serialize)] pub struct OperatorGroup { pub op_type: String, pub avg_duration_us: f64, - pub avg_mte2_time_us: f64, + pub avg_memory_move_time_us: f64, } pub fn group_times_by_op_type(operators: &Vec) -> HashMap { @@ -29,7 +29,7 @@ pub fn group_times_by_op_type(operators: &Vec) -> HashMap) -> HashMap for Operator { name: value.name, op_type: value.type_field, duration_us: value.duration_us, - mte2_time_us: value.aiv_mte2_time_us, + memory_move_time_us: value.aiv_mte2_time_us + value.aiv_mte3_time_us, } } } @@ -64,7 +64,7 @@ impl From for Operator { name: value.name, op_type: value.type_field, duration_us: value.duration_us, - mte2_time_us: value.aiv_mte2_time_us, + memory_move_time_us: value.aiv_mte2_time_us + value.aiv_mte3_time_us, } } } diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs index 678b3b6c47..bc61d6806c 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs @@ -4,6 +4,7 @@ use std::fs::File; use std::io::BufReader; use csv::Reader; +use serde::de::DeserializeOwned; use serde::Deserialize; use thiserror::Error; use crate::operator::Operator; @@ -15,6 +16,9 @@ where D: serde::Deserializer<'de>, { let s = String::deserialize(deserializer)?; + if s.starts_with("N/A") { // 如果 csv 有 N/A, 返回-1，在之后的操作中会过滤掉负数的行 + return Ok(-1.0); + } // 只保留数字、小数点、负号 let cleaned: String = s .chars() @@ -44,7 +48,11 @@ pub struct OperatorFromKernelDetail { #[serde(rename = "aiv_mte2_time(us)", deserialize_with = "parse_micros_f64")] pub aiv_mte2_time_us: f64, + + #[serde(rename = "aiv_mte3_time(us)", deserialize_with = "parse_micros_f64")] + pub aiv_mte3_time_us: f64, } + // 设置来自 op_summary_{时间戳}.csv 文件的解析 #[derive(Debug, Deserialize)] pub struct OperatorFromOpSummary { @@ -59,6 +67,9 @@ pub struct OperatorFromOpSummary { #[serde(rename = "aiv_mte2_time(us)", deserialize_with = "parse_micros_f64")] pub aiv_mte2_time_us: f64, + + #[serde(rename = "aiv_mte3_time(us)", deserialize_with = "parse_micros_f64")] + pub aiv_mte3_time_us: f64, } #[derive(Error, Debug)] @@ -79,6 +90,51 @@ pub enum ParseFile { OpSummary, } +// 定义一个Trait，该Trait提供对共通属性的基本访问。 +pub trait OperatorInfo { + fn name(&self) -> &str; + fn type_field(&self) -> &str; + fn duration_us(&self) -> f64; + fn aiv_mte2_time_us(&self) -> f64; + fn aiv_mte3_time_us(&self) -> f64; +} + +impl OperatorInfo for OperatorFromKernelDetail { + fn name(&self) -> &str { + &self.name + } + fn type_field(&self) -> &str { + &self.type_field + } + fn duration_us(&self) -> f64 { + self.duration_us + } + fn aiv_mte2_time_us(&self) -> f64 { + self.aiv_mte2_time_us + } + fn aiv_mte3_time_us(&self) -> f64 { + self.aiv_mte3_time_us + } +} + +impl OperatorInfo for OperatorFromOpSummary { + fn name(&self) -> &str { + &self.name + } + fn type_field(&self) -> &str { + &self.type_field + } + fn duration_us(&self) -> f64 { + self.duration_us + } + fn aiv_mte2_time_us(&self) -> f64 { + self.aiv_mte2_time_us + } + fn aiv_mte3_time_us(&self) -> f64 { + self.aiv_mte3_time_us + } +} + pub fn parse_operator(path: &str, parse_file: ParseFile) -> Result, ParseError> { let file = File::open(path)?; let reader = BufReader::new(file); @@ -86,47 +142,34 @@ pub fn parse_operator(path: &str, parse_file: ParseFile) -> Result // 解析 CSV 文件 let result = match parse_file { ParseFile::KernelDetail => { - parse_kernel_detail(reader)? + parse_csv::<_, OperatorFromKernelDetail>(reader)? }, ParseFile::OpSummary => { - parse_op_summary(reader)? + parse_csv::<_, OperatorFromOpSummary>(reader)? }, }; Ok(result) // 返回解析后的 Operator 列表 } -pub fn parse_kernel_detail(reader: R) -> Result, ParseError> { - let mut reader = Reader::from_reader(reader); - let mut operators = Vec::new(); - - for (row_count, result) in reader.deserialize().enumerate() { - let op_detail: OperatorFromKernelDetail = result?; - if check_csv_injection(&op_detail.name, row_count) || - check_csv_injection(&op_detail.type_field, row_count) { - continue; - } - let operator: Operator = op_detail.into(); // 转换 - operators.push(operator); - } - - if operators.len() < 1 { - return Err(ParseError::EmptyData); - } - Ok(operators) +fn valid_operator(row_count: usize, operator: &T) -> bool { + operator.name() == "" || check_csv_injection(&operator.name(), row_count) || + operator.type_field() == "" || check_csv_injection(&operator.type_field(), row_count) || + operator.duration_us() < 0.0 || operator.aiv_mte2_time_us() < 0.0 || + operator.aiv_mte3_time_us() < 0.0 } -pub fn parse_op_summary(reader: R) -> Result, ParseError> { +pub fn parse_csv(reader: R) -> Result, ParseError> +where T: OperatorInfo + DeserializeOwned + Into, { let mut reader = Reader::from_reader(reader); let mut operators = Vec::new(); for (row_count, result) in reader.deserialize().enumerate() { - let op_summary: OperatorFromOpSummary = result?; - if check_csv_injection(&op_summary.name, row_count) || - check_csv_injection(&op_summary.type_field, row_count) { + let op_detail: T = result?; + if valid_operator(row_count + 1, &op_detail) { continue; } - let operator: Operator = op_summary.into(); // 转换 + let operator: Operator = op_detail.into(); // 转换 operators.push(operator); } @@ -135,6 +178,3 @@ pub fn parse_op_summary(reader: R) -> Result, Pa } Ok(operators) } - - - diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/utils.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/utils.rs index a96f00dc42..2138b733d5 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/utils.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/utils.rs @@ -5,7 +5,6 @@ pub fn check_csv_injection(field: &str, row_count: usize) -> bool { if "=+@-".contains(first_char) { let e_field = escape_special_chars(field); eprintln!("Line {}, Potential CSV formula injection detected, filtered: {}", row_count, &e_field); - // 可选择跳过或转义 return true; } } diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/parser_test.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/parser_test.rs index f9f5c5290a..404c3dcae0 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/parser_test.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/parser_test.rs @@ -4,50 +4,104 @@ #[cfg(test)] mod parser_tests { use std::io::Cursor; - use csv_parser::parser::{parse_kernel_detail, parse_op_summary}; + use csv_parser::parser::{parse_csv, OperatorFromKernelDetail, OperatorFromOpSummary}; #[test] fn test_parse_valid_kernel_detail_csv() { - let csv_data = "Name,Type,Duration(us),aiv_mte2_time(us)\nconv2d,MatMul,120.0,20\nrelu,Active,30,3\nmatmul,MatMul,200,20"; + let csv_data = "Name,Type,Duration(us),aiv_mte2_time(us),aiv_mte3_time(us),Other\nconv2d,MatMul,120.0,20,200,\nrelu,Active,30,3,4,\nmatmul,MatMul,200,20,40,"; let cursor = Cursor::new(csv_data); - let result = parse_kernel_detail(cursor) - .unwrap(); - + let result = parse_csv::<_, OperatorFromKernelDetail>(cursor).unwrap(); assert_eq!(result.len(), 3); assert_eq!(result[0].name, "conv2d"); assert_eq!(result[0].duration_us, 120.0); - assert_eq!(result[0].mte2_time_us, 20.0); + assert_eq!(result[0].memory_move_time_us, 220.0); } #[test] fn test_parse_valid_op_summary_csv() { - let csv_data = "Op Name,Op Type,Task Duration(us),aiv_mte2_time(us)\nconv2d,MatMul,120.0,20\nrelu,Active,30,3\nmatmul,MatMul,200,20"; + let csv_data = "Op Name,Op Type,Task Duration(us),aiv_mte2_time(us),aiv_mte3_time(us),Other\nconv2d,MatMul,120.0,20,200,\nrelu,Active,30,3,4,\nmatmul,MatMul,200,20,40,"; let cursor = Cursor::new(csv_data); - let result = parse_op_summary(cursor) - .unwrap(); - + let result = parse_csv::<_, OperatorFromOpSummary>(cursor).unwrap(); assert_eq!(result.len(), 3); assert_eq!(result[0].name, "conv2d"); assert_eq!(result[0].duration_us, 120.0); - assert_eq!(result[0].mte2_time_us, 20.0); + assert_eq!(result[0].memory_move_time_us, 220.0); + } + + #[test] + fn test_parse_with_duplicate_mte2_column() { + let csv_data = "Name,Type,Duration(us),aiv_mte2_time(us),aiv_mte2_time(us)\nconv2d,MatMul,1,1,0\nrelu,Active,30,3,4"; + let cursor = Cursor::new(csv_data); + let result = parse_csv::<_, OperatorFromKernelDetail>(cursor); + assert!(result.is_err()); + assert_eq!(result.unwrap_err().to_string(), "CSV format error: CSV deserialize error: record 1 (line: 2, byte: 59): duplicate field `aiv_mte2_time(us)`"); + } + + #[test] + fn test_parse_with_missing_mte3_column() { + let csv_data = "Name,Type,Duration(us),aiv_mte2_time(us)\nconv2d,MatMul,1,1\nrelu,Active,30,3"; + let cursor = Cursor::new(csv_data); + let result = parse_csv::<_, OperatorFromKernelDetail>(cursor); + assert!(result.is_err()); + assert_eq!(result.unwrap_err().to_string(), "CSV format error: CSV deserialize error: record 1 (line: 2, byte: 41): missing field `aiv_mte3_time(us)`"); + } + + #[test] + fn test_parse_with_na_duration_column() { + let csv_data = "Name,Type,Duration(us),aiv_mte2_time(us),aiv_mte3_time(us)\nconv2d,MatMul,N/A,N/A,N/A\nrelu,Active,1,1,1"; + let cursor = Cursor::new(csv_data); + let result = parse_csv::<_, OperatorFromKernelDetail>(cursor).unwrap(); + assert_eq!(result[0].name, "relu"); } #[test] fn test_parse_with_invalid_duration() { - let csv_data = "Name,Type,Duration(us),aiv_mte2_time(us)\nconv2d,MatMul,abc,1\nrelu,Active,30,3"; + let csv_data = "Name,Type,Duration(us),aiv_mte2_time(us),aiv_mte3_time(us)\nconv2d,MatMul,abc,1,0\nrelu,Active,30,3,4"; let cursor = Cursor::new(csv_data); - let result = parse_kernel_detail(cursor); + let result = parse_csv::<_, OperatorFromKernelDetail>(cursor); assert!(result.is_err()); - assert_eq!(result.unwrap_err().to_string(), "CSV format error: CSV deserialize error: record 1 (line: 2, byte: 41): empty or invalid number"); + assert_eq!(result.unwrap_err().to_string(), "CSV format error: CSV deserialize error: record 1 (line: 2, byte: 59): empty or invalid number"); } #[test] - fn test_csv_injection_filter() { - let csv_data = "Op Name,Op Type,Task Duration(us),aiv_mte2_time(us)\n=CMD|' /C calc'!A0,A,100,10\nsafe_op,Op,50,5"; + fn test_parse_with_blank_duration_kernel_detail_csv() { + let csv_data = "Name,Type,Duration(us),aiv_mte2_time(us),aiv_mte3_time(us)\nconv2d,MatMul,1,1,0\nrelu,Active,,3,4"; let cursor = Cursor::new(csv_data); - let result = parse_op_summary(cursor) - .unwrap(); + let result = parse_csv::<_, OperatorFromKernelDetail>(cursor); + assert!(result.is_err()); + assert_eq!(result.unwrap_err().to_string(), "CSV format error: CSV deserialize error: record 2 (line: 3, byte: 79): empty or invalid number"); + } + + #[test] + fn test_parse_with_blank_type_kernel_detail_csv() { + let csv_data = "Name,Type,Duration(us),aiv_mte2_time(us),aiv_mte3_time(us)\nconv2d,,1,1,0\nrelu,Active,30,3,4"; + let cursor = Cursor::new(csv_data); + let result = parse_csv::<_, OperatorFromKernelDetail>(cursor).unwrap(); + assert_eq!(result[0].op_type, "Active"); + } + #[test] + fn test_parse_with_blank_duration_op_summary_csv() { + let csv_data = "Op Name,Op Type,Task Duration(us),aiv_mte2_time(us),aiv_mte3_time(us)\nconv2d,MatMul,120.0,20,200\nrelu,Active,,3,4"; + let cursor = Cursor::new(csv_data); + let result = parse_csv::<_, OperatorFromOpSummary>(cursor); + assert!(result.is_err()); + assert_eq!(result.unwrap_err().to_string(), "CSV format error: CSV deserialize error: record 2 (line: 3, byte: 97): empty or invalid number"); + } + + #[test] + fn test_parse_with_blank_name_op_summary_csv() { + let csv_data = "Op Name,Op Type,Task Duration(us),aiv_mte2_time(us),aiv_mte3_time(us)\n,MatMul,120.0,20,200\nrelu,Active,30,3,4"; + let cursor = Cursor::new(csv_data); + let result = parse_csv::<_, OperatorFromOpSummary>(cursor).unwrap(); + assert_eq!(result[0].name, "relu"); + } + + #[test] + fn test_csv_injection_filter() { + let csv_data = "Op Name,Op Type,Task Duration(us),aiv_mte2_time(us),aiv_mte3_time(us)\n=CMD|' /C calc'!A0,A,100,10,20\nsafe_op,Op,50,5,6"; + let cursor = Cursor::new(csv_data); + let result = parse_csv::<_, OperatorFromOpSummary>(cursor).unwrap(); assert_eq!(result.len(), 1); assert_eq!(result[0].name, "safe_op"); // 第一行因以 '=' 开头被过滤 @@ -55,9 +109,9 @@ mod parser_tests { #[test] fn test_empty_csv() { - let csv_data = "Op Name,Op Type,Task Duration(us),aiv_mte2_time(us)"; // 无数据行 + let csv_data = "Op Name,Op Type,Task Duration(us),aiv_mte2_time(us),aiv_mte3_time(us)"; // 无数据行 let cursor = Cursor::new(csv_data); - let result = parse_op_summary(cursor); + let result = parse_csv::<_, OperatorFromOpSummary>(cursor); assert!(result.is_err()); assert_eq!(result.unwrap_err().to_string(), "File is empty or has invalid data"); } diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/test.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/test.rs index 5d030890a9..5376ce0034 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/test.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/test.rs @@ -42,10 +42,10 @@ mod tests { match parse_operator_csv(path) { Ok(ops) => { assert_eq!(ops.len(), 2); - assert_eq!(ops.get("MatMul").unwrap().avg_duration_us, 160.0); - assert_eq!(ops.get("MatMul").unwrap().avg_mte2_time_us, 20.0); + assert_eq!(ops.get("MatMul").unwrap().avg_duration_us, 120.0); + assert_eq!(ops.get("MatMul").unwrap().avg_memory_move_time_us, 23.0); assert_eq!(ops.get("Active").unwrap().avg_duration_us, 30.0); - assert_eq!(ops.get("Active").unwrap().avg_mte2_time_us, 3.0); + assert_eq!(ops.get("Active").unwrap().avg_memory_move_time_us, 7.0); }, Err(err) => panic!("Error occurred while parsing csv: {}", err), } diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/valid/kernel_detail.csv b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/valid/kernel_detail.csv index 48a8cc2198..40b48ac58d 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/valid/kernel_detail.csv +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/valid/kernel_detail.csv @@ -1,4 +1,4 @@ -Name,Type,Duration(us),aiv_mte2_time(us), -conv2d,MatMul,120.0,20, -relu,Active,30,3, -matmul,MatMul,200,20, \ No newline at end of file +Name,Type,Duration(us),aiv_mte2_time(us),aiv_mte3_time(us), +conv2d,MatMul,120.0,20,3, +relu,Active,30,3,4, +matmul,MatMul,200,N/A,5, \ No newline at end of file diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/valid/op_summary_20250812203130.csv b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/valid/op_summary_20250812203130.csv index aecbb02435..6202219399 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/valid/op_summary_20250812203130.csv +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/valid/op_summary_20250812203130.csv @@ -1,4 +1,4 @@ -Op Name,Op Type,Task Duration(us),aiv_mte2_time(us), -conv2d,MatMul,120.0,20, -relu,Active,30,3, -matmul,MatMul,200,20 \ No newline at end of file +Op Name,Op Type,Task Duration(us),aiv_mte2_time(us),aiv_mte3_time(us),other +conv2d,MatMul,120.0,20,1, +relu,Active,30,3,2, +matmul,MatMul,200,20,3, \ No newline at end of file -- Gitee From c27d456a2f6b6434f0cb4c706d55a0d419982139 Mon Sep 17 00:00:00 2001 From: hummel mao Date: Tue, 19 Aug 2025 21:36:24 +0800 Subject: [PATCH 11/11] =?UTF-8?q?fix(csv=5Fparser):=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E5=AE=9E=E9=99=85=E4=BD=BF=E7=94=A8=E4=B8=AD=E9=81=87=E5=88=B0?= =?UTF-8?q?=E7=9A=84=E5=AD=97=E7=AC=A6=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ModelVis/rust/csv_parser/src/lib.rs | 6 ++++-- .../ModelVis/rust/csv_parser/src/parser.rs | 2 +- .../invalid/{kernel_detail.csv => kernel_details.csv} | 0 .../invalid/{kernel_detail.txt => kernel_details.txt} | 0 .../ModelVis/rust/csv_parser/tests/parser_test.rs | 10 +++++----- .../ModelVis/rust/csv_parser/tests/test.rs | 8 ++++---- .../valid/{kernel_detail.csv => kernel_details.csv} | 0 .../tests/valid/op_summary_20250812203130.csv | 2 +- 8 files changed, 15 insertions(+), 13 deletions(-) rename plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/invalid/{kernel_detail.csv => kernel_details.csv} (100%) rename plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/invalid/{kernel_detail.txt => kernel_details.txt} (100%) rename plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/valid/{kernel_detail.csv => kernel_details.csv} (100%) diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs index b37e085fb5..d6b3d216d1 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/lib.rs @@ -11,9 +11,11 @@ pub mod operator; pub mod parser; mod utils; +const KERNEL_DETAIL_CSV: &str = "kernel_details.csv"; + pub fn is_valid_csv(name: &str) -> bool { // 情况1：精确匹配 - if name == "kernel_detail.csv" { + if name == KERNEL_DETAIL_CSV { return true; } @@ -26,7 +28,7 @@ pub fn is_valid_csv(name: &str) -> bool { } pub fn get_parse_file(name: &str) -> ParseFile { - if name == "kernel_detail.csv" { + if name == KERNEL_DETAIL_CSV { return ParseFile::KernelDetail; } ParseFile::OpSummary diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs index bc61d6806c..fc77e21d28 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/src/parser.rs @@ -59,7 +59,7 @@ pub struct OperatorFromOpSummary { #[serde(rename = "Op Name")] pub name: String, - #[serde(rename = "Op Type")] + #[serde(rename = "OP Type")] pub type_field: String, #[serde(rename = "Task Duration(us)", deserialize_with = "parse_micros_f64")] diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/invalid/kernel_detail.csv b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/invalid/kernel_details.csv similarity index 100% rename from plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/invalid/kernel_detail.csv rename to plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/invalid/kernel_details.csv diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/invalid/kernel_detail.txt b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/invalid/kernel_details.txt similarity index 100% rename from plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/invalid/kernel_detail.txt rename to plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/invalid/kernel_details.txt diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/parser_test.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/parser_test.rs index 404c3dcae0..34c0a407d6 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/parser_test.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/parser_test.rs @@ -19,7 +19,7 @@ mod parser_tests { #[test] fn test_parse_valid_op_summary_csv() { - let csv_data = "Op Name,Op Type,Task Duration(us),aiv_mte2_time(us),aiv_mte3_time(us),Other\nconv2d,MatMul,120.0,20,200,\nrelu,Active,30,3,4,\nmatmul,MatMul,200,20,40,"; + let csv_data = "Op Name,OP Type,Task Duration(us),aiv_mte2_time(us),aiv_mte3_time(us),Other\nconv2d,MatMul,120.0,20,200,\nrelu,Active,30,3,4,\nmatmul,MatMul,200,20,40,"; let cursor = Cursor::new(csv_data); let result = parse_csv::<_, OperatorFromOpSummary>(cursor).unwrap(); assert_eq!(result.len(), 3); @@ -82,7 +82,7 @@ mod parser_tests { #[test] fn test_parse_with_blank_duration_op_summary_csv() { - let csv_data = "Op Name,Op Type,Task Duration(us),aiv_mte2_time(us),aiv_mte3_time(us)\nconv2d,MatMul,120.0,20,200\nrelu,Active,,3,4"; + let csv_data = "Op Name,OP Type,Task Duration(us),aiv_mte2_time(us),aiv_mte3_time(us)\nconv2d,MatMul,120.0,20,200\nrelu,Active,,3,4"; let cursor = Cursor::new(csv_data); let result = parse_csv::<_, OperatorFromOpSummary>(cursor); assert!(result.is_err()); @@ -91,7 +91,7 @@ mod parser_tests { #[test] fn test_parse_with_blank_name_op_summary_csv() { - let csv_data = "Op Name,Op Type,Task Duration(us),aiv_mte2_time(us),aiv_mte3_time(us)\n,MatMul,120.0,20,200\nrelu,Active,30,3,4"; + let csv_data = "Op Name,OP Type,Task Duration(us),aiv_mte2_time(us),aiv_mte3_time(us)\n,MatMul,120.0,20,200\nrelu,Active,30,3,4"; let cursor = Cursor::new(csv_data); let result = parse_csv::<_, OperatorFromOpSummary>(cursor).unwrap(); assert_eq!(result[0].name, "relu"); @@ -99,7 +99,7 @@ mod parser_tests { #[test] fn test_csv_injection_filter() { - let csv_data = "Op Name,Op Type,Task Duration(us),aiv_mte2_time(us),aiv_mte3_time(us)\n=CMD|' /C calc'!A0,A,100,10,20\nsafe_op,Op,50,5,6"; + let csv_data = "Op Name,OP Type,Task Duration(us),aiv_mte2_time(us),aiv_mte3_time(us)\n=CMD|' /C calc'!A0,A,100,10,20\nsafe_op,Op,50,5,6"; let cursor = Cursor::new(csv_data); let result = parse_csv::<_, OperatorFromOpSummary>(cursor).unwrap(); assert_eq!(result.len(), 1); @@ -109,7 +109,7 @@ mod parser_tests { #[test] fn test_empty_csv() { - let csv_data = "Op Name,Op Type,Task Duration(us),aiv_mte2_time(us),aiv_mte3_time(us)"; // 无数据行 + let csv_data = "Op Name,OP Type,Task Duration(us),aiv_mte2_time(us),aiv_mte3_time(us)"; // 无数据行 let cursor = Cursor::new(csv_data); let result = parse_csv::<_, OperatorFromOpSummary>(cursor); assert!(result.is_err()); diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/test.rs b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/test.rs index 5376ce0034..58ae1d7a16 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/test.rs +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/test.rs @@ -11,7 +11,7 @@ mod tests { // 获取当前工作目录 let current_dir = env::current_dir().expect("Failed to get current directory"); - let filename = current_dir.join(r#"tests\valid\kernel_detail.csv"#); + let filename = current_dir.join(r#"tests\valid\kernel_details.csv"#); let path = filename.to_str().expect("Failed to convert path to string"); println!("{}", path); let name = get_filename(path).unwrap(); @@ -27,7 +27,7 @@ mod tests { fn test_invalid_path() { // 获取当前工作目录 let current_dir = env::current_dir().expect("Failed to get current directory"); - let filename = current_dir.join(r#"tests\invalid\kernel_detail.txt"#); + let filename = current_dir.join(r#"tests\invalid\kernel_details.txt"#); let path = filename.to_str().expect("Failed to convert path to string"); let name = get_filename(path).unwrap(); assert_eq!(is_valid_csv(&name), false); @@ -37,7 +37,7 @@ mod tests { fn test_parse_valid_csv() { // 获取当前工作目录 let current_dir = env::current_dir().expect("Failed to get current directory"); - let filename = current_dir.join(r#"tests\valid\kernel_detail.csv"#); + let filename = current_dir.join(r#"tests\valid\kernel_details.csv"#); let path = filename.to_str().expect("Failed to convert path to string"); match parse_operator_csv(path) { Ok(ops) => { @@ -55,7 +55,7 @@ mod tests { fn test_parse_invalid_csv() { // 获取当前工作目录 let current_dir = env::current_dir().expect("Failed to get current directory"); - let filename = current_dir.join(r#"tests\invalid\kernel_detail.csv"#); + let filename = current_dir.join(r#"tests\invalid\kernel_details.csv"#); let path = filename.to_str().expect("Failed to convert path to string"); let result = parse_operator_csv(path); assert!(result.is_err()); diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/valid/kernel_detail.csv b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/valid/kernel_details.csv similarity index 100% rename from plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/valid/kernel_detail.csv rename to plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/valid/kernel_details.csv diff --git a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/valid/op_summary_20250812203130.csv b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/valid/op_summary_20250812203130.csv index 6202219399..fd05c1f235 100644 --- a/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/valid/op_summary_20250812203130.csv +++ b/plugins/mindstudio-insight-plugins/ModelVis/rust/csv_parser/tests/valid/op_summary_20250812203130.csv @@ -1,4 +1,4 @@ -Op Name,Op Type,Task Duration(us),aiv_mte2_time(us),aiv_mte3_time(us),other +Op Name,OP Type,Task Duration(us),aiv_mte2_time(us),aiv_mte3_time(us),other conv2d,MatMul,120.0,20,1, relu,Active,30,3,2, matmul,MatMul,200,20,3, \ No newline at end of file -- Gitee