diff --git a/docs/lite/docs/source_zh_cn/tools/benchmark_tool.md b/docs/lite/docs/source_zh_cn/tools/benchmark_tool.md index 7b81829e3b9da280615a925e64ff052c1e3da5ba..e2e3b7d55893c19ec9f1fb1e95a4dfadc272dc1e 100644 --- a/docs/lite/docs/source_zh_cn/tools/benchmark_tool.md +++ b/docs/lite/docs/source_zh_cn/tools/benchmark_tool.md @@ -344,4 +344,238 @@ call benchmark.exe [--modelFile=] [--accuracyThreshold==2.0.2 + - `onnx`>=1.17.0 + - `onnxruntime`>=1.19.2 + - `ast`>=1.6.3 + +2、运行示例 + +``` + python create_bin.py --input_model "/workspace/project/example/model_example.onnx" --save_path "data_example" +``` + + + +3、代码细节 + +- 导入依赖库 + +```python +import numpy as np +import onnx +import os +import onnxruntime as rt +import argparse +import ast +``` + +- 构造随机输入并保存，使用`onnxruntime`推理得到标杆输出 + +```python +def generate_random_input_data_and_run_model(args): + """ + generate random input data and save, and run model + """ + output_file = os.path.join(args.save_path, 'model_onnx.out') + + # Init + if not os.path.exists(os.path.join(args.save_path)): + print(f"mkdir: {args.save_path}") + os.makedirs(args.save_path) + model = onnx.load(args.input_model) + graph = model.graph + org_output_num = len(graph.output) + sess = rt.InferenceSession(args.input_model) + input_tensors = sess.get_inputs() + input_dict = {} + + if(args.input_data_file): # User specifies input + input_data_files = args.input_data_file.split(':') + assert len(input_data_files) == len(input_tensors), "Shape of input data is not compatiable with the model!" + for i, input_tensor in enumerate(input_tensors): + tensor_type = input_tensor.type[7:-1] + if tensor_type == "float": + tensor_type = "float32" + if "int" in input_tensor.type: + tensor_type = "int32" + + dtype = np.dtype(tensor_type) + with open(input_data_files[i],"rb") as file: + input_data = np.fromfile(file, dtype=dtype) + assert len(input_data) == np.prod(input_tensor.shape), "Shape of input data is not compatiable with the model!" + input_data = input_data.reshape(input_tensor.shape) + input_dict[input_tensor.name] = input_data + else: # Generate random input and save + for input_tensor in input_tensors: + input_info = { + "input_name": input_tensor.name, + "type": input_tensor.type, + "shape": input_tensor.shape, + } + print(input_info) + + + for i, input_tensor in enumerate(input_tensors): + tensor_type = input_tensor.type[7:-1] + if tensor_type == "float": + tensor_type = "float32" + + shape_info = input_tensor.shape + if args.input_shape: + stable_shape = ast.literal_eval(args.input_shape) + shape_info = stable_shape[i] + + # generate random data and save + if "int" in tensor_type: + input_data = np.random.uniform(low=0, high=20, size=shape_info).astype(tensor_type) + input_mindir_data = input_data.astype(np.int32) + input_data.astype(np.int32).flatten().tofile(os.path.join(args.save_path, f"input.bin{i}")) + else: + input_data = np.random.uniform(low=-1, high=1, size=shape_info).astype(tensor_type) + input_mindir_data = input_data.astype(tensor_type) + input_data.flatten().tofile(os.path.join(args.save_path, f"input.bin{i}")) + + input_dict[input_tensor.name] = input_data + + # run model + + res = sess.run(None, input_dict) + return res, sess +``` + + + +- 按照上述标杆约定格式，保存输出数据。 + +```python +def save_output(args, sess, res): + opened = 0 + output_file = os.path.join(args.save_path, 'model_onnx.out') + for i in range(len(sess.get_outputs())): + output_name = sess.get_outputs()[i].name + output_shape = sess.get_outputs()[i].shape + output_type = sess.get_outputs()[i].type + output_info = { + "output_name": output_name, + "type": output_type, + "shape": output_shape, + } + print(output_info) + + + if str(sess.get_outputs()[i].shape) == "[]": + output_shape = [1] + + result = np.array(res[i]) + np.save(os.path.join(args.save_path, f"output{i}_{output_name}.npy"), result) + + mode = 'w' if opened == 0 else 'a' + with open(output_file, mode) as text_file: + opened = 1 + if len(output_shape) == 0: + output_shape.append(len(result)) + + print(f"output len: {output_name}, shape: {output_shape}, {len(output_shape)}") + text_file.write(f"{output_name} {len(output_shape)} ") + text_file.write(" ".join([str(s) for s in result.shape])) + text_file.write('\n') + print(f"result shape: {len(result.flatten())}") + + for k in result.flatten(): + text_file.write(f"{k} ") + text_file.write('\n') +``` + + + +- 主函数 + +```python +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Run ONNX model with random input data.") + parser.add_argument('--input_model', type=str, required=True, help="The ONNX model file.") + parser.add_argument('--save_path', type=str, required=True, help="Path to the data directory.") + parser.add_argument('--input_shape', type=str, help="Shape of inputs (e.g."[[1,2],[3],[4,5]]")) + parser.add_argument('--input_data_file', type=str, help='Input your specified data.(e.g."input.bin0:input.bin1")') + args = parser.parse_args() + + res, sess = generate_random_input_data_and_run_model(args) + + save_output(args, sess, res) +``` + +4、其他使用场景 + +支持用户指定输入测试数据，用户提供的标杆数据必须和模型输出拥有相同的数据大小，才能跟模型输出进行对比，以得到推理精度误差量。 + +``` +python create_bin.py --input_model "/workspace/project/example/model_example.onnx" --save_path "data_example" --input_data_file /data/input.bin0:/data/input.bin1" +``` + + + +### 文件结构 + +``` +model_root +├── create_bin.py +├── model_name.onnx +└── data_path + ├── input.bin0 // benchmark 所用的输入文件 + └── input.bin1 + │ ... + ├── output0.npy // 模型预测的最终结果 + └── output1.npy + │ ... + └── model.onnx.out // onnxruntime 输出的标杆结果 +```