From 88d4987269e40a455f4ecf60231472ce33e850c7 Mon Sep 17 00:00:00 2001 From: hehongzhe <935062458@qq.com> Date: Mon, 20 Jan 2025 17:25:35 +0800 Subject: [PATCH] =?UTF-8?q?msprof-analyze=E5=8C=85=E5=90=8D=E5=92=8C?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=E8=A7=84=E8=8C=83=E6=95=B4=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 8 +- profiler/MANIFEST.in | 7 - profiler/{ => msprof_analyze}/LICENSE | 0 profiler/msprof_analyze/MANIFEST.in | 8 + profiler/{ => msprof_analyze}/OWNERS | 24 +- profiler/{ => msprof_analyze}/README.md | 496 ++--- profiler/{ => msprof_analyze}/__init__.py | 0 .../{ => msprof_analyze}/advisor/README.md | 4 +- .../{ => msprof_analyze}/advisor/__init__.py | 0 .../advisor/advisor_backend/__init__.py | 0 .../advisor/advisor_backend/advice_base.py | 0 .../advice_factory/__init__.py | 0 .../advice_factory/advice_factory.py | 0 .../advice_factory/cluster_advice_factory.py | 0 .../advice_factory/compute_advice_factory.py | 0 .../advice_factory/overall_advice_factory.py | 0 .../advice_factory/timeline_advice_factory.py | 0 .../cluster_advice/__init__.py | 0 .../cluster_advice/cluster_advice_base.py | 4 +- .../cluster_advice/cluster_pipeline_advice.py | 4 +- .../cluster_advice/kernel_cluster_advice.py | 6 +- .../cluster_advice/slow_link_advice.py | 2 +- .../cluster_advice/slow_rank_advice.py | 2 +- .../common_func_advisor/__init__.py | 0 .../common_func_advisor/constant.py | 0 .../common_func_advisor/trace_view_json.py | 4 +- .../trace_view_preprocessor.py | 2 +- .../compute_advice/__init__.py | 0 .../compute_advice/compute_advice_base.py | 2 +- .../compute_advice/npu_fused/__init__.py | 0 .../compute_advice/npu_fused/csv_analyzer.py | 0 .../compute_advice/npu_fused/json_analyzer.py | 0 .../compute_advice/npu_fused/op_perf.py | 0 .../compute_advice/npu_fused_advice.py | 0 .../compute_advice/npu_slow_advice.py | 0 .../advisor/advisor_backend/interface.py | 0 .../advisor/advisor_backend/logger.py | 0 .../overall_advice/__init__.py | 0 .../overall_advice/overall_summary_advice.py | 6 +- .../prof_bean_advisor/__init__.py | 0 .../cluster_step_trace_time_bean.py | 0 .../timeline_advice/__init__.py | 0 .../timeline_advice/op_schedule_advice.py | 0 .../timeline_advice/optimizer_advice.py | 0 .../timeline_advice/timeline_advice_base.py | 2 +- .../advisor/analyzer/__init__.py | 0 .../advisor/analyzer/analyzer_controller.py | 1900 ++++++++--------- .../advisor/analyzer/base_analyzer.py | 18 +- .../advisor/analyzer/cluster/__init__.py | 0 .../communication_retransmission_analyzer.py | 10 +- .../communication_retransmission_checker.py | 14 +- .../analyzer/cluster/slow_link_analyzer.py | 14 +- .../analyzer/cluster/slow_rank_analyzer.py | 14 +- .../analyzer/communication/__init__.py | 0 .../communication/alignment/__init__.py | 0 .../alignment/byte_alignment_analyzer.py | 14 +- .../alignment/byte_alignment_checker.py | 20 +- .../communication/bandwidth/__init__.py | 0 .../base_communication_analyzer.py | 44 +- .../communication/contention/__init__.py | 0 .../bandwidth_contention_analyzer.py | 14 +- .../bandwidth_contention_checker.py | 22 +- .../communication/environment/__init__.py | 0 .../analyzer/communication/packet/__init__.py | 0 .../communication/packet/packet_analyzer.py | 12 +- .../communication/packet/packet_checker.py | 16 +- .../communication/retransmission/__init__.py | 0 .../communication_retransmission_analyzer.py | 12 +- .../communication_retransmission_checker.py | 16 +- .../advisor/analyzer/comparison/__init__.py | 0 .../comparison/comparison_analyzer.py | 8 +- .../analyzer/comparison/comparison_checker.py | 10 +- .../advisor/analyzer/computation/__init__.py | 0 .../computation/ai_core_freq/__init__.py | 0 .../ai_core_freq/ai_core_freq_analyzer.py | 16 +- .../ai_core_freq/ai_core_freq_checker.py | 14 +- .../analyzer/computation/aicpu/__init__.py | 0 .../computation/aicpu/aicpu_checker.py | 16 +- .../analyzer/computation/bound/__init__.py | 0 .../computation/bound/block_dim_checker.py | 12 +- .../bound/operator_bound_checker.py | 14 +- .../computation/op_compile/__init__.py | 0 .../op_compile/dynamic_shape_checker.py | 14 +- .../analyzer/computation/operator_checker.py | 20 +- .../pp_stage_computation_analyzer.py | 238 +-- .../computation/profiling_analyzer.py | 20 +- .../advisor/analyzer/dataloader/__init__.py | 0 .../dataloader/dataloader_analyzer.py | 12 +- .../analyzer/dataloader/dataloader_checker.py | 10 +- .../advisor/analyzer/graph_fusion/__init__.py | 0 .../graph_fusion/graph_fusion_analyzer.py | 12 +- .../graph_fusion/graph_fusion_checker.py | 16 +- .../advisor/analyzer/memory/__init__.py | 0 .../analyzer/memory/memory_analyzer.py | 12 +- .../advisor/analyzer/memory/memory_checker.py | 6 +- .../advisor/analyzer/overall/__init__.py | 0 .../overall/environment_variable_analyzer.py | 10 +- .../overall/environment_variable_checker.py | 20 +- .../overall/overall_summary_analyzer.py | 16 +- .../advisor/analyzer/schedule/__init__.py | 0 .../schedule/conjectured_gc/__init__.py | 0 .../conjectured_gc/conjectured_gc_analyzer.py | 12 +- .../conjectured_gc/conjectured_gc_checker.py | 14 +- .../analyzer/schedule/dispatch/__init__.py | 0 .../dispatch/timeline_op_dispatch_analyzer.py | 18 +- .../analyzer/schedule/free_event/__init__.py | 0 .../analyzer/schedule/fusion_ops/__init__.py | 0 .../fusion_ops/fusion_ops_analyzer.py | 24 +- .../fusion_ops/timeline_api_stack_checker.py | 14 +- .../advisor/analyzer/schedule/gc/__init__.py | 0 .../analyzer/schedule/gc/gc_analyzer.py | 12 +- .../analyzer/schedule/gc/gc_checker.py | 16 +- .../analyzer/schedule/syncbn/__init__.py | 0 .../schedule/syncbn/syncbn_analyzer.py | 90 +- .../schedule/syncbn/syncbn_checker.py | 10 +- .../schedule/synchronize_stream/__init__.py | 0 .../synchronize_stream_analyzer.py | 94 +- .../synchronize_stream_checker.py | 20 +- .../schedule/timeline_base_checker.py | 8 +- .../advisor/cluster_perf_analysis.ipynb | 2 +- .../advisor/common/__init__.py | 0 .../advisor/common/analyzer_scopes.py | 0 .../advisor/common/async_analysis_status.py | 54 +- .../advisor/common/enum_params_parser.py | 208 +- .../advisor/common/graph/__init__.py | 0 .../advisor/common/graph/graph.py | 2 +- .../advisor/common/graph/graph_match.py | 0 .../advisor/common/graph/graph_parser.py | 4 +- .../advisor/common/profiling/__init__.py | 0 .../advisor/common/profiling/ge_info.py | 6 +- .../advisor/common/profiling/msprof.py | 4 +- .../advisor/common/profiling/op_summary.py | 6 +- .../advisor/common/profiling/tasktime.py | 4 +- .../advisor/common/timeline/__init__.py | 0 .../advisor/common/timeline/event.py | 0 .../advisor/common/timeline/fusion_ops_db.py | 14 +- .../common/timeline/fusion_ops_rule.py | 2 +- .../timeline/fusion_ops_rule_handler.py | 6 +- .../advisor/common/version_control.py | 0 .../advisor/computation_analysis.ipynb | 6 +- .../advisor/config/__init__.py | 0 .../advisor/config/config.ini | 2 +- .../advisor/config/config.py | 6 +- .../advisor/config/enum_parameters.yaml | 116 +- .../config/profiling_data_version_config.yaml | 0 .../advisor/dataset/__init__.py | 0 .../advisor/dataset/ai_core_freq/__init__.py | 0 .../ai_core_freq/ai_core_freq_dataset.py | 10 +- .../advisor/dataset/cluster/__init__.py | 0 .../dataset/cluster/cluster_dataset.py | 14 +- .../cluster/cluster_step_trace_time_bean.py | 0 .../dataset/cluster/hccl_collection.py | 0 .../advisor/dataset/communication/__init__.py | 0 .../communication/communication_dataset.py | 10 +- .../communication/hccl_detail_dataset.py | 6 +- .../advisor/dataset/dataset.py | 2 +- .../dataset/environment_variable_dataset.py | 4 +- .../advisor/dataset/graph_dataset.py | 8 +- .../advisor/dataset/profiling/__init__.py | 0 .../advisor/dataset/profiling/builder_base.py | 4 +- .../advisor/dataset/profiling/db_manager.py | 0 .../advisor/dataset/profiling/device_info.py | 6 +- .../dataset/profiling/info_collection.py | 2 +- .../dataset/profiling/profiling_dataset.py | 20 +- .../dataset/profiling/profiling_parser.py | 6 +- .../advisor/dataset/timeline_event_dataset.py | 8 +- .../dataset/timeline_op_collector/__init__.py | 0 .../timeline_op_collector.py | 10 +- .../advisor/display/__init__.py | 0 .../advisor/display/html/__init__.py | 0 .../display/html/priority_background_color.py | 38 +- .../advisor/display/html/render.py | 6 +- .../display/html/templates/__init__.py | 0 .../display/html/templates/affinity_api.html | 0 .../html/templates/ai_core_frequency.html | 0 .../html/templates/byte_alignment.html | 0 .../html/templates/cluster_analysis.html | 0 ...communication_retransmission_analysis.html | 0 .../display/html/templates/comparison.html | 48 +- .../html/templates/compute_analysis.html | 0 .../display/html/templates/contention.html | 0 .../html/templates/environment_variable.html | 0 .../display/html/templates/fusion.html | 0 .../advisor/display/html/templates/gc.html | 0 .../advisor/display/html/templates/main.html | 0 .../display/html/templates/memory.html | 42 +- .../html/templates/operator_ai_cpu.html | 0 .../html/templates/operator_block_dim.html | 0 .../html/templates/operator_dispatch.html | 0 .../templates/operator_dynamic_shape.html | 0 .../html/templates/operator_no_bound.html | 0 .../html/templates/overall_analysis.html | 0 .../html/templates/packet_analysis.html | 0 .../pp_stage_computation_analysis.html | 36 +- .../html/templates/slow_dataloader.html | 42 +- .../html/templates/sync_batchnorm.html | 66 +- .../html/templates/synchronize_stream.html | 52 +- .../html/templates/timeline_analysis.html | 0 .../advisor/display/prompt/__init__.py | 0 .../advisor/display/prompt/base_prompt.py | 2 +- .../advisor/display/prompt/cn/__init__.py | 0 .../display/prompt/cn/ai_core_freq_prompt.py | 0 .../display/prompt/cn/block_dim_prompt.py | 0 .../display/prompt/cn/dynamic_shape_prompt.py | 0 .../prompt/cn/environment_variable_prompt.py | 0 .../display/prompt/cn/graph_fusion_prompt.py | 0 .../display/prompt/cn/op_dispatch_prompt.py | 0 .../prompt/cn/operator_bound_prompt.py | 0 .../display/prompt/cn/operator_prompt.py | 0 .../cn/overall_summary_advice_prompt.py | 0 .../prompt/cn/overall_summary_prompt.py | 0 .../prompt/cn/timeline_fusion_ops_prompt.py | 0 .../advisor/display/prompt/en/__init__.py | 0 .../display/prompt/en/ai_core_freq_prompt.py | 0 .../display/prompt/en/block_dim_prompt.py | 0 .../display/prompt/en/dynamic_shape_prompt.py | 0 .../prompt/en/environment_variable_prompt.py | 0 .../display/prompt/en/graph_fusion_prompt.py | 0 .../display/prompt/en/op_dispatch_prompt.py | 0 .../prompt/en/operator_bound_prompt.py | 0 .../display/prompt/en/operator_prompt.py | 0 .../en/overall_summary_advice_prompt.py | 0 .../prompt/en/overall_summary_prompt.py | 0 .../prompt/en/timeline_fusion_ops_prompt.py | 0 .../Samples of AI CPU Operator Replacement.md | 0 ...mples of Fused Operator API Replacement.md | 0 .../advisor/doc/img/Equal.png | Bin .../advisor/doc/img/Mul.png | Bin .../advisor/doc/img/Pytorch_dtype.png | Bin .../advisor/doc/img/TensorEqual.png | Bin .../advisor/doc/img/index by index.png | Bin .../advisor/doc/img/index_put_by_mask.png | Bin .../doc/img/single_op_time_CANN63RC2.png | Bin .../doc/img/single_op_time_CANN70RC1.png | Bin ...2\345\211\215\350\200\227\346\227\266.png" | Bin ...2\345\220\216\350\200\227\346\227\266.png" | Bin .../fusion_operators_api_analysis.ipynb | 2 +- .../advisor/img/advisor_result.PNG | Bin .../{ => msprof_analyze}/advisor/img/all.png | Bin .../advisor/img/bandwidth.png | Bin .../advisor/img/block_dim.png | Bin .../advisor/img/byte_alignment.png | Bin .../advisor/img/cluster.png | Bin .../advisor/img/cluster_1.png | Bin .../advisor/img/cluster_2.png | Bin .../advisor/img/cluster_3.png | Bin .../advisor/img/cluster_4.png | Bin .../advisor/img/cluster_5.png | Bin .../advisor/img/communication.png | Bin .../advisor/img/comparison.png | Bin .../advisor/img/comparison1.png | Bin .../advisor/img/comparison2.png | Bin .../advisor/img/comparison3.png | Bin .../advisor/img/computation.png | Bin .../advisor/img/computation_1.png | Bin .../advisor/img/computation_2.png | Bin .../advisor/img/dataloader.png | Bin .../advisor/img/env_var.png | Bin .../{ => msprof_analyze}/advisor/img/gc.png | Bin .../advisor/img/jupyter_report.PNG | Bin .../advisor/img/memory.png | Bin .../advisor/img/op_no_bound.png | Bin .../advisor/img/overall.png | Bin .../advisor/img/overall_0.png | Bin .../advisor/img/schedule.png | Bin .../advisor/img/schedule_1.png | Bin .../advisor/img/schedule_2.png | Bin .../advisor/img/schedule_3.png | Bin .../advisor/interface/__init__.py | 0 .../advisor/interface/interface.py | 46 +- .../advisor/result/__init__.py | 0 .../advisor/result/item.py | 0 .../advisor/result/result.py | 12 +- .../advisor/rules/__init__.py | 0 .../advisor/rules/cn/__init__.py | 0 .../advisor/rules/cn/aicpu_rules.yaml | 2 +- .../rules/cn/bandwidth_contention.yaml | 0 .../advisor/rules/cn/byte_alignment.yaml | 0 .../advisor/rules/cn/conjectured_gc.yaml | 0 .../advisor/rules/cn/dataloader.yaml | 0 .../rules/cn/environment_variable_info.yaml | 0 .../advisor/rules/cn/gc.yaml | 0 .../advisor/rules/cn/memory.yaml | 0 .../advisor/rules/cn/packet.yaml | 0 .../advisor/rules/cn/rdma_analysis.yaml | 0 .../advisor/rules/cn/sync_batchnorm.yaml | 0 .../advisor/rules/cn/synchronize.yaml | 0 .../advisor/rules/en/__init__.py | 0 .../advisor/rules/en/aicpu_rules.yaml | 2 +- .../rules/en/bandwidth_contention.yaml | 0 .../advisor/rules/en/byte_alignment.yaml | 0 .../advisor/rules/en/conjectured_gc.yaml | 0 .../advisor/rules/en/dataloader.yaml | 0 .../rules/en/environment_variable_info.yaml | 0 .../advisor/rules/en/gc.yaml | 0 .../advisor/rules/en/memory.yaml | 0 .../advisor/rules/en/packet.yaml | 0 .../advisor/rules/en/rdma_analysis.yaml | 0 .../advisor/rules/en/sync_batchnorm.yaml | 0 .../advisor/rules/en/synchronize.yaml | 0 .../advisor/rules/op_fusion_pass.yaml | 0 .../advisor/rules/timeline_fusion_ops.yaml | 0 .../advisor/utils/__init__.py | 0 .../advisor/utils/file.py | 6 +- .../{ => msprof_analyze}/advisor/utils/log.py | 2 +- .../advisor/utils/tools.py | 0 .../advisor/utils/utils.py | 6 +- .../{ => msprof_analyze}/advisor/version.py | 0 .../affinity_cpu_bind/README.md | 0 .../affinity_cpu_bind/bind_core.py | 2 +- profiler/{ => msprof_analyze}/cli/__init__.py | 0 .../{ => msprof_analyze}/cli/analyze_cli.py | 14 +- .../{ => msprof_analyze}/cli/cluster_cli.py | 8 +- .../{ => msprof_analyze}/cli/compare_cli.py | 10 +- .../{ => msprof_analyze}/cli/complete_cli.py | 2 +- profiler/{ => msprof_analyze}/cli/entrance.py | 10 +- .../cluster_analyse/README.md | 314 +-- .../cluster_analyse/__init__.py | 0 .../cluster_analyse/analysis/__init__.py | 0 .../analysis/analysis_facade.py | 2 +- .../cluster_analyse/analysis/base_analysis.py | 4 +- .../analysis/comm_matrix_analysis.py | 2 +- .../analysis/communication_analysis.py | 2 +- .../analysis/host_info_analysis.py | 4 +- .../analysis/step_trace_time_analysis.py | 4 +- .../cluster_analyse/cluster_analysis.py | 10 +- .../cluster_data_preprocess/__init__.py | 0 .../data_preprocessor.py | 0 .../mindspore_data_preprocessor.py | 0 .../pytorch_data_preprocessor.py | 4 +- .../cluster_kernels_analysis/README.md | 0 .../cluster_kernels_analysis/__init__.py | 0 .../cluster_prof_Info_analysis.py | 4 +- .../cluster_analyse/cluster_utils/__init__.py | 0 .../cluster_utils/data_transfer_adapter.py | 2 +- .../cluster_utils/parallel_algorithm.py | 0 .../parallel_strategy_calculator.py | 0 .../cluster_analyse/common_func/__init__.py | 0 .../cluster_analyse/common_func/db_manager.py | 6 +- .../common_func/empty_class.py | 0 .../common_func/path_manager.py | 2 +- .../common_func/table_constant.py | 0 .../common_func/tables_config.py | 0 .../cluster_analyse/common_func/utils.py | 0 .../communication_group/__init__.py | 0 .../base_communication_group.py | 2 +- .../communication_db_group.py | 2 +- .../communication_group_generator.py | 2 +- .../communication_json_group.py | 2 +- .../cluster_analyse/prof_bean/__init__.py | 0 .../prof_bean/communication_bandwidth_bean.py | 0 .../prof_bean/communication_time_bean.py | 0 .../prof_bean/step_trace_time_bean.py | 0 .../cluster_analyse/resources/.keep | 0 .../compare_tools/README.md | 4 +- .../compare_tools/__init__.py | 0 .../compare_tools/compare_backend/__init__.py | 0 .../compare_backend/comparator/__init__.py | 0 .../comparator/api_compare_comparator.py | 2 +- .../comparator/base_comparator.py | 0 .../comparator/communication_comparator.py | 2 +- .../comparator/kernel_compare_comparator.py | 2 +- .../comparator/kernel_type_comparator.py | 2 +- .../comparator/module_comparetor.py | 4 +- .../comparator/module_statistic_comparator.py | 0 .../comparator/operator_comparator.py | 0 .../operator_statistic_comparator.py | 0 .../comparator/overall_metrics_comparator.py | 2 +- .../overall_performance_comparator.py | 2 +- .../compare_backend/compare_bean/__init__.py | 0 .../compare_bean/api_compare_bean.py | 2 +- .../compare_bean/communication_bean.py | 2 +- .../compare_bean/kernel_compare_bean.py | 2 +- .../compare_bean/kernel_type_compare_bean.py | 2 +- .../compare_bean/memory_compare_bean.py | 2 +- .../compare_bean/memory_statistic_bean.py | 2 +- .../compare_bean/module_compare_bean.py | 2 +- .../compare_bean/module_statistic_bean.py | 2 +- .../compare_bean/operator_compare_bean.py | 2 +- .../compare_bean/operator_statistic_bean.py | 2 +- .../compare_bean/origin_data_bean/__init__.py | 0 .../origin_data_bean/compare_event.py | 2 +- .../origin_data_bean/kernel_details_bean.py | 2 +- .../origin_data_bean/memory_record_bean.py | 0 .../origin_data_bean/op_stastic_bean.py | 2 +- .../origin_data_bean/operator_memory_bean.py | 0 .../origin_data_bean/trace_event_bean.py | 2 +- .../compare_bean/overall_metrics_bean.py | 2 +- .../compare_bean/profiling_info.py | 2 +- .../compare_config/__init__.py | 0 .../compare_config/compare_config.ini | 0 .../compare_config/compare_config.py | 2 +- .../compare_backend/comparison_generator.py | 6 +- .../compare_backend/data_prepare/__init__.py | 0 .../data_prepare/module_data_prepare.py | 2 +- .../data_prepare/operator_data_prepare.py | 4 +- .../data_prepare/sequence_pre_matching.py | 2 +- .../compare_backend/disaggregate/__init__.py | 0 .../disaggregate/overall_perf_interface.py | 6 +- .../compare_backend/generator/__init__.py | 0 .../generator/base_generator.py | 0 .../generator/detail_performance_generator.py | 4 +- .../overall_performance_generator.py | 2 +- .../compare_backend/interface/__init__.py | 0 .../interface/compare_interface.py | 2 +- .../interface/overall_interface.py | 2 +- .../profiling_parser/__init__.py | 0 .../profiling_parser/base_profiling_parser.py | 8 +- .../profiling_parser/gpu_profiling_parser.py | 4 +- .../profiling_parser/npu_profiling_parser.py | 8 +- .../compare_backend/utils/__init__.py | 0 .../compare_backend/utils/args_manager.py | 8 +- .../compare_backend/utils/common_func.py | 2 +- .../compare_backend/utils/compare_args.py | 0 .../compare_backend/utils/excel_config.py | 2 +- .../compare_backend/utils/module_node.py | 0 .../compare_backend/utils/name_function.py | 0 .../compare_backend/utils/singleton.py | 0 .../compare_backend/utils/torch_op_node.py | 2 +- .../compare_backend/utils/tree_builder.py | 0 .../compare_backend/view/__init__.py | 0 .../compare_backend/view/base_view.py | 0 .../compare_backend/view/excel_view.py | 2 +- .../compare_backend/view/screen_view.py | 0 .../view/work_sheet_creator.py | 0 .../compare_interface/__init__.py | 0 .../compare_interface/comparison_interface.py | 6 +- .../compare_tools/img/OverallMetrics.png | Bin .../compare_tools/img/config.PNG | Bin .../compare_tools/performance_compare.py | 8 +- .../{ => msprof_analyze}/config/config.ini | 0 .../{ => msprof_analyze}/example/__init__.py | 0 .../example/mstx_torch_plugin/README.md | 0 .../example/mstx_torch_plugin/__init__.py | 0 .../example/mstx_torch_plugin/img/result.png | Bin .../mstx_torch_plugin/mstx_torch_plugin.py | 0 .../{ => msprof_analyze}/example/setup.py | 0 .../prof_common/__init__.py | 2 +- .../prof_common/additional_args_manager.py | 2 +- .../prof_common/analyze_dict.py | 0 .../prof_common/constant.py | 0 .../prof_common/file_manager.py | 10 +- .../prof_common/logger.py | 2 +- .../prof_common/path_manager.py | 4 +- .../{ => msprof_analyze}/prof_common/utils.py | 0 .../{ => msprof_analyze}/requirements.txt | 0 .../requirements/build.txt | 0 .../requirements/tests.txt | 0 profiler/{ => msprof_analyze}/setup.cfg | 0 profiler/{ => msprof_analyze}/setup.py | 16 +- .../{ => msprof_analyze}/test/__init__.py | 0 .../cluster_communication.json | 0 .../cluster_step_trace_time.csv | 0 .../test/resource/event_list.json | 0 .../test/resource/pipeline_view.png | Bin .../test/resource/test.csv | 0 profiler/{ => msprof_analyze}/test/run_st.py | 0 profiler/{ => msprof_analyze}/test/run_ut.py | 2 +- .../{ => msprof_analyze}/test/st/__init__.py | 0 .../test/st/advisor/__init__.py | 0 ...st_advisor_cmd_single_ascend_pt_compare.py | 2 +- .../test/st/advisor/utils.py | 0 .../test/st/cluster_analyse/__init__.py | 0 ...ter_communication_analyzer_bandwidth_db.py | 0 ...luster_communication_analyzer_matrix_db.py | 0 .../cluster_communication_analyzer_time_db.py | 0 .../cluster_step_trace_time_db.py | 0 .../test_cluster_analyse_pytorch_db.py | 6 +- .../test_cluster_analyse_pytorch_text.py | 2 +- .../test/st/compare_tools/__init__.py | 0 ..._cmd_pytorch_npu_npu_enable_api_compare.py | 4 +- ...ch_npu_npu_enable_communication_compare.py | 4 +- ...d_pytorch_npu_npu_enable_kernel_compare.py | 4 +- ...d_pytorch_npu_npu_enable_memory_compare.py | 4 +- ...pytorch_npu_npu_enable_operator_compare.py | 4 +- ...ls_cmd_pytorch_npu_npu_enable_profiling.py | 4 +- ...st_compare_tools_cmd_pytorch_npu_vs_npu.py | 4 +- ...mpare_tools_cmd_pytorch_npu_vs_npu_step.py | 4 +- ...t_advisor_cmd_cluster_ascend_pt_compare.py | 4 +- ...dvisor_cmd_cluster_ascend_pt_no_compare.py | 4 +- ...advisor_cmd_single_ascend_pt_no_compare.py | 4 +- .../{ => msprof_analyze}/test/st/utils.py | 0 .../test_cluster_advice_base.py | 0 .../test_cluster_pipeline_advice.py | 0 .../test_kernel_cluster_advice.py | 2 +- .../cluster_advice/test_slow_link_advice.py | 0 .../cluster_advice/test_slow_rank_advice.py | 0 .../compute_advice/kernel_details.csv | 0 .../compute_advice/test_npu_slow_advice.py | 0 .../compute_advice/test_npufused_advice.py | 0 .../test_cluster_step_trace_time_bean.py | 0 .../timeline_advice/test_opsche_advice.py | 0 .../timeline_advice/test_optimizer_advice.py | 0 .../timeline_advice/trace_view.json | 0 .../advisor/advisor_backend/tools/__init__.py | 0 .../ut/advisor/advisor_backend/tools/tool.py | 0 .../test/ut/advisor/common/__init__.py | 0 .../advisor/common/test_enum_params_parser.py | 110 +- .../test_bandwidth_contention_advice.py | 4 +- .../test_byte_alignment_analyzer.py | 4 +- .../test_packet_advice.py | 4 +- .../test_rdma_retransmission_advice.py | 4 +- .../compute_advice/test_frequency_advice.py | 6 +- .../test_pp_stage_computation_analyzer.py | 4 +- .../test_conjectured_gc_advice.py | 4 +- .../test_dataloader_checker.py | 6 +- .../timeline_advice/test_gc_checker.py | 4 +- .../timeline_advice/test_memory_op_checker.py | 6 +- .../timeline_advice/test_syncbn_checker.py | 6 +- .../test_synchronize_stream.py | 10 +- .../test_timeline_op_collector.py | 282 +-- .../test_timeline_op_compile_checker.py | 14 +- .../advisor/timeline_advice/trace_view.json | 0 .../test_pytorch_data_preprocessor.py | 0 .../test_step_trace_time_analysis.py | 2 +- .../test_parallel_strategy_calculator.py | 0 .../common_func/test_file_manager.py | 2 +- .../common_func/test_path_manager.py | 2 +- .../test_communication_group_generator.py | 2 +- .../prof_bean/test_step_trace_time_bean.py | 0 .../test/ut/compare_tools/__init__.py | 0 .../test_communication_comparator.py | 0 .../comparator/test_operator_comparator.py | 0 .../test_operator_statistic_comparator.py | 0 .../ut/compare_tools/compare_bean/__init__.py | 0 .../compare_bean/origin_data_bean/__init__.py | 0 .../origin_data_bean/test_compare_event.py | 0 .../test_kernel_details_bean.py | 0 .../test_memory_record_bean.py | 0 .../test_operator_memory_bean.py | 0 .../origin_data_bean/test_trace_event_bean.py | 0 .../compare_bean/test_communication_bean.py | 0 .../compare_bean/test_memory_compare_bean.py | 0 .../test_memory_statistic_bean.py | 0 .../test_operator_compare_bean.py | 0 .../test_operator_statistic_bean.py | 0 .../compare_bean/test_profiling_info.py | 0 .../profiling_parser/__init__.py | 0 .../test_base_profiling_parser.py | 0 .../test_gpu_profiling_parser.py | 0 .../test_npu_profiling_parser.py | 0 .../compare_tools/utils/test_name_function.py | 0 .../compare_tools/utils/test_tree_builder.py | 0 .../test/ut/compare_tools/view/__init__.py | 0 .../ut/compare_tools/view/test_excel_view.py | 0 .../ut/compare_tools/view/test_screen_view.py | 0 .../view/test_worker_sheet_creator.py | 0 profiler/{ => msprof_analyze}/version.txt | 0 ...347\275\221URL\350\257\264\346\230\216.md" | 20 +- 549 files changed, 2826 insertions(+), 2823 deletions(-) delete mode 100644 profiler/MANIFEST.in rename profiler/{ => msprof_analyze}/LICENSE (100%) create mode 100644 profiler/msprof_analyze/MANIFEST.in rename profiler/{ => msprof_analyze}/OWNERS (92%) rename profiler/{ => msprof_analyze}/README.md (96%) rename profiler/{ => msprof_analyze}/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/README.md (98%) rename profiler/{ => msprof_analyze}/advisor/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/advice_base.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/advice_factory/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/advice_factory/advice_factory.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/advice_factory/cluster_advice_factory.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/advice_factory/compute_advice_factory.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/advice_factory/overall_advice_factory.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/advice_factory/timeline_advice_factory.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/cluster_advice/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/cluster_advice/cluster_advice_base.py (94%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py (99%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/cluster_advice/kernel_cluster_advice.py (95%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/cluster_advice/slow_link_advice.py (98%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/cluster_advice/slow_rank_advice.py (98%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/common_func_advisor/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/common_func_advisor/constant.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/common_func_advisor/trace_view_json.py (98%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/common_func_advisor/trace_view_preprocessor.py (99%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/compute_advice/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/compute_advice/compute_advice_base.py (98%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/compute_advice/npu_fused/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/compute_advice/npu_fused/csv_analyzer.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/compute_advice/npu_fused/json_analyzer.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/compute_advice/npu_fused/op_perf.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/compute_advice/npu_fused_advice.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/compute_advice/npu_slow_advice.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/interface.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/logger.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/overall_advice/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/overall_advice/overall_summary_advice.py (97%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/prof_bean_advisor/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/prof_bean_advisor/cluster_step_trace_time_bean.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/timeline_advice/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/timeline_advice/op_schedule_advice.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/timeline_advice/optimizer_advice.py (100%) rename profiler/{ => msprof_analyze}/advisor/advisor_backend/timeline_advice/timeline_advice_base.py (98%) rename profiler/{ => msprof_analyze}/advisor/analyzer/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/analyzer_controller.py (95%) rename profiler/{ => msprof_analyze}/advisor/analyzer/base_analyzer.py (92%) rename profiler/{ => msprof_analyze}/advisor/analyzer/cluster/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/cluster/communication_retransmission_analyzer.py (79%) rename profiler/{ => msprof_analyze}/advisor/analyzer/cluster/communication_retransmission_checker.py (91%) rename profiler/{ => msprof_analyze}/advisor/analyzer/cluster/slow_link_analyzer.py (94%) rename profiler/{ => msprof_analyze}/advisor/analyzer/cluster/slow_rank_analyzer.py (95%) rename profiler/{ => msprof_analyze}/advisor/analyzer/communication/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/communication/alignment/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/communication/alignment/byte_alignment_analyzer.py (75%) rename profiler/{ => msprof_analyze}/advisor/analyzer/communication/alignment/byte_alignment_checker.py (89%) rename profiler/{ => msprof_analyze}/advisor/analyzer/communication/bandwidth/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/communication/base_communication_analyzer.py (90%) rename profiler/{ => msprof_analyze}/advisor/analyzer/communication/contention/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/communication/contention/bandwidth_contention_analyzer.py (77%) rename profiler/{ => msprof_analyze}/advisor/analyzer/communication/contention/bandwidth_contention_checker.py (89%) rename profiler/{ => msprof_analyze}/advisor/analyzer/communication/environment/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/communication/packet/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/communication/packet/packet_analyzer.py (78%) rename profiler/{ => msprof_analyze}/advisor/analyzer/communication/packet/packet_checker.py (91%) rename profiler/{ => msprof_analyze}/advisor/analyzer/communication/retransmission/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/communication/retransmission/communication_retransmission_analyzer.py (77%) rename profiler/{ => msprof_analyze}/advisor/analyzer/communication/retransmission/communication_retransmission_checker.py (91%) rename profiler/{ => msprof_analyze}/advisor/analyzer/comparison/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/comparison/comparison_analyzer.py (86%) rename profiler/{ => msprof_analyze}/advisor/analyzer/comparison/comparison_checker.py (93%) rename profiler/{ => msprof_analyze}/advisor/analyzer/computation/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/computation/ai_core_freq/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py (76%) rename profiler/{ => msprof_analyze}/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py (91%) rename profiler/{ => msprof_analyze}/advisor/analyzer/computation/aicpu/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/computation/aicpu/aicpu_checker.py (95%) rename profiler/{ => msprof_analyze}/advisor/analyzer/computation/bound/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/computation/bound/block_dim_checker.py (91%) rename profiler/{ => msprof_analyze}/advisor/analyzer/computation/bound/operator_bound_checker.py (86%) rename profiler/{ => msprof_analyze}/advisor/analyzer/computation/op_compile/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/computation/op_compile/dynamic_shape_checker.py (87%) rename profiler/{ => msprof_analyze}/advisor/analyzer/computation/operator_checker.py (94%) rename profiler/{ => msprof_analyze}/advisor/analyzer/computation/pp_stage_computation_analyzer.py (88%) rename profiler/{ => msprof_analyze}/advisor/analyzer/computation/profiling_analyzer.py (83%) rename profiler/{ => msprof_analyze}/advisor/analyzer/dataloader/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/dataloader/dataloader_analyzer.py (77%) rename profiler/{ => msprof_analyze}/advisor/analyzer/dataloader/dataloader_checker.py (91%) rename profiler/{ => msprof_analyze}/advisor/analyzer/graph_fusion/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/graph_fusion/graph_fusion_analyzer.py (82%) rename profiler/{ => msprof_analyze}/advisor/analyzer/graph_fusion/graph_fusion_checker.py (94%) rename profiler/{ => msprof_analyze}/advisor/analyzer/memory/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/memory/memory_analyzer.py (79%) rename profiler/{ => msprof_analyze}/advisor/analyzer/memory/memory_checker.py (93%) rename profiler/{ => msprof_analyze}/advisor/analyzer/overall/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/overall/environment_variable_analyzer.py (81%) rename profiler/{ => msprof_analyze}/advisor/analyzer/overall/environment_variable_checker.py (84%) rename profiler/{ => msprof_analyze}/advisor/analyzer/overall/overall_summary_analyzer.py (94%) rename profiler/{ => msprof_analyze}/advisor/analyzer/schedule/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/schedule/conjectured_gc/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/schedule/conjectured_gc/conjectured_gc_analyzer.py (75%) rename profiler/{ => msprof_analyze}/advisor/analyzer/schedule/conjectured_gc/conjectured_gc_checker.py (94%) rename profiler/{ => msprof_analyze}/advisor/analyzer/schedule/dispatch/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py (87%) rename profiler/{ => msprof_analyze}/advisor/analyzer/schedule/free_event/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/schedule/fusion_ops/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py (93%) rename profiler/{ => msprof_analyze}/advisor/analyzer/schedule/fusion_ops/timeline_api_stack_checker.py (93%) rename profiler/{ => msprof_analyze}/advisor/analyzer/schedule/gc/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/schedule/gc/gc_analyzer.py (79%) rename profiler/{ => msprof_analyze}/advisor/analyzer/schedule/gc/gc_checker.py (89%) rename profiler/{ => msprof_analyze}/advisor/analyzer/schedule/syncbn/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/schedule/syncbn/syncbn_analyzer.py (76%) rename profiler/{ => msprof_analyze}/advisor/analyzer/schedule/syncbn/syncbn_checker.py (89%) rename profiler/{ => msprof_analyze}/advisor/analyzer/schedule/synchronize_stream/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py (79%) rename profiler/{ => msprof_analyze}/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py (86%) rename profiler/{ => msprof_analyze}/advisor/analyzer/schedule/timeline_base_checker.py (92%) rename profiler/{ => msprof_analyze}/advisor/cluster_perf_analysis.ipynb (99%) rename profiler/{ => msprof_analyze}/advisor/common/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/common/analyzer_scopes.py (100%) rename profiler/{ => msprof_analyze}/advisor/common/async_analysis_status.py (97%) rename profiler/{ => msprof_analyze}/advisor/common/enum_params_parser.py (92%) rename profiler/{ => msprof_analyze}/advisor/common/graph/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/common/graph/graph.py (97%) rename profiler/{ => msprof_analyze}/advisor/common/graph/graph_match.py (100%) rename profiler/{ => msprof_analyze}/advisor/common/graph/graph_parser.py (99%) rename profiler/{ => msprof_analyze}/advisor/common/profiling/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/common/profiling/ge_info.py (90%) rename profiler/{ => msprof_analyze}/advisor/common/profiling/msprof.py (96%) rename profiler/{ => msprof_analyze}/advisor/common/profiling/op_summary.py (92%) rename profiler/{ => msprof_analyze}/advisor/common/profiling/tasktime.py (94%) rename profiler/{ => msprof_analyze}/advisor/common/timeline/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/common/timeline/event.py (100%) rename profiler/{ => msprof_analyze}/advisor/common/timeline/fusion_ops_db.py (96%) rename profiler/{ => msprof_analyze}/advisor/common/timeline/fusion_ops_rule.py (98%) rename profiler/{ => msprof_analyze}/advisor/common/timeline/fusion_ops_rule_handler.py (98%) rename profiler/{ => msprof_analyze}/advisor/common/version_control.py (100%) rename profiler/{ => msprof_analyze}/advisor/computation_analysis.ipynb (99%) rename profiler/{ => msprof_analyze}/advisor/config/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/config/config.ini (93%) rename profiler/{ => msprof_analyze}/advisor/config/config.py (96%) rename profiler/{ => msprof_analyze}/advisor/config/enum_parameters.yaml (94%) rename profiler/{ => msprof_analyze}/advisor/config/profiling_data_version_config.yaml (100%) rename profiler/{ => msprof_analyze}/advisor/dataset/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/dataset/ai_core_freq/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py (93%) rename profiler/{ => msprof_analyze}/advisor/dataset/cluster/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/dataset/cluster/cluster_dataset.py (94%) rename profiler/{ => msprof_analyze}/advisor/dataset/cluster/cluster_step_trace_time_bean.py (100%) rename profiler/{ => msprof_analyze}/advisor/dataset/cluster/hccl_collection.py (100%) rename profiler/{ => msprof_analyze}/advisor/dataset/communication/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/dataset/communication/communication_dataset.py (92%) rename profiler/{ => msprof_analyze}/advisor/dataset/communication/hccl_detail_dataset.py (93%) rename profiler/{ => msprof_analyze}/advisor/dataset/dataset.py (96%) rename profiler/{ => msprof_analyze}/advisor/dataset/environment_variable_dataset.py (92%) rename profiler/{ => msprof_analyze}/advisor/dataset/graph_dataset.py (87%) rename profiler/{ => msprof_analyze}/advisor/dataset/profiling/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/dataset/profiling/builder_base.py (91%) rename profiler/{ => msprof_analyze}/advisor/dataset/profiling/db_manager.py (100%) rename profiler/{ => msprof_analyze}/advisor/dataset/profiling/device_info.py (92%) rename profiler/{ => msprof_analyze}/advisor/dataset/profiling/info_collection.py (99%) rename profiler/{ => msprof_analyze}/advisor/dataset/profiling/profiling_dataset.py (86%) rename profiler/{ => msprof_analyze}/advisor/dataset/profiling/profiling_parser.py (95%) rename profiler/{ => msprof_analyze}/advisor/dataset/timeline_event_dataset.py (96%) rename profiler/{ => msprof_analyze}/advisor/dataset/timeline_op_collector/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/dataset/timeline_op_collector/timeline_op_collector.py (97%) rename profiler/{ => msprof_analyze}/advisor/display/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/html/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/html/priority_background_color.py (97%) rename profiler/{ => msprof_analyze}/advisor/display/html/render.py (95%) rename profiler/{ => msprof_analyze}/advisor/display/html/templates/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/html/templates/affinity_api.html (100%) rename profiler/{ => msprof_analyze}/advisor/display/html/templates/ai_core_frequency.html (100%) rename profiler/{ => msprof_analyze}/advisor/display/html/templates/byte_alignment.html (100%) rename profiler/{ => msprof_analyze}/advisor/display/html/templates/cluster_analysis.html (100%) rename profiler/{ => msprof_analyze}/advisor/display/html/templates/communication_retransmission_analysis.html (100%) rename profiler/{ => msprof_analyze}/advisor/display/html/templates/comparison.html (97%) rename profiler/{ => msprof_analyze}/advisor/display/html/templates/compute_analysis.html (100%) rename profiler/{ => msprof_analyze}/advisor/display/html/templates/contention.html (100%) rename profiler/{ => msprof_analyze}/advisor/display/html/templates/environment_variable.html (100%) rename profiler/{ => msprof_analyze}/advisor/display/html/templates/fusion.html (100%) rename profiler/{ => msprof_analyze}/advisor/display/html/templates/gc.html (100%) rename profiler/{ => msprof_analyze}/advisor/display/html/templates/main.html (100%) rename profiler/{ => msprof_analyze}/advisor/display/html/templates/memory.html (96%) rename profiler/{ => msprof_analyze}/advisor/display/html/templates/operator_ai_cpu.html (100%) rename profiler/{ => msprof_analyze}/advisor/display/html/templates/operator_block_dim.html (100%) rename profiler/{ => msprof_analyze}/advisor/display/html/templates/operator_dispatch.html (100%) rename profiler/{ => msprof_analyze}/advisor/display/html/templates/operator_dynamic_shape.html (100%) rename profiler/{ => msprof_analyze}/advisor/display/html/templates/operator_no_bound.html (100%) rename profiler/{ => msprof_analyze}/advisor/display/html/templates/overall_analysis.html (100%) rename profiler/{ => msprof_analyze}/advisor/display/html/templates/packet_analysis.html (100%) rename profiler/{ => msprof_analyze}/advisor/display/html/templates/pp_stage_computation_analysis.html (97%) rename profiler/{ => msprof_analyze}/advisor/display/html/templates/slow_dataloader.html (96%) rename profiler/{ => msprof_analyze}/advisor/display/html/templates/sync_batchnorm.html (97%) rename profiler/{ => msprof_analyze}/advisor/display/html/templates/synchronize_stream.html (97%) rename profiler/{ => msprof_analyze}/advisor/display/html/templates/timeline_analysis.html (100%) rename profiler/{ => msprof_analyze}/advisor/display/prompt/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/prompt/base_prompt.py (95%) rename profiler/{ => msprof_analyze}/advisor/display/prompt/cn/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/prompt/cn/ai_core_freq_prompt.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/prompt/cn/block_dim_prompt.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/prompt/cn/dynamic_shape_prompt.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/prompt/cn/environment_variable_prompt.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/prompt/cn/graph_fusion_prompt.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/prompt/cn/op_dispatch_prompt.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/prompt/cn/operator_bound_prompt.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/prompt/cn/operator_prompt.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/prompt/cn/overall_summary_advice_prompt.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/prompt/cn/overall_summary_prompt.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/prompt/cn/timeline_fusion_ops_prompt.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/prompt/en/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/prompt/en/ai_core_freq_prompt.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/prompt/en/block_dim_prompt.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/prompt/en/dynamic_shape_prompt.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/prompt/en/environment_variable_prompt.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/prompt/en/graph_fusion_prompt.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/prompt/en/op_dispatch_prompt.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/prompt/en/operator_bound_prompt.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/prompt/en/operator_prompt.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/prompt/en/overall_summary_advice_prompt.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/prompt/en/overall_summary_prompt.py (100%) rename profiler/{ => msprof_analyze}/advisor/display/prompt/en/timeline_fusion_ops_prompt.py (100%) rename profiler/{ => msprof_analyze}/advisor/doc/Samples of AI CPU Operator Replacement.md (100%) rename profiler/{ => msprof_analyze}/advisor/doc/Samples of Fused Operator API Replacement.md (100%) rename profiler/{ => msprof_analyze}/advisor/doc/img/Equal.png (100%) rename profiler/{ => msprof_analyze}/advisor/doc/img/Mul.png (100%) rename profiler/{ => msprof_analyze}/advisor/doc/img/Pytorch_dtype.png (100%) rename profiler/{ => msprof_analyze}/advisor/doc/img/TensorEqual.png (100%) rename profiler/{ => msprof_analyze}/advisor/doc/img/index by index.png (100%) rename profiler/{ => msprof_analyze}/advisor/doc/img/index_put_by_mask.png (100%) rename profiler/{ => msprof_analyze}/advisor/doc/img/single_op_time_CANN63RC2.png (100%) rename profiler/{ => msprof_analyze}/advisor/doc/img/single_op_time_CANN70RC1.png (100%) rename "profiler/advisor/doc/img/\346\233\277\346\215\242\345\211\215\350\200\227\346\227\266.png" => "profiler/msprof_analyze/advisor/doc/img/\346\233\277\346\215\242\345\211\215\350\200\227\346\227\266.png" (100%) rename "profiler/advisor/doc/img/\346\233\277\346\215\242\345\220\216\350\200\227\346\227\266.png" => "profiler/msprof_analyze/advisor/doc/img/\346\233\277\346\215\242\345\220\216\350\200\227\346\227\266.png" (100%) rename profiler/{ => msprof_analyze}/advisor/fusion_operators_api_analysis.ipynb (99%) rename profiler/{ => msprof_analyze}/advisor/img/advisor_result.PNG (100%) rename profiler/{ => msprof_analyze}/advisor/img/all.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/bandwidth.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/block_dim.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/byte_alignment.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/cluster.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/cluster_1.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/cluster_2.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/cluster_3.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/cluster_4.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/cluster_5.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/communication.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/comparison.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/comparison1.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/comparison2.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/comparison3.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/computation.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/computation_1.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/computation_2.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/dataloader.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/env_var.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/gc.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/jupyter_report.PNG (100%) rename profiler/{ => msprof_analyze}/advisor/img/memory.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/op_no_bound.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/overall.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/overall_0.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/schedule.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/schedule_1.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/schedule_2.png (100%) rename profiler/{ => msprof_analyze}/advisor/img/schedule_3.png (100%) rename profiler/{ => msprof_analyze}/advisor/interface/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/interface/interface.py (69%) rename profiler/{ => msprof_analyze}/advisor/result/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/result/item.py (100%) rename profiler/{ => msprof_analyze}/advisor/result/result.py (95%) rename profiler/{ => msprof_analyze}/advisor/rules/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/rules/cn/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/rules/cn/aicpu_rules.yaml (97%) rename profiler/{ => msprof_analyze}/advisor/rules/cn/bandwidth_contention.yaml (100%) rename profiler/{ => msprof_analyze}/advisor/rules/cn/byte_alignment.yaml (100%) rename profiler/{ => msprof_analyze}/advisor/rules/cn/conjectured_gc.yaml (100%) rename profiler/{ => msprof_analyze}/advisor/rules/cn/dataloader.yaml (100%) rename profiler/{ => msprof_analyze}/advisor/rules/cn/environment_variable_info.yaml (100%) rename profiler/{ => msprof_analyze}/advisor/rules/cn/gc.yaml (100%) rename profiler/{ => msprof_analyze}/advisor/rules/cn/memory.yaml (100%) rename profiler/{ => msprof_analyze}/advisor/rules/cn/packet.yaml (100%) rename profiler/{ => msprof_analyze}/advisor/rules/cn/rdma_analysis.yaml (100%) rename profiler/{ => msprof_analyze}/advisor/rules/cn/sync_batchnorm.yaml (100%) rename profiler/{ => msprof_analyze}/advisor/rules/cn/synchronize.yaml (100%) rename profiler/{ => msprof_analyze}/advisor/rules/en/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/rules/en/aicpu_rules.yaml (97%) rename profiler/{ => msprof_analyze}/advisor/rules/en/bandwidth_contention.yaml (100%) rename profiler/{ => msprof_analyze}/advisor/rules/en/byte_alignment.yaml (100%) rename profiler/{ => msprof_analyze}/advisor/rules/en/conjectured_gc.yaml (100%) rename profiler/{ => msprof_analyze}/advisor/rules/en/dataloader.yaml (100%) rename profiler/{ => msprof_analyze}/advisor/rules/en/environment_variable_info.yaml (100%) rename profiler/{ => msprof_analyze}/advisor/rules/en/gc.yaml (100%) rename profiler/{ => msprof_analyze}/advisor/rules/en/memory.yaml (100%) rename profiler/{ => msprof_analyze}/advisor/rules/en/packet.yaml (100%) rename profiler/{ => msprof_analyze}/advisor/rules/en/rdma_analysis.yaml (100%) rename profiler/{ => msprof_analyze}/advisor/rules/en/sync_batchnorm.yaml (100%) rename profiler/{ => msprof_analyze}/advisor/rules/en/synchronize.yaml (100%) rename profiler/{ => msprof_analyze}/advisor/rules/op_fusion_pass.yaml (100%) rename profiler/{ => msprof_analyze}/advisor/rules/timeline_fusion_ops.yaml (100%) rename profiler/{ => msprof_analyze}/advisor/utils/__init__.py (100%) rename profiler/{ => msprof_analyze}/advisor/utils/file.py (93%) rename profiler/{ => msprof_analyze}/advisor/utils/log.py (97%) rename profiler/{ => msprof_analyze}/advisor/utils/tools.py (100%) rename profiler/{ => msprof_analyze}/advisor/utils/utils.py (98%) rename profiler/{ => msprof_analyze}/advisor/version.py (100%) rename profiler/{ => msprof_analyze}/affinity_cpu_bind/README.md (100%) rename profiler/{ => msprof_analyze}/affinity_cpu_bind/bind_core.py (99%) rename profiler/{ => msprof_analyze}/cli/__init__.py (100%) rename profiler/{ => msprof_analyze}/cli/analyze_cli.py (93%) rename profiler/{ => msprof_analyze}/cli/cluster_cli.py (86%) rename profiler/{ => msprof_analyze}/cli/compare_cli.py (90%) rename profiler/{ => msprof_analyze}/cli/complete_cli.py (96%) rename profiler/{ => msprof_analyze}/cli/entrance.py (87%) rename profiler/{ => msprof_analyze}/cluster_analyse/README.md (98%) rename profiler/{ => msprof_analyze}/cluster_analyse/__init__.py (100%) rename profiler/{ => msprof_analyze}/cluster_analyse/analysis/__init__.py (100%) rename profiler/{ => msprof_analyze}/cluster_analyse/analysis/analysis_facade.py (97%) rename profiler/{ => msprof_analyze}/cluster_analyse/analysis/base_analysis.py (96%) rename profiler/{ => msprof_analyze}/cluster_analyse/analysis/comm_matrix_analysis.py (99%) rename profiler/{ => msprof_analyze}/cluster_analyse/analysis/communication_analysis.py (99%) rename profiler/{ => msprof_analyze}/cluster_analyse/analysis/host_info_analysis.py (97%) rename profiler/{ => msprof_analyze}/cluster_analyse/analysis/step_trace_time_analysis.py (98%) rename profiler/{ => msprof_analyze}/cluster_analyse/cluster_analysis.py (95%) rename profiler/{ => msprof_analyze}/cluster_analyse/cluster_data_preprocess/__init__.py (100%) rename profiler/{ => msprof_analyze}/cluster_analyse/cluster_data_preprocess/data_preprocessor.py (100%) rename profiler/{ => msprof_analyze}/cluster_analyse/cluster_data_preprocess/mindspore_data_preprocessor.py (100%) rename profiler/{ => msprof_analyze}/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py (94%) rename profiler/{ => msprof_analyze}/cluster_analyse/cluster_kernels_analysis/README.md (100%) rename profiler/{ => msprof_analyze}/cluster_analyse/cluster_kernels_analysis/__init__.py (100%) rename profiler/{ => msprof_analyze}/cluster_analyse/cluster_kernels_analysis/cluster_prof_Info_analysis.py (99%) rename profiler/{ => msprof_analyze}/cluster_analyse/cluster_utils/__init__.py (100%) rename profiler/{ => msprof_analyze}/cluster_analyse/cluster_utils/data_transfer_adapter.py (99%) rename profiler/{ => msprof_analyze}/cluster_analyse/cluster_utils/parallel_algorithm.py (100%) rename profiler/{ => msprof_analyze}/cluster_analyse/cluster_utils/parallel_strategy_calculator.py (100%) rename profiler/{ => msprof_analyze}/cluster_analyse/common_func/__init__.py (100%) rename profiler/{ => msprof_analyze}/cluster_analyse/common_func/db_manager.py (97%) rename profiler/{ => msprof_analyze}/cluster_analyse/common_func/empty_class.py (100%) rename profiler/{ => msprof_analyze}/cluster_analyse/common_func/path_manager.py (98%) rename profiler/{ => msprof_analyze}/cluster_analyse/common_func/table_constant.py (100%) rename profiler/{ => msprof_analyze}/cluster_analyse/common_func/tables_config.py (100%) rename profiler/{ => msprof_analyze}/cluster_analyse/common_func/utils.py (100%) rename profiler/{ => msprof_analyze}/cluster_analyse/communication_group/__init__.py (100%) rename profiler/{ => msprof_analyze}/cluster_analyse/communication_group/base_communication_group.py (99%) rename profiler/{ => msprof_analyze}/cluster_analyse/communication_group/communication_db_group.py (99%) rename profiler/{ => msprof_analyze}/cluster_analyse/communication_group/communication_group_generator.py (96%) rename profiler/{ => msprof_analyze}/cluster_analyse/communication_group/communication_json_group.py (96%) rename profiler/{ => msprof_analyze}/cluster_analyse/prof_bean/__init__.py (100%) rename profiler/{ => msprof_analyze}/cluster_analyse/prof_bean/communication_bandwidth_bean.py (100%) rename profiler/{ => msprof_analyze}/cluster_analyse/prof_bean/communication_time_bean.py (100%) rename profiler/{ => msprof_analyze}/cluster_analyse/prof_bean/step_trace_time_bean.py (100%) rename profiler/{ => msprof_analyze}/cluster_analyse/resources/.keep (100%) rename profiler/{ => msprof_analyze}/compare_tools/README.md (98%) rename profiler/{ => msprof_analyze}/compare_tools/__init__.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/__init__.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/comparator/__init__.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/comparator/api_compare_comparator.py (97%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/comparator/base_comparator.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/comparator/communication_comparator.py (96%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/comparator/kernel_compare_comparator.py (97%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/comparator/kernel_type_comparator.py (96%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/comparator/module_comparetor.py (95%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/comparator/module_statistic_comparator.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/comparator/operator_comparator.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/comparator/operator_statistic_comparator.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/comparator/overall_metrics_comparator.py (96%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/comparator/overall_performance_comparator.py (99%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/compare_bean/__init__.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/compare_bean/api_compare_bean.py (97%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/compare_bean/communication_bean.py (98%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py (98%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/compare_bean/kernel_type_compare_bean.py (97%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/compare_bean/memory_compare_bean.py (97%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/compare_bean/memory_statistic_bean.py (97%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/compare_bean/module_compare_bean.py (98%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/compare_bean/module_statistic_bean.py (99%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/compare_bean/operator_compare_bean.py (97%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/compare_bean/operator_statistic_bean.py (97%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/compare_bean/origin_data_bean/__init__.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/compare_bean/origin_data_bean/compare_event.py (98%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py (98%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/compare_bean/origin_data_bean/memory_record_bean.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/compare_bean/origin_data_bean/op_stastic_bean.py (93%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/compare_bean/origin_data_bean/operator_memory_bean.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py (99%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py (99%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/compare_bean/profiling_info.py (99%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/compare_config/__init__.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/compare_config/compare_config.ini (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/compare_config/compare_config.py (97%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/comparison_generator.py (94%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/data_prepare/__init__.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/data_prepare/module_data_prepare.py (98%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/data_prepare/operator_data_prepare.py (96%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/data_prepare/sequence_pre_matching.py (99%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/disaggregate/__init__.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/disaggregate/overall_perf_interface.py (96%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/generator/__init__.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/generator/base_generator.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/generator/detail_performance_generator.py (98%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/generator/overall_performance_generator.py (96%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/interface/__init__.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/interface/compare_interface.py (98%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/interface/overall_interface.py (95%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/profiling_parser/__init__.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py (98%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py (98%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py (98%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/utils/__init__.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/utils/args_manager.py (97%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/utils/common_func.py (98%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/utils/compare_args.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/utils/excel_config.py (99%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/utils/module_node.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/utils/name_function.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/utils/singleton.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/utils/torch_op_node.py (98%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/utils/tree_builder.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/view/__init__.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/view/base_view.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/view/excel_view.py (95%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/view/screen_view.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_backend/view/work_sheet_creator.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_interface/__init__.py (100%) rename profiler/{ => msprof_analyze}/compare_tools/compare_interface/comparison_interface.py (92%) rename profiler/{ => msprof_analyze}/compare_tools/img/OverallMetrics.png (100%) rename profiler/{ => msprof_analyze}/compare_tools/img/config.PNG (100%) rename profiler/{ => msprof_analyze}/compare_tools/performance_compare.py (94%) rename profiler/{ => msprof_analyze}/config/config.ini (100%) rename profiler/{ => msprof_analyze}/example/__init__.py (100%) rename profiler/{ => msprof_analyze}/example/mstx_torch_plugin/README.md (100%) rename profiler/{ => msprof_analyze}/example/mstx_torch_plugin/__init__.py (100%) rename profiler/{ => msprof_analyze}/example/mstx_torch_plugin/img/result.png (100%) rename profiler/{ => msprof_analyze}/example/mstx_torch_plugin/mstx_torch_plugin.py (100%) rename profiler/{ => msprof_analyze}/example/setup.py (100%) rename profiler/{ => msprof_analyze}/prof_common/__init__.py (93%) rename profiler/{ => msprof_analyze}/prof_common/additional_args_manager.py (96%) rename profiler/{ => msprof_analyze}/prof_common/analyze_dict.py (100%) rename profiler/{ => msprof_analyze}/prof_common/constant.py (100%) rename profiler/{ => msprof_analyze}/prof_common/file_manager.py (95%) rename profiler/{ => msprof_analyze}/prof_common/logger.py (96%) rename profiler/{ => msprof_analyze}/prof_common/path_manager.py (98%) rename profiler/{ => msprof_analyze}/prof_common/utils.py (100%) rename profiler/{ => msprof_analyze}/requirements.txt (100%) rename profiler/{ => msprof_analyze}/requirements/build.txt (100%) rename profiler/{ => msprof_analyze}/requirements/tests.txt (100%) rename profiler/{ => msprof_analyze}/setup.cfg (100%) rename profiler/{ => msprof_analyze}/setup.py (84%) rename profiler/{ => msprof_analyze}/test/__init__.py (100%) rename profiler/{ => msprof_analyze}/test/resource/advisor/cluster_analysis_output/cluster_communication.json (100%) rename profiler/{ => msprof_analyze}/test/resource/advisor/cluster_analysis_output/cluster_step_trace_time.csv (100%) rename profiler/{ => msprof_analyze}/test/resource/event_list.json (100%) rename profiler/{ => msprof_analyze}/test/resource/pipeline_view.png (100%) rename profiler/{ => msprof_analyze}/test/resource/test.csv (100%) rename profiler/{ => msprof_analyze}/test/run_st.py (100%) rename profiler/{ => msprof_analyze}/test/run_ut.py (97%) rename profiler/{ => msprof_analyze}/test/st/__init__.py (100%) rename profiler/{ => msprof_analyze}/test/st/advisor/__init__.py (100%) rename profiler/{ => msprof_analyze}/test/st/advisor/test_advisor_cmd_single_ascend_pt_compare.py (99%) rename profiler/{ => msprof_analyze}/test/st/advisor/utils.py (100%) rename profiler/{ => msprof_analyze}/test/st/cluster_analyse/__init__.py (100%) rename profiler/{ => msprof_analyze}/test/st/cluster_analyse/cluster_communication_analyzer_bandwidth_db.py (100%) rename profiler/{ => msprof_analyze}/test/st/cluster_analyse/cluster_communication_analyzer_matrix_db.py (100%) rename profiler/{ => msprof_analyze}/test/st/cluster_analyse/cluster_communication_analyzer_time_db.py (100%) rename profiler/{ => msprof_analyze}/test/st/cluster_analyse/cluster_step_trace_time_db.py (100%) rename profiler/{ => msprof_analyze}/test/st/cluster_analyse/test_cluster_analyse_pytorch_db.py (98%) rename profiler/{ => msprof_analyze}/test/st/cluster_analyse/test_cluster_analyse_pytorch_text.py (99%) rename profiler/{ => msprof_analyze}/test/st/compare_tools/__init__.py (100%) rename profiler/{ => msprof_analyze}/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_api_compare.py (94%) rename profiler/{ => msprof_analyze}/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_communication_compare.py (95%) rename profiler/{ => msprof_analyze}/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_kernel_compare.py (94%) rename profiler/{ => msprof_analyze}/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_memory_compare.py (94%) rename profiler/{ => msprof_analyze}/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_operator_compare.py (98%) rename profiler/{ => msprof_analyze}/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_profiling.py (95%) rename profiler/{ => msprof_analyze}/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_vs_npu.py (96%) rename profiler/{ => msprof_analyze}/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_vs_npu_step.py (97%) rename profiler/{ => msprof_analyze}/test/st/dev/test_advisor_cmd_cluster_ascend_pt_compare.py (99%) rename profiler/{ => msprof_analyze}/test/st/dev/test_advisor_cmd_cluster_ascend_pt_no_compare.py (99%) rename profiler/{ => msprof_analyze}/test/st/dev/test_advisor_cmd_single_ascend_pt_no_compare.py (99%) rename profiler/{ => msprof_analyze}/test/st/utils.py (100%) rename profiler/{ => msprof_analyze}/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_advice_base.py (100%) rename profiler/{ => msprof_analyze}/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_pipeline_advice.py (100%) rename profiler/{ => msprof_analyze}/test/ut/advisor/advisor_backend/cluster_advice/test_kernel_cluster_advice.py (98%) rename profiler/{ => msprof_analyze}/test/ut/advisor/advisor_backend/cluster_advice/test_slow_link_advice.py (100%) rename profiler/{ => msprof_analyze}/test/ut/advisor/advisor_backend/cluster_advice/test_slow_rank_advice.py (100%) rename profiler/{ => msprof_analyze}/test/ut/advisor/advisor_backend/compute_advice/kernel_details.csv (100%) rename profiler/{ => msprof_analyze}/test/ut/advisor/advisor_backend/compute_advice/test_npu_slow_advice.py (100%) rename profiler/{ => msprof_analyze}/test/ut/advisor/advisor_backend/compute_advice/test_npufused_advice.py (100%) rename profiler/{ => msprof_analyze}/test/ut/advisor/advisor_backend/prof_bean_advisor/test_cluster_step_trace_time_bean.py (100%) rename profiler/{ => msprof_analyze}/test/ut/advisor/advisor_backend/timeline_advice/test_opsche_advice.py (100%) rename profiler/{ => msprof_analyze}/test/ut/advisor/advisor_backend/timeline_advice/test_optimizer_advice.py (100%) rename profiler/{ => msprof_analyze}/test/ut/advisor/advisor_backend/timeline_advice/trace_view.json (100%) rename profiler/{ => msprof_analyze}/test/ut/advisor/advisor_backend/tools/__init__.py (100%) rename profiler/{ => msprof_analyze}/test/ut/advisor/advisor_backend/tools/tool.py (100%) rename profiler/{ => msprof_analyze}/test/ut/advisor/common/__init__.py (100%) rename profiler/{ => msprof_analyze}/test/ut/advisor/common/test_enum_params_parser.py (84%) rename profiler/{ => msprof_analyze}/test/ut/advisor/communication_advice/test_bandwidth_contention_advice.py (98%) rename profiler/{ => msprof_analyze}/test/ut/advisor/communication_advice/test_byte_alignment_analyzer.py (97%) rename profiler/{ => msprof_analyze}/test/ut/advisor/communication_advice/test_packet_advice.py (98%) rename profiler/{ => msprof_analyze}/test/ut/advisor/communication_advice/test_rdma_retransmission_advice.py (98%) rename profiler/{ => msprof_analyze}/test/ut/advisor/compute_advice/test_frequency_advice.py (96%) rename profiler/{ => msprof_analyze}/test/ut/advisor/compute_advice/test_pp_stage_computation_analyzer.py (90%) rename profiler/{ => msprof_analyze}/test/ut/advisor/timeline_advice/test_conjectured_gc_advice.py (98%) rename profiler/{ => msprof_analyze}/test/ut/advisor/timeline_advice/test_dataloader_checker.py (90%) rename profiler/{ => msprof_analyze}/test/ut/advisor/timeline_advice/test_gc_checker.py (97%) rename profiler/{ => msprof_analyze}/test/ut/advisor/timeline_advice/test_memory_op_checker.py (88%) rename profiler/{ => msprof_analyze}/test/ut/advisor/timeline_advice/test_syncbn_checker.py (88%) rename profiler/{ => msprof_analyze}/test/ut/advisor/timeline_advice/test_synchronize_stream.py (85%) rename profiler/{ => msprof_analyze}/test/ut/advisor/timeline_advice/test_timeline_op_collector.py (94%) rename profiler/{ => msprof_analyze}/test/ut/advisor/timeline_advice/test_timeline_op_compile_checker.py (74%) rename profiler/{ => msprof_analyze}/test/ut/advisor/timeline_advice/trace_view.json (100%) rename profiler/{ => msprof_analyze}/test/ut/cluster_analyse/cluster_data_preprocess/test_pytorch_data_preprocessor.py (100%) rename profiler/{ => msprof_analyze}/test/ut/cluster_analyse/cluster_data_preprocess/test_step_trace_time_analysis.py (98%) rename profiler/{ => msprof_analyze}/test/ut/cluster_analyse/cluster_utils/test_parallel_strategy_calculator.py (100%) rename profiler/{ => msprof_analyze}/test/ut/cluster_analyse/common_func/test_file_manager.py (97%) rename profiler/{ => msprof_analyze}/test/ut/cluster_analyse/common_func/test_path_manager.py (98%) rename profiler/{ => msprof_analyze}/test/ut/cluster_analyse/communication_group/test_communication_group_generator.py (98%) rename profiler/{ => msprof_analyze}/test/ut/cluster_analyse/prof_bean/test_step_trace_time_bean.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/__init__.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/comparator/test_communication_comparator.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/comparator/test_operator_comparator.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/comparator/test_operator_statistic_comparator.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/compare_bean/__init__.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/compare_bean/origin_data_bean/__init__.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/compare_bean/origin_data_bean/test_compare_event.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/compare_bean/origin_data_bean/test_kernel_details_bean.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/compare_bean/origin_data_bean/test_memory_record_bean.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/compare_bean/origin_data_bean/test_operator_memory_bean.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/compare_bean/origin_data_bean/test_trace_event_bean.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/compare_bean/test_communication_bean.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/compare_bean/test_memory_compare_bean.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/compare_bean/test_memory_statistic_bean.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/compare_bean/test_operator_compare_bean.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/compare_bean/test_operator_statistic_bean.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/compare_bean/test_profiling_info.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/profiling_parser/__init__.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/utils/test_name_function.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/utils/test_tree_builder.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/view/__init__.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/view/test_excel_view.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/view/test_screen_view.py (100%) rename profiler/{ => msprof_analyze}/test/ut/compare_tools/view/test_worker_sheet_creator.py (100%) rename profiler/{ => msprof_analyze}/version.txt (100%) diff --git a/README.md b/README.md index 064720548f..de8aafa79f 100644 --- a/README.md +++ b/README.md @@ -32,19 +32,19 @@ ## [性能工具](./profiler) -1. [compare_tools（性能比对工具）](./profiler/compare_tools) +1. [compare_tools（性能比对工具）](./profiler/msprof_analyze/compare_tools) 提供 NPU 与 GPU 性能拆解功能以及算子、通信、内存性能的比对功能。 -2. [cluster_analyse（集群分析工具）](./profiler/cluster_analyse) +2. [cluster_analyse（集群分析工具）](./profiler/msprof_analyze/cluster_analyse) 提供多机多卡的集群分析能力（基于通信域的通信分析和迭代耗时分析）, 当前需要配合 MindStudio Insight 的集群分析功能使用。 -3. [advisor](./profiler/advisor) +3. [advisor](./profiler/msprof_analyze/advisor) 将 Ascend PyTorch Profiler 或者 msprof 采集的 PyTorch 场景性能数据进行分析，并输出性能调优建议。 -4. [bind_core](./profiler/affinity_cpu_bind) +4. [bind_core](./profiler/msprof_analyze/affinity_cpu_bind) 绑核脚本，支持非侵入修改工程代码，实现一键式绑核功能。 diff --git a/profiler/MANIFEST.in b/profiler/MANIFEST.in deleted file mode 100644 index 0550da458f..0000000000 --- a/profiler/MANIFEST.in +++ /dev/null @@ -1,7 +0,0 @@ -recursive-include profiler/advisor/ * -recursive-include profiler/cli/ * -recursive-include profiler/prof_common/ * -recursive-include profiler/compare_tools/ * -recursive-include profiler/cluster_analyse/ * -global-exclude */__pycache__/* -global-exclude *.pyc diff --git a/profiler/LICENSE b/profiler/msprof_analyze/LICENSE similarity index 100% rename from profiler/LICENSE rename to profiler/msprof_analyze/LICENSE diff --git a/profiler/msprof_analyze/MANIFEST.in b/profiler/msprof_analyze/MANIFEST.in new file mode 100644 index 0000000000..c99898e731 --- /dev/null +++ b/profiler/msprof_analyze/MANIFEST.in @@ -0,0 +1,8 @@ +recursive-include msprof_analyze/advisor/ * +recursive-include msprof_analyze/cli/ * +recursive-include msprof_analyze/prof_common/ * +recursive-include msprof_analyze/compare_tools/ * +recursive-include msprof_analyze/cluster_analyse/ * +global-exclude */__pycache__/* +global-exclude *.pyc + diff --git a/profiler/OWNERS b/profiler/msprof_analyze/OWNERS similarity index 92% rename from profiler/OWNERS rename to profiler/msprof_analyze/OWNERS index 5649903c38..7524470824 100644 --- a/profiler/OWNERS +++ b/profiler/msprof_analyze/OWNERS @@ -1,12 +1,12 @@ -options: - no_parent_owners: true -approvers: -- xhahn -- aerfaliang -- chenhao_1209 -- feng123www -- sunboquan -reviewers: -- sunboquan -- Seanesmhxocism -- wjchuee +options: + no_parent_owners: true +approvers: +- xhahn +- aerfaliang +- chenhao_1209 +- feng123www +- sunboquan +reviewers: +- sunboquan +- Seanesmhxocism +- wjchuee diff --git a/profiler/README.md b/profiler/msprof_analyze/README.md similarity index 96% rename from profiler/README.md rename to profiler/msprof_analyze/README.md index 37cb75bfb4..2c59288366 100644 --- a/profiler/README.md +++ b/profiler/msprof_analyze/README.md @@ -1,248 +1,248 @@ -# 性能工具 - -MindStudio Training Tools工具针对训练&大模型场景，提供端到端性能调优工具msprof-analyze：用户采集到性能数据后，由MindStudio Training Tools的性能工具msprof-analyze提供统计、分析以及相关的调优建议。 - -## NPU性能数据采集 - -目前MindStudio Training Tools工具主要支持对Ascend PyTorch Profiler接口采集的性能数据进行分析，请参考官方文档：[Ascend PyTorch Profiler数据采集与分析](https://www.hiascend.com/document/detail/zh/canncommercial/80RC1/devaids/auxiliarydevtool/atlasprofiling_16_0006.html)。 - -### 环境和依赖 - -- 硬件环境请参见《[昇腾产品形态说明](https://gitee.com/link?target=https%3A%2F%2Fwww.hiascend.com%2Fdocument%2Fdetail%2Fzh%2Fcanncommercial%2F80RC22%2Fquickstart%2Fquickstart%2Fquickstart_18_0002.html)》。 -- 软件环境请参见《[CANN 软件安装指南](https://gitee.com/link?target=https%3A%2F%2Fwww.hiascend.com%2Fdocument%2Fdetail%2Fzh%2Fcanncommercial%2F80RC22%2Fsoftwareinst%2Finstg%2Finstg_0000.html%3FMode%3DPmIns%26OS%3DUbuntu%26Software%3DcannToolKit)》安装昇腾设备开发或运行环境，即toolkit软件包。 - -以上环境依赖请根据实际环境选择适配的版本。 - -### 版本配套说明 - -- Ascend PyTorch Profiler接口支持AscendPyTorch 1.11.0或更高版本，支持的PyTorch和CANN以及PyTorch和Python软件版本配套关系请参见《[Ascend Extension for PyTorch插件](https://gitee.com/ascend/pytorch)》。 -- Ascend PyTorch Profiler接口支持的固件驱动版本与配套CANN软件支持的固件驱动版本相同，开发者可通过“[昇腾社区-固件与驱动](https://gitee.com/link?target=https%3A%2F%2Fwww.hiascend.com%2Fhardware%2Ffirmware-drivers%2Fcommunity%3Fproduct%3D2%26model%3D28%26cann%3D8.0.RC3.alpha003%26driver%3D1.0.25.alpha)”页面根据产品型号与CANN软件版本获取配套的固件与驱动。 - -### 采集方式一：通过with语句进行采集 - -```python -import torch_npu -experimental_config = torch_npu.profiler._ExperimentalConfig( - aic_metrics=torch_npu.profiler.AiCMetrics.PipeUtilization, - profiler_level=torch_npu.profiler.ProfilerLevel.Level1, - l2_cache=False -) -with torch_npu.profiler.profile( - activities=[ - torch_npu.profiler.ProfilerActivity.CPU, - torch_npu.profiler.ProfilerActivity.NPU - ], - record_shapes=True, - profile_memory=True, - with_stack=True, - experimental_config=experimental_config, - schedule=torch_npu.profiler.schedule(wait=10, warmup=0, active=1, repeat=1), - on_trace_ready=torch_npu.profiler.tensorboard_trace_handler("./profiling_data") -) as prof: - # 模型训练代码 - for epoch, data in enumerate(dataloader): - train_model_one_step(model, data) - prof.step() -``` - -### 采集方式二：start，stop方式进行采集 - -```python -import torch_npu -experimental_config = torch_npu.profiler._ExperimentalConfig( - aic_metrics=torch_npu.profiler.AiCMetrics.PipeUtilization, - profiler_level=torch_npu.profiler.ProfilerLevel.Level1, - l2_cache=False -) -prof = torch_npu.profiler.profile( - activities=[ - torch_npu.profiler.ProfilerActivity.CPU, - torch_npu.profiler.ProfilerActivity.NPU - ], - record_shapes=True, - profile_memory=True, - with_stack=True, - experimental_config=experimental_config, - on_trace_ready=torch_npu.profiler.tensorboard_trace_handler("./profiling_data")) -# 模型训练代码 -for epoch, data in enumerate(dataloader): - if epoch == 11: - prof.start() - train_model_one_step(model, data) - prof.step() - if epoch == 11: - prof.stop() -``` - -### NPU性能数据目录结构 - -ascend pytorch profiler数据目录结构如下： - -``` -|- ascend_pytorch_profiling - |- * _ascend_pt - |- ASCEND_PROFILER_OUTPUT - |- trace_view.json - |- FRAMEWORK - |- PROF_XXX - |- profiler_info.json - |- * _ascend_pt -``` - -## 安装 - -性能工具的安装方式包括：**pip安装**、**下载whl包安装**和**源代码编译安装**。 - -### pip安装 - -```shell -pip install msprof-analyze -``` - -使用`pip install msprof-analyze==版本号`可安装指定版本的包，支持1.2.1及之后版本，版本号参见“**下载whl包安装**”。 - -pip命令会自动安装最新的包及其配套依赖。 - -提示如下信息则表示安装成功。 - -```bash -Successfully installed msprof-analyze-{version} -``` - -### 下载whl包安装 - -1. whl包获取。 - - 请通过下表链接下载profiler工具whl包。 - - | profiler版本 | 发布日期 | 下载链接 | 校验码 | - |------------|------------|-------------------------------------------------------------------------------------------------------------------------------------------| ------------------------------------------------------------ | - | 1.3.3 | 2024-12-26 | [msprof_analyze-1.3.3-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.3.3/msprof_analyze-1.3.3-py3-none-any.whl) | 27676f2eee636bd0c65243f81e292c7f9d30d7f985c772ac9cbaf10b54d3584e | - | 1.3.2 | 2024-12-20 | [msprof_analyze-1.3.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.3.2/msprof_analyze-1.3.2-py3-none-any.whl) | ceb227e751ec3a204135be13801f1deee6a66c347f1bb3cdaef596872874df06 | - | 1.3.1 | 2024-12-04 | [msprof_analyze-1.3.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.3.1/msprof_analyze-1.3.1-py3-none-any.whl) | eae5548804314110a649caae537f2c63320fc70ec41ce1167f67c1d674d8798e | - | 1.3.0 | 2024-10-12 | [msprof_analyze-1.3.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.3.0/msprof_analyze-1.3.0-py3-none-any.whl) | 8b09758c6b5181bb656a95857c32852f898c370e7f1041e5a08e4f10d5004d48 | - | 1.2.5 | 2024-09-25 | [msprof_analyze-1.2.5-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.5/msprof_analyze-1.2.5-py3-none-any.whl) | aea8ae8deac07b5b4980bd2240da27d0eec93b9ace9ea9eb2e3a05ae9072018b | - | 1.2.4 | 2024-09-19 | [msprof_analyze-1.2.4-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.4/msprof_analyze-1.2.4-py3-none-any.whl) | 7c392e72c3347c4034fd3fdfcccb1f7936c24d9c3eb217e2cc05bae1347e5ab7 | - | 1.2.3 | 2024-08-29 | [msprof_analyze-1.2.3-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.3/msprof_analyze-1.2.3-py3-none-any.whl) | 354a55747f64ba1ec6ee6fe0f05a53e84e1b403ee0341ec40cc216dd25fda14c | - | 1.2.2 | 2024-08-23 | [msprof_analyze-1.2.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.2/msprof_analyze-1.2.2-py3-none-any.whl) | ed92a8e4eaf5ada8a2b4079072ec0cc42501b1b1f2eb00c8fdcb077fecb4ae02 | - | 1.2.1 | 2024-08-14 | [msprof_analyze-1.2.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.1/msprof_analyze-1.2.1-py3-none-any.whl) | 7acd477417bfb3ea29029dadf175d019ad3212403b7e11dc1f87e84c2412c078 | - | 1.2.0 | 2024-07-25 | [msprof_analyze-1.2.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.0/msprof_analyze-1.2.0-py3-none-any.whl) | 6a4366e3beca40b4a8305080e6e441d6ecafb5c05489e5905ac0265787555f37 | - | 1.1.2 | 2024-07-12 | [msprof_analyze-1.1.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.1.2/msprof_analyze-1.1.2-py3-none-any.whl) | af62125b1f9348bf491364e03af712fc6d0282ccee3fb07458bc9bbef82dacc6 | - | 1.1.1 | 2024-06-20 | [msprof_analyze-1.1.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.1.1/msprof_analyze-1.1.1-py3-none-any.whl) | 76aad967a3823151421153d368d4d2f8e5cfbcb356033575e0b8ec5acea8e5e4 | - | 1.1.0 | 2024-05-28 | [msprof_analyze-1.1.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.1.0/msprof_analyze-1.1.0-py3-none-any.whl) | b339f70e7d1e45e81f289332ca64990a744d0e7ce6fdd84a8d82e814fa400698 | - | 1.0 | 2024-05-10 | [msprof_analyze-1.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.0/msprof_analyze-1.0-py3-none-any.whl) | 95b2f41c8c8e8afe4887b738c8cababcb4f412e1874483b6adae4a025fcbb7d4 | - -2. whl包校验。 - - 1. 根据以上下载链接下载whl包到Linux安装环境。 - - 2. 进入whl包所在目录，执行如下命令。 - - ``` - sha256sum {name}.whl - ``` - - {name}为whl包名称。 - - 若回显呈现对应版本whl包一致的**校验码**，则表示下载了正确的性能工具whl安装包。示例如下： - - ```bash - sha256sum msprof_analyze-1.0-py3-none-any.whl - xx *msprof_analyze-1.0-py3-none-any.whl - ``` - -3. whl包安装。 - - 执行如下命令进行安装。 - - ```bash - pip3 install ./msprof_analyze-{version}-py3-none-any.whl - ``` - - 提示如下信息则表示安装成功。 - - ```bash - Successfully installed msprof_analyze-{version} - ``` - -### 源代码编译安装 - -1. 安装依赖。 - - 编译前需要安装wheel。 - - ```bash - pip3 install wheel - ``` - -2. 下载源码。 - - ```bash - git clone https://gitee.com/ascend/mstt.git - ``` - -3. 编译whl包。 - - ```bash - cd mstt/profiler - pip3 install -r requirements.txt && python3 setup.py bdist_wheel - ``` - - 以上命令执行完成后在mstt/profiler/dist目录下生成性能工具whl安装包`msprof_analyze-{version}-py3-none-any.whl`。 - -4. 安装。 - - 执行如下命令进行性能工具安装。 - - ```bash - cd dist - pip3 install ./msprof_analyze-{version}-py3-none-any.whl - ``` - -## 卸载和更新 - -若需要更新工具，请先卸载旧版本后再重新安装新版本，如下操作： - -1. 卸载 - - ```bash - pip3 uninstall msprof-analyze - ``` - -2. 更新 - - ```bash - pip3 install ./msprof_analyze-{version}-py3-none-any.whl - ``` - -## 工具使用 - -```bash -msprof-analyze advisor [-h] -``` - -```bash -msprof-analyze compare [-h] -``` - -```bash -msprof-analyze cluster [-h] -``` - -```bash -msprof-analyze auto-completion [-h] -``` - -``` -msprof-analyze [-h] [-v] -``` - -| 参数 | 说明 | -| -------------------- | ------------------------------------------------------------ | -| advisor | [advisor](./advisor/README.md)。将Ascend PyTorch Profiler或者msprof采集的PyThon场景性能数据进行分析，并输出性能调优建议。 | -| compare | [compare_tools（性能比对工具）](./compare_tools/README.md)。提供NPU与GPU性能拆解功能以及算子、通信、内存性能的比对功能。 | -| cluster | [cluster_analyse（集群分析工具）](./cluster_analyse/README.md)。提供多机多卡的集群分析能力（基于通信域的通信分析和迭代耗时分析）, 当前需要配合Ascend Insight的集群分析功能使用。 | -| auto-completion | 自动补全。配置后在当前视图下配置msprof-analyze工具所有的子参数时，可以使用Tab将所有子参数自动补全。 | -| -v，-V
--version | 查看版本号。 | -| -h，-H
--help | 命令行参数帮助信息。 | - +# 性能工具 + +MindStudio Training Tools工具针对训练&大模型场景，提供端到端性能调优工具msprof-analyze：用户采集到性能数据后，由MindStudio Training Tools的性能工具msprof-analyze提供统计、分析以及相关的调优建议。 + +## NPU性能数据采集 + +目前MindStudio Training Tools工具主要支持对Ascend PyTorch Profiler接口采集的性能数据进行分析，请参考官方文档：[Ascend PyTorch Profiler数据采集与分析](https://www.hiascend.com/document/detail/zh/canncommercial/80RC1/devaids/auxiliarydevtool/atlasprofiling_16_0006.html)。 + +### 环境和依赖 + +- 硬件环境请参见《[昇腾产品形态说明](https://gitee.com/link?target=https%3A%2F%2Fwww.hiascend.com%2Fdocument%2Fdetail%2Fzh%2Fcanncommercial%2F80RC22%2Fquickstart%2Fquickstart%2Fquickstart_18_0002.html)》。 +- 软件环境请参见《[CANN 软件安装指南](https://gitee.com/link?target=https%3A%2F%2Fwww.hiascend.com%2Fdocument%2Fdetail%2Fzh%2Fcanncommercial%2F80RC22%2Fsoftwareinst%2Finstg%2Finstg_0000.html%3FMode%3DPmIns%26OS%3DUbuntu%26Software%3DcannToolKit)》安装昇腾设备开发或运行环境，即toolkit软件包。 + +以上环境依赖请根据实际环境选择适配的版本。 + +### 版本配套说明 + +- Ascend PyTorch Profiler接口支持AscendPyTorch 1.11.0或更高版本，支持的PyTorch和CANN以及PyTorch和Python软件版本配套关系请参见《[Ascend Extension for PyTorch插件](https://gitee.com/ascend/pytorch)》。 +- Ascend PyTorch Profiler接口支持的固件驱动版本与配套CANN软件支持的固件驱动版本相同，开发者可通过“[昇腾社区-固件与驱动](https://gitee.com/link?target=https%3A%2F%2Fwww.hiascend.com%2Fhardware%2Ffirmware-drivers%2Fcommunity%3Fproduct%3D2%26model%3D28%26cann%3D8.0.RC3.alpha003%26driver%3D1.0.25.alpha)”页面根据产品型号与CANN软件版本获取配套的固件与驱动。 + +### 采集方式一：通过with语句进行采集 + +```python +import torch_npu +experimental_config = torch_npu.profiler._ExperimentalConfig( + aic_metrics=torch_npu.profiler.AiCMetrics.PipeUtilization, + profiler_level=torch_npu.profiler.ProfilerLevel.Level1, + l2_cache=False +) +with torch_npu.profiler.profile( + activities=[ + torch_npu.profiler.ProfilerActivity.CPU, + torch_npu.profiler.ProfilerActivity.NPU + ], + record_shapes=True, + profile_memory=True, + with_stack=True, + experimental_config=experimental_config, + schedule=torch_npu.profiler.schedule(wait=10, warmup=0, active=1, repeat=1), + on_trace_ready=torch_npu.profiler.tensorboard_trace_handler("./profiling_data") +) as prof: + # 模型训练代码 + for epoch, data in enumerate(dataloader): + train_model_one_step(model, data) + prof.step() +``` + +### 采集方式二：start，stop方式进行采集 + +```python +import torch_npu +experimental_config = torch_npu.profiler._ExperimentalConfig( + aic_metrics=torch_npu.profiler.AiCMetrics.PipeUtilization, + profiler_level=torch_npu.profiler.ProfilerLevel.Level1, + l2_cache=False +) +prof = torch_npu.profiler.profile( + activities=[ + torch_npu.profiler.ProfilerActivity.CPU, + torch_npu.profiler.ProfilerActivity.NPU + ], + record_shapes=True, + profile_memory=True, + with_stack=True, + experimental_config=experimental_config, + on_trace_ready=torch_npu.profiler.tensorboard_trace_handler("./profiling_data")) +# 模型训练代码 +for epoch, data in enumerate(dataloader): + if epoch == 11: + prof.start() + train_model_one_step(model, data) + prof.step() + if epoch == 11: + prof.stop() +``` + +### NPU性能数据目录结构 + +ascend pytorch profiler数据目录结构如下： + +``` +|- ascend_pytorch_profiling + |- * _ascend_pt + |- ASCEND_PROFILER_OUTPUT + |- trace_view.json + |- FRAMEWORK + |- PROF_XXX + |- profiler_info.json + |- * _ascend_pt +``` + +## 安装 + +性能工具的安装方式包括：**pip安装**、**下载whl包安装**和**源代码编译安装**。 + +### pip安装 + +```shell +pip install msprof-analyze +``` + +使用`pip install msprof-analyze==版本号`可安装指定版本的包，支持1.2.1及之后版本，版本号参见“**下载whl包安装**”。 + +pip命令会自动安装最新的包及其配套依赖。 + +提示如下信息则表示安装成功。 + +```bash +Successfully installed msprof-analyze-{version} +``` + +### 下载whl包安装 + +1. whl包获取。 + + 请通过下表链接下载profiler工具whl包。 + + | profiler版本 | 发布日期 | 下载链接 | 校验码 | + |------------|------------|-------------------------------------------------------------------------------------------------------------------------------------------| ------------------------------------------------------------ | + | 1.3.3 | 2024-12-26 | [msprof_analyze-1.3.3-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.3.3/msprof_analyze-1.3.3-py3-none-any.whl) | 27676f2eee636bd0c65243f81e292c7f9d30d7f985c772ac9cbaf10b54d3584e | + | 1.3.2 | 2024-12-20 | [msprof_analyze-1.3.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.3.2/msprof_analyze-1.3.2-py3-none-any.whl) | ceb227e751ec3a204135be13801f1deee6a66c347f1bb3cdaef596872874df06 | + | 1.3.1 | 2024-12-04 | [msprof_analyze-1.3.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.3.1/msprof_analyze-1.3.1-py3-none-any.whl) | eae5548804314110a649caae537f2c63320fc70ec41ce1167f67c1d674d8798e | + | 1.3.0 | 2024-10-12 | [msprof_analyze-1.3.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.3.0/msprof_analyze-1.3.0-py3-none-any.whl) | 8b09758c6b5181bb656a95857c32852f898c370e7f1041e5a08e4f10d5004d48 | + | 1.2.5 | 2024-09-25 | [msprof_analyze-1.2.5-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.5/msprof_analyze-1.2.5-py3-none-any.whl) | aea8ae8deac07b5b4980bd2240da27d0eec93b9ace9ea9eb2e3a05ae9072018b | + | 1.2.4 | 2024-09-19 | [msprof_analyze-1.2.4-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.4/msprof_analyze-1.2.4-py3-none-any.whl) | 7c392e72c3347c4034fd3fdfcccb1f7936c24d9c3eb217e2cc05bae1347e5ab7 | + | 1.2.3 | 2024-08-29 | [msprof_analyze-1.2.3-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.3/msprof_analyze-1.2.3-py3-none-any.whl) | 354a55747f64ba1ec6ee6fe0f05a53e84e1b403ee0341ec40cc216dd25fda14c | + | 1.2.2 | 2024-08-23 | [msprof_analyze-1.2.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.2/msprof_analyze-1.2.2-py3-none-any.whl) | ed92a8e4eaf5ada8a2b4079072ec0cc42501b1b1f2eb00c8fdcb077fecb4ae02 | + | 1.2.1 | 2024-08-14 | [msprof_analyze-1.2.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.1/msprof_analyze-1.2.1-py3-none-any.whl) | 7acd477417bfb3ea29029dadf175d019ad3212403b7e11dc1f87e84c2412c078 | + | 1.2.0 | 2024-07-25 | [msprof_analyze-1.2.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.0/msprof_analyze-1.2.0-py3-none-any.whl) | 6a4366e3beca40b4a8305080e6e441d6ecafb5c05489e5905ac0265787555f37 | + | 1.1.2 | 2024-07-12 | [msprof_analyze-1.1.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.1.2/msprof_analyze-1.1.2-py3-none-any.whl) | af62125b1f9348bf491364e03af712fc6d0282ccee3fb07458bc9bbef82dacc6 | + | 1.1.1 | 2024-06-20 | [msprof_analyze-1.1.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.1.1/msprof_analyze-1.1.1-py3-none-any.whl) | 76aad967a3823151421153d368d4d2f8e5cfbcb356033575e0b8ec5acea8e5e4 | + | 1.1.0 | 2024-05-28 | [msprof_analyze-1.1.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.1.0/msprof_analyze-1.1.0-py3-none-any.whl) | b339f70e7d1e45e81f289332ca64990a744d0e7ce6fdd84a8d82e814fa400698 | + | 1.0 | 2024-05-10 | [msprof_analyze-1.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.0/msprof_analyze-1.0-py3-none-any.whl) | 95b2f41c8c8e8afe4887b738c8cababcb4f412e1874483b6adae4a025fcbb7d4 | + +2. whl包校验。 + + 1. 根据以上下载链接下载whl包到Linux安装环境。 + + 2. 进入whl包所在目录，执行如下命令。 + + ``` + sha256sum {name}.whl + ``` + + {name}为whl包名称。 + + 若回显呈现对应版本whl包一致的**校验码**，则表示下载了正确的性能工具whl安装包。示例如下： + + ```bash + sha256sum msprof_analyze-1.0-py3-none-any.whl + xx *msprof_analyze-1.0-py3-none-any.whl + ``` + +3. whl包安装。 + + 执行如下命令进行安装。 + + ```bash + pip3 install ./msprof_analyze-{version}-py3-none-any.whl + ``` + + 提示如下信息则表示安装成功。 + + ```bash + Successfully installed msprof_analyze-{version} + ``` + +### 源代码编译安装 + +1. 安装依赖。 + + 编译前需要安装wheel。 + + ```bash + pip3 install wheel + ``` + +2. 下载源码。 + + ```bash + git clone https://gitee.com/ascend/mstt.git + ``` + +3. 编译whl包。 + + ```bash + cd mstt/profiler/msprof_analyze + pip3 install -r requirements.txt && python3 setup.py bdist_wheel + ``` + + 以上命令执行完成后在mstt/profiler/msprof_analyze/dist目录下生成性能工具whl安装包`msprof_analyze-{version}-py3-none-any.whl`。 + +4. 安装。 + + 执行如下命令进行性能工具安装。 + + ```bash + cd dist + pip3 install ./msprof_analyze-{version}-py3-none-any.whl + ``` + +## 卸载和更新 + +若需要更新工具，请先卸载旧版本后再重新安装新版本，如下操作： + +1. 卸载 + + ```bash + pip3 uninstall msprof-analyze + ``` + +2. 更新 + + ```bash + pip3 install ./msprof_analyze-{version}-py3-none-any.whl + ``` + +## 工具使用 + +```bash +msprof-analyze advisor [-h] +``` + +```bash +msprof-analyze compare [-h] +``` + +```bash +msprof-analyze cluster [-h] +``` + +```bash +msprof-analyze auto-completion [-h] +``` + +``` +msprof-analyze [-h] [-v] +``` + +| 参数 | 说明 | +| -------------------- | ------------------------------------------------------------ | +| advisor | [advisor](./advisor/README.md)。将Ascend PyTorch Profiler或者msprof采集的PyThon场景性能数据进行分析，并输出性能调优建议。 | +| compare | [compare_tools（性能比对工具）](./compare_tools/README.md)。提供NPU与GPU性能拆解功能以及算子、通信、内存性能的比对功能。 | +| cluster | [cluster_analyse（集群分析工具）](./cluster_analyse/README.md)。提供多机多卡的集群分析能力（基于通信域的通信分析和迭代耗时分析）, 当前需要配合Ascend Insight的集群分析功能使用。 | +| auto-completion | 自动补全。配置后在当前视图下配置msprof-analyze工具所有的子参数时，可以使用Tab将所有子参数自动补全。 | +| -v，-V
--version | 查看版本号。 | +| -h，-H
--help | 命令行参数帮助信息。 | + diff --git a/profiler/__init__.py b/profiler/msprof_analyze/__init__.py similarity index 100% rename from profiler/__init__.py rename to profiler/msprof_analyze/__init__.py diff --git a/profiler/advisor/README.md b/profiler/msprof_analyze/advisor/README.md similarity index 98% rename from profiler/advisor/README.md rename to profiler/msprof_analyze/advisor/README.md index 0e162977b1..2c9e055a11 100644 --- a/profiler/advisor/README.md +++ b/profiler/msprof_analyze/advisor/README.md @@ -241,7 +241,7 @@ computation模块从device计算性能维度进行分析，能够识别AI CPU、 ![op_no_bound](./img/op_no_bound.png) -上图中torch_npu.npu.set_compile_mode接口介绍请参见[torch_npu.npu.set_compile_mode](https://www.hiascend.com/document/detail/zh/Pytorch/60RC2/apiref/apilist/ptaoplist_000880.html)；AICPU算子替换样例可参考《[Samples of AI CPU Operator Replacement](https://gitee.com/ascend/mstt/blob/master/profiler/advisor/doc/Samples%20of%20AI%20CPU%20Operator%20Replacement.md)》。 +上图中torch_npu.npu.set_compile_mode接口介绍请参见[torch_npu.npu.set_compile_mode](https://www.hiascend.com/document/detail/zh/Pytorch/60RC2/apiref/apilist/ptaoplist_000880.html)；AICPU算子替换样例可参考《[Samples of AI CPU Operator Replacement](https://gitee.com/ascend/mstt/blob/master/profiler/msprof_analyze/advisor/doc/Samples%20of%20AI%20CPU%20Operator%20Replacement.md)》。 当存在pp stage（流水线并行）时，computation会按stage分析，每个stage就是一个流水线切分，比如0\~7卡为stage-0、8\~15卡为stage-1。 @@ -266,7 +266,7 @@ schedule模块包GC Analysis、含亲和API、aclOpCompile、SyncBatchNorm、Syn - `gc.set_threshold(threshold0, thresholdl, threshold2)`：这个函数用于设置垃圾回收的阈值。垃圾回收器将所有对象分为三代（0代、1代和2代），每一代的对象在经历垃圾回收后会被移到下一代。`threshold0`控制第0代的垃圾回收频率，`threshold1`控制第1代的垃圾回收频率，`threshold2`控制第2代的垃圾回收频率。将`threshold0`设为0可以禁用垃圾回收。 - `gc.disable ()`：这个函数用于禁用自动垃圾回收。调用`gc.disable ()`后，垃圾回收器将不会自动运行，直到手动调用`gc.enable（）`。 -如下图示例，Affinity API Issues提示存在可以替换的亲和API并给出对应的堆栈，用户可以根据堆栈找到需要修改的代码，并给出修改案例（[API instruction](https://gitee.com/ascend/mstt/blob/master/profiler/advisor/doc/Samples%20of%20Fused%20Operator%20API%20Replacement.md)）。 +如下图示例，Affinity API Issues提示存在可以替换的亲和API并给出对应的堆栈，用户可以根据堆栈找到需要修改的代码，并给出修改案例（[API instruction](https://gitee.com/ascend/mstt/blob/master/profiler/msprof_analyze/advisor/doc/Samples%20of%20Fused%20Operator%20API%20Replacement.md)）。 ![schedule_3](./img/schedule_3.png) diff --git a/profiler/advisor/__init__.py b/profiler/msprof_analyze/advisor/__init__.py similarity index 100% rename from profiler/advisor/__init__.py rename to profiler/msprof_analyze/advisor/__init__.py diff --git a/profiler/advisor/advisor_backend/__init__.py b/profiler/msprof_analyze/advisor/advisor_backend/__init__.py similarity index 100% rename from profiler/advisor/advisor_backend/__init__.py rename to profiler/msprof_analyze/advisor/advisor_backend/__init__.py diff --git a/profiler/advisor/advisor_backend/advice_base.py b/profiler/msprof_analyze/advisor/advisor_backend/advice_base.py similarity index 100% rename from profiler/advisor/advisor_backend/advice_base.py rename to profiler/msprof_analyze/advisor/advisor_backend/advice_base.py diff --git a/profiler/advisor/advisor_backend/advice_factory/__init__.py b/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/__init__.py similarity index 100% rename from profiler/advisor/advisor_backend/advice_factory/__init__.py rename to profiler/msprof_analyze/advisor/advisor_backend/advice_factory/__init__.py diff --git a/profiler/advisor/advisor_backend/advice_factory/advice_factory.py b/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/advice_factory.py similarity index 100% rename from profiler/advisor/advisor_backend/advice_factory/advice_factory.py rename to profiler/msprof_analyze/advisor/advisor_backend/advice_factory/advice_factory.py diff --git a/profiler/advisor/advisor_backend/advice_factory/cluster_advice_factory.py b/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/cluster_advice_factory.py similarity index 100% rename from profiler/advisor/advisor_backend/advice_factory/cluster_advice_factory.py rename to profiler/msprof_analyze/advisor/advisor_backend/advice_factory/cluster_advice_factory.py diff --git a/profiler/advisor/advisor_backend/advice_factory/compute_advice_factory.py b/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/compute_advice_factory.py similarity index 100% rename from profiler/advisor/advisor_backend/advice_factory/compute_advice_factory.py rename to profiler/msprof_analyze/advisor/advisor_backend/advice_factory/compute_advice_factory.py diff --git a/profiler/advisor/advisor_backend/advice_factory/overall_advice_factory.py b/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/overall_advice_factory.py similarity index 100% rename from profiler/advisor/advisor_backend/advice_factory/overall_advice_factory.py rename to profiler/msprof_analyze/advisor/advisor_backend/advice_factory/overall_advice_factory.py diff --git a/profiler/advisor/advisor_backend/advice_factory/timeline_advice_factory.py b/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/timeline_advice_factory.py similarity index 100% rename from profiler/advisor/advisor_backend/advice_factory/timeline_advice_factory.py rename to profiler/msprof_analyze/advisor/advisor_backend/advice_factory/timeline_advice_factory.py diff --git a/profiler/advisor/advisor_backend/cluster_advice/__init__.py b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/__init__.py similarity index 100% rename from profiler/advisor/advisor_backend/cluster_advice/__init__.py rename to profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/__init__.py diff --git a/profiler/advisor/advisor_backend/cluster_advice/cluster_advice_base.py b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_advice_base.py similarity index 94% rename from profiler/advisor/advisor_backend/cluster_advice/cluster_advice_base.py rename to profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_advice_base.py index a9dbf4c050..d5013fffb4 100644 --- a/profiler/advisor/advisor_backend/cluster_advice/cluster_advice_base.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_advice_base.py @@ -19,8 +19,8 @@ from abc import abstractmethod from advice_base import AdviceBase from cluster_analysis import Interface -from profiler.advisor.advisor_backend.logger import Logger -from profiler.prof_common.constant import Constant +from msprof_analyze.advisor.advisor_backend.logger import Logger +from msprof_analyze.prof_common.constant import Constant logger = Logger() diff --git a/profiler/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py similarity index 99% rename from profiler/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py rename to profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py index 5213d715e2..81fcf88ec0 100644 --- a/profiler/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py @@ -32,8 +32,8 @@ from common_func_advisor.constant import Constant from common_func_advisor.trace_view_preprocessor import FineTraceViewData from common_func_advisor.trace_view_preprocessor import TraceViewPreProcessor -from profiler.advisor.advisor_backend.logger import Logger -from profiler.prof_common.file_manager import FileManager +from msprof_analyze.advisor.advisor_backend.logger import Logger +from msprof_analyze.prof_common.file_manager import FileManager logger = Logger() diff --git a/profiler/advisor/advisor_backend/cluster_advice/kernel_cluster_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/kernel_cluster_advice.py similarity index 95% rename from profiler/advisor/advisor_backend/cluster_advice/kernel_cluster_advice.py rename to profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/kernel_cluster_advice.py index 89a9ccf772..e3b264463b 100644 --- a/profiler/advisor/advisor_backend/cluster_advice/kernel_cluster_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/kernel_cluster_advice.py @@ -18,9 +18,9 @@ import pandas as pd from common_func_advisor.constant import Constant as AdvisorConstant from cluster_advice.cluster_advice_base import ClusterAdviceBase from cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor -from profiler.prof_common.file_manager import FileManager -from profiler.prof_common.constant import Constant -from profiler.prof_common.path_manager import PathManager +from msprof_analyze.prof_common.file_manager import FileManager +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.prof_common.path_manager import PathManager class KernelClusterAdvice(ClusterAdviceBase): diff --git a/profiler/advisor/advisor_backend/cluster_advice/slow_link_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/slow_link_advice.py similarity index 98% rename from profiler/advisor/advisor_backend/cluster_advice/slow_link_advice.py rename to profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/slow_link_advice.py index 065e9a9ca2..564a844df3 100644 --- a/profiler/advisor/advisor_backend/cluster_advice/slow_link_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/slow_link_advice.py @@ -17,7 +17,7 @@ import os from collections import defaultdict from common_func_advisor.constant import Constant from cluster_advice.cluster_advice_base import ClusterAdviceBase -from profiler.prof_common.file_manager import FileManager +from msprof_analyze.prof_common.file_manager import FileManager class SlowLinkAdvice(ClusterAdviceBase): diff --git a/profiler/advisor/advisor_backend/cluster_advice/slow_rank_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/slow_rank_advice.py similarity index 98% rename from profiler/advisor/advisor_backend/cluster_advice/slow_rank_advice.py rename to profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/slow_rank_advice.py index c8a067d0ee..af644ed4b8 100644 --- a/profiler/advisor/advisor_backend/cluster_advice/slow_rank_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/slow_rank_advice.py @@ -18,7 +18,7 @@ from collections import defaultdict from common_func_advisor.constant import Constant from cluster_advice.cluster_advice_base import ClusterAdviceBase from prof_bean_advisor.cluster_step_trace_time_bean import ClusterStepTraceTimeBean -from profiler.prof_common.file_manager import FileManager +from msprof_analyze.prof_common.file_manager import FileManager class SlowRankAdvice(ClusterAdviceBase): diff --git a/profiler/advisor/advisor_backend/common_func_advisor/__init__.py b/profiler/msprof_analyze/advisor/advisor_backend/common_func_advisor/__init__.py similarity index 100% rename from profiler/advisor/advisor_backend/common_func_advisor/__init__.py rename to profiler/msprof_analyze/advisor/advisor_backend/common_func_advisor/__init__.py diff --git a/profiler/advisor/advisor_backend/common_func_advisor/constant.py b/profiler/msprof_analyze/advisor/advisor_backend/common_func_advisor/constant.py similarity index 100% rename from profiler/advisor/advisor_backend/common_func_advisor/constant.py rename to profiler/msprof_analyze/advisor/advisor_backend/common_func_advisor/constant.py diff --git a/profiler/advisor/advisor_backend/common_func_advisor/trace_view_json.py b/profiler/msprof_analyze/advisor/advisor_backend/common_func_advisor/trace_view_json.py similarity index 98% rename from profiler/advisor/advisor_backend/common_func_advisor/trace_view_json.py rename to profiler/msprof_analyze/advisor/advisor_backend/common_func_advisor/trace_view_json.py index 7fb75f065e..6b57ee83fc 100644 --- a/profiler/advisor/advisor_backend/common_func_advisor/trace_view_json.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/common_func_advisor/trace_view_json.py @@ -23,8 +23,8 @@ from typing import List import pandas as pd -from profiler.advisor.advisor_backend.logger import Logger -from profiler.prof_common.file_manager import FileManager +from msprof_analyze.advisor.advisor_backend.logger import Logger +from msprof_analyze.prof_common.file_manager import FileManager logger = Logger() diff --git a/profiler/advisor/advisor_backend/common_func_advisor/trace_view_preprocessor.py b/profiler/msprof_analyze/advisor/advisor_backend/common_func_advisor/trace_view_preprocessor.py similarity index 99% rename from profiler/advisor/advisor_backend/common_func_advisor/trace_view_preprocessor.py rename to profiler/msprof_analyze/advisor/advisor_backend/common_func_advisor/trace_view_preprocessor.py index fb57e37e9b..028ee8d4b1 100644 --- a/profiler/advisor/advisor_backend/common_func_advisor/trace_view_preprocessor.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/common_func_advisor/trace_view_preprocessor.py @@ -20,7 +20,7 @@ from typing import Optional from dataclasses import dataclass from common_func_advisor.constant import Constant -from profiler.advisor.advisor_backend.logger import Logger +from msprof_analyze.advisor.advisor_backend.logger import Logger logger = Logger() diff --git a/profiler/advisor/advisor_backend/compute_advice/__init__.py b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/__init__.py similarity index 100% rename from profiler/advisor/advisor_backend/compute_advice/__init__.py rename to profiler/msprof_analyze/advisor/advisor_backend/compute_advice/__init__.py diff --git a/profiler/advisor/advisor_backend/compute_advice/compute_advice_base.py b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/compute_advice_base.py similarity index 98% rename from profiler/advisor/advisor_backend/compute_advice/compute_advice_base.py rename to profiler/msprof_analyze/advisor/advisor_backend/compute_advice/compute_advice_base.py index 8cc0d010fc..67e1513d81 100644 --- a/profiler/advisor/advisor_backend/compute_advice/compute_advice_base.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/compute_advice_base.py @@ -19,7 +19,7 @@ import os import logging from advice_base import AdviceBase -from profiler.prof_common.file_manager import FileManager +from msprof_analyze.prof_common.file_manager import FileManager logger = logging.getLogger() diff --git a/profiler/advisor/advisor_backend/compute_advice/npu_fused/__init__.py b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/__init__.py similarity index 100% rename from profiler/advisor/advisor_backend/compute_advice/npu_fused/__init__.py rename to profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/__init__.py diff --git a/profiler/advisor/advisor_backend/compute_advice/npu_fused/csv_analyzer.py b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/csv_analyzer.py similarity index 100% rename from profiler/advisor/advisor_backend/compute_advice/npu_fused/csv_analyzer.py rename to profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/csv_analyzer.py diff --git a/profiler/advisor/advisor_backend/compute_advice/npu_fused/json_analyzer.py b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/json_analyzer.py similarity index 100% rename from profiler/advisor/advisor_backend/compute_advice/npu_fused/json_analyzer.py rename to profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/json_analyzer.py diff --git a/profiler/advisor/advisor_backend/compute_advice/npu_fused/op_perf.py b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/op_perf.py similarity index 100% rename from profiler/advisor/advisor_backend/compute_advice/npu_fused/op_perf.py rename to profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/op_perf.py diff --git a/profiler/advisor/advisor_backend/compute_advice/npu_fused_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused_advice.py similarity index 100% rename from profiler/advisor/advisor_backend/compute_advice/npu_fused_advice.py rename to profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused_advice.py diff --git a/profiler/advisor/advisor_backend/compute_advice/npu_slow_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_slow_advice.py similarity index 100% rename from profiler/advisor/advisor_backend/compute_advice/npu_slow_advice.py rename to profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_slow_advice.py diff --git a/profiler/advisor/advisor_backend/interface.py b/profiler/msprof_analyze/advisor/advisor_backend/interface.py similarity index 100% rename from profiler/advisor/advisor_backend/interface.py rename to profiler/msprof_analyze/advisor/advisor_backend/interface.py diff --git a/profiler/advisor/advisor_backend/logger.py b/profiler/msprof_analyze/advisor/advisor_backend/logger.py similarity index 100% rename from profiler/advisor/advisor_backend/logger.py rename to profiler/msprof_analyze/advisor/advisor_backend/logger.py diff --git a/profiler/advisor/advisor_backend/overall_advice/__init__.py b/profiler/msprof_analyze/advisor/advisor_backend/overall_advice/__init__.py similarity index 100% rename from profiler/advisor/advisor_backend/overall_advice/__init__.py rename to profiler/msprof_analyze/advisor/advisor_backend/overall_advice/__init__.py diff --git a/profiler/advisor/advisor_backend/overall_advice/overall_summary_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/overall_advice/overall_summary_advice.py similarity index 97% rename from profiler/advisor/advisor_backend/overall_advice/overall_summary_advice.py rename to profiler/msprof_analyze/advisor/advisor_backend/overall_advice/overall_summary_advice.py index 4441f8a981..407f636335 100644 --- a/profiler/advisor/advisor_backend/overall_advice/overall_summary_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/overall_advice/overall_summary_advice.py @@ -18,9 +18,9 @@ import logging from compare_interface.comparison_interface import ComparisonInterface from advisor_backend.advice_base import AdviceBase -from profiler.advisor.display.prompt.base_prompt import BasePrompt -from profiler.prof_common.additional_args_manager import AdditionalArgsManager -from profiler.prof_common.constant import Constant +from msprof_analyze.advisor.display.prompt.base_prompt import BasePrompt +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.prof_common.constant import Constant logger = logging.getLogger() diff --git a/profiler/advisor/advisor_backend/prof_bean_advisor/__init__.py b/profiler/msprof_analyze/advisor/advisor_backend/prof_bean_advisor/__init__.py similarity index 100% rename from profiler/advisor/advisor_backend/prof_bean_advisor/__init__.py rename to profiler/msprof_analyze/advisor/advisor_backend/prof_bean_advisor/__init__.py diff --git a/profiler/advisor/advisor_backend/prof_bean_advisor/cluster_step_trace_time_bean.py b/profiler/msprof_analyze/advisor/advisor_backend/prof_bean_advisor/cluster_step_trace_time_bean.py similarity index 100% rename from profiler/advisor/advisor_backend/prof_bean_advisor/cluster_step_trace_time_bean.py rename to profiler/msprof_analyze/advisor/advisor_backend/prof_bean_advisor/cluster_step_trace_time_bean.py diff --git a/profiler/advisor/advisor_backend/timeline_advice/__init__.py b/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/__init__.py similarity index 100% rename from profiler/advisor/advisor_backend/timeline_advice/__init__.py rename to profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/__init__.py diff --git a/profiler/advisor/advisor_backend/timeline_advice/op_schedule_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/op_schedule_advice.py similarity index 100% rename from profiler/advisor/advisor_backend/timeline_advice/op_schedule_advice.py rename to profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/op_schedule_advice.py diff --git a/profiler/advisor/advisor_backend/timeline_advice/optimizer_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/optimizer_advice.py similarity index 100% rename from profiler/advisor/advisor_backend/timeline_advice/optimizer_advice.py rename to profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/optimizer_advice.py diff --git a/profiler/advisor/advisor_backend/timeline_advice/timeline_advice_base.py b/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/timeline_advice_base.py similarity index 98% rename from profiler/advisor/advisor_backend/timeline_advice/timeline_advice_base.py rename to profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/timeline_advice_base.py index 98842cf84b..e712799a96 100644 --- a/profiler/advisor/advisor_backend/timeline_advice/timeline_advice_base.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/timeline_advice_base.py @@ -21,7 +21,7 @@ from collections import defaultdict from advice_base import AdviceBase -from profiler.prof_common.file_manager import FileManager +from msprof_analyze.prof_common.file_manager import FileManager logger = logging.getLogger() logger.setLevel(logging.INFO) diff --git a/profiler/advisor/analyzer/__init__.py b/profiler/msprof_analyze/advisor/analyzer/__init__.py similarity index 100% rename from profiler/advisor/analyzer/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/__init__.py diff --git a/profiler/advisor/analyzer/analyzer_controller.py b/profiler/msprof_analyze/advisor/analyzer/analyzer_controller.py similarity index 95% rename from profiler/advisor/analyzer/analyzer_controller.py rename to profiler/msprof_analyze/advisor/analyzer/analyzer_controller.py index 1a5a28b637..1e9790e1da 100644 --- a/profiler/advisor/analyzer/analyzer_controller.py +++ b/profiler/msprof_analyze/advisor/analyzer/analyzer_controller.py @@ -1,950 +1,950 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import copy -import logging -import json -import sys -import os -import platform -import multiprocessing as mp -from multiprocessing import Manager -from pathlib import Path - -import psutil - -sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "compare_tools")) -sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "cluster_analyse")) - -from profiler.prof_common.additional_args_manager import AdditionalArgsManager -from profiler.advisor.analyzer.cluster.slow_rank_analyzer import SlowRankAnalyzer -from profiler.advisor.analyzer.cluster.slow_link_analyzer import SlowLinkAnalyzer -from profiler.advisor.analyzer.computation.pp_stage_computation_analyzer import PPStageComputationAnalyzer -from profiler.advisor.analyzer.overall.overall_summary_analyzer import OverallSummaryAnalyzer -from profiler.advisor.config.config import Config -from profiler.advisor.common.analyzer_scopes import SupportedScopes -from profiler.advisor.common.async_analysis_status import AsyncAnalysisStatus -from profiler.advisor.common.enum_params_parser import EnumParamsParser -from profiler.advisor.utils.utils import Timer, safe_index_value, safe_division, safe_index, convert_to_int -from profiler.advisor.interface.interface import Interface -from profiler.cluster_analyse.cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor -from profiler.cluster_analyse.cluster_data_preprocess.mindspore_data_preprocessor import MindsporeDataPreprocessor -from profiler.prof_common.path_manager import PathManager -from profiler.prof_common.constant import Constant - -# 以spawn模式启动多进程，避免fork主进程资源。如果主进程逻辑较为复杂，fork可能会导致异常。 -mp.set_start_method("spawn", force=True) -logger = logging.getLogger() - - -class AsyncParams: - """处理用户异步请求的输入参数，包括cli arguments和环境变量两类参数.""" - user_valid_arguments = {} - user_valid_envs = {} - user_non_enum_params = {} - user_invalid_values = [] - user_total_params = {} - - @staticmethod - def parse_async_list_params(key, value, option_values, key_type, value_type): - if isinstance(value, list): - value_list = value - else: - value_list = [_.strip(" ") for _ in str(value).split(",")] - - if sorted(value_list) not in [sorted(option) for option in option_values]: - AsyncParams.user_invalid_values.append( - {"key": key, "invalid value": value, "optional values": option_values, - "required value type": value_type}) - return - if key_type == EnumParamsParser.ENVS: - AsyncParams.user_valid_envs[key.upper()] = ",".join(value_list) - elif key_type == EnumParamsParser.ARGUMENTS: - AsyncParams.user_valid_arguments[key] = value_list - - @staticmethod - def parse_async_int_params(key, value, option_values, key_type, value_type): - if convert_to_int(value) not in option_values: - AsyncParams.user_invalid_values.append( - {"key": key, "invalid value": value, "optional values": option_values, - "required value type": value_type}) - return - - if key_type == EnumParamsParser.ENVS: - AsyncParams.user_valid_envs[key.upper()] = str(convert_to_int(value)) - elif key_type == EnumParamsParser.ARGUMENTS: - AsyncParams.user_valid_arguments[key] = convert_to_int(value) - - @staticmethod - def parse_async_str_params(key, value, option_values, key_type, value_type): - if str(value) not in option_values: - AsyncParams.user_invalid_values.append( - {"key": key, "invalid value": value, "optional values": option_values, - "required value type": value_type}) - return - if key_type == EnumParamsParser.ENVS: - AsyncParams.user_valid_envs[key.upper()] = str(value) - elif key_type == EnumParamsParser.ARGUMENTS: - AsyncParams.user_valid_arguments[key] = str(value) - - @staticmethod - def parse_async_boolean_params(key, value, option_values, key_type, value_type): - - if str(value).lower() not in ["true", "false"]: - AsyncParams.user_invalid_values.append( - {"key": key, "invalid value": value, "optional values": option_values, - "required value type": value_type}) - return - - if key_type == EnumParamsParser.ENVS: - AsyncParams.user_valid_envs[key.upper()] = str(value) - elif key_type == EnumParamsParser.ARGUMENTS: - AsyncParams.user_valid_arguments[key] = str(value).lower() == "true" - - @staticmethod - def parse_params(user_async_params): - params_parser = EnumParamsParser() - valid_env_keys = [key.lower() for key in params_parser.get_envs_keys()] - valid_arg_keys = [key.lower() for key in params_parser.get_arguments_keys()] - - for key, value in user_async_params.items(): - key = key.lower() - if key not in valid_env_keys + valid_arg_keys: - AsyncParams.user_non_enum_params[key] = value - continue - - if key in valid_env_keys: - # 环境变量均大写，异步调用入参到analyzer controller时支持用户使用小写配置环境变量 - option_values = params_parser.get_options(key.upper()) - value_type = params_parser.get_value_type(key.upper()) - key_type = params_parser.ENVS - else: - option_values = params_parser.get_options(key) - value_type = params_parser.get_value_type(key) - key_type = params_parser.ARGUMENTS - - if hasattr(AsyncParams, f"parse_async_{value_type}_params"): - getattr(AsyncParams, f"parse_async_{value_type}_params")(key, value, option_values, key_type, - value_type) - - AsyncParams.user_total_params["async_analysis_env"] = AsyncParams.user_valid_envs - AsyncParams.user_total_params.update(AsyncParams.user_valid_arguments) - AsyncParams.user_total_params.update(AsyncParams.user_non_enum_params) - - -class AnalyzerController: - CLUSTER_RANK_THRESHOLD = 2 - SDMA_SUPPORT_SCOPES = [SupportedScopes.BANDWIDTH_CONTENTION_DETECTION, SupportedScopes.BYTE_ALIGNMENT_DETECTION] - RDMA_SUPPORT_SCOPES = [SupportedScopes.PACKET] - COMMUNICATION_MAPPING = { - SlowLinkAnalyzer.SDMA: SDMA_SUPPORT_SCOPES, - SlowLinkAnalyzer.RDMA: RDMA_SUPPORT_SCOPES - } - - def __init__(self): - self.dimensions = Interface.all_dimension - self.kwargs = {} - self.args_manager = None - self.slow_rank_analyzer = None - self.slow_link_analyzer = None - self.cluster_local_data_map = {} - self.default_rank_id = None - self.rank_id_map = {} - self._is_cluster = False - self.analysis_process_resp = Manager().dict() - - @staticmethod - def _set_analysis_process_priority(pid): - # 将分析进程优先级设置为最低，避免因为分析进程阻塞其他任务进程，unix上19表示最低优先级 - unix_process_lowest_priority = 19 - windows_platform = "windows" - linux_platform = "linux" - p = psutil.Process(pid) - if platform.system().lower() == windows_platform: - p.nice(psutil.BELOW_NORMAL_PRIORITY_CLASS) - elif platform.system().lower() == linux_platform: - p.nice(unix_process_lowest_priority) - - @staticmethod - def _check_profiling_path_valid(profiling_path): - PathManager.input_path_common_check(profiling_path) - - if not Path(profiling_path).exists(): - logger.error("Profiling path is not existed. Invalid profiling path: %s", profiling_path) - return False - - return True - - - @staticmethod - def _get_step_rank_for_cluster_statistic_diff(target_cluster_statistic_data, benchmark_cluster_statistic_data, - headers, dimension, get_max=False): - if dimension not in headers: - logger.error("Error dimension %s for cluster statistics data, optionals are %s.", dimension, headers) - return None, None, None - - dimension_index = safe_index_value(headers, dimension) - diff_record = [] - # 对比目标profiling和benchmark profiling 每张卡的计算和下发和带宽，取计算、下发、带宽差异最大的卡进行下一步分析 - for target_row_data, benchmark_row_data in zip(target_cluster_statistic_data, benchmark_cluster_statistic_data): - target_data = safe_index(target_row_data, dimension_index) - benchmark_data = safe_index(benchmark_row_data, dimension_index) - - if not isinstance(target_data, (int, float)) or not isinstance(benchmark_data, (int, float)): - continue - diff_record.append(target_data - benchmark_data) - - if SlowRankAnalyzer.compute_max_gap_ratio(diff_record, safe_division(sum(diff_record), len( - diff_record))) < SlowRankAnalyzer.RATIO_THRESHOLD: - return None, None, None - - value = max(diff_record) if get_max else min(diff_record) - value_index = safe_index_value(diff_record, value) - - step_value_index = safe_index_value(headers, "step") - rank_id_value_index = safe_index_value(headers, "rank_id") - - step = safe_index(safe_index(target_cluster_statistic_data, value_index, []), step_value_index) - benchmark_step = safe_index(safe_index(benchmark_cluster_statistic_data, value_index, []), step_value_index) - target_rank_id = safe_index(safe_index(target_cluster_statistic_data, value_index, []), rank_id_value_index) - benchmark_rank_id = safe_index(safe_index(benchmark_cluster_statistic_data, value_index, []), - rank_id_value_index) - - if target_rank_id != benchmark_rank_id: - logger.error( - "Rank ids of target profiling must keep the same as benchmark profiling, skip cluster comparison") - return None, None, None - - return step, benchmark_step, target_rank_id - - @staticmethod - def _init_async_analysis_env(kwargs): - envs = kwargs.get("async_analysis_env", {}) - for key, value in envs.items(): - os.environ[key] = value - - def format_async_analysis_params(self, pid, async_resp, dimensions, kwargs): - - AsyncParams.parse_params(kwargs) - dimensions = AsyncParams.user_total_params.get("analysis_dimensions") or dimensions - - if AsyncParams.user_invalid_values: - error_msg = "Got invalid arguments as follows: \n " - for index, invalid_value in enumerate(AsyncParams.user_invalid_values): - error_msg += f"{index + 1}. Key '{invalid_value.get('key')}', " \ - f"invalid value '{invalid_value.get('invalid value')}', " \ - f"optional valid values '{invalid_value.get('optional values')}', " \ - f"required value type '{invalid_value.get('required value type')}'.\n " - self._update_analysis_process_resp(pid, async_resp, error_msg=error_msg, - status_code=AsyncAnalysisStatus.BAD_REQUEST_STATUS_CODE, - status=AsyncAnalysisStatus.FAILED) - raise ValueError(error_msg) - - logger.warning("User parameters for async analysis is as follows:\n %s", - json.dumps(AsyncParams.user_total_params, indent=4)) - return dimensions, AsyncParams.user_total_params - - def do_analysis(self, dimensions, **kwargs): - pid = os.getpid() - resp = {"id": pid} - self.args_manager = AdditionalArgsManager() - self.args_manager.init(kwargs) - output_path = kwargs.get("output_path") - - AnalyzerController._set_analysis_process_priority(pid) - if kwargs.get("is_async_analysis"): - del kwargs["is_async_analysis"] - dimensions, kwargs = self.format_async_analysis_params(pid, resp, dimensions, kwargs) - AnalyzerController._init_async_analysis_env(kwargs) - - try: - if output_path: - - PathManager.check_input_directory_path(output_path) - if os.path.exists(output_path): - PathManager.check_path_owner_consistent([output_path]) - else: - PathManager.make_dir_safety(output_path) - - Config().set_config("_work_path", output_path) - Config().set_log_path(f"mstt_advisor_{Timer().strftime}.xlsx") - - self._do_analysis(dimensions, pid=pid, async_resp=resp, **kwargs) - except Exception as e: - self._update_analysis_process_resp(pid, resp, status_code=AsyncAnalysisStatus.INNER_ERROR_STATUS_CODE, - status=AsyncAnalysisStatus.FAILED, error_msg=str(e)) - logger.error(e) - raise RuntimeError("Do analysis error.") from e - - def async_do_analysis(self, dimensions, **kwargs): - """ Deploy a online service to start async analysis job, wrap this api by flask or tornado and so on, - then could query the analysis status by restful api. - You can view file 'profiler/advisor/config/enum_parameters.yaml' to obtain detailed information for - all the args listed below. - - Args: - dimensions: analysis dimension, normally set as Interface.all_dimension, support specific dimension analysis - such as ['computation'] or ['computation', 'schedule'] - cann_version: cann version of your runtime, inpact on the analysis of affinity api and AICPU operators - profiling_type: profiling type of your runtime - profiling_version: profiling version of your runtime, inpact on the analysis of affinity api - analysis_dimensions: can overwite dimensions. - advisor_analyze_processes: number of processes to use while the training params pipeline parallel(pp) >1, - can reduce the time of analysis. - disable_profiling_comparison: disable comparison of operators(including npu computation operator and - cpu torch aten operator), can reduce the time of analysis. - disable_affinity_api: disable analysis of affinity api, normally set as 'True' while you training job - has been trained on NPU for a long time and suddenly shows performance degradation. - output_path: analysis output path(including html and xlsx). - - Example: - >>> # initialize a global analyzer controller - >>> analyzer_controller = AnalyzerController() - >>> analysis_kwargs = dict(advisor_analyze_processes=2, disable_profiling_comparison=True) - >>> - >>> async_analysis_process = analyzer_controller.async_do_analysis( - >>> Interface.all_dimension, **analysis_kwargs) - >>> - >>> - >>> # query the job status every second - >>> while True: - >>> response = analyzer_controller.get_response_by_pid(async_analysis_process.pid) - >>> print(f'analysis response is {response}') - >>> if response.get("status") in ["success", "failed"]: - >>> async_analysis_process.join() - >>> break - >>> time.sleep(1) - """ - kwargs["is_async_analysis"] = True - - async_analysis_process = mp.Process(target=self.do_analysis, args=(dimensions,), kwargs=kwargs, - name="Async advisor performance analysis") - async_analysis_process.start() - self._update_analysis_process_resp(async_analysis_process.pid, {"id": async_analysis_process.pid}, - status_code=AsyncAnalysisStatus.NON_FAILED_STATUS_CODE, - status=AsyncAnalysisStatus.ANALYZING) - return async_analysis_process - - def get_response_by_pid(self, pid): - def _is_pid_exists(pid): - try: - psutil.Process(pid) - return True - except psutil.NoSuchProcess: - return False - - pid_not_exist_response = dict(id=pid, status_code=AsyncAnalysisStatus.NOT_FOUND_STATUS_CODE, - status=AsyncAnalysisStatus.FAILED, - error_msg="The advisor task id does not exist") - if pid not in self.analysis_process_resp: - return pid_not_exist_response - - response = self.analysis_process_resp.get(pid) - if response.get("status") not in [AsyncAnalysisStatus.FAILED, - AsyncAnalysisStatus.SUCCESS] and not _is_pid_exists(pid): - return pid_not_exist_response - return response - - def single_rank_analysis(self, profiling_path, benchmark_profiling_path=None): - job_list = [] - - profiling_path = self._get_profiling_path_by_rank(profiling_path) - benchmark_profiling_path = self._get_profiling_path_by_rank(benchmark_profiling_path) - - # 单卡场景无集群分析 - for dim in [Interface.CLUSTER]: - if dim in self.dimensions: - self.dimensions.remove(dim) - - for dimension in self.dimensions: - dimension_analysis_func_name = f"{dimension}_analysis" - if not hasattr(self, dimension_analysis_func_name): - continue - logger.info("Start %s analysis", dimension) - job_list += getattr(self, dimension_analysis_func_name)(profiling_path) - - if benchmark_profiling_path: - # kernel/api 比对 - compare_profiling_list = [ - dict(profiling_path=profiling_path, benchmark_profiling_path=benchmark_profiling_path, - compare_mode=Constant.KERNEL_COMPARE), - dict(profiling_path=profiling_path, benchmark_profiling_path=benchmark_profiling_path, - compare_mode=Constant.API_COMPARE) - ] - - job_list += self._profiling_comparison(compare_profiling_list) - else: - self.overall(profiling_path) - - return job_list - - def do_cluster_analysis(self, profiling_path, benchmark_profiling_path=None): - job_list = [] - - # 单集群profiling分析：下发、通信、计算、显存/内存 - for dimension in self.dimensions: - dimension_analysis_func_name = f"cluster_{dimension}_analysis" - if not hasattr(self, dimension_analysis_func_name): - continue - logger.info("Start cluster %s analysis", dimension) - job_list += getattr(self, dimension_analysis_func_name)(profiling_path) - - self.overall(profiling_path) - - if benchmark_profiling_path: - # 两个集群profiling比对分析 - job_list += self._cluster_profiling_comparison(profiling_path, benchmark_profiling_path) - return job_list - - def overall(self, profiling_path): - from profiler.advisor.analyzer.overall.environment_variable_analyzer import EnvironmentVariableAnalyzer - env_analyzer = EnvironmentVariableAnalyzer(profiling_path) - env_analyzer.optimize() - - if self._is_cluster: - self.slow_rank_analyzer.optimize(template_key=Interface.OVERALL) - self.slow_link_analyzer.optimize(template_key=Interface.OVERALL) - else: - overall_analyzer = OverallSummaryAnalyzer(profiling_path) - overall_analyzer.optimize() - - def schedule_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, benchmark_step=None, - **kwargs): - # 任意单卡的下发分析 - - input_kwargs = copy.deepcopy(self.kwargs) - job_list = [] - - input_kwargs["profiling_path"] = profiling_path - input_kwargs["benchmark_profiling_path"] = benchmark_profiling_path - input_kwargs["step"] = step - input_kwargs["benchmark_step"] = benchmark_step - input_kwargs["rank"] = kwargs.get("rank") - input_kwargs["step_duration"] = kwargs.get("step_duration") - - for dimension in [Interface.SCHEDULE]: - for scope in Interface.get_scope(dimension): - interface = Interface(**input_kwargs) - job_list.append((dimension, scope, interface, input_kwargs)) - return job_list - - def computation_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, - benchmark_step=None, stage=None, **kwargs): - # 任意单卡的计算分析 - - input_kwargs = copy.deepcopy(self.kwargs) - input_kwargs["profiling_path"] = profiling_path - input_kwargs["benchmark_profiling_path"] = benchmark_profiling_path - input_kwargs["step"] = step - input_kwargs["benchmark_step"] = benchmark_step - input_kwargs["stage"] = stage - input_kwargs["rank"] = kwargs.get("rank") - input_kwargs["step_duration"] = kwargs.get("step_duration") - job_list = [] - - for dimension in [Interface.COMPUTATION]: - for scope in Interface.get_scope(dimension): - if scope == SupportedScopes.STAGE_COMPUTE: - continue - interface = Interface(**input_kwargs) - job_list.append((dimension, scope, interface, input_kwargs)) - return job_list - - def memory_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, benchmark_step=None, **kwargs): - # 任意单卡的内存分析 - - input_kwargs = copy.deepcopy(self.kwargs) - job_list = [] - - input_kwargs["profiling_path"] = profiling_path - input_kwargs["benchmark_profiling_path"] = benchmark_profiling_path - input_kwargs["step"] = step - input_kwargs["benchmark_step"] = benchmark_step - input_kwargs["rank"] = kwargs.get("rank") - input_kwargs["step_duration"] = kwargs.get("step_duration") - - for dimension in [Interface.MEMORY]: - for scope in Interface.get_scope(dimension): - interface = Interface(**input_kwargs) - job_list.append((dimension, scope, interface, input_kwargs)) - return job_list - - def communication_analysis(self, profiling_path, benchmark_profiling_path=None, **kwargs): - - job_list = [] - supported_trans_type = [SlowLinkAnalyzer.SDMA, SlowLinkAnalyzer.RDMA] - step = kwargs.get("step", None) - benchmark_step = kwargs.get("benchmark_step", None) - bandwidth_type = kwargs.get("bandwidth_type", None) - scope = kwargs.get("scope", None) - if bandwidth_type is not None and bandwidth_type not in supported_trans_type: - logger.error("Error transit type %s, optionals are %s", bandwidth_type, supported_trans_type) - return job_list - - job_list += self._communication_analysis(profiling_path=profiling_path, - benchmark_profiling_path=benchmark_profiling_path, - step=step, benchmark_step=benchmark_step, - scope=scope, bandwidth_type=bandwidth_type) - - return job_list - - def cluster_schedule_analysis(self, profiling_path): - # 目标集群profiling数据下发分析，不包含两个集群profiling数据的比对分析 - - job_list = [] - global_step_rank = self.slow_rank_analyzer.get_global_step_rank(SlowRankAnalyzer.FREE) - - info_msg = "For cluster schedule analysis, " - slow_rank_id = global_step_rank.get("maximum", {}).get("rank_id") - if slow_rank_id is not None: - info_msg += f"maximum free for rank {slow_rank_id}" - else: - slow_rank_id = self.default_rank_id - info_msg += f"no slow rank with free time, analysis for default rank {slow_rank_id}" - - fast_rank_id = global_step_rank.get("minimum", {}).get("rank_id") - - slow_step = global_step_rank.get("maximum", {}).get("step") - fast_step = global_step_rank.get("minimum", {}).get("step") - - if slow_step is not None: - info_msg += f" and step {slow_step}" - logger.info(info_msg) - - kwargs = dict(profiling_path=self._get_profiling_path_by_rank(profiling_path, slow_rank_id), - benchmark_profiling_path=self._get_profiling_path_by_rank(profiling_path, fast_rank_id), - step=slow_step, benchmark_step=fast_step, - rank=slow_rank_id, benchmark_rank=fast_rank_id, - compare_mode=Constant.API_COMPARE, - step_duration=self.slow_rank_analyzer.get_step_duration(slow_rank_id, slow_step)) - - job_list += self.schedule_analysis(**kwargs) - - rank_id_valid = slow_rank_id is not None and fast_rank_id is not None and fast_rank_id != slow_rank_id - if not self.kwargs.get("benchmark_profiling_path") and rank_id_valid: - # 当用户指定benchmark profiling path时，不进行目标集群profiling的内部快慢卡对比 - logger.info("Enable schedule comparison of fast and slow rank/step") - job_list += self._profiling_comparison([kwargs]) - return job_list - - def cluster_communication_analysis(self, profiling_path): - job_list = [] - - for dimension in [Interface.COMMUNICATION]: - for scope in Interface.get_scope(dimension): - analyzer_class = Interface.get_analyzer(dimension, scope) - if hasattr(analyzer_class, "requires_cluster_dataset") and getattr(analyzer_class, - "requires_cluster_dataset"): - - # 如果不依赖数据集，或者依赖的是ClusterDataset，则不用根据带宽确定需要分析的特定rank - kwargs = copy.deepcopy(self.kwargs) - kwargs["profiling_path"] = profiling_path - interface = Interface(**kwargs) - job_list.append((dimension, scope, interface, kwargs)) - else: - # 非ClusterDataset场景，需要根据带宽大小分析特定的rank - for bandwidth_type in [SlowLinkAnalyzer.SDMA, SlowLinkAnalyzer.RDMA]: - global_step_rank = self.slow_link_analyzer.get_global_step_rank(bandwidth_type) - # 获取带宽最小的卡进行分析 - target_rank_id = global_step_rank.get("minimum", {}).get("rank_id") - if target_rank_id is None: - target_rank_id = self.default_rank_id - step = global_step_rank.get("minimum", {}).get("step") - analysis_profiling_path = self._get_profiling_path_by_rank(profiling_path, target_rank_id) - - info_msg = f"Minimum {bandwidth_type} bandwidth for rank {target_rank_id} " - if step: - info_msg += f"and step {step}" - logger.info(info_msg) - - job_list += self.communication_analysis(analysis_profiling_path, step=step, - bandwidth_type=bandwidth_type, scope=scope) - - return job_list - - def cluster_computation_analysis(self, profiling_path): - # 目标集群profiling数据计算分析，不包含两个集群profiling数据的比对分析；如果有pp stage，则对不同stage进行计算分析 - - job_list = [] - global_step_rank = self.slow_rank_analyzer.get_global_step_rank(SlowRankAnalyzer.COMPUTE) - stage_step_rank = self.slow_rank_analyzer.get_stage_step_rank(SlowRankAnalyzer.COMPUTE) - - if stage_step_rank: - job_list = self._stage_computation_analysis(profiling_path, stage_step_rank, job_list) - else: - job_list = self._global_computation_analysis(profiling_path, global_step_rank, job_list) - return job_list - - def cluster_memory_analysis(self, profiling_path): - # 目标集群profiling数据内存分析，当前memory识别的两个算子，导致的问题都是大的free，因此选择FREE最慢的卡进行分析 - - job_list = [] - global_step_rank = self.slow_rank_analyzer.get_global_step_rank(SlowRankAnalyzer.FREE) - - info_msg = "For cluster memory analysis, " - slow_rank_id = global_step_rank.get("maximum", {}).get("rank_id") - if slow_rank_id is not None: - info_msg += f"maximum free for rank {slow_rank_id}" - else: - slow_rank_id = self.default_rank_id - info_msg += f"no slow rank with free time, analysis for default rank {slow_rank_id}" - - slow_step = global_step_rank.get("maximum", {}).get("step") - if slow_step is not None: - info_msg += f" and step {slow_step}" - logger.info(info_msg) - - analysis_profiling_path = self._get_profiling_path_by_rank(profiling_path, slow_rank_id) - step_duration = self.slow_rank_analyzer.get_step_duration(slow_rank_id, slow_step) - job_list += self.memory_analysis(analysis_profiling_path, step=slow_step, rank=slow_rank_id, - step_duration=step_duration) - return job_list - - def _do_analysis(self, dimensions, pid=0, async_resp=None, **kwargs): - self.dimensions = dimensions - self.kwargs = kwargs - result_list = [] - profiling_path = PathManager.get_realpath(self.kwargs.get("profiling_path")) - benchmark_profiling_path = self.kwargs.get("benchmark_profiling_path") - PathManager.check_path_owner_consistent([profiling_path]) - if benchmark_profiling_path: - benchmark_profiling_path = PathManager.get_realpath(benchmark_profiling_path) - PathManager.check_path_owner_consistent([benchmark_profiling_path]) - - if not self._check_profiling_path_valid(profiling_path): - error_msg = f"Got invalid argument '-d/--profiling_path' {profiling_path}, skip analysis" - self._update_analysis_process_resp(pid, async_resp, error_msg=error_msg, - status_code=AsyncAnalysisStatus.BAD_REQUEST_STATUS_CODE, - status=AsyncAnalysisStatus.FAILED) - logger.error(error_msg) - return - - - if benchmark_profiling_path and not self._check_profiling_path_valid(benchmark_profiling_path): - error_msg = (f"Got invalid argument '-bp/--benchmark_profiling_path' {benchmark_profiling_path}, " - f"skip analysis") - self._update_analysis_process_resp(pid, async_resp, error_msg=error_msg, - status_code=AsyncAnalysisStatus.BAD_REQUEST_STATUS_CODE, - status=AsyncAnalysisStatus.FAILED) - logger.error(error_msg) - return - - self._is_cluster = self._is_cluster_profiling(profiling_path) - if benchmark_profiling_path: - # 构建benchmark profiling的map，用于根据rank获取profiling路径，否则无法进行比对 - is_benchmark_cluster = self._is_cluster_profiling(benchmark_profiling_path) - is_comparison_path_valid = (self._is_cluster and is_benchmark_cluster) or ( - not self._is_cluster and not is_benchmark_cluster) - if not is_comparison_path_valid: - error_msg = f"Only support profiling comparison for '1 npu vs 1 gpu/npu' and 'multi npus vs multi npus'" - self._update_analysis_process_resp(pid, async_resp, error_msg=error_msg, - status_code=AsyncAnalysisStatus.BAD_REQUEST_STATUS_CODE, - status=AsyncAnalysisStatus.FAILED) - logger.error(error_msg) - return - - if not self._is_cluster: - job_list = self.single_rank_analysis(profiling_path, benchmark_profiling_path) - else: - self.slow_rank_analyzer = SlowRankAnalyzer(profiling_path, output_path=self.kwargs.get("output_path")) - self.slow_link_analyzer = SlowLinkAnalyzer(profiling_path, output_path=self.kwargs.get("output_path")) - job_list = self.do_cluster_analysis(profiling_path, benchmark_profiling_path) - - for i, (dimension, scope, interface, kwargs) in enumerate(job_list[::-1]): - result_list.append( - interface.get_result(dimension, scope, render_html=i == len(job_list) - 1, output_dict=False, - **kwargs) - ) - - for result in result_list[::-1]: - if result and hasattr(result, "show"): - result.show() - break - self._get_analysis_finished_resp(pid, async_resp) - - def _get_scopes(self, scope=None, bandwidth_type=SlowLinkAnalyzer.SDMA): - """ - Args: - scope: analyzer type - bandwidth_type: analysis standard - Returns: - scope lists - """ - scopes = [] - if scope: - if scope in self.COMMUNICATION_MAPPING.get(bandwidth_type, self.SDMA_SUPPORT_SCOPES): - scopes.append(scope) - return scopes - for dimension in [Interface.COMMUNICATION]: - for scope_ in Interface.get_scope(dimension): - if scope_ in self.SDMA_SUPPORT_SCOPES or scope_ in self.RDMA_SUPPORT_SCOPES: - scopes.append(scope_) - return scopes - - def _communication_analysis(self, **child_kwargs): - kwargs = copy.deepcopy(self.kwargs) - job_list = [] - - kwargs["profiling_path"] = child_kwargs.get("profiling_path", "") - kwargs["benchmark_profiling_path"] = child_kwargs.get("benchmark_profiling_path", "") - kwargs["step"] = child_kwargs.get("step", -1) - kwargs["benchmark_step"] = child_kwargs.get("benchmark_step", -1) - bandwidth_type = child_kwargs.get("bandwidth_type", SlowLinkAnalyzer.SDMA) - scope = child_kwargs.get("scope", None) - - for scope_ in self._get_scopes(scope, bandwidth_type): - interface = Interface(**kwargs) - job_list.append((Interface.COMMUNICATION, scope_, interface, kwargs)) - - return job_list - - def _profiling_comparison(self, compare_profiling_list): - job_list = [] - disable_profiling_comparison = os.getenv(Constant.DISABLE_PROFILING_COMPARISON) - if disable_profiling_comparison is not None and disable_profiling_comparison.lower() == "true": - logger.info( - "Skip profiling comparison due to longer processing time due to env 'DISABLE_PROFILING_COMPARISON'") - return job_list - - for index, _kwargs in enumerate(compare_profiling_list): - kwargs = copy.deepcopy(self.kwargs) - kwargs.update(_kwargs) - compare_profiling_list[index] = kwargs - - compare_kwargs = { - "profiling_path": kwargs.get("profiling_path"), - "compare_profiling_list": compare_profiling_list, - } - - interface = Interface(**compare_kwargs) - job_list.append((Interface.COMPARISON, SupportedScopes.COMPARISON, interface, compare_kwargs)) - - return job_list - - def _cluster_profiling_comparison(self, profiling_path, benchmark_profiling_path): - # 从计算、下发和通信三个维度对集群profiling数据进行对比 - - job_list = [] - benchmark_profiling_path = self._get_profiling_path_by_rank(benchmark_profiling_path) - benchmark_slow_rank_analyzer = SlowRankAnalyzer(benchmark_profiling_path) - benchmark_slow_link_analyzer = SlowLinkAnalyzer(benchmark_profiling_path) - - # 计算和下发分析 - job_list += self._cluster_data_comparison(profiling_path, - benchmark_profiling_path, - self.slow_rank_analyzer, - benchmark_slow_rank_analyzer, - get_max=True) - - # 通信分析 - job_list += self._cluster_data_comparison(profiling_path, - benchmark_profiling_path, - self.slow_link_analyzer, - benchmark_slow_link_analyzer, - get_max=False) - return job_list - - def _cluster_data_comparison(self, profiling_path, benchmark_profiling_path, target_cluster_analyzer, - benchmark_cluster_analyzer, get_max=False): - # #low rank/slow link结果逐行对比获取差值最大的rank和step进行单卡分析 - job_list = [] - - if isinstance(target_cluster_analyzer, SlowRankAnalyzer): - comparison_dims = [SlowRankAnalyzer.COMPUTE, SlowRankAnalyzer.FREE] - comparison_modes = [Constant.KERNEL_COMPARE, Constant.API_COMPARE] - elif isinstance(target_cluster_analyzer, SlowLinkAnalyzer): - comparison_dims = [SlowLinkAnalyzer.SDMA_BANDWIDTH, SlowLinkAnalyzer.RDMA_BANDWIDTH] - comparison_modes = [None, None] - else: - return job_list - - target_data = target_cluster_analyzer.format_datas.get("data", []) - benchmark_data = benchmark_cluster_analyzer.format_datas.get("data", []) - headers = benchmark_cluster_analyzer.format_datas.get("headers", []) - - if len(target_data) != len(benchmark_data): - logger.warning( - "The product of ranks and steps of Benchmark profiling is not equals to target profiling, " - "skip cluster comparison.") - return job_list - - compare_profiling_list = [] - for dimension, compare_mode in zip(comparison_dims, comparison_modes): - step, benchmark_step, rank_id_for_comparison = AnalyzerController._get_step_rank_for_cluster_statistic_diff( - target_data, - benchmark_data, - headers, - dimension, - get_max=get_max - ) - - rank_profiling_path = self._get_profiling_path_by_rank(profiling_path, rank_id_for_comparison) - rank_benchmark_profiling_path = self._get_profiling_path_by_rank( - benchmark_profiling_path, - rank_id_for_comparison - ) - - if rank_id_for_comparison is None: - # rank id为空则无法获取对应rank的profiling路径，无法进行比较 - continue - - compare_profiling_list.append( - dict(profiling_path=rank_profiling_path, benchmark_profiling_path=rank_benchmark_profiling_path, - step=step, benchmark_step=benchmark_step, - rank=rank_id_for_comparison, benchmark_rank=rank_id_for_comparison, compare_mode=compare_mode) - ) - - if not compare_profiling_list: - return job_list - - job_list += self._profiling_comparison(compare_profiling_list) - return job_list - - def _is_cluster_profiling(self, profiling_path): - if os.path.isfile(profiling_path): - return False - path_list = [os.path.join(profiling_path, dir_name) for dir_name in os.listdir(profiling_path)] - ascend_pt_dirs = [path for path in path_list if os.path.isdir(path) and path.endswith("ascend_pt")] - ascend_ms_dirs = [path for path in path_list if os.path.isdir(path) and path.endswith("ascend_ms")] - if ascend_ms_dirs and ascend_pt_dirs: - logger.error("Cannot analyze pytorch and mindspore meantime.") - return False - if not ascend_pt_dirs and not ascend_ms_dirs: - return False - if ascend_ms_dirs and not ascend_pt_dirs: - data_processor = MindsporeDataPreprocessor(ascend_ms_dirs) - elif ascend_pt_dirs and not ascend_ms_dirs: - data_processor = PytorchDataPreprocessor(ascend_pt_dirs) - - self.cluster_local_data_map[profiling_path] = data_processor.get_data_map() - - if not self.cluster_local_data_map or not self.cluster_local_data_map.get(profiling_path): - return False - - self.default_rank_id = list(self.cluster_local_data_map[profiling_path].keys())[0] - - return len(self.cluster_local_data_map[profiling_path]) >= self.CLUSTER_RANK_THRESHOLD - - def _get_profiling_path_by_rank(self, profiling_path, rank_id=None): - - if not profiling_path: - return profiling_path - - return self._get_target_profiling_path_for_local(profiling_path, rank_id) - - def _get_target_profiling_path_for_local(self, profiling_path, rank_id): - rank_id_map = self.cluster_local_data_map.get(profiling_path, {}) - if rank_id is None or not rank_id_map: - return profiling_path - - if rank_id in rank_id_map: - return rank_id_map.get(rank_id) - - local_first_rank_id = sorted(list(map(int, rank_id_map.keys())))[0] - logger.warning("Target rank id %s does not exist in local profiling data %s, use rank %s for analysis", - rank_id, profiling_path, local_first_rank_id) - return rank_id_map.get(local_first_rank_id) - - def _update_analysis_process_resp(self, pid, resp, **kwargs): - if kwargs: - resp.update(kwargs) - self.analysis_process_resp[pid] = resp - - def _get_analysis_finished_resp(self, pid, resp): - advisor_output_file_prefix = f"mstt_advisor_{Timer().strftime}" - html_path = os.path.join(Config().work_path, f"{advisor_output_file_prefix}.html") - xlsx_path = os.path.join(Config().work_path, "log", f"{advisor_output_file_prefix}.xlsx") - if os.path.exists(html_path) and os.path.exists(xlsx_path): - result_files = {"html": html_path, "xlsx": xlsx_path} - self._update_analysis_process_resp(pid, resp, status_code=AsyncAnalysisStatus.NON_FAILED_STATUS_CODE, - status=AsyncAnalysisStatus.SUCCESS, result_files=result_files) - else: - self._update_analysis_process_resp(pid, resp, status_code=AsyncAnalysisStatus.BAD_REQUEST_STATUS_CODE, - status=AsyncAnalysisStatus.FAILED, - error_msg="No optimization suggestions, please check your input path.") - - def _stage_computation_analysis(self, profiling_path, stage_step_rank, job_list): - # 对不同pp stage取min max进行分析 - logger.info("Steps and ranks to be analyzed of different pipeline parallel stages are %s", - json.dumps(stage_step_rank)) - - stages_profiling_path = [] - for stage, step_rank_info in stage_step_rank.items(): - rank_id = step_rank_info.get("maximum", {}).get("rank_id") - step = step_rank_info.get("maximum", {}).get("step") - benchmark_rank_id = step_rank_info.get("minimum", {}).get("rank_id") - benchmark_step = step_rank_info.get("minimum", {}).get("step") - - info_msg = f"For {stage}, slow rank is {rank_id}" - if step: - info_msg += f", step is {step}" - logger.info(info_msg) - - stages_profiling_path.append( - dict( - stage=stage, rank=rank_id, step=step, benchmark_rank=benchmark_rank_id, - benchmark_step=benchmark_step, - profiling_path=self._get_profiling_path_by_rank(profiling_path, rank_id), - benchmark_profiling_path=self._get_profiling_path_by_rank(profiling_path, benchmark_rank_id), - compare_mode=Constant.KERNEL_COMPARE, - step_duration=self.slow_rank_analyzer.get_step_duration(rank_id, step) - ) - ) - Interface.add_analyzer(Interface.COMPUTATION, SupportedScopes.STAGE_COMPUTE, PPStageComputationAnalyzer) - compute_analysis_kwargs = {"stages_profiling_path": stages_profiling_path, "profiling_path": profiling_path} - - job_list.append((Interface.COMPUTATION, SupportedScopes.STAGE_COMPUTE, Interface(**compute_analysis_kwargs), - compute_analysis_kwargs)) - if not self.kwargs.get("benchmark_profiling_path"): - logger.info("Enable computation comparison of fast and slow rank/step in different pp stages") - job_list += self._profiling_comparison(stages_profiling_path) - return job_list - - def _global_computation_analysis(self, profiling_path, global_step_rank, job_list): - # 不区分stage，对所有卡取Min max进行分析 - logger.info("Without pipeline parallel stage, steps and ranks to be analyzed are %s", - json.dumps(global_step_rank)) - slow_rank_id = global_step_rank.get("maximum", {}).get("rank_id") - if slow_rank_id is not None: - info_msg = f"Maximum computation time for rank {slow_rank_id}" - else: - slow_rank_id = self.default_rank_id - info_msg = f"No slow rank with computation time, analysis for default rank {slow_rank_id}" - slow_step = global_step_rank.get("maximum", {}).get("step") - # 如果没有标杆profiling数据的rank id，说明没有快慢卡问题，直接对默认rank id进行分析，因此这里取值为None - fast_rank_id = global_step_rank.get("minimum", {}).get("rank_id") - fast_step = global_step_rank.get("minimum", {}).get("step") - - if slow_step is not None: - info_msg += f" and step {slow_step}, " - if fast_rank_id is not None: - info_msg += f"minimum computation time for rank {fast_rank_id}" - if fast_step is not None: - info_msg += f" and step {fast_step}" - logger.info(info_msg) - - kwargs = dict(profiling_path=self._get_profiling_path_by_rank(profiling_path, slow_rank_id), - benchmark_profiling_path=self._get_profiling_path_by_rank(profiling_path, fast_rank_id), - step=slow_step, benchmark_step=fast_step, rank=slow_rank_id, benchmark_rank=fast_rank_id, - compare_mode=Constant.KERNEL_COMPARE, - step_duration=self.slow_rank_analyzer.get_step_duration(slow_rank_id, slow_step)) - - job_list += self.computation_analysis(**kwargs) - - rank_id_valid = slow_rank_id is not None and fast_rank_id is not None and fast_rank_id != slow_rank_id - if not self.kwargs.get("benchmark_profiling_path") and rank_id_valid: - # 当用户指定benchmark profiling path时，不进行目标集群profiling的内部快慢卡对比 - logger.info("Enable computation comparison of fast and slow rank/step") - job_list += self._profiling_comparison([kwargs]) - return job_list +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import copy +import logging +import json +import sys +import os +import platform +import multiprocessing as mp +from multiprocessing import Manager +from pathlib import Path + +import psutil + +sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "compare_tools")) +sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "cluster_analyse")) + +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.advisor.analyzer.cluster.slow_rank_analyzer import SlowRankAnalyzer +from msprof_analyze.advisor.analyzer.cluster.slow_link_analyzer import SlowLinkAnalyzer +from msprof_analyze.advisor.analyzer.computation.pp_stage_computation_analyzer import PPStageComputationAnalyzer +from msprof_analyze.advisor.analyzer.overall.overall_summary_analyzer import OverallSummaryAnalyzer +from msprof_analyze.advisor.config.config import Config +from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes +from msprof_analyze.advisor.common.async_analysis_status import AsyncAnalysisStatus +from msprof_analyze.advisor.common.enum_params_parser import EnumParamsParser +from msprof_analyze.advisor.utils.utils import Timer, safe_index_value, safe_division, safe_index, convert_to_int +from msprof_analyze.advisor.interface.interface import Interface +from msprof_analyze.cluster_analyse.cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor +from msprof_analyze.cluster_analyse.cluster_data_preprocess.mindspore_data_preprocessor import MindsporeDataPreprocessor +from msprof_analyze.prof_common.path_manager import PathManager +from msprof_analyze.prof_common.constant import Constant + +# 以spawn模式启动多进程，避免fork主进程资源。如果主进程逻辑较为复杂，fork可能会导致异常。 +mp.set_start_method("spawn", force=True) +logger = logging.getLogger() + + +class AsyncParams: + """处理用户异步请求的输入参数，包括cli arguments和环境变量两类参数.""" + user_valid_arguments = {} + user_valid_envs = {} + user_non_enum_params = {} + user_invalid_values = [] + user_total_params = {} + + @staticmethod + def parse_async_list_params(key, value, option_values, key_type, value_type): + if isinstance(value, list): + value_list = value + else: + value_list = [_.strip(" ") for _ in str(value).split(",")] + + if sorted(value_list) not in [sorted(option) for option in option_values]: + AsyncParams.user_invalid_values.append( + {"key": key, "invalid value": value, "optional values": option_values, + "required value type": value_type}) + return + if key_type == EnumParamsParser.ENVS: + AsyncParams.user_valid_envs[key.upper()] = ",".join(value_list) + elif key_type == EnumParamsParser.ARGUMENTS: + AsyncParams.user_valid_arguments[key] = value_list + + @staticmethod + def parse_async_int_params(key, value, option_values, key_type, value_type): + if convert_to_int(value) not in option_values: + AsyncParams.user_invalid_values.append( + {"key": key, "invalid value": value, "optional values": option_values, + "required value type": value_type}) + return + + if key_type == EnumParamsParser.ENVS: + AsyncParams.user_valid_envs[key.upper()] = str(convert_to_int(value)) + elif key_type == EnumParamsParser.ARGUMENTS: + AsyncParams.user_valid_arguments[key] = convert_to_int(value) + + @staticmethod + def parse_async_str_params(key, value, option_values, key_type, value_type): + if str(value) not in option_values: + AsyncParams.user_invalid_values.append( + {"key": key, "invalid value": value, "optional values": option_values, + "required value type": value_type}) + return + if key_type == EnumParamsParser.ENVS: + AsyncParams.user_valid_envs[key.upper()] = str(value) + elif key_type == EnumParamsParser.ARGUMENTS: + AsyncParams.user_valid_arguments[key] = str(value) + + @staticmethod + def parse_async_boolean_params(key, value, option_values, key_type, value_type): + + if str(value).lower() not in ["true", "false"]: + AsyncParams.user_invalid_values.append( + {"key": key, "invalid value": value, "optional values": option_values, + "required value type": value_type}) + return + + if key_type == EnumParamsParser.ENVS: + AsyncParams.user_valid_envs[key.upper()] = str(value) + elif key_type == EnumParamsParser.ARGUMENTS: + AsyncParams.user_valid_arguments[key] = str(value).lower() == "true" + + @staticmethod + def parse_params(user_async_params): + params_parser = EnumParamsParser() + valid_env_keys = [key.lower() for key in params_parser.get_envs_keys()] + valid_arg_keys = [key.lower() for key in params_parser.get_arguments_keys()] + + for key, value in user_async_params.items(): + key = key.lower() + if key not in valid_env_keys + valid_arg_keys: + AsyncParams.user_non_enum_params[key] = value + continue + + if key in valid_env_keys: + # 环境变量均大写，异步调用入参到analyzer controller时支持用户使用小写配置环境变量 + option_values = params_parser.get_options(key.upper()) + value_type = params_parser.get_value_type(key.upper()) + key_type = params_parser.ENVS + else: + option_values = params_parser.get_options(key) + value_type = params_parser.get_value_type(key) + key_type = params_parser.ARGUMENTS + + if hasattr(AsyncParams, f"parse_async_{value_type}_params"): + getattr(AsyncParams, f"parse_async_{value_type}_params")(key, value, option_values, key_type, + value_type) + + AsyncParams.user_total_params["async_analysis_env"] = AsyncParams.user_valid_envs + AsyncParams.user_total_params.update(AsyncParams.user_valid_arguments) + AsyncParams.user_total_params.update(AsyncParams.user_non_enum_params) + + +class AnalyzerController: + CLUSTER_RANK_THRESHOLD = 2 + SDMA_SUPPORT_SCOPES = [SupportedScopes.BANDWIDTH_CONTENTION_DETECTION, SupportedScopes.BYTE_ALIGNMENT_DETECTION] + RDMA_SUPPORT_SCOPES = [SupportedScopes.PACKET] + COMMUNICATION_MAPPING = { + SlowLinkAnalyzer.SDMA: SDMA_SUPPORT_SCOPES, + SlowLinkAnalyzer.RDMA: RDMA_SUPPORT_SCOPES + } + + def __init__(self): + self.dimensions = Interface.all_dimension + self.kwargs = {} + self.args_manager = None + self.slow_rank_analyzer = None + self.slow_link_analyzer = None + self.cluster_local_data_map = {} + self.default_rank_id = None + self.rank_id_map = {} + self._is_cluster = False + self.analysis_process_resp = Manager().dict() + + @staticmethod + def _set_analysis_process_priority(pid): + # 将分析进程优先级设置为最低，避免因为分析进程阻塞其他任务进程，unix上19表示最低优先级 + unix_process_lowest_priority = 19 + windows_platform = "windows" + linux_platform = "linux" + p = psutil.Process(pid) + if platform.system().lower() == windows_platform: + p.nice(psutil.BELOW_NORMAL_PRIORITY_CLASS) + elif platform.system().lower() == linux_platform: + p.nice(unix_process_lowest_priority) + + @staticmethod + def _check_profiling_path_valid(profiling_path): + PathManager.input_path_common_check(profiling_path) + + if not Path(profiling_path).exists(): + logger.error("Profiling path is not existed. Invalid profiling path: %s", profiling_path) + return False + + return True + + + @staticmethod + def _get_step_rank_for_cluster_statistic_diff(target_cluster_statistic_data, benchmark_cluster_statistic_data, + headers, dimension, get_max=False): + if dimension not in headers: + logger.error("Error dimension %s for cluster statistics data, optionals are %s.", dimension, headers) + return None, None, None + + dimension_index = safe_index_value(headers, dimension) + diff_record = [] + # 对比目标profiling和benchmark profiling 每张卡的计算和下发和带宽，取计算、下发、带宽差异最大的卡进行下一步分析 + for target_row_data, benchmark_row_data in zip(target_cluster_statistic_data, benchmark_cluster_statistic_data): + target_data = safe_index(target_row_data, dimension_index) + benchmark_data = safe_index(benchmark_row_data, dimension_index) + + if not isinstance(target_data, (int, float)) or not isinstance(benchmark_data, (int, float)): + continue + diff_record.append(target_data - benchmark_data) + + if SlowRankAnalyzer.compute_max_gap_ratio(diff_record, safe_division(sum(diff_record), len( + diff_record))) < SlowRankAnalyzer.RATIO_THRESHOLD: + return None, None, None + + value = max(diff_record) if get_max else min(diff_record) + value_index = safe_index_value(diff_record, value) + + step_value_index = safe_index_value(headers, "step") + rank_id_value_index = safe_index_value(headers, "rank_id") + + step = safe_index(safe_index(target_cluster_statistic_data, value_index, []), step_value_index) + benchmark_step = safe_index(safe_index(benchmark_cluster_statistic_data, value_index, []), step_value_index) + target_rank_id = safe_index(safe_index(target_cluster_statistic_data, value_index, []), rank_id_value_index) + benchmark_rank_id = safe_index(safe_index(benchmark_cluster_statistic_data, value_index, []), + rank_id_value_index) + + if target_rank_id != benchmark_rank_id: + logger.error( + "Rank ids of target profiling must keep the same as benchmark profiling, skip cluster comparison") + return None, None, None + + return step, benchmark_step, target_rank_id + + @staticmethod + def _init_async_analysis_env(kwargs): + envs = kwargs.get("async_analysis_env", {}) + for key, value in envs.items(): + os.environ[key] = value + + def format_async_analysis_params(self, pid, async_resp, dimensions, kwargs): + + AsyncParams.parse_params(kwargs) + dimensions = AsyncParams.user_total_params.get("analysis_dimensions") or dimensions + + if AsyncParams.user_invalid_values: + error_msg = "Got invalid arguments as follows: \n " + for index, invalid_value in enumerate(AsyncParams.user_invalid_values): + error_msg += f"{index + 1}. Key '{invalid_value.get('key')}', " \ + f"invalid value '{invalid_value.get('invalid value')}', " \ + f"optional valid values '{invalid_value.get('optional values')}', " \ + f"required value type '{invalid_value.get('required value type')}'.\n " + self._update_analysis_process_resp(pid, async_resp, error_msg=error_msg, + status_code=AsyncAnalysisStatus.BAD_REQUEST_STATUS_CODE, + status=AsyncAnalysisStatus.FAILED) + raise ValueError(error_msg) + + logger.warning("User parameters for async analysis is as follows:\n %s", + json.dumps(AsyncParams.user_total_params, indent=4)) + return dimensions, AsyncParams.user_total_params + + def do_analysis(self, dimensions, **kwargs): + pid = os.getpid() + resp = {"id": pid} + self.args_manager = AdditionalArgsManager() + self.args_manager.init(kwargs) + output_path = kwargs.get("output_path") + + AnalyzerController._set_analysis_process_priority(pid) + if kwargs.get("is_async_analysis"): + del kwargs["is_async_analysis"] + dimensions, kwargs = self.format_async_analysis_params(pid, resp, dimensions, kwargs) + AnalyzerController._init_async_analysis_env(kwargs) + + try: + if output_path: + + PathManager.check_input_directory_path(output_path) + if os.path.exists(output_path): + PathManager.check_path_owner_consistent([output_path]) + else: + PathManager.make_dir_safety(output_path) + + Config().set_config("_work_path", output_path) + Config().set_log_path(f"mstt_advisor_{Timer().strftime}.xlsx") + + self._do_analysis(dimensions, pid=pid, async_resp=resp, **kwargs) + except Exception as e: + self._update_analysis_process_resp(pid, resp, status_code=AsyncAnalysisStatus.INNER_ERROR_STATUS_CODE, + status=AsyncAnalysisStatus.FAILED, error_msg=str(e)) + logger.error(e) + raise RuntimeError("Do analysis error.") from e + + def async_do_analysis(self, dimensions, **kwargs): + """ Deploy a online service to start async analysis job, wrap this api by flask or tornado and so on, + then could query the analysis status by restful api. + You can view file 'profiler/msprof_analyze/advisor/config/enum_parameters.yaml' to obtain detailed information for + all the args listed below. + + Args: + dimensions: analysis dimension, normally set as Interface.all_dimension, support specific dimension analysis + such as ['computation'] or ['computation', 'schedule'] + cann_version: cann version of your runtime, inpact on the analysis of affinity api and AICPU operators + profiling_type: profiling type of your runtime + profiling_version: profiling version of your runtime, inpact on the analysis of affinity api + analysis_dimensions: can overwite dimensions. + advisor_analyze_processes: number of processes to use while the training params pipeline parallel(pp) >1, + can reduce the time of analysis. + disable_profiling_comparison: disable comparison of operators(including npu computation operator and + cpu torch aten operator), can reduce the time of analysis. + disable_affinity_api: disable analysis of affinity api, normally set as 'True' while you training job + has been trained on NPU for a long time and suddenly shows performance degradation. + output_path: analysis output path(including html and xlsx). + + Example: + >>> # initialize a global analyzer controller + >>> analyzer_controller = AnalyzerController() + >>> analysis_kwargs = dict(advisor_analyze_processes=2, disable_profiling_comparison=True) + >>> + >>> async_analysis_process = analyzer_controller.async_do_analysis( + >>> Interface.all_dimension, **analysis_kwargs) + >>> + >>> + >>> # query the job status every second + >>> while True: + >>> response = analyzer_controller.get_response_by_pid(async_analysis_process.pid) + >>> print(f'analysis response is {response}') + >>> if response.get("status") in ["success", "failed"]: + >>> async_analysis_process.join() + >>> break + >>> time.sleep(1) + """ + kwargs["is_async_analysis"] = True + + async_analysis_process = mp.Process(target=self.do_analysis, args=(dimensions,), kwargs=kwargs, + name="Async advisor performance analysis") + async_analysis_process.start() + self._update_analysis_process_resp(async_analysis_process.pid, {"id": async_analysis_process.pid}, + status_code=AsyncAnalysisStatus.NON_FAILED_STATUS_CODE, + status=AsyncAnalysisStatus.ANALYZING) + return async_analysis_process + + def get_response_by_pid(self, pid): + def _is_pid_exists(pid): + try: + psutil.Process(pid) + return True + except psutil.NoSuchProcess: + return False + + pid_not_exist_response = dict(id=pid, status_code=AsyncAnalysisStatus.NOT_FOUND_STATUS_CODE, + status=AsyncAnalysisStatus.FAILED, + error_msg="The advisor task id does not exist") + if pid not in self.analysis_process_resp: + return pid_not_exist_response + + response = self.analysis_process_resp.get(pid) + if response.get("status") not in [AsyncAnalysisStatus.FAILED, + AsyncAnalysisStatus.SUCCESS] and not _is_pid_exists(pid): + return pid_not_exist_response + return response + + def single_rank_analysis(self, profiling_path, benchmark_profiling_path=None): + job_list = [] + + profiling_path = self._get_profiling_path_by_rank(profiling_path) + benchmark_profiling_path = self._get_profiling_path_by_rank(benchmark_profiling_path) + + # 单卡场景无集群分析 + for dim in [Interface.CLUSTER]: + if dim in self.dimensions: + self.dimensions.remove(dim) + + for dimension in self.dimensions: + dimension_analysis_func_name = f"{dimension}_analysis" + if not hasattr(self, dimension_analysis_func_name): + continue + logger.info("Start %s analysis", dimension) + job_list += getattr(self, dimension_analysis_func_name)(profiling_path) + + if benchmark_profiling_path: + # kernel/api 比对 + compare_profiling_list = [ + dict(profiling_path=profiling_path, benchmark_profiling_path=benchmark_profiling_path, + compare_mode=Constant.KERNEL_COMPARE), + dict(profiling_path=profiling_path, benchmark_profiling_path=benchmark_profiling_path, + compare_mode=Constant.API_COMPARE) + ] + + job_list += self._profiling_comparison(compare_profiling_list) + else: + self.overall(profiling_path) + + return job_list + + def do_cluster_analysis(self, profiling_path, benchmark_profiling_path=None): + job_list = [] + + # 单集群profiling分析：下发、通信、计算、显存/内存 + for dimension in self.dimensions: + dimension_analysis_func_name = f"cluster_{dimension}_analysis" + if not hasattr(self, dimension_analysis_func_name): + continue + logger.info("Start cluster %s analysis", dimension) + job_list += getattr(self, dimension_analysis_func_name)(profiling_path) + + self.overall(profiling_path) + + if benchmark_profiling_path: + # 两个集群profiling比对分析 + job_list += self._cluster_profiling_comparison(profiling_path, benchmark_profiling_path) + return job_list + + def overall(self, profiling_path): + from msprof_analyze.advisor.analyzer.overall.environment_variable_analyzer import EnvironmentVariableAnalyzer + env_analyzer = EnvironmentVariableAnalyzer(profiling_path) + env_analyzer.optimize() + + if self._is_cluster: + self.slow_rank_analyzer.optimize(template_key=Interface.OVERALL) + self.slow_link_analyzer.optimize(template_key=Interface.OVERALL) + else: + overall_analyzer = OverallSummaryAnalyzer(profiling_path) + overall_analyzer.optimize() + + def schedule_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, benchmark_step=None, + **kwargs): + # 任意单卡的下发分析 + + input_kwargs = copy.deepcopy(self.kwargs) + job_list = [] + + input_kwargs["profiling_path"] = profiling_path + input_kwargs["benchmark_profiling_path"] = benchmark_profiling_path + input_kwargs["step"] = step + input_kwargs["benchmark_step"] = benchmark_step + input_kwargs["rank"] = kwargs.get("rank") + input_kwargs["step_duration"] = kwargs.get("step_duration") + + for dimension in [Interface.SCHEDULE]: + for scope in Interface.get_scope(dimension): + interface = Interface(**input_kwargs) + job_list.append((dimension, scope, interface, input_kwargs)) + return job_list + + def computation_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, + benchmark_step=None, stage=None, **kwargs): + # 任意单卡的计算分析 + + input_kwargs = copy.deepcopy(self.kwargs) + input_kwargs["profiling_path"] = profiling_path + input_kwargs["benchmark_profiling_path"] = benchmark_profiling_path + input_kwargs["step"] = step + input_kwargs["benchmark_step"] = benchmark_step + input_kwargs["stage"] = stage + input_kwargs["rank"] = kwargs.get("rank") + input_kwargs["step_duration"] = kwargs.get("step_duration") + job_list = [] + + for dimension in [Interface.COMPUTATION]: + for scope in Interface.get_scope(dimension): + if scope == SupportedScopes.STAGE_COMPUTE: + continue + interface = Interface(**input_kwargs) + job_list.append((dimension, scope, interface, input_kwargs)) + return job_list + + def memory_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, benchmark_step=None, **kwargs): + # 任意单卡的内存分析 + + input_kwargs = copy.deepcopy(self.kwargs) + job_list = [] + + input_kwargs["profiling_path"] = profiling_path + input_kwargs["benchmark_profiling_path"] = benchmark_profiling_path + input_kwargs["step"] = step + input_kwargs["benchmark_step"] = benchmark_step + input_kwargs["rank"] = kwargs.get("rank") + input_kwargs["step_duration"] = kwargs.get("step_duration") + + for dimension in [Interface.MEMORY]: + for scope in Interface.get_scope(dimension): + interface = Interface(**input_kwargs) + job_list.append((dimension, scope, interface, input_kwargs)) + return job_list + + def communication_analysis(self, profiling_path, benchmark_profiling_path=None, **kwargs): + + job_list = [] + supported_trans_type = [SlowLinkAnalyzer.SDMA, SlowLinkAnalyzer.RDMA] + step = kwargs.get("step", None) + benchmark_step = kwargs.get("benchmark_step", None) + bandwidth_type = kwargs.get("bandwidth_type", None) + scope = kwargs.get("scope", None) + if bandwidth_type is not None and bandwidth_type not in supported_trans_type: + logger.error("Error transit type %s, optionals are %s", bandwidth_type, supported_trans_type) + return job_list + + job_list += self._communication_analysis(profiling_path=profiling_path, + benchmark_profiling_path=benchmark_profiling_path, + step=step, benchmark_step=benchmark_step, + scope=scope, bandwidth_type=bandwidth_type) + + return job_list + + def cluster_schedule_analysis(self, profiling_path): + # 目标集群profiling数据下发分析，不包含两个集群profiling数据的比对分析 + + job_list = [] + global_step_rank = self.slow_rank_analyzer.get_global_step_rank(SlowRankAnalyzer.FREE) + + info_msg = "For cluster schedule analysis, " + slow_rank_id = global_step_rank.get("maximum", {}).get("rank_id") + if slow_rank_id is not None: + info_msg += f"maximum free for rank {slow_rank_id}" + else: + slow_rank_id = self.default_rank_id + info_msg += f"no slow rank with free time, analysis for default rank {slow_rank_id}" + + fast_rank_id = global_step_rank.get("minimum", {}).get("rank_id") + + slow_step = global_step_rank.get("maximum", {}).get("step") + fast_step = global_step_rank.get("minimum", {}).get("step") + + if slow_step is not None: + info_msg += f" and step {slow_step}" + logger.info(info_msg) + + kwargs = dict(profiling_path=self._get_profiling_path_by_rank(profiling_path, slow_rank_id), + benchmark_profiling_path=self._get_profiling_path_by_rank(profiling_path, fast_rank_id), + step=slow_step, benchmark_step=fast_step, + rank=slow_rank_id, benchmark_rank=fast_rank_id, + compare_mode=Constant.API_COMPARE, + step_duration=self.slow_rank_analyzer.get_step_duration(slow_rank_id, slow_step)) + + job_list += self.schedule_analysis(**kwargs) + + rank_id_valid = slow_rank_id is not None and fast_rank_id is not None and fast_rank_id != slow_rank_id + if not self.kwargs.get("benchmark_profiling_path") and rank_id_valid: + # 当用户指定benchmark profiling path时，不进行目标集群profiling的内部快慢卡对比 + logger.info("Enable schedule comparison of fast and slow rank/step") + job_list += self._profiling_comparison([kwargs]) + return job_list + + def cluster_communication_analysis(self, profiling_path): + job_list = [] + + for dimension in [Interface.COMMUNICATION]: + for scope in Interface.get_scope(dimension): + analyzer_class = Interface.get_analyzer(dimension, scope) + if hasattr(analyzer_class, "requires_cluster_dataset") and getattr(analyzer_class, + "requires_cluster_dataset"): + + # 如果不依赖数据集，或者依赖的是ClusterDataset，则不用根据带宽确定需要分析的特定rank + kwargs = copy.deepcopy(self.kwargs) + kwargs["profiling_path"] = profiling_path + interface = Interface(**kwargs) + job_list.append((dimension, scope, interface, kwargs)) + else: + # 非ClusterDataset场景，需要根据带宽大小分析特定的rank + for bandwidth_type in [SlowLinkAnalyzer.SDMA, SlowLinkAnalyzer.RDMA]: + global_step_rank = self.slow_link_analyzer.get_global_step_rank(bandwidth_type) + # 获取带宽最小的卡进行分析 + target_rank_id = global_step_rank.get("minimum", {}).get("rank_id") + if target_rank_id is None: + target_rank_id = self.default_rank_id + step = global_step_rank.get("minimum", {}).get("step") + analysis_profiling_path = self._get_profiling_path_by_rank(profiling_path, target_rank_id) + + info_msg = f"Minimum {bandwidth_type} bandwidth for rank {target_rank_id} " + if step: + info_msg += f"and step {step}" + logger.info(info_msg) + + job_list += self.communication_analysis(analysis_profiling_path, step=step, + bandwidth_type=bandwidth_type, scope=scope) + + return job_list + + def cluster_computation_analysis(self, profiling_path): + # 目标集群profiling数据计算分析，不包含两个集群profiling数据的比对分析；如果有pp stage，则对不同stage进行计算分析 + + job_list = [] + global_step_rank = self.slow_rank_analyzer.get_global_step_rank(SlowRankAnalyzer.COMPUTE) + stage_step_rank = self.slow_rank_analyzer.get_stage_step_rank(SlowRankAnalyzer.COMPUTE) + + if stage_step_rank: + job_list = self._stage_computation_analysis(profiling_path, stage_step_rank, job_list) + else: + job_list = self._global_computation_analysis(profiling_path, global_step_rank, job_list) + return job_list + + def cluster_memory_analysis(self, profiling_path): + # 目标集群profiling数据内存分析，当前memory识别的两个算子，导致的问题都是大的free，因此选择FREE最慢的卡进行分析 + + job_list = [] + global_step_rank = self.slow_rank_analyzer.get_global_step_rank(SlowRankAnalyzer.FREE) + + info_msg = "For cluster memory analysis, " + slow_rank_id = global_step_rank.get("maximum", {}).get("rank_id") + if slow_rank_id is not None: + info_msg += f"maximum free for rank {slow_rank_id}" + else: + slow_rank_id = self.default_rank_id + info_msg += f"no slow rank with free time, analysis for default rank {slow_rank_id}" + + slow_step = global_step_rank.get("maximum", {}).get("step") + if slow_step is not None: + info_msg += f" and step {slow_step}" + logger.info(info_msg) + + analysis_profiling_path = self._get_profiling_path_by_rank(profiling_path, slow_rank_id) + step_duration = self.slow_rank_analyzer.get_step_duration(slow_rank_id, slow_step) + job_list += self.memory_analysis(analysis_profiling_path, step=slow_step, rank=slow_rank_id, + step_duration=step_duration) + return job_list + + def _do_analysis(self, dimensions, pid=0, async_resp=None, **kwargs): + self.dimensions = dimensions + self.kwargs = kwargs + result_list = [] + profiling_path = PathManager.get_realpath(self.kwargs.get("profiling_path")) + benchmark_profiling_path = self.kwargs.get("benchmark_profiling_path") + PathManager.check_path_owner_consistent([profiling_path]) + if benchmark_profiling_path: + benchmark_profiling_path = PathManager.get_realpath(benchmark_profiling_path) + PathManager.check_path_owner_consistent([benchmark_profiling_path]) + + if not self._check_profiling_path_valid(profiling_path): + error_msg = f"Got invalid argument '-d/--profiling_path' {profiling_path}, skip analysis" + self._update_analysis_process_resp(pid, async_resp, error_msg=error_msg, + status_code=AsyncAnalysisStatus.BAD_REQUEST_STATUS_CODE, + status=AsyncAnalysisStatus.FAILED) + logger.error(error_msg) + return + + + if benchmark_profiling_path and not self._check_profiling_path_valid(benchmark_profiling_path): + error_msg = (f"Got invalid argument '-bp/--benchmark_profiling_path' {benchmark_profiling_path}, " + f"skip analysis") + self._update_analysis_process_resp(pid, async_resp, error_msg=error_msg, + status_code=AsyncAnalysisStatus.BAD_REQUEST_STATUS_CODE, + status=AsyncAnalysisStatus.FAILED) + logger.error(error_msg) + return + + self._is_cluster = self._is_cluster_profiling(profiling_path) + if benchmark_profiling_path: + # 构建benchmark profiling的map，用于根据rank获取profiling路径，否则无法进行比对 + is_benchmark_cluster = self._is_cluster_profiling(benchmark_profiling_path) + is_comparison_path_valid = (self._is_cluster and is_benchmark_cluster) or ( + not self._is_cluster and not is_benchmark_cluster) + if not is_comparison_path_valid: + error_msg = f"Only support profiling comparison for '1 npu vs 1 gpu/npu' and 'multi npus vs multi npus'" + self._update_analysis_process_resp(pid, async_resp, error_msg=error_msg, + status_code=AsyncAnalysisStatus.BAD_REQUEST_STATUS_CODE, + status=AsyncAnalysisStatus.FAILED) + logger.error(error_msg) + return + + if not self._is_cluster: + job_list = self.single_rank_analysis(profiling_path, benchmark_profiling_path) + else: + self.slow_rank_analyzer = SlowRankAnalyzer(profiling_path, output_path=self.kwargs.get("output_path")) + self.slow_link_analyzer = SlowLinkAnalyzer(profiling_path, output_path=self.kwargs.get("output_path")) + job_list = self.do_cluster_analysis(profiling_path, benchmark_profiling_path) + + for i, (dimension, scope, interface, kwargs) in enumerate(job_list[::-1]): + result_list.append( + interface.get_result(dimension, scope, render_html=i == len(job_list) - 1, output_dict=False, + **kwargs) + ) + + for result in result_list[::-1]: + if result and hasattr(result, "show"): + result.show() + break + self._get_analysis_finished_resp(pid, async_resp) + + def _get_scopes(self, scope=None, bandwidth_type=SlowLinkAnalyzer.SDMA): + """ + Args: + scope: analyzer type + bandwidth_type: analysis standard + Returns: + scope lists + """ + scopes = [] + if scope: + if scope in self.COMMUNICATION_MAPPING.get(bandwidth_type, self.SDMA_SUPPORT_SCOPES): + scopes.append(scope) + return scopes + for dimension in [Interface.COMMUNICATION]: + for scope_ in Interface.get_scope(dimension): + if scope_ in self.SDMA_SUPPORT_SCOPES or scope_ in self.RDMA_SUPPORT_SCOPES: + scopes.append(scope_) + return scopes + + def _communication_analysis(self, **child_kwargs): + kwargs = copy.deepcopy(self.kwargs) + job_list = [] + + kwargs["profiling_path"] = child_kwargs.get("profiling_path", "") + kwargs["benchmark_profiling_path"] = child_kwargs.get("benchmark_profiling_path", "") + kwargs["step"] = child_kwargs.get("step", -1) + kwargs["benchmark_step"] = child_kwargs.get("benchmark_step", -1) + bandwidth_type = child_kwargs.get("bandwidth_type", SlowLinkAnalyzer.SDMA) + scope = child_kwargs.get("scope", None) + + for scope_ in self._get_scopes(scope, bandwidth_type): + interface = Interface(**kwargs) + job_list.append((Interface.COMMUNICATION, scope_, interface, kwargs)) + + return job_list + + def _profiling_comparison(self, compare_profiling_list): + job_list = [] + disable_profiling_comparison = os.getenv(Constant.DISABLE_PROFILING_COMPARISON) + if disable_profiling_comparison is not None and disable_profiling_comparison.lower() == "true": + logger.info( + "Skip profiling comparison due to longer processing time due to env 'DISABLE_PROFILING_COMPARISON'") + return job_list + + for index, _kwargs in enumerate(compare_profiling_list): + kwargs = copy.deepcopy(self.kwargs) + kwargs.update(_kwargs) + compare_profiling_list[index] = kwargs + + compare_kwargs = { + "profiling_path": kwargs.get("profiling_path"), + "compare_profiling_list": compare_profiling_list, + } + + interface = Interface(**compare_kwargs) + job_list.append((Interface.COMPARISON, SupportedScopes.COMPARISON, interface, compare_kwargs)) + + return job_list + + def _cluster_profiling_comparison(self, profiling_path, benchmark_profiling_path): + # 从计算、下发和通信三个维度对集群profiling数据进行对比 + + job_list = [] + benchmark_profiling_path = self._get_profiling_path_by_rank(benchmark_profiling_path) + benchmark_slow_rank_analyzer = SlowRankAnalyzer(benchmark_profiling_path) + benchmark_slow_link_analyzer = SlowLinkAnalyzer(benchmark_profiling_path) + + # 计算和下发分析 + job_list += self._cluster_data_comparison(profiling_path, + benchmark_profiling_path, + self.slow_rank_analyzer, + benchmark_slow_rank_analyzer, + get_max=True) + + # 通信分析 + job_list += self._cluster_data_comparison(profiling_path, + benchmark_profiling_path, + self.slow_link_analyzer, + benchmark_slow_link_analyzer, + get_max=False) + return job_list + + def _cluster_data_comparison(self, profiling_path, benchmark_profiling_path, target_cluster_analyzer, + benchmark_cluster_analyzer, get_max=False): + # #low rank/slow link结果逐行对比获取差值最大的rank和step进行单卡分析 + job_list = [] + + if isinstance(target_cluster_analyzer, SlowRankAnalyzer): + comparison_dims = [SlowRankAnalyzer.COMPUTE, SlowRankAnalyzer.FREE] + comparison_modes = [Constant.KERNEL_COMPARE, Constant.API_COMPARE] + elif isinstance(target_cluster_analyzer, SlowLinkAnalyzer): + comparison_dims = [SlowLinkAnalyzer.SDMA_BANDWIDTH, SlowLinkAnalyzer.RDMA_BANDWIDTH] + comparison_modes = [None, None] + else: + return job_list + + target_data = target_cluster_analyzer.format_datas.get("data", []) + benchmark_data = benchmark_cluster_analyzer.format_datas.get("data", []) + headers = benchmark_cluster_analyzer.format_datas.get("headers", []) + + if len(target_data) != len(benchmark_data): + logger.warning( + "The product of ranks and steps of Benchmark profiling is not equals to target profiling, " + "skip cluster comparison.") + return job_list + + compare_profiling_list = [] + for dimension, compare_mode in zip(comparison_dims, comparison_modes): + step, benchmark_step, rank_id_for_comparison = AnalyzerController._get_step_rank_for_cluster_statistic_diff( + target_data, + benchmark_data, + headers, + dimension, + get_max=get_max + ) + + rank_profiling_path = self._get_profiling_path_by_rank(profiling_path, rank_id_for_comparison) + rank_benchmark_profiling_path = self._get_profiling_path_by_rank( + benchmark_profiling_path, + rank_id_for_comparison + ) + + if rank_id_for_comparison is None: + # rank id为空则无法获取对应rank的profiling路径，无法进行比较 + continue + + compare_profiling_list.append( + dict(profiling_path=rank_profiling_path, benchmark_profiling_path=rank_benchmark_profiling_path, + step=step, benchmark_step=benchmark_step, + rank=rank_id_for_comparison, benchmark_rank=rank_id_for_comparison, compare_mode=compare_mode) + ) + + if not compare_profiling_list: + return job_list + + job_list += self._profiling_comparison(compare_profiling_list) + return job_list + + def _is_cluster_profiling(self, profiling_path): + if os.path.isfile(profiling_path): + return False + path_list = [os.path.join(profiling_path, dir_name) for dir_name in os.listdir(profiling_path)] + ascend_pt_dirs = [path for path in path_list if os.path.isdir(path) and path.endswith("ascend_pt")] + ascend_ms_dirs = [path for path in path_list if os.path.isdir(path) and path.endswith("ascend_ms")] + if ascend_ms_dirs and ascend_pt_dirs: + logger.error("Cannot analyze pytorch and mindspore meantime.") + return False + if not ascend_pt_dirs and not ascend_ms_dirs: + return False + if ascend_ms_dirs and not ascend_pt_dirs: + data_processor = MindsporeDataPreprocessor(ascend_ms_dirs) + elif ascend_pt_dirs and not ascend_ms_dirs: + data_processor = PytorchDataPreprocessor(ascend_pt_dirs) + + self.cluster_local_data_map[profiling_path] = data_processor.get_data_map() + + if not self.cluster_local_data_map or not self.cluster_local_data_map.get(profiling_path): + return False + + self.default_rank_id = list(self.cluster_local_data_map[profiling_path].keys())[0] + + return len(self.cluster_local_data_map[profiling_path]) >= self.CLUSTER_RANK_THRESHOLD + + def _get_profiling_path_by_rank(self, profiling_path, rank_id=None): + + if not profiling_path: + return profiling_path + + return self._get_target_profiling_path_for_local(profiling_path, rank_id) + + def _get_target_profiling_path_for_local(self, profiling_path, rank_id): + rank_id_map = self.cluster_local_data_map.get(profiling_path, {}) + if rank_id is None or not rank_id_map: + return profiling_path + + if rank_id in rank_id_map: + return rank_id_map.get(rank_id) + + local_first_rank_id = sorted(list(map(int, rank_id_map.keys())))[0] + logger.warning("Target rank id %s does not exist in local profiling data %s, use rank %s for analysis", + rank_id, profiling_path, local_first_rank_id) + return rank_id_map.get(local_first_rank_id) + + def _update_analysis_process_resp(self, pid, resp, **kwargs): + if kwargs: + resp.update(kwargs) + self.analysis_process_resp[pid] = resp + + def _get_analysis_finished_resp(self, pid, resp): + advisor_output_file_prefix = f"mstt_advisor_{Timer().strftime}" + html_path = os.path.join(Config().work_path, f"{advisor_output_file_prefix}.html") + xlsx_path = os.path.join(Config().work_path, "log", f"{advisor_output_file_prefix}.xlsx") + if os.path.exists(html_path) and os.path.exists(xlsx_path): + result_files = {"html": html_path, "xlsx": xlsx_path} + self._update_analysis_process_resp(pid, resp, status_code=AsyncAnalysisStatus.NON_FAILED_STATUS_CODE, + status=AsyncAnalysisStatus.SUCCESS, result_files=result_files) + else: + self._update_analysis_process_resp(pid, resp, status_code=AsyncAnalysisStatus.BAD_REQUEST_STATUS_CODE, + status=AsyncAnalysisStatus.FAILED, + error_msg="No optimization suggestions, please check your input path.") + + def _stage_computation_analysis(self, profiling_path, stage_step_rank, job_list): + # 对不同pp stage取min max进行分析 + logger.info("Steps and ranks to be analyzed of different pipeline parallel stages are %s", + json.dumps(stage_step_rank)) + + stages_profiling_path = [] + for stage, step_rank_info in stage_step_rank.items(): + rank_id = step_rank_info.get("maximum", {}).get("rank_id") + step = step_rank_info.get("maximum", {}).get("step") + benchmark_rank_id = step_rank_info.get("minimum", {}).get("rank_id") + benchmark_step = step_rank_info.get("minimum", {}).get("step") + + info_msg = f"For {stage}, slow rank is {rank_id}" + if step: + info_msg += f", step is {step}" + logger.info(info_msg) + + stages_profiling_path.append( + dict( + stage=stage, rank=rank_id, step=step, benchmark_rank=benchmark_rank_id, + benchmark_step=benchmark_step, + profiling_path=self._get_profiling_path_by_rank(profiling_path, rank_id), + benchmark_profiling_path=self._get_profiling_path_by_rank(profiling_path, benchmark_rank_id), + compare_mode=Constant.KERNEL_COMPARE, + step_duration=self.slow_rank_analyzer.get_step_duration(rank_id, step) + ) + ) + Interface.add_analyzer(Interface.COMPUTATION, SupportedScopes.STAGE_COMPUTE, PPStageComputationAnalyzer) + compute_analysis_kwargs = {"stages_profiling_path": stages_profiling_path, "profiling_path": profiling_path} + + job_list.append((Interface.COMPUTATION, SupportedScopes.STAGE_COMPUTE, Interface(**compute_analysis_kwargs), + compute_analysis_kwargs)) + if not self.kwargs.get("benchmark_profiling_path"): + logger.info("Enable computation comparison of fast and slow rank/step in different pp stages") + job_list += self._profiling_comparison(stages_profiling_path) + return job_list + + def _global_computation_analysis(self, profiling_path, global_step_rank, job_list): + # 不区分stage，对所有卡取Min max进行分析 + logger.info("Without pipeline parallel stage, steps and ranks to be analyzed are %s", + json.dumps(global_step_rank)) + slow_rank_id = global_step_rank.get("maximum", {}).get("rank_id") + if slow_rank_id is not None: + info_msg = f"Maximum computation time for rank {slow_rank_id}" + else: + slow_rank_id = self.default_rank_id + info_msg = f"No slow rank with computation time, analysis for default rank {slow_rank_id}" + slow_step = global_step_rank.get("maximum", {}).get("step") + # 如果没有标杆profiling数据的rank id，说明没有快慢卡问题，直接对默认rank id进行分析，因此这里取值为None + fast_rank_id = global_step_rank.get("minimum", {}).get("rank_id") + fast_step = global_step_rank.get("minimum", {}).get("step") + + if slow_step is not None: + info_msg += f" and step {slow_step}, " + if fast_rank_id is not None: + info_msg += f"minimum computation time for rank {fast_rank_id}" + if fast_step is not None: + info_msg += f" and step {fast_step}" + logger.info(info_msg) + + kwargs = dict(profiling_path=self._get_profiling_path_by_rank(profiling_path, slow_rank_id), + benchmark_profiling_path=self._get_profiling_path_by_rank(profiling_path, fast_rank_id), + step=slow_step, benchmark_step=fast_step, rank=slow_rank_id, benchmark_rank=fast_rank_id, + compare_mode=Constant.KERNEL_COMPARE, + step_duration=self.slow_rank_analyzer.get_step_duration(slow_rank_id, slow_step)) + + job_list += self.computation_analysis(**kwargs) + + rank_id_valid = slow_rank_id is not None and fast_rank_id is not None and fast_rank_id != slow_rank_id + if not self.kwargs.get("benchmark_profiling_path") and rank_id_valid: + # 当用户指定benchmark profiling path时，不进行目标集群profiling的内部快慢卡对比 + logger.info("Enable computation comparison of fast and slow rank/step") + job_list += self._profiling_comparison([kwargs]) + return job_list diff --git a/profiler/advisor/analyzer/base_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/base_analyzer.py similarity index 92% rename from profiler/advisor/analyzer/base_analyzer.py rename to profiler/msprof_analyze/advisor/analyzer/base_analyzer.py index 0391eb88a2..ee7835cf53 100644 --- a/profiler/advisor/analyzer/base_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/base_analyzer.py @@ -18,15 +18,15 @@ from functools import wraps from typing import Dict, List, Union from abc import abstractmethod, ABCMeta -from profiler.prof_common.constant import Constant -from profiler.advisor.common.enum_params_parser import EnumParamsParser -from profiler.advisor.common.version_control import VersionControl -from profiler.advisor.dataset.dataset import Dataset -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor -from profiler.advisor.utils.utils import safe_division -from profiler.prof_common.file_manager import FileManager +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.advisor.common.enum_params_parser import EnumParamsParser +from msprof_analyze.advisor.common.version_control import VersionControl +from msprof_analyze.advisor.dataset.dataset import Dataset +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.display.html.render import HTMLRender +from msprof_analyze.advisor.display.html.priority_background_color import PriorityBackgroundColor +from msprof_analyze.advisor.utils.utils import safe_division +from msprof_analyze.prof_common.file_manager import FileManager logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/cluster/__init__.py b/profiler/msprof_analyze/advisor/analyzer/cluster/__init__.py similarity index 100% rename from profiler/advisor/analyzer/cluster/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/cluster/__init__.py diff --git a/profiler/advisor/analyzer/cluster/communication_retransmission_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/cluster/communication_retransmission_analyzer.py similarity index 79% rename from profiler/advisor/analyzer/cluster/communication_retransmission_analyzer.py rename to profiler/msprof_analyze/advisor/analyzer/cluster/communication_retransmission_analyzer.py index 78cea38ada..826f13ec1e 100644 --- a/profiler/advisor/analyzer/cluster/communication_retransmission_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/cluster/communication_retransmission_analyzer.py @@ -14,11 +14,11 @@ # limitations under the License. import logging -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.analyzer.cluster.communication_retransmission_checker import CommunicationRetransmissionChecker -from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.dataset.cluster.cluster_dataset import ClusterCommunicationDataset +from msprof_analyze.advisor.analyzer.base_analyzer import BaseAnalyzer +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.analyzer.cluster.communication_retransmission_checker import CommunicationRetransmissionChecker +from msprof_analyze.advisor.display.html.render import HTMLRender +from msprof_analyze.advisor.dataset.cluster.cluster_dataset import ClusterCommunicationDataset logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/cluster/communication_retransmission_checker.py b/profiler/msprof_analyze/advisor/analyzer/cluster/communication_retransmission_checker.py similarity index 91% rename from profiler/advisor/analyzer/cluster/communication_retransmission_checker.py rename to profiler/msprof_analyze/advisor/analyzer/cluster/communication_retransmission_checker.py index 79c8402ca7..bd63da7ee9 100644 --- a/profiler/advisor/analyzer/cluster/communication_retransmission_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/cluster/communication_retransmission_checker.py @@ -16,13 +16,13 @@ import logging import os from typing import Dict, List from collections import defaultdict -from profiler.advisor.dataset.cluster.cluster_dataset import ClusterCommunicationDataset -from profiler.advisor.display.prompt.base_prompt import BasePrompt -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.prof_common.additional_args_manager import AdditionalArgsManager -from profiler.prof_common.file_manager import FileManager -from profiler.advisor.dataset.cluster.hccl_collection import HcclInfo +from msprof_analyze.advisor.dataset.cluster.cluster_dataset import ClusterCommunicationDataset +from msprof_analyze.advisor.display.prompt.base_prompt import BasePrompt +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.result.item import OptimizeItem, OptimizeRecord +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.prof_common.file_manager import FileManager +from msprof_analyze.advisor.dataset.cluster.hccl_collection import HcclInfo logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/cluster/slow_link_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/cluster/slow_link_analyzer.py similarity index 94% rename from profiler/advisor/analyzer/cluster/slow_link_analyzer.py rename to profiler/msprof_analyze/advisor/analyzer/cluster/slow_link_analyzer.py index 2a70580ec9..062b6548e1 100644 --- a/profiler/advisor/analyzer/cluster/slow_link_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/cluster/slow_link_analyzer.py @@ -17,13 +17,13 @@ from collections import defaultdict from typing import Dict, List import logging -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.prof_common.constant import Constant -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.advisor.dataset.cluster.cluster_dataset import ClusterCommunicationDataset -from profiler.advisor.utils.utils import safe_index_value, convert_to_int -from profiler.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.advisor.analyzer.base_analyzer import BaseAnalyzer +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.result.item import OptimizeItem, OptimizeRecord +from msprof_analyze.advisor.dataset.cluster.cluster_dataset import ClusterCommunicationDataset +from msprof_analyze.advisor.utils.utils import safe_index_value, convert_to_int +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/cluster/slow_rank_analyzer.py similarity index 95% rename from profiler/advisor/analyzer/cluster/slow_rank_analyzer.py rename to profiler/msprof_analyze/advisor/analyzer/cluster/slow_rank_analyzer.py index e0f13ac2d7..7073fc2874 100644 --- a/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/cluster/slow_rank_analyzer.py @@ -15,13 +15,13 @@ import logging -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.prof_common.constant import Constant -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.advisor.dataset.cluster.cluster_dataset import ClusterStepTraceTimeDataset -from profiler.advisor.utils.utils import safe_index_value, safe_division, convert_to_int, safe_index, convert_to_float -from profiler.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.advisor.analyzer.base_analyzer import BaseAnalyzer +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.result.item import OptimizeItem, OptimizeRecord +from msprof_analyze.advisor.dataset.cluster.cluster_dataset import ClusterStepTraceTimeDataset +from msprof_analyze.advisor.utils.utils import safe_index_value, safe_division, convert_to_int, safe_index, convert_to_float +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/communication/__init__.py b/profiler/msprof_analyze/advisor/analyzer/communication/__init__.py similarity index 100% rename from profiler/advisor/analyzer/communication/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/communication/__init__.py diff --git a/profiler/advisor/analyzer/communication/alignment/__init__.py b/profiler/msprof_analyze/advisor/analyzer/communication/alignment/__init__.py similarity index 100% rename from profiler/advisor/analyzer/communication/alignment/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/communication/alignment/__init__.py diff --git a/profiler/advisor/analyzer/communication/alignment/byte_alignment_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/communication/alignment/byte_alignment_analyzer.py similarity index 75% rename from profiler/advisor/analyzer/communication/alignment/byte_alignment_analyzer.py rename to profiler/msprof_analyze/advisor/analyzer/communication/alignment/byte_alignment_analyzer.py index fe622dfb5e..0021365a4e 100644 --- a/profiler/advisor/analyzer/communication/alignment/byte_alignment_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/communication/alignment/byte_alignment_analyzer.py @@ -14,13 +14,13 @@ # limitations under the License. import logging -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.analyzer.communication.alignment.byte_alignment_checker import ByteAlignmentChecker -from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor -from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.dataset.profiling.profiling_dataset import ProfilingDataset -from profiler.advisor.dataset.communication.hccl_detail_dataset import HcclDetailDataset -from profiler.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.analyzer.base_analyzer import BaseAnalyzer +from msprof_analyze.advisor.analyzer.communication.alignment.byte_alignment_checker import ByteAlignmentChecker +from msprof_analyze.advisor.display.html.priority_background_color import PriorityBackgroundColor +from msprof_analyze.advisor.display.html.render import HTMLRender +from msprof_analyze.advisor.dataset.profiling.profiling_dataset import ProfilingDataset +from msprof_analyze.advisor.dataset.communication.hccl_detail_dataset import HcclDetailDataset +from msprof_analyze.advisor.result.result import OptimizeResult logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/communication/alignment/byte_alignment_checker.py b/profiler/msprof_analyze/advisor/analyzer/communication/alignment/byte_alignment_checker.py similarity index 89% rename from profiler/advisor/analyzer/communication/alignment/byte_alignment_checker.py rename to profiler/msprof_analyze/advisor/analyzer/communication/alignment/byte_alignment_checker.py index cf01c96873..4f8bf95091 100644 --- a/profiler/advisor/analyzer/communication/alignment/byte_alignment_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/communication/alignment/byte_alignment_checker.py @@ -15,16 +15,16 @@ import logging import os from typing import List -from profiler.advisor.dataset.communication.hccl_detail_dataset import HcclDetailDataset -from profiler.advisor.dataset.profiling.info_collection import HcclTask -from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor -from profiler.advisor.display.prompt.base_prompt import BasePrompt -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.prof_common.additional_args_manager import AdditionalArgsManager -from profiler.prof_common.file_manager import FileManager -from profiler.advisor.utils.utils import safe_division -from profiler.prof_common.constant import Constant +from msprof_analyze.advisor.dataset.communication.hccl_detail_dataset import HcclDetailDataset +from msprof_analyze.advisor.dataset.profiling.info_collection import HcclTask +from msprof_analyze.advisor.display.html.priority_background_color import PriorityBackgroundColor +from msprof_analyze.advisor.display.prompt.base_prompt import BasePrompt +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.result.item import OptimizeItem, OptimizeRecord +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.prof_common.file_manager import FileManager +from msprof_analyze.advisor.utils.utils import safe_division +from msprof_analyze.prof_common.constant import Constant logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/communication/bandwidth/__init__.py b/profiler/msprof_analyze/advisor/analyzer/communication/bandwidth/__init__.py similarity index 100% rename from profiler/advisor/analyzer/communication/bandwidth/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/communication/bandwidth/__init__.py diff --git a/profiler/advisor/analyzer/communication/base_communication_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/communication/base_communication_analyzer.py similarity index 90% rename from profiler/advisor/analyzer/communication/base_communication_analyzer.py rename to profiler/msprof_analyze/advisor/analyzer/communication/base_communication_analyzer.py index be97e07fc0..73724ee299 100644 --- a/profiler/advisor/analyzer/communication/base_communication_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/communication/base_communication_analyzer.py @@ -1,22 +1,22 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer - - -class BaseCommunicationAnalyzer(BaseAnalyzer): - requires_cluster_dataset = True - - def __init__(self, collection_path, n_processes: int = 1, **kwargs): - super().__init__(collection_path, n_processes, **kwargs) +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from msprof_analyze.advisor.analyzer.base_analyzer import BaseAnalyzer + + +class BaseCommunicationAnalyzer(BaseAnalyzer): + requires_cluster_dataset = True + + def __init__(self, collection_path, n_processes: int = 1, **kwargs): + super().__init__(collection_path, n_processes, **kwargs) diff --git a/profiler/advisor/analyzer/communication/contention/__init__.py b/profiler/msprof_analyze/advisor/analyzer/communication/contention/__init__.py similarity index 100% rename from profiler/advisor/analyzer/communication/contention/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/communication/contention/__init__.py diff --git a/profiler/advisor/analyzer/communication/contention/bandwidth_contention_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/communication/contention/bandwidth_contention_analyzer.py similarity index 77% rename from profiler/advisor/analyzer/communication/contention/bandwidth_contention_analyzer.py rename to profiler/msprof_analyze/advisor/analyzer/communication/contention/bandwidth_contention_analyzer.py index fdc57adb79..aa144b884f 100644 --- a/profiler/advisor/analyzer/communication/contention/bandwidth_contention_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/communication/contention/bandwidth_contention_analyzer.py @@ -14,13 +14,13 @@ # limitations under the License. import logging -from profiler.advisor.analyzer.communication.base_communication_analyzer import BaseCommunicationAnalyzer -from profiler.advisor.analyzer.communication.contention.bandwidth_contention_checker import BandwidthContentionChecker -from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor -from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.dataset.communication.communication_dataset import CommunicationDataset -from profiler.advisor.dataset.profiling.profiling_dataset import ProfilingDataset -from profiler.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.analyzer.communication.base_communication_analyzer import BaseCommunicationAnalyzer +from msprof_analyze.advisor.analyzer.communication.contention.bandwidth_contention_checker import BandwidthContentionChecker +from msprof_analyze.advisor.display.html.priority_background_color import PriorityBackgroundColor +from msprof_analyze.advisor.display.html.render import HTMLRender +from msprof_analyze.advisor.dataset.communication.communication_dataset import CommunicationDataset +from msprof_analyze.advisor.dataset.profiling.profiling_dataset import ProfilingDataset +from msprof_analyze.advisor.result.result import OptimizeResult logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/communication/contention/bandwidth_contention_checker.py b/profiler/msprof_analyze/advisor/analyzer/communication/contention/bandwidth_contention_checker.py similarity index 89% rename from profiler/advisor/analyzer/communication/contention/bandwidth_contention_checker.py rename to profiler/msprof_analyze/advisor/analyzer/communication/contention/bandwidth_contention_checker.py index 0ba02f07da..2f2d2e5df8 100644 --- a/profiler/advisor/analyzer/communication/contention/bandwidth_contention_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/communication/contention/bandwidth_contention_checker.py @@ -15,17 +15,17 @@ import logging import os from typing import List -from profiler.advisor.dataset.communication.communication_dataset import CommunicationDataset -from profiler.advisor.dataset.profiling.profiling_dataset import ProfilingDataset -from profiler.advisor.display.prompt.base_prompt import BasePrompt -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.prof_common.additional_args_manager import AdditionalArgsManager -from profiler.prof_common.file_manager import FileManager -from profiler.advisor.utils.utils import convert_to_float -from profiler.advisor.dataset.cluster.hccl_collection import HcclInfo -from profiler.advisor.dataset.profiling.info_collection import OpInfo -from profiler.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.advisor.dataset.communication.communication_dataset import CommunicationDataset +from msprof_analyze.advisor.dataset.profiling.profiling_dataset import ProfilingDataset +from msprof_analyze.advisor.display.prompt.base_prompt import BasePrompt +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.result.item import OptimizeItem, OptimizeRecord +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.prof_common.file_manager import FileManager +from msprof_analyze.advisor.utils.utils import convert_to_float +from msprof_analyze.advisor.dataset.cluster.hccl_collection import HcclInfo +from msprof_analyze.advisor.dataset.profiling.info_collection import OpInfo +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/communication/environment/__init__.py b/profiler/msprof_analyze/advisor/analyzer/communication/environment/__init__.py similarity index 100% rename from profiler/advisor/analyzer/communication/environment/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/communication/environment/__init__.py diff --git a/profiler/advisor/analyzer/communication/packet/__init__.py b/profiler/msprof_analyze/advisor/analyzer/communication/packet/__init__.py similarity index 100% rename from profiler/advisor/analyzer/communication/packet/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/communication/packet/__init__.py diff --git a/profiler/advisor/analyzer/communication/packet/packet_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/communication/packet/packet_analyzer.py similarity index 78% rename from profiler/advisor/analyzer/communication/packet/packet_analyzer.py rename to profiler/msprof_analyze/advisor/analyzer/communication/packet/packet_analyzer.py index 33b5265009..76a3590b39 100644 --- a/profiler/advisor/analyzer/communication/packet/packet_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/communication/packet/packet_analyzer.py @@ -14,12 +14,12 @@ # limitations under the License. import logging -from profiler.advisor.analyzer.communication.base_communication_analyzer import BaseCommunicationAnalyzer -from profiler.advisor.analyzer.communication.packet.packet_checker import PacketChecker -from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor -from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.dataset.communication.communication_dataset import CommunicationDataset -from profiler.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.analyzer.communication.base_communication_analyzer import BaseCommunicationAnalyzer +from msprof_analyze.advisor.analyzer.communication.packet.packet_checker import PacketChecker +from msprof_analyze.advisor.display.html.priority_background_color import PriorityBackgroundColor +from msprof_analyze.advisor.display.html.render import HTMLRender +from msprof_analyze.advisor.dataset.communication.communication_dataset import CommunicationDataset +from msprof_analyze.advisor.result.result import OptimizeResult logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/communication/packet/packet_checker.py b/profiler/msprof_analyze/advisor/analyzer/communication/packet/packet_checker.py similarity index 91% rename from profiler/advisor/analyzer/communication/packet/packet_checker.py rename to profiler/msprof_analyze/advisor/analyzer/communication/packet/packet_checker.py index c53a0f9d0b..6f76b09ff9 100644 --- a/profiler/advisor/analyzer/communication/packet/packet_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/communication/packet/packet_checker.py @@ -14,14 +14,14 @@ # limitations under the License. import logging import os -from profiler.advisor.dataset.communication.communication_dataset import CommunicationDataset -from profiler.advisor.display.prompt.base_prompt import BasePrompt -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.prof_common.additional_args_manager import AdditionalArgsManager -from profiler.prof_common.file_manager import FileManager -from profiler.advisor.utils.utils import convert_to_float -from profiler.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.advisor.dataset.communication.communication_dataset import CommunicationDataset +from msprof_analyze.advisor.display.prompt.base_prompt import BasePrompt +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.result.item import OptimizeItem, OptimizeRecord +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.prof_common.file_manager import FileManager +from msprof_analyze.advisor.utils.utils import convert_to_float +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/communication/retransmission/__init__.py b/profiler/msprof_analyze/advisor/analyzer/communication/retransmission/__init__.py similarity index 100% rename from profiler/advisor/analyzer/communication/retransmission/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/communication/retransmission/__init__.py diff --git a/profiler/advisor/analyzer/communication/retransmission/communication_retransmission_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/communication/retransmission/communication_retransmission_analyzer.py similarity index 77% rename from profiler/advisor/analyzer/communication/retransmission/communication_retransmission_analyzer.py rename to profiler/msprof_analyze/advisor/analyzer/communication/retransmission/communication_retransmission_analyzer.py index bd5d76a40f..73d798c5c4 100644 --- a/profiler/advisor/analyzer/communication/retransmission/communication_retransmission_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/communication/retransmission/communication_retransmission_analyzer.py @@ -14,13 +14,13 @@ # limitations under the License. import logging -from profiler.advisor.analyzer.communication.base_communication_analyzer import BaseCommunicationAnalyzer -from profiler.advisor.analyzer.communication.retransmission.communication_retransmission_checker import \ +from msprof_analyze.advisor.analyzer.communication.base_communication_analyzer import BaseCommunicationAnalyzer +from msprof_analyze.advisor.analyzer.communication.retransmission.communication_retransmission_checker import \ CommunicationRetransmissionChecker -from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor -from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.dataset.cluster.cluster_dataset import ClusterCommunicationDataset -from profiler.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.display.html.priority_background_color import PriorityBackgroundColor +from msprof_analyze.advisor.display.html.render import HTMLRender +from msprof_analyze.advisor.dataset.cluster.cluster_dataset import ClusterCommunicationDataset +from msprof_analyze.advisor.result.result import OptimizeResult logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/communication/retransmission/communication_retransmission_checker.py b/profiler/msprof_analyze/advisor/analyzer/communication/retransmission/communication_retransmission_checker.py similarity index 91% rename from profiler/advisor/analyzer/communication/retransmission/communication_retransmission_checker.py rename to profiler/msprof_analyze/advisor/analyzer/communication/retransmission/communication_retransmission_checker.py index 8220ac67bc..f27dbb13e8 100644 --- a/profiler/advisor/analyzer/communication/retransmission/communication_retransmission_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/communication/retransmission/communication_retransmission_checker.py @@ -16,14 +16,14 @@ import logging import os from typing import Dict, List from collections import defaultdict -from profiler.advisor.dataset.cluster.cluster_dataset import ClusterCommunicationDataset -from profiler.advisor.display.prompt.base_prompt import BasePrompt -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.prof_common.additional_args_manager import AdditionalArgsManager -from profiler.prof_common.file_manager import FileManager -from profiler.advisor.dataset.cluster.hccl_collection import HcclInfo -from profiler.prof_common.constant import Constant +from msprof_analyze.advisor.dataset.cluster.cluster_dataset import ClusterCommunicationDataset +from msprof_analyze.advisor.display.prompt.base_prompt import BasePrompt +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.result.item import OptimizeItem, OptimizeRecord +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.prof_common.file_manager import FileManager +from msprof_analyze.advisor.dataset.cluster.hccl_collection import HcclInfo +from msprof_analyze.prof_common.constant import Constant logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/comparison/__init__.py b/profiler/msprof_analyze/advisor/analyzer/comparison/__init__.py similarity index 100% rename from profiler/advisor/analyzer/comparison/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/comparison/__init__.py diff --git a/profiler/advisor/analyzer/comparison/comparison_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/comparison/comparison_analyzer.py similarity index 86% rename from profiler/advisor/analyzer/comparison/comparison_analyzer.py rename to profiler/msprof_analyze/advisor/analyzer/comparison/comparison_analyzer.py index 323f7afc2a..cd84644d78 100644 --- a/profiler/advisor/analyzer/comparison/comparison_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/comparison/comparison_analyzer.py @@ -14,10 +14,10 @@ # limitations under the License. import logging -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.analyzer.comparison.comparison_checker import ComparisonChecker -from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.analyzer.base_analyzer import BaseAnalyzer +from msprof_analyze.advisor.analyzer.comparison.comparison_checker import ComparisonChecker +from msprof_analyze.advisor.display.html.render import HTMLRender +from msprof_analyze.advisor.result.result import OptimizeResult logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/comparison/comparison_checker.py b/profiler/msprof_analyze/advisor/analyzer/comparison/comparison_checker.py similarity index 93% rename from profiler/advisor/analyzer/comparison/comparison_checker.py rename to profiler/msprof_analyze/advisor/analyzer/comparison/comparison_checker.py index cef04c3002..83cf40d8a4 100644 --- a/profiler/advisor/analyzer/comparison/comparison_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/comparison/comparison_checker.py @@ -14,11 +14,11 @@ # limitations under the License. import logging -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.advisor.utils.utils import safe_index_value, convert_to_float, convert_to_int -from profiler.compare_tools.compare_interface.comparison_interface import ComparisonInterface -from profiler.prof_common.constant import Constant +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.result.item import OptimizeItem, OptimizeRecord +from msprof_analyze.advisor.utils.utils import safe_index_value, convert_to_float, convert_to_int +from msprof_analyze.compare_tools.compare_interface.comparison_interface import ComparisonInterface +from msprof_analyze.prof_common.constant import Constant logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/computation/__init__.py b/profiler/msprof_analyze/advisor/analyzer/computation/__init__.py similarity index 100% rename from profiler/advisor/analyzer/computation/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/computation/__init__.py diff --git a/profiler/advisor/analyzer/computation/ai_core_freq/__init__.py b/profiler/msprof_analyze/advisor/analyzer/computation/ai_core_freq/__init__.py similarity index 100% rename from profiler/advisor/analyzer/computation/ai_core_freq/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/computation/ai_core_freq/__init__.py diff --git a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py similarity index 76% rename from profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py rename to profiler/msprof_analyze/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py index 38ebc336c3..b4d9736259 100644 --- a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py @@ -14,14 +14,14 @@ # limitations under the License. import logging -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.analyzer.computation.ai_core_freq.ai_core_freq_checker import AICoreFreqChecker -from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor -from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.dataset.timeline_event_dataset import ComputationAnalysisDataset -from profiler.advisor.dataset.profiling.device_info import DeviceInfoParser -from profiler.advisor.config.config import Config +from msprof_analyze.advisor.analyzer.base_analyzer import BaseAnalyzer +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.analyzer.computation.ai_core_freq.ai_core_freq_checker import AICoreFreqChecker +from msprof_analyze.advisor.display.html.priority_background_color import PriorityBackgroundColor +from msprof_analyze.advisor.display.html.render import HTMLRender +from msprof_analyze.advisor.dataset.timeline_event_dataset import ComputationAnalysisDataset +from msprof_analyze.advisor.dataset.profiling.device_info import DeviceInfoParser +from msprof_analyze.advisor.config.config import Config logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py b/profiler/msprof_analyze/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py similarity index 91% rename from profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py rename to profiler/msprof_analyze/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py index b203514208..7e07e2f3b5 100644 --- a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py @@ -14,13 +14,13 @@ # limitations under the License. import logging -from profiler.advisor.config.config import Config -from profiler.advisor.dataset.timeline_event_dataset import ComputationAnalysisDataset -from profiler.advisor.display.prompt.base_prompt import BasePrompt -from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.utils.utils import convert_to_float -from profiler.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.advisor.config.config import Config +from msprof_analyze.advisor.dataset.timeline_event_dataset import ComputationAnalysisDataset +from msprof_analyze.advisor.display.prompt.base_prompt import BasePrompt +from msprof_analyze.advisor.result.item import OptimizeItem, OptimizeRecord +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.utils.utils import convert_to_float +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/computation/aicpu/__init__.py b/profiler/msprof_analyze/advisor/analyzer/computation/aicpu/__init__.py similarity index 100% rename from profiler/advisor/analyzer/computation/aicpu/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/computation/aicpu/__init__.py diff --git a/profiler/advisor/analyzer/computation/aicpu/aicpu_checker.py b/profiler/msprof_analyze/advisor/analyzer/computation/aicpu/aicpu_checker.py similarity index 95% rename from profiler/advisor/analyzer/computation/aicpu/aicpu_checker.py rename to profiler/msprof_analyze/advisor/analyzer/computation/aicpu/aicpu_checker.py index e7d87e1fd9..269e9b435c 100644 --- a/profiler/advisor/analyzer/computation/aicpu/aicpu_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/computation/aicpu/aicpu_checker.py @@ -17,14 +17,14 @@ import os from functools import partial from typing import List, Dict, Optional -from profiler.advisor.analyzer.computation.operator_checker import OperatorChecker, logger -from profiler.advisor.analyzer.schedule.fusion_ops.timeline_api_stack_checker import OpStackFinder -from profiler.advisor.dataset.dataset import Dataset -from profiler.advisor.dataset.profiling.profiling_dataset import ProfilingDataset -from profiler.advisor.dataset.timeline_event_dataset import ComputationAnalysisDataset -from profiler.prof_common.additional_args_manager import AdditionalArgsManager -from profiler.prof_common.file_manager import FileManager -from profiler.prof_common.constant import Constant +from msprof_analyze.advisor.analyzer.computation.operator_checker import OperatorChecker, logger +from msprof_analyze.advisor.analyzer.schedule.fusion_ops.timeline_api_stack_checker import OpStackFinder +from msprof_analyze.advisor.dataset.dataset import Dataset +from msprof_analyze.advisor.dataset.profiling.profiling_dataset import ProfilingDataset +from msprof_analyze.advisor.dataset.timeline_event_dataset import ComputationAnalysisDataset +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.prof_common.file_manager import FileManager +from msprof_analyze.prof_common.constant import Constant class AicpuChecker(OperatorChecker): diff --git a/profiler/advisor/analyzer/computation/bound/__init__.py b/profiler/msprof_analyze/advisor/analyzer/computation/bound/__init__.py similarity index 100% rename from profiler/advisor/analyzer/computation/bound/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/computation/bound/__init__.py diff --git a/profiler/advisor/analyzer/computation/bound/block_dim_checker.py b/profiler/msprof_analyze/advisor/analyzer/computation/bound/block_dim_checker.py similarity index 91% rename from profiler/advisor/analyzer/computation/bound/block_dim_checker.py rename to profiler/msprof_analyze/advisor/analyzer/computation/bound/block_dim_checker.py index 67eb8fad9f..5044f5532e 100644 --- a/profiler/advisor/analyzer/computation/bound/block_dim_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/computation/bound/block_dim_checker.py @@ -15,12 +15,12 @@ import logging from typing import List -from profiler.advisor.analyzer.computation.operator_checker import OperatorChecker -from profiler.advisor.display.prompt.base_prompt import BasePrompt -from profiler.prof_common.constant import Constant -from profiler.advisor.config.config import Config -from profiler.advisor.dataset.profiling.profiling_dataset import ProfilingDataset -from profiler.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.advisor.analyzer.computation.operator_checker import OperatorChecker +from msprof_analyze.advisor.display.prompt.base_prompt import BasePrompt +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.advisor.config.config import Config +from msprof_analyze.advisor.dataset.profiling.profiling_dataset import ProfilingDataset +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/computation/bound/operator_bound_checker.py b/profiler/msprof_analyze/advisor/analyzer/computation/bound/operator_bound_checker.py similarity index 86% rename from profiler/advisor/analyzer/computation/bound/operator_bound_checker.py rename to profiler/msprof_analyze/advisor/analyzer/computation/bound/operator_bound_checker.py index 36c514bd38..444843ca35 100644 --- a/profiler/advisor/analyzer/computation/bound/operator_bound_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/computation/bound/operator_bound_checker.py @@ -15,13 +15,13 @@ import logging from typing import List -from profiler.advisor.analyzer.computation.operator_checker import OperatorChecker -from profiler.advisor.display.prompt.base_prompt import BasePrompt -from profiler.prof_common.additional_args_manager import AdditionalArgsManager -from profiler.prof_common.constant import Constant -from profiler.advisor.config.config import Config -from profiler.advisor.dataset.profiling.profiling_dataset import ProfilingDataset -from profiler.advisor.utils.utils import to_percent +from msprof_analyze.advisor.analyzer.computation.operator_checker import OperatorChecker +from msprof_analyze.advisor.display.prompt.base_prompt import BasePrompt +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.advisor.config.config import Config +from msprof_analyze.advisor.dataset.profiling.profiling_dataset import ProfilingDataset +from msprof_analyze.advisor.utils.utils import to_percent logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/computation/op_compile/__init__.py b/profiler/msprof_analyze/advisor/analyzer/computation/op_compile/__init__.py similarity index 100% rename from profiler/advisor/analyzer/computation/op_compile/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/computation/op_compile/__init__.py diff --git a/profiler/advisor/analyzer/computation/op_compile/dynamic_shape_checker.py b/profiler/msprof_analyze/advisor/analyzer/computation/op_compile/dynamic_shape_checker.py similarity index 87% rename from profiler/advisor/analyzer/computation/op_compile/dynamic_shape_checker.py rename to profiler/msprof_analyze/advisor/analyzer/computation/op_compile/dynamic_shape_checker.py index 8481821332..d31b2afc2c 100644 --- a/profiler/advisor/analyzer/computation/op_compile/dynamic_shape_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/computation/op_compile/dynamic_shape_checker.py @@ -16,13 +16,13 @@ import copy import logging from typing import List -from profiler.advisor.analyzer.computation.operator_checker import OperatorChecker -from profiler.advisor.config.config import Config -from profiler.advisor.dataset.profiling.info_collection import OpInfo -from profiler.advisor.display.prompt.base_prompt import BasePrompt -from profiler.advisor.result.item import OptimizeItem, StatisticsItem, OptimizeRecord -from profiler.prof_common.additional_args_manager import AdditionalArgsManager -from profiler.prof_common.file_manager import FileManager +from msprof_analyze.advisor.analyzer.computation.operator_checker import OperatorChecker +from msprof_analyze.advisor.config.config import Config +from msprof_analyze.advisor.dataset.profiling.info_collection import OpInfo +from msprof_analyze.advisor.display.prompt.base_prompt import BasePrompt +from msprof_analyze.advisor.result.item import OptimizeItem, StatisticsItem, OptimizeRecord +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.prof_common.file_manager import FileManager logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/computation/operator_checker.py b/profiler/msprof_analyze/advisor/analyzer/computation/operator_checker.py similarity index 94% rename from profiler/advisor/analyzer/computation/operator_checker.py rename to profiler/msprof_analyze/advisor/analyzer/computation/operator_checker.py index 5feb47a9d3..54e98887ec 100644 --- a/profiler/advisor/analyzer/computation/operator_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/computation/operator_checker.py @@ -17,16 +17,16 @@ import logging from textwrap import fill from typing import List -from profiler.advisor.display.prompt.base_prompt import BasePrompt -from profiler.prof_common.constant import Constant -from profiler.advisor.common.enum_params_parser import EnumParamsParser -from profiler.advisor.common.version_control import VersionControl -from profiler.advisor.config.config import Config -from profiler.advisor.dataset.profiling.info_collection import OpInfo -from profiler.advisor.dataset.profiling.profiling_dataset import ProfilingDataset -from profiler.advisor.result.item import OptimizeItem, StatisticsItem, OptimizeRecord -from profiler.advisor.utils.utils import safe_division, convert_to_float -from profiler.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.advisor.display.prompt.base_prompt import BasePrompt +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.advisor.common.enum_params_parser import EnumParamsParser +from msprof_analyze.advisor.common.version_control import VersionControl +from msprof_analyze.advisor.config.config import Config +from msprof_analyze.advisor.dataset.profiling.info_collection import OpInfo +from msprof_analyze.advisor.dataset.profiling.profiling_dataset import ProfilingDataset +from msprof_analyze.advisor.result.item import OptimizeItem, StatisticsItem, OptimizeRecord +from msprof_analyze.advisor.utils.utils import safe_division, convert_to_float +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/computation/pp_stage_computation_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/computation/pp_stage_computation_analyzer.py similarity index 88% rename from profiler/advisor/analyzer/computation/pp_stage_computation_analyzer.py rename to profiler/msprof_analyze/advisor/analyzer/computation/pp_stage_computation_analyzer.py index cc27722657..c5d40e004d 100644 --- a/profiler/advisor/analyzer/computation/pp_stage_computation_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/computation/pp_stage_computation_analyzer.py @@ -1,119 +1,119 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -import os -from multiprocessing import Manager - -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.common.analyzer_scopes import SupportedScopes -from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor -from profiler.advisor.interface.interface import Interface -from profiler.advisor.utils.utils import ParallelJob, get_analyze_processes -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.result.item import OptimizeItem, OptimizeRecord - -logger = logging.getLogger() - - -class PPStageComputationAnalyzer(BaseAnalyzer): - - def __init__(self, collection_path, **kwargs): - super().__init__(collection_path, **kwargs) - self.collection_path = collection_path - self._stages_rendered_html = Manager().list() - self._multiprocess_result = Manager().dict() - # html render不能序列化，无法用多进程，放到optimize里面初始化 - self.html_render = None - self.result = None - - @staticmethod - def _get_valid_sheet_name(sheet_name, prefix): - if not sheet_name.lower().startswith(prefix.lower()): - sheet_name = f"{prefix} {sheet_name}" - return sheet_name - - def optimize(self, stages_profiling_path, **kwargs): - pp_stage_processes = min(get_analyze_processes(), len(stages_profiling_path)) - if pp_stage_processes <= 1: - for stage_profiling_path in stages_profiling_path: - self._optimize(**stage_profiling_path) - else: - logger.info("Start to parallel analysis of pp stages, number of processes is %s", pp_stage_processes) - parallel_stage_analysis_job = ParallelJob(self._optimize, stages_profiling_path, - "Computation analysis of Pipeline parallel stages") - parallel_stage_analysis_job.start(pp_stage_processes) - self._merge_multiprocess_result() - - self.make_render() - self.html_render = HTMLRender() - return self.result - - def make_render(self): - HTMLRender().render_template(key="computation", - template_dir="templates", - template_name="pp_stage_computation_analysis.html", - stages_rendered_html=list(self._stages_rendered_html), - priority_background_color=PriorityBackgroundColor.high) - - def get_priority(self, max_mem_op_dur=None): - pass - - def _optimize(self, profiling_path, **kwargs): - stage_html_record = dict(stage=kwargs.get("stage"), rank=kwargs.get("rank"), step=kwargs.get("step")) - kwargs["add_render_list"] = False - - # stage 并行分析时，避免调用本身，即SupportedScopes.STAGE_COMPUTE - scopes = Interface.get_scope(Interface.COMPUTATION) - stage_analyzer_list = [Interface.get_analyzer(Interface.COMPUTATION, scope) - for scope in scopes - if scope != SupportedScopes.STAGE_COMPUTE] - - for analyzer_cls in stage_analyzer_list: - analyzer = analyzer_cls(collection_path=profiling_path, **kwargs) - result = analyzer.optimize(**kwargs) - if hasattr(result, "data") and result.data: - self.result = result - if hasattr(analyzer, "html") and analyzer.html: - if "html_list" not in stage_html_record: - stage_html_record["html_list"] = [] - stage_html_record["html_list"].append(analyzer.html) - self._stages_rendered_html.append(stage_html_record) - self._multiprocess_result[f"rank {kwargs.get('rank')}".capitalize()] = result.data - - def _merge_multiprocess_result(self): - self.result = OptimizeResult() - for key, result_data in self._multiprocess_result.items(): - problem_data = result_data.get("problems", {}).get("data", []) - if not problem_data: - continue - - for row in problem_data: - if len(row) < 3: - continue - issue_name, desc, suggestion = row[:3] - sheet_name = PPStageComputationAnalyzer._get_valid_sheet_name(issue_name, key) - optimization_item = OptimizeItem(sheet_name, desc, [suggestion]) - self.result.add(OptimizeRecord(optimization_item)) - del result_data["problems"] - - for issue_name, issue_details in result_data.items(): - headers = issue_details.get("headers", []) - data = issue_details.get("data", []) - sheet_name = PPStageComputationAnalyzer._get_valid_sheet_name(issue_name, key) - self.result.add_detail(sheet_name, headers=headers) - - for row in data: - self.result.add_detail(sheet_name, detail=row) +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import os +from multiprocessing import Manager + +from msprof_analyze.advisor.analyzer.base_analyzer import BaseAnalyzer +from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes +from msprof_analyze.advisor.display.html.render import HTMLRender +from msprof_analyze.advisor.display.html.priority_background_color import PriorityBackgroundColor +from msprof_analyze.advisor.interface.interface import Interface +from msprof_analyze.advisor.utils.utils import ParallelJob, get_analyze_processes +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.result.item import OptimizeItem, OptimizeRecord + +logger = logging.getLogger() + + +class PPStageComputationAnalyzer(BaseAnalyzer): + + def __init__(self, collection_path, **kwargs): + super().__init__(collection_path, **kwargs) + self.collection_path = collection_path + self._stages_rendered_html = Manager().list() + self._multiprocess_result = Manager().dict() + # html render不能序列化，无法用多进程，放到optimize里面初始化 + self.html_render = None + self.result = None + + @staticmethod + def _get_valid_sheet_name(sheet_name, prefix): + if not sheet_name.lower().startswith(prefix.lower()): + sheet_name = f"{prefix} {sheet_name}" + return sheet_name + + def optimize(self, stages_profiling_path, **kwargs): + pp_stage_processes = min(get_analyze_processes(), len(stages_profiling_path)) + if pp_stage_processes <= 1: + for stage_profiling_path in stages_profiling_path: + self._optimize(**stage_profiling_path) + else: + logger.info("Start to parallel analysis of pp stages, number of processes is %s", pp_stage_processes) + parallel_stage_analysis_job = ParallelJob(self._optimize, stages_profiling_path, + "Computation analysis of Pipeline parallel stages") + parallel_stage_analysis_job.start(pp_stage_processes) + self._merge_multiprocess_result() + + self.make_render() + self.html_render = HTMLRender() + return self.result + + def make_render(self): + HTMLRender().render_template(key="computation", + template_dir="templates", + template_name="pp_stage_computation_analysis.html", + stages_rendered_html=list(self._stages_rendered_html), + priority_background_color=PriorityBackgroundColor.high) + + def get_priority(self, max_mem_op_dur=None): + pass + + def _optimize(self, profiling_path, **kwargs): + stage_html_record = dict(stage=kwargs.get("stage"), rank=kwargs.get("rank"), step=kwargs.get("step")) + kwargs["add_render_list"] = False + + # stage 并行分析时，避免调用本身，即SupportedScopes.STAGE_COMPUTE + scopes = Interface.get_scope(Interface.COMPUTATION) + stage_analyzer_list = [Interface.get_analyzer(Interface.COMPUTATION, scope) + for scope in scopes + if scope != SupportedScopes.STAGE_COMPUTE] + + for analyzer_cls in stage_analyzer_list: + analyzer = analyzer_cls(collection_path=profiling_path, **kwargs) + result = analyzer.optimize(**kwargs) + if hasattr(result, "data") and result.data: + self.result = result + if hasattr(analyzer, "html") and analyzer.html: + if "html_list" not in stage_html_record: + stage_html_record["html_list"] = [] + stage_html_record["html_list"].append(analyzer.html) + self._stages_rendered_html.append(stage_html_record) + self._multiprocess_result[f"rank {kwargs.get('rank')}".capitalize()] = result.data + + def _merge_multiprocess_result(self): + self.result = OptimizeResult() + for key, result_data in self._multiprocess_result.items(): + problem_data = result_data.get("problems", {}).get("data", []) + if not problem_data: + continue + + for row in problem_data: + if len(row) < 3: + continue + issue_name, desc, suggestion = row[:3] + sheet_name = PPStageComputationAnalyzer._get_valid_sheet_name(issue_name, key) + optimization_item = OptimizeItem(sheet_name, desc, [suggestion]) + self.result.add(OptimizeRecord(optimization_item)) + del result_data["problems"] + + for issue_name, issue_details in result_data.items(): + headers = issue_details.get("headers", []) + data = issue_details.get("data", []) + sheet_name = PPStageComputationAnalyzer._get_valid_sheet_name(issue_name, key) + self.result.add_detail(sheet_name, headers=headers) + + for row in data: + self.result.add_detail(sheet_name, detail=row) diff --git a/profiler/advisor/analyzer/computation/profiling_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/computation/profiling_analyzer.py similarity index 83% rename from profiler/advisor/analyzer/computation/profiling_analyzer.py rename to profiler/msprof_analyze/advisor/analyzer/computation/profiling_analyzer.py index ccf6711399..665c157045 100644 --- a/profiler/advisor/analyzer/computation/profiling_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/computation/profiling_analyzer.py @@ -15,16 +15,16 @@ import logging from abc import ABC -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.analyzer.computation.aicpu.aicpu_checker import AicpuChecker -from profiler.advisor.analyzer.computation.bound.block_dim_checker import BlockDimChecker -from profiler.advisor.analyzer.computation.bound.operator_bound_checker import OperatorBoundChecker -from profiler.advisor.analyzer.computation.op_compile.dynamic_shape_checker import DynamicShapeChecker -from profiler.advisor.analyzer.computation.operator_checker import OperatorChecker -from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor -from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.dataset.profiling.profiling_dataset import ProfilingDataset +from msprof_analyze.advisor.analyzer.base_analyzer import BaseAnalyzer +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.analyzer.computation.aicpu.aicpu_checker import AicpuChecker +from msprof_analyze.advisor.analyzer.computation.bound.block_dim_checker import BlockDimChecker +from msprof_analyze.advisor.analyzer.computation.bound.operator_bound_checker import OperatorBoundChecker +from msprof_analyze.advisor.analyzer.computation.op_compile.dynamic_shape_checker import DynamicShapeChecker +from msprof_analyze.advisor.analyzer.computation.operator_checker import OperatorChecker +from msprof_analyze.advisor.display.html.priority_background_color import PriorityBackgroundColor +from msprof_analyze.advisor.display.html.render import HTMLRender +from msprof_analyze.advisor.dataset.profiling.profiling_dataset import ProfilingDataset logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/dataloader/__init__.py b/profiler/msprof_analyze/advisor/analyzer/dataloader/__init__.py similarity index 100% rename from profiler/advisor/analyzer/dataloader/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/dataloader/__init__.py diff --git a/profiler/advisor/analyzer/dataloader/dataloader_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/dataloader/dataloader_analyzer.py similarity index 77% rename from profiler/advisor/analyzer/dataloader/dataloader_analyzer.py rename to profiler/msprof_analyze/advisor/analyzer/dataloader/dataloader_analyzer.py index dd377eb1cd..54471bab18 100644 --- a/profiler/advisor/analyzer/dataloader/dataloader_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/dataloader/dataloader_analyzer.py @@ -16,12 +16,12 @@ import logging from typing import List, Dict, Any -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.analyzer.dataloader.dataloader_checker import DataloaderChecker -from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor -from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset +from msprof_analyze.advisor.analyzer.base_analyzer import BaseAnalyzer +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.analyzer.dataloader.dataloader_checker import DataloaderChecker +from msprof_analyze.advisor.display.html.priority_background_color import PriorityBackgroundColor +from msprof_analyze.advisor.display.html.render import HTMLRender +from msprof_analyze.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/dataloader/dataloader_checker.py b/profiler/msprof_analyze/advisor/analyzer/dataloader/dataloader_checker.py similarity index 91% rename from profiler/advisor/analyzer/dataloader/dataloader_checker.py rename to profiler/msprof_analyze/advisor/analyzer/dataloader/dataloader_checker.py index 7fade10cde..45efecc728 100644 --- a/profiler/advisor/analyzer/dataloader/dataloader_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/dataloader/dataloader_checker.py @@ -17,11 +17,11 @@ import re import logging import yaml -from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.prof_common.additional_args_manager import AdditionalArgsManager -from profiler.prof_common.file_manager import FileManager +from msprof_analyze.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.result.item import OptimizeItem, OptimizeRecord +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.prof_common.file_manager import FileManager logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/graph_fusion/__init__.py b/profiler/msprof_analyze/advisor/analyzer/graph_fusion/__init__.py similarity index 100% rename from profiler/advisor/analyzer/graph_fusion/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/graph_fusion/__init__.py diff --git a/profiler/advisor/analyzer/graph_fusion/graph_fusion_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/graph_fusion/graph_fusion_analyzer.py similarity index 82% rename from profiler/advisor/analyzer/graph_fusion/graph_fusion_analyzer.py rename to profiler/msprof_analyze/advisor/analyzer/graph_fusion/graph_fusion_analyzer.py index 7b8bb6a20f..c3323aa020 100644 --- a/profiler/advisor/analyzer/graph_fusion/graph_fusion_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/graph_fusion/graph_fusion_analyzer.py @@ -15,12 +15,12 @@ from typing import List from functools import partial -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.dataset.graph_dataset import GraphDataset -from profiler.advisor.analyzer.graph_fusion.graph_fusion_checker import GraphFusionRules -from profiler.advisor.dataset.profiling.profiling_dataset import ProfilingDataset -from profiler.advisor.display.html.render import HTMLRender +from msprof_analyze.advisor.analyzer.base_analyzer import BaseAnalyzer +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.dataset.graph_dataset import GraphDataset +from msprof_analyze.advisor.analyzer.graph_fusion.graph_fusion_checker import GraphFusionRules +from msprof_analyze.advisor.dataset.profiling.profiling_dataset import ProfilingDataset +from msprof_analyze.advisor.display.html.render import HTMLRender class FusionOPAnalyzer(BaseAnalyzer): diff --git a/profiler/advisor/analyzer/graph_fusion/graph_fusion_checker.py b/profiler/msprof_analyze/advisor/analyzer/graph_fusion/graph_fusion_checker.py similarity index 94% rename from profiler/advisor/analyzer/graph_fusion/graph_fusion_checker.py rename to profiler/msprof_analyze/advisor/analyzer/graph_fusion/graph_fusion_checker.py index bc5f1eff68..1e2a66695d 100644 --- a/profiler/advisor/analyzer/graph_fusion/graph_fusion_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/graph_fusion/graph_fusion_checker.py @@ -17,14 +17,14 @@ from typing import List from tqdm import tqdm -from profiler.advisor.display.prompt.base_prompt import BasePrompt -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.result.item import OptimizeItem, OptimizeRecord, StatisticsItem -from profiler.advisor.common.graph.graph import Graph -from profiler.advisor.common.graph.graph_parser import QueryGraphParser -from profiler.advisor.dataset.graph_dataset import GraphDataset -from profiler.advisor.common.graph.graph_match import find_isomorphisms -from profiler.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.advisor.display.prompt.base_prompt import BasePrompt +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.result.item import OptimizeItem, OptimizeRecord, StatisticsItem +from msprof_analyze.advisor.common.graph.graph import Graph +from msprof_analyze.advisor.common.graph.graph_parser import QueryGraphParser +from msprof_analyze.advisor.dataset.graph_dataset import GraphDataset +from msprof_analyze.advisor.common.graph.graph_match import find_isomorphisms +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/memory/__init__.py b/profiler/msprof_analyze/advisor/analyzer/memory/__init__.py similarity index 100% rename from profiler/advisor/analyzer/memory/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/memory/__init__.py diff --git a/profiler/advisor/analyzer/memory/memory_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/memory/memory_analyzer.py similarity index 79% rename from profiler/advisor/analyzer/memory/memory_analyzer.py rename to profiler/msprof_analyze/advisor/analyzer/memory/memory_analyzer.py index 799094ef55..9fe274174c 100644 --- a/profiler/advisor/analyzer/memory/memory_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/memory/memory_analyzer.py @@ -14,12 +14,12 @@ # limitations under the License. import logging -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.analyzer.memory.memory_checker import MemoryOpsChecker -from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset -from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor +from msprof_analyze.advisor.analyzer.base_analyzer import BaseAnalyzer +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.analyzer.memory.memory_checker import MemoryOpsChecker +from msprof_analyze.advisor.display.html.render import HTMLRender +from msprof_analyze.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset +from msprof_analyze.advisor.display.html.priority_background_color import PriorityBackgroundColor logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/memory/memory_checker.py b/profiler/msprof_analyze/advisor/analyzer/memory/memory_checker.py similarity index 93% rename from profiler/advisor/analyzer/memory/memory_checker.py rename to profiler/msprof_analyze/advisor/analyzer/memory/memory_checker.py index 682a19f696..7288bf8a5e 100644 --- a/profiler/advisor/analyzer/memory/memory_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/memory/memory_checker.py @@ -17,9 +17,9 @@ import re import logging import yaml -from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset, MemCollector -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.result.item import OptimizeItem, OptimizeRecord +from msprof_analyze.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset, MemCollector +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.result.item import OptimizeItem, OptimizeRecord logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/overall/__init__.py b/profiler/msprof_analyze/advisor/analyzer/overall/__init__.py similarity index 100% rename from profiler/advisor/analyzer/overall/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/overall/__init__.py diff --git a/profiler/advisor/analyzer/overall/environment_variable_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/overall/environment_variable_analyzer.py similarity index 81% rename from profiler/advisor/analyzer/overall/environment_variable_analyzer.py rename to profiler/msprof_analyze/advisor/analyzer/overall/environment_variable_analyzer.py index b15ac2d944..c32b8e5ec2 100644 --- a/profiler/advisor/analyzer/overall/environment_variable_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/overall/environment_variable_analyzer.py @@ -14,11 +14,11 @@ # limitations under the License. import logging -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.prof_common.path_manager import PathManager -from profiler.advisor.dataset.environment_variable_dataset import EnvironmentVariableDataset -from profiler.advisor.analyzer.overall.environment_variable_checker import EnvironmentVariableChecker -from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor +from msprof_analyze.advisor.analyzer.base_analyzer import BaseAnalyzer +from msprof_analyze.prof_common.path_manager import PathManager +from msprof_analyze.advisor.dataset.environment_variable_dataset import EnvironmentVariableDataset +from msprof_analyze.advisor.analyzer.overall.environment_variable_checker import EnvironmentVariableChecker +from msprof_analyze.advisor.display.html.priority_background_color import PriorityBackgroundColor logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/overall/environment_variable_checker.py b/profiler/msprof_analyze/advisor/analyzer/overall/environment_variable_checker.py similarity index 84% rename from profiler/advisor/analyzer/overall/environment_variable_checker.py rename to profiler/msprof_analyze/advisor/analyzer/overall/environment_variable_checker.py index 286af31791..05093b3896 100644 --- a/profiler/advisor/analyzer/overall/environment_variable_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/overall/environment_variable_checker.py @@ -14,16 +14,16 @@ # limitations under the License. import os -from profiler.advisor.display.prompt.base_prompt import BasePrompt -from profiler.prof_common.additional_args_manager import AdditionalArgsManager -from profiler.prof_common.file_manager import FileManager -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.result.item import OptimizeItem -from profiler.advisor.result.item import OptimizeRecord -from profiler.advisor.common.analyzer_scopes import SupportedScopes -from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.utils.utils import convert_to_int_with_exception -from profiler.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.advisor.display.prompt.base_prompt import BasePrompt +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.prof_common.file_manager import FileManager +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.result.item import OptimizeItem +from msprof_analyze.advisor.result.item import OptimizeRecord +from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes +from msprof_analyze.advisor.display.html.render import HTMLRender +from msprof_analyze.advisor.utils.utils import convert_to_int_with_exception +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager class EnvironmentVariableChecker: diff --git a/profiler/advisor/analyzer/overall/overall_summary_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/overall/overall_summary_analyzer.py similarity index 94% rename from profiler/advisor/analyzer/overall/overall_summary_analyzer.py rename to profiler/msprof_analyze/advisor/analyzer/overall/overall_summary_analyzer.py index d5b4674f6f..1bfaf8d611 100644 --- a/profiler/advisor/analyzer/overall/overall_summary_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/overall/overall_summary_analyzer.py @@ -15,14 +15,14 @@ import logging import os -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.display.prompt.base_prompt import BasePrompt -from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.advisor.result.result import OptimizeResult -from profiler.compare_tools.compare_interface.comparison_interface import ComparisonInterface -from profiler.prof_common.additional_args_manager import AdditionalArgsManager -from profiler.prof_common.constant import Constant +from msprof_analyze.advisor.analyzer.base_analyzer import BaseAnalyzer +from msprof_analyze.advisor.display.html.render import HTMLRender +from msprof_analyze.advisor.display.prompt.base_prompt import BasePrompt +from msprof_analyze.advisor.result.item import OptimizeItem, OptimizeRecord +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.compare_tools.compare_interface.comparison_interface import ComparisonInterface +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.prof_common.constant import Constant class OverallSummaryAnalyzer(BaseAnalyzer): diff --git a/profiler/advisor/analyzer/schedule/__init__.py b/profiler/msprof_analyze/advisor/analyzer/schedule/__init__.py similarity index 100% rename from profiler/advisor/analyzer/schedule/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/schedule/__init__.py diff --git a/profiler/advisor/analyzer/schedule/conjectured_gc/__init__.py b/profiler/msprof_analyze/advisor/analyzer/schedule/conjectured_gc/__init__.py similarity index 100% rename from profiler/advisor/analyzer/schedule/conjectured_gc/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/schedule/conjectured_gc/__init__.py diff --git a/profiler/advisor/analyzer/schedule/conjectured_gc/conjectured_gc_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/schedule/conjectured_gc/conjectured_gc_analyzer.py similarity index 75% rename from profiler/advisor/analyzer/schedule/conjectured_gc/conjectured_gc_analyzer.py rename to profiler/msprof_analyze/advisor/analyzer/schedule/conjectured_gc/conjectured_gc_analyzer.py index d9ed42dce0..93f3593087 100644 --- a/profiler/advisor/analyzer/schedule/conjectured_gc/conjectured_gc_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/schedule/conjectured_gc/conjectured_gc_analyzer.py @@ -13,12 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.analyzer.schedule.conjectured_gc.conjectured_gc_checker import ConjecturedGcChecker -from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset -from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor +from msprof_analyze.advisor.analyzer.base_analyzer import BaseAnalyzer +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.analyzer.schedule.conjectured_gc.conjectured_gc_checker import ConjecturedGcChecker +from msprof_analyze.advisor.display.html.render import HTMLRender +from msprof_analyze.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset +from msprof_analyze.advisor.display.html.priority_background_color import PriorityBackgroundColor class ConjecturedGcAnalyzer(BaseAnalyzer): diff --git a/profiler/advisor/analyzer/schedule/conjectured_gc/conjectured_gc_checker.py b/profiler/msprof_analyze/advisor/analyzer/schedule/conjectured_gc/conjectured_gc_checker.py similarity index 94% rename from profiler/advisor/analyzer/schedule/conjectured_gc/conjectured_gc_checker.py rename to profiler/msprof_analyze/advisor/analyzer/schedule/conjectured_gc/conjectured_gc_checker.py index cac9d62034..05d2e79969 100644 --- a/profiler/advisor/analyzer/schedule/conjectured_gc/conjectured_gc_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/schedule/conjectured_gc/conjectured_gc_checker.py @@ -14,13 +14,13 @@ # limitations under the License. import os -from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.advisor.utils.utils import convert_to_float, convert_to_int, safe_division -from profiler.prof_common.additional_args_manager import AdditionalArgsManager -from profiler.prof_common.constant import Constant -from profiler.prof_common.file_manager import FileManager +from msprof_analyze.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.result.item import OptimizeItem, OptimizeRecord +from msprof_analyze.advisor.utils.utils import convert_to_float, convert_to_int, safe_division +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.prof_common.file_manager import FileManager diff --git a/profiler/advisor/analyzer/schedule/dispatch/__init__.py b/profiler/msprof_analyze/advisor/analyzer/schedule/dispatch/__init__.py similarity index 100% rename from profiler/advisor/analyzer/schedule/dispatch/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/schedule/dispatch/__init__.py diff --git a/profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py similarity index 87% rename from profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py rename to profiler/msprof_analyze/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py index 01bc8a5f6e..c1669d0184 100644 --- a/profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py @@ -16,15 +16,15 @@ # limitations under the License. import logging -from profiler.advisor.display.prompt.base_prompt import BasePrompt -from profiler.prof_common.constant import Constant -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset -from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor -from profiler.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.advisor.display.prompt.base_prompt import BasePrompt +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.advisor.analyzer.base_analyzer import BaseAnalyzer +from msprof_analyze.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset +from msprof_analyze.advisor.result.item import OptimizeItem, OptimizeRecord +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.display.html.render import HTMLRender +from msprof_analyze.advisor.display.html.priority_background_color import PriorityBackgroundColor +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/schedule/free_event/__init__.py b/profiler/msprof_analyze/advisor/analyzer/schedule/free_event/__init__.py similarity index 100% rename from profiler/advisor/analyzer/schedule/free_event/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/schedule/free_event/__init__.py diff --git a/profiler/advisor/analyzer/schedule/fusion_ops/__init__.py b/profiler/msprof_analyze/advisor/analyzer/schedule/fusion_ops/__init__.py similarity index 100% rename from profiler/advisor/analyzer/schedule/fusion_ops/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/schedule/fusion_ops/__init__.py diff --git a/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py similarity index 93% rename from profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py rename to profiler/msprof_analyze/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py index 7b786f8b89..1c5c325be1 100644 --- a/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py @@ -19,18 +19,18 @@ import re from tqdm import tqdm -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.display.prompt.base_prompt import BasePrompt -from profiler.prof_common.constant import Constant -from profiler.advisor.common.analyzer_scopes import SupportedScopes -from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.advisor.config.config import Config -from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset -from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.advisor.utils.utils import format_timeline_result -from profiler.advisor.common.timeline.fusion_ops_db import init_timeline_ops_db -from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor -from profiler.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.advisor.analyzer.base_analyzer import BaseAnalyzer +from msprof_analyze.advisor.display.prompt.base_prompt import BasePrompt +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes +from msprof_analyze.advisor.common.timeline.event import TimelineEvent +from msprof_analyze.advisor.config.config import Config +from msprof_analyze.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset +from msprof_analyze.advisor.result.item import OptimizeItem, OptimizeRecord +from msprof_analyze.advisor.utils.utils import format_timeline_result +from msprof_analyze.advisor.common.timeline.fusion_ops_db import init_timeline_ops_db +from msprof_analyze.advisor.display.html.priority_background_color import PriorityBackgroundColor +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/schedule/fusion_ops/timeline_api_stack_checker.py b/profiler/msprof_analyze/advisor/analyzer/schedule/fusion_ops/timeline_api_stack_checker.py similarity index 93% rename from profiler/advisor/analyzer/schedule/fusion_ops/timeline_api_stack_checker.py rename to profiler/msprof_analyze/advisor/analyzer/schedule/fusion_ops/timeline_api_stack_checker.py index 17c494dbdc..51469a1f3c 100644 --- a/profiler/advisor/analyzer/schedule/fusion_ops/timeline_api_stack_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/schedule/fusion_ops/timeline_api_stack_checker.py @@ -15,13 +15,13 @@ import logging from typing import List -from profiler.prof_common.constant import Constant -from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.advisor.dataset.timeline_event_dataset import ComputationAnalysisDataset -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.advisor.utils.utils import get_analyze_processes, ParallelJob -from profiler.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.advisor.common.timeline.event import TimelineEvent +from msprof_analyze.advisor.dataset.timeline_event_dataset import ComputationAnalysisDataset +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.result.item import OptimizeItem, OptimizeRecord +from msprof_analyze.advisor.utils.utils import get_analyze_processes, ParallelJob +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/schedule/gc/__init__.py b/profiler/msprof_analyze/advisor/analyzer/schedule/gc/__init__.py similarity index 100% rename from profiler/advisor/analyzer/schedule/gc/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/schedule/gc/__init__.py diff --git a/profiler/advisor/analyzer/schedule/gc/gc_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/schedule/gc/gc_analyzer.py similarity index 79% rename from profiler/advisor/analyzer/schedule/gc/gc_analyzer.py rename to profiler/msprof_analyze/advisor/analyzer/schedule/gc/gc_analyzer.py index df30ad68d9..7e04a0597f 100644 --- a/profiler/advisor/analyzer/schedule/gc/gc_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/schedule/gc/gc_analyzer.py @@ -14,12 +14,12 @@ # limitations under the License. import logging -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.analyzer.schedule.gc.gc_checker import GcChecker -from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset -from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor +from msprof_analyze.advisor.analyzer.base_analyzer import BaseAnalyzer +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.analyzer.schedule.gc.gc_checker import GcChecker +from msprof_analyze.advisor.display.html.render import HTMLRender +from msprof_analyze.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset +from msprof_analyze.advisor.display.html.priority_background_color import PriorityBackgroundColor logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/schedule/gc/gc_checker.py b/profiler/msprof_analyze/advisor/analyzer/schedule/gc/gc_checker.py similarity index 89% rename from profiler/advisor/analyzer/schedule/gc/gc_checker.py rename to profiler/msprof_analyze/advisor/analyzer/schedule/gc/gc_checker.py index 78afef725c..92a0d208f8 100644 --- a/profiler/advisor/analyzer/schedule/gc/gc_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/schedule/gc/gc_checker.py @@ -15,14 +15,14 @@ import logging import os -from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset -from profiler.advisor.display.prompt.base_prompt import BasePrompt -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.advisor.utils.utils import convert_to_float, convert_to_int -from profiler.prof_common.additional_args_manager import AdditionalArgsManager -from profiler.prof_common.constant import Constant -from profiler.prof_common.file_manager import FileManager +from msprof_analyze.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset +from msprof_analyze.advisor.display.prompt.base_prompt import BasePrompt +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.result.item import OptimizeItem, OptimizeRecord +from msprof_analyze.advisor.utils.utils import convert_to_float, convert_to_int +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.prof_common.file_manager import FileManager logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/schedule/syncbn/__init__.py b/profiler/msprof_analyze/advisor/analyzer/schedule/syncbn/__init__.py similarity index 100% rename from profiler/advisor/analyzer/schedule/syncbn/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/schedule/syncbn/__init__.py diff --git a/profiler/advisor/analyzer/schedule/syncbn/syncbn_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/schedule/syncbn/syncbn_analyzer.py similarity index 76% rename from profiler/advisor/analyzer/schedule/syncbn/syncbn_analyzer.py rename to profiler/msprof_analyze/advisor/analyzer/schedule/syncbn/syncbn_analyzer.py index 3942f04a86..48506da626 100644 --- a/profiler/advisor/analyzer/schedule/syncbn/syncbn_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/schedule/syncbn/syncbn_analyzer.py @@ -1,46 +1,46 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging - -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.analyzer.schedule.syncbn.syncbn_checker import SyncBNChecker -from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor -from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset - -logger = logging.getLogger() - - -class SyncBNAnalyzer(BaseAnalyzer): - dataset_cls_list = [ScheduleAnalysisDataset] - - def __init__(self, collection_path, **kwargs): - super().__init__(collection_path, **kwargs) - self.result = OptimizeResult() - self.html_render = HTMLRender() - key = ScheduleAnalysisDataset.get_key() - self.timeline_event_dataset = self.get_first_data_by_key(self.dataset_list, key) - - @BaseAnalyzer.check_data((ScheduleAnalysisDataset.get_key(),)) - def optimize(self, **kwargs): - syncbn_checker = SyncBNChecker() - syncbn_checker.check_syncbn(self.timeline_event_dataset) - syncbn_checker.make_record(self.result) - syncbn_checker.make_render(self.html_render, priority=self.get_priority(), rank=kwargs.get("rank")) - return self.result - - def get_priority(self, max_mem_op_dur=None): +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from msprof_analyze.advisor.analyzer.base_analyzer import BaseAnalyzer +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.analyzer.schedule.syncbn.syncbn_checker import SyncBNChecker +from msprof_analyze.advisor.display.html.priority_background_color import PriorityBackgroundColor +from msprof_analyze.advisor.display.html.render import HTMLRender +from msprof_analyze.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset + +logger = logging.getLogger() + + +class SyncBNAnalyzer(BaseAnalyzer): + dataset_cls_list = [ScheduleAnalysisDataset] + + def __init__(self, collection_path, **kwargs): + super().__init__(collection_path, **kwargs) + self.result = OptimizeResult() + self.html_render = HTMLRender() + key = ScheduleAnalysisDataset.get_key() + self.timeline_event_dataset = self.get_first_data_by_key(self.dataset_list, key) + + @BaseAnalyzer.check_data((ScheduleAnalysisDataset.get_key(),)) + def optimize(self, **kwargs): + syncbn_checker = SyncBNChecker() + syncbn_checker.check_syncbn(self.timeline_event_dataset) + syncbn_checker.make_record(self.result) + syncbn_checker.make_render(self.html_render, priority=self.get_priority(), rank=kwargs.get("rank")) + return self.result + + def get_priority(self, max_mem_op_dur=None): return PriorityBackgroundColor.high \ No newline at end of file diff --git a/profiler/advisor/analyzer/schedule/syncbn/syncbn_checker.py b/profiler/msprof_analyze/advisor/analyzer/schedule/syncbn/syncbn_checker.py similarity index 89% rename from profiler/advisor/analyzer/schedule/syncbn/syncbn_checker.py rename to profiler/msprof_analyze/advisor/analyzer/schedule/syncbn/syncbn_checker.py index 28661aa09b..64bf40ee23 100644 --- a/profiler/advisor/analyzer/schedule/syncbn/syncbn_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/schedule/syncbn/syncbn_checker.py @@ -15,11 +15,11 @@ import logging import os -from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.prof_common.additional_args_manager import AdditionalArgsManager -from profiler.prof_common.file_manager import FileManager +from msprof_analyze.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.result.item import OptimizeItem, OptimizeRecord +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.prof_common.file_manager import FileManager logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/schedule/synchronize_stream/__init__.py b/profiler/msprof_analyze/advisor/analyzer/schedule/synchronize_stream/__init__.py similarity index 100% rename from profiler/advisor/analyzer/schedule/synchronize_stream/__init__.py rename to profiler/msprof_analyze/advisor/analyzer/schedule/synchronize_stream/__init__.py diff --git a/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py similarity index 79% rename from profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py rename to profiler/msprof_analyze/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py index 0dceaa1afd..4b0218d572 100644 --- a/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py @@ -1,47 +1,47 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging - -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.analyzer.schedule.synchronize_stream.synchronize_stream_checker import SynchronizeStreamChecker -from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset -from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.result.result import OptimizeResult - -logger = logging.getLogger() - - -class SynchronizeStreamAnalyzer(BaseAnalyzer): - dataset_cls_list = [ScheduleAnalysisDataset] - - def __init__(self, collection_path, **kwargs): - super().__init__(collection_path, **kwargs) - self.result = OptimizeResult() - self.html_render = HTMLRender() - - key = ScheduleAnalysisDataset.get_key() - self.timeline_event_dataset = self.get_first_data_by_key(self.dataset_list, key) - - @BaseAnalyzer.check_data((ScheduleAnalysisDataset.get_key(),)) - def optimize(self, **kwargs): - synchronize_stream_checker = SynchronizeStreamChecker() - synchronize_stream_checker.check_synchronize(self.timeline_event_dataset) - synchronize_stream_checker.make_record(self.result) - synchronize_stream_checker.make_render(self.html_render, priority=self.get_priority(synchronize_stream_checker), - rank=kwargs.get("rank")) - return self.result - - def get_priority(self, max_mem_op_dur): - return max_mem_op_dur.priority +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from msprof_analyze.advisor.analyzer.base_analyzer import BaseAnalyzer +from msprof_analyze.advisor.analyzer.schedule.synchronize_stream.synchronize_stream_checker import SynchronizeStreamChecker +from msprof_analyze.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset +from msprof_analyze.advisor.display.html.render import HTMLRender +from msprof_analyze.advisor.result.result import OptimizeResult + +logger = logging.getLogger() + + +class SynchronizeStreamAnalyzer(BaseAnalyzer): + dataset_cls_list = [ScheduleAnalysisDataset] + + def __init__(self, collection_path, **kwargs): + super().__init__(collection_path, **kwargs) + self.result = OptimizeResult() + self.html_render = HTMLRender() + + key = ScheduleAnalysisDataset.get_key() + self.timeline_event_dataset = self.get_first_data_by_key(self.dataset_list, key) + + @BaseAnalyzer.check_data((ScheduleAnalysisDataset.get_key(),)) + def optimize(self, **kwargs): + synchronize_stream_checker = SynchronizeStreamChecker() + synchronize_stream_checker.check_synchronize(self.timeline_event_dataset) + synchronize_stream_checker.make_record(self.result) + synchronize_stream_checker.make_render(self.html_render, priority=self.get_priority(synchronize_stream_checker), + rank=kwargs.get("rank")) + return self.result + + def get_priority(self, max_mem_op_dur): + return max_mem_op_dur.priority diff --git a/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py b/profiler/msprof_analyze/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py similarity index 86% rename from profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py rename to profiler/msprof_analyze/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py index 745882cbdd..1b9c074304 100644 --- a/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py @@ -15,16 +15,16 @@ import logging import os -from profiler.advisor.analyzer.schedule.timeline_base_checker import TimelineBaseChecker -from profiler.prof_common.additional_args_manager import AdditionalArgsManager -from profiler.prof_common.constant import Constant -from profiler.advisor.config.config import Config -from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset -from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.advisor.utils.utils import format_timeline_result, safe_division -from profiler.prof_common.file_manager import FileManager +from msprof_analyze.advisor.analyzer.schedule.timeline_base_checker import TimelineBaseChecker +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.advisor.config.config import Config +from msprof_analyze.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset +from msprof_analyze.advisor.display.html.priority_background_color import PriorityBackgroundColor +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.result.item import OptimizeItem, OptimizeRecord +from msprof_analyze.advisor.utils.utils import format_timeline_result, safe_division +from msprof_analyze.prof_common.file_manager import FileManager logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/schedule/timeline_base_checker.py b/profiler/msprof_analyze/advisor/analyzer/schedule/timeline_base_checker.py similarity index 92% rename from profiler/advisor/analyzer/schedule/timeline_base_checker.py rename to profiler/msprof_analyze/advisor/analyzer/schedule/timeline_base_checker.py index 7d3cdf7e90..99ca99e99e 100644 --- a/profiler/advisor/analyzer/schedule/timeline_base_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/schedule/timeline_base_checker.py @@ -16,10 +16,10 @@ from abc import ABC, abstractmethod import multiprocessing import logging -from profiler.prof_common.constant import Constant -from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset -from profiler.advisor.result.result import OptimizeResult +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.advisor.common.timeline.event import TimelineEvent +from msprof_analyze.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset +from msprof_analyze.advisor.result.result import OptimizeResult logger = logging.getLogger() diff --git a/profiler/advisor/cluster_perf_analysis.ipynb b/profiler/msprof_analyze/advisor/cluster_perf_analysis.ipynb similarity index 99% rename from profiler/advisor/cluster_perf_analysis.ipynb rename to profiler/msprof_analyze/advisor/cluster_perf_analysis.ipynb index 7ee0b24e85..af9d4d6c2c 100644 --- a/profiler/advisor/cluster_perf_analysis.ipynb +++ b/profiler/msprof_analyze/advisor/cluster_perf_analysis.ipynb @@ -23,7 +23,7 @@ "metadata": {}, "outputs": [], "source": [ - "from profiler.advisor.interface.interface import Interface\n", + "from msprof_analyze.advisor.interface.interface import Interface\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "from prettytable import PrettyTable, ALL\n", diff --git a/profiler/advisor/common/__init__.py b/profiler/msprof_analyze/advisor/common/__init__.py similarity index 100% rename from profiler/advisor/common/__init__.py rename to profiler/msprof_analyze/advisor/common/__init__.py diff --git a/profiler/advisor/common/analyzer_scopes.py b/profiler/msprof_analyze/advisor/common/analyzer_scopes.py similarity index 100% rename from profiler/advisor/common/analyzer_scopes.py rename to profiler/msprof_analyze/advisor/common/analyzer_scopes.py diff --git a/profiler/advisor/common/async_analysis_status.py b/profiler/msprof_analyze/advisor/common/async_analysis_status.py similarity index 97% rename from profiler/advisor/common/async_analysis_status.py rename to profiler/msprof_analyze/advisor/common/async_analysis_status.py index 98bb458105..2d314b5cb0 100644 --- a/profiler/advisor/common/async_analysis_status.py +++ b/profiler/msprof_analyze/advisor/common/async_analysis_status.py @@ -1,27 +1,27 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - - -class AsyncAnalysisStatus: - FAILED = "failed" - SUCCESS = "success" - ANALYZING = "analyzing" - - BAD_REQUEST_STATUS_CODE = 400 - NOT_FOUND_STATUS_CODE = 404 - INNER_ERROR_STATUS_CODE = 500 - NON_FAILED_STATUS_CODE = 200 +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + + +class AsyncAnalysisStatus: + FAILED = "failed" + SUCCESS = "success" + ANALYZING = "analyzing" + + BAD_REQUEST_STATUS_CODE = 400 + NOT_FOUND_STATUS_CODE = 404 + INNER_ERROR_STATUS_CODE = 500 + NON_FAILED_STATUS_CODE = 200 diff --git a/profiler/advisor/common/enum_params_parser.py b/profiler/msprof_analyze/advisor/common/enum_params_parser.py similarity index 92% rename from profiler/advisor/common/enum_params_parser.py rename to profiler/msprof_analyze/advisor/common/enum_params_parser.py index 2f0fe3a2a3..7158af929f 100644 --- a/profiler/advisor/common/enum_params_parser.py +++ b/profiler/msprof_analyze/advisor/common/enum_params_parser.py @@ -1,104 +1,104 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import logging -import typing - -from profiler.advisor.common.timeline.event import AdvisorDict -from profiler.advisor.utils.utils import singleton -from profiler.prof_common.file_manager import FileManager - -logger = logging.getLogger() - - -@singleton -class EnumParamsParser(): - # 枚举变量抽象成yaml文件，统一管理，便于第三方服务对接advisor时调用当前类查询所有枚举变量参数的默认值和可选值 - - ARGUMENTS = "arguments" - ENVS = "envs" - OPTIONS = "options" - DEFAULT = "default" - TYPE = "type" - STR_TYPE = "str" - LIST_TYPE = "list" - INT_TYPE = "int" - BOOLEAN_TYPE = "boolean" - - def __init__(self): - enum_params_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "config", - "enum_parameters.yaml") - self.enum_params = FileManager.read_yaml_file(enum_params_path) - self._set_value() - - def get_keys(self): - return list(self.get_arguments_keys()) + list(self.get_envs_keys()) - - def get_arguments_keys(self): - return list(self.enum_params.get(self.ARGUMENTS, {}).keys()) - - def get_envs_keys(self): - return list(self.enum_params.get(self.ENVS, {}).keys()) - - def get_options(self, key, filter_func=None): - options = [] - for param_type in [self.ARGUMENTS, self.ENVS]: - if key not in self.enum_params.get(param_type, {}): - continue - options = self.enum_params.get(param_type, {}).get(key, {}).get(self.OPTIONS, []) - - if not options: - logger.error("Key %s not exists, optionals are %s", key, self.get_keys()) - - if filter_func is not None and callable(filter_func): - options = [value for value in options if filter_func(value)] - - return options - - def get_value_type(self, key): - for param_type in [self.ARGUMENTS, self.ENVS]: - if key not in self.enum_params.get(param_type, {}): - continue - value_type = self.enum_params.get(param_type, {}).get(key, {}).get(self.TYPE, self.STR_TYPE) - return value_type - return self.STR_TYPE - - def get_default(self, key): - default_value = None - for param_type in [self.ARGUMENTS, self.ENVS]: - if key not in self.enum_params.get(param_type, {}): - continue - default_value = self.enum_params.get(param_type, {}).get(key, {}).get(self.DEFAULT, []) - - if not default_value: - logger.error("Key %s not exists, optionals are %s", key, self.get_keys()) - - return default_value - - def _set_value(self): - - for key in self.get_keys(): - - if not hasattr(self, key): - setattr(self, str(key), AdvisorDict()) - - options = self.get_options(key) - - for value in options: - if not isinstance(value, typing.Hashable): - continue - getattr(self, key)[str(value)] = value +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import logging +import typing + +from msprof_analyze.advisor.common.timeline.event import AdvisorDict +from msprof_analyze.advisor.utils.utils import singleton +from msprof_analyze.prof_common.file_manager import FileManager + +logger = logging.getLogger() + + +@singleton +class EnumParamsParser(): + # 枚举变量抽象成yaml文件，统一管理，便于第三方服务对接advisor时调用当前类查询所有枚举变量参数的默认值和可选值 + + ARGUMENTS = "arguments" + ENVS = "envs" + OPTIONS = "options" + DEFAULT = "default" + TYPE = "type" + STR_TYPE = "str" + LIST_TYPE = "list" + INT_TYPE = "int" + BOOLEAN_TYPE = "boolean" + + def __init__(self): + enum_params_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "config", + "enum_parameters.yaml") + self.enum_params = FileManager.read_yaml_file(enum_params_path) + self._set_value() + + def get_keys(self): + return list(self.get_arguments_keys()) + list(self.get_envs_keys()) + + def get_arguments_keys(self): + return list(self.enum_params.get(self.ARGUMENTS, {}).keys()) + + def get_envs_keys(self): + return list(self.enum_params.get(self.ENVS, {}).keys()) + + def get_options(self, key, filter_func=None): + options = [] + for param_type in [self.ARGUMENTS, self.ENVS]: + if key not in self.enum_params.get(param_type, {}): + continue + options = self.enum_params.get(param_type, {}).get(key, {}).get(self.OPTIONS, []) + + if not options: + logger.error("Key %s not exists, optionals are %s", key, self.get_keys()) + + if filter_func is not None and callable(filter_func): + options = [value for value in options if filter_func(value)] + + return options + + def get_value_type(self, key): + for param_type in [self.ARGUMENTS, self.ENVS]: + if key not in self.enum_params.get(param_type, {}): + continue + value_type = self.enum_params.get(param_type, {}).get(key, {}).get(self.TYPE, self.STR_TYPE) + return value_type + return self.STR_TYPE + + def get_default(self, key): + default_value = None + for param_type in [self.ARGUMENTS, self.ENVS]: + if key not in self.enum_params.get(param_type, {}): + continue + default_value = self.enum_params.get(param_type, {}).get(key, {}).get(self.DEFAULT, []) + + if not default_value: + logger.error("Key %s not exists, optionals are %s", key, self.get_keys()) + + return default_value + + def _set_value(self): + + for key in self.get_keys(): + + if not hasattr(self, key): + setattr(self, str(key), AdvisorDict()) + + options = self.get_options(key) + + for value in options: + if not isinstance(value, typing.Hashable): + continue + getattr(self, key)[str(value)] = value diff --git a/profiler/advisor/common/graph/__init__.py b/profiler/msprof_analyze/advisor/common/graph/__init__.py similarity index 100% rename from profiler/advisor/common/graph/__init__.py rename to profiler/msprof_analyze/advisor/common/graph/__init__.py diff --git a/profiler/advisor/common/graph/graph.py b/profiler/msprof_analyze/advisor/common/graph/graph.py similarity index 97% rename from profiler/advisor/common/graph/graph.py rename to profiler/msprof_analyze/advisor/common/graph/graph.py index ad68da9174..b237e8d594 100644 --- a/profiler/advisor/common/graph/graph.py +++ b/profiler/msprof_analyze/advisor/common/graph/graph.py @@ -19,7 +19,7 @@ from typing import Dict, List, Tuple, Callable, Any, Optional, Union import networkx as nx -from profiler.advisor.common.graph.graph_parser import HostGraphNode, QueryGraphNode +from msprof_analyze.advisor.common.graph.graph_parser import HostGraphNode, QueryGraphNode logger = logging.getLogger() diff --git a/profiler/advisor/common/graph/graph_match.py b/profiler/msprof_analyze/advisor/common/graph/graph_match.py similarity index 100% rename from profiler/advisor/common/graph/graph_match.py rename to profiler/msprof_analyze/advisor/common/graph/graph_match.py diff --git a/profiler/advisor/common/graph/graph_parser.py b/profiler/msprof_analyze/advisor/common/graph/graph_parser.py similarity index 99% rename from profiler/advisor/common/graph/graph_parser.py rename to profiler/msprof_analyze/advisor/common/graph/graph_parser.py index bc0b04a357..5a35a971f0 100644 --- a/profiler/advisor/common/graph/graph_parser.py +++ b/profiler/msprof_analyze/advisor/common/graph/graph_parser.py @@ -21,8 +21,8 @@ from collections import deque from dataclasses import dataclass from typing import List, Tuple, Dict -from profiler.prof_common.file_manager import FileManager -from profiler.advisor.utils.file import FileOpen +from msprof_analyze.prof_common.file_manager import FileManager +from msprof_analyze.advisor.utils.file import FileOpen logger = logging.getLogger() diff --git a/profiler/advisor/common/profiling/__init__.py b/profiler/msprof_analyze/advisor/common/profiling/__init__.py similarity index 100% rename from profiler/advisor/common/profiling/__init__.py rename to profiler/msprof_analyze/advisor/common/profiling/__init__.py diff --git a/profiler/advisor/common/profiling/ge_info.py b/profiler/msprof_analyze/advisor/common/profiling/ge_info.py similarity index 90% rename from profiler/advisor/common/profiling/ge_info.py rename to profiler/msprof_analyze/advisor/common/profiling/ge_info.py index 927f5d3e71..f255684290 100644 --- a/profiler/advisor/common/profiling/ge_info.py +++ b/profiler/msprof_analyze/advisor/common/profiling/ge_info.py @@ -21,9 +21,9 @@ from typing import Any, List from sqlalchemy import text from sqlalchemy.exc import SQLAlchemyError -from profiler.advisor.dataset.profiling.db_manager import ConnectionManager -from profiler.advisor.dataset.profiling.profiling_parser import ProfilingParser -from profiler.advisor.utils.utils import check_path_valid +from msprof_analyze.advisor.dataset.profiling.db_manager import ConnectionManager +from msprof_analyze.advisor.dataset.profiling.profiling_parser import ProfilingParser +from msprof_analyze.advisor.utils.utils import check_path_valid logger = logging.getLogger() diff --git a/profiler/advisor/common/profiling/msprof.py b/profiler/msprof_analyze/advisor/common/profiling/msprof.py similarity index 96% rename from profiler/advisor/common/profiling/msprof.py rename to profiler/msprof_analyze/advisor/common/profiling/msprof.py index 76f51b672d..e4d537ddc7 100644 --- a/profiler/advisor/common/profiling/msprof.py +++ b/profiler/msprof_analyze/advisor/common/profiling/msprof.py @@ -17,8 +17,8 @@ import logging from typing import Dict, List -from profiler.advisor.dataset.profiling.info_collection import TaskInfo, HcclOp -from profiler.advisor.dataset.profiling.profiling_parser import ProfilingParser +from msprof_analyze.advisor.dataset.profiling.info_collection import TaskInfo, HcclOp +from msprof_analyze.advisor.dataset.profiling.profiling_parser import ProfilingParser logger = logging.getLogger() diff --git a/profiler/advisor/common/profiling/op_summary.py b/profiler/msprof_analyze/advisor/common/profiling/op_summary.py similarity index 92% rename from profiler/advisor/common/profiling/op_summary.py rename to profiler/msprof_analyze/advisor/common/profiling/op_summary.py index b1b416888f..f4659705bd 100644 --- a/profiler/advisor/common/profiling/op_summary.py +++ b/profiler/msprof_analyze/advisor/common/profiling/op_summary.py @@ -18,9 +18,9 @@ import logging from decimal import Decimal from typing import List, Any -from profiler.advisor.dataset.profiling.info_collection import OpInfo -from profiler.advisor.dataset.profiling.profiling_parser import ProfilingParser -from profiler.advisor.utils.utils import format_excel_title, lazy_property +from msprof_analyze.advisor.dataset.profiling.info_collection import OpInfo +from msprof_analyze.advisor.dataset.profiling.profiling_parser import ProfilingParser +from msprof_analyze.advisor.utils.utils import format_excel_title, lazy_property logger = logging.getLogger() diff --git a/profiler/advisor/common/profiling/tasktime.py b/profiler/msprof_analyze/advisor/common/profiling/tasktime.py similarity index 94% rename from profiler/advisor/common/profiling/tasktime.py rename to profiler/msprof_analyze/advisor/common/profiling/tasktime.py index 7891f89039..2d0474be60 100644 --- a/profiler/advisor/common/profiling/tasktime.py +++ b/profiler/msprof_analyze/advisor/common/profiling/tasktime.py @@ -17,8 +17,8 @@ import logging from typing import Dict, List -from profiler.advisor.dataset.profiling.info_collection import TaskInfo -from profiler.advisor.dataset.profiling.profiling_parser import ProfilingParser +from msprof_analyze.advisor.dataset.profiling.info_collection import TaskInfo +from msprof_analyze.advisor.dataset.profiling.profiling_parser import ProfilingParser logger = logging.getLogger() diff --git a/profiler/advisor/common/timeline/__init__.py b/profiler/msprof_analyze/advisor/common/timeline/__init__.py similarity index 100% rename from profiler/advisor/common/timeline/__init__.py rename to profiler/msprof_analyze/advisor/common/timeline/__init__.py diff --git a/profiler/advisor/common/timeline/event.py b/profiler/msprof_analyze/advisor/common/timeline/event.py similarity index 100% rename from profiler/advisor/common/timeline/event.py rename to profiler/msprof_analyze/advisor/common/timeline/event.py diff --git a/profiler/advisor/common/timeline/fusion_ops_db.py b/profiler/msprof_analyze/advisor/common/timeline/fusion_ops_db.py similarity index 96% rename from profiler/advisor/common/timeline/fusion_ops_db.py rename to profiler/msprof_analyze/advisor/common/timeline/fusion_ops_db.py index e2228f3480..87812d8c73 100644 --- a/profiler/advisor/common/timeline/fusion_ops_db.py +++ b/profiler/msprof_analyze/advisor/common/timeline/fusion_ops_db.py @@ -17,13 +17,13 @@ import logging import os -from profiler.prof_common.constant import Constant -from profiler.advisor.common.enum_params_parser import EnumParamsParser -from profiler.advisor.common.timeline.fusion_ops_rule import OpRule -from profiler.advisor.common.timeline.fusion_ops_rule_handler import TimelineOpRuleHandler -from profiler.advisor.utils.log import get_log_level -from profiler.advisor.utils.utils import get_file_path_by_walk -from profiler.prof_common.file_manager import FileManager +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.advisor.common.enum_params_parser import EnumParamsParser +from msprof_analyze.advisor.common.timeline.fusion_ops_rule import OpRule +from msprof_analyze.advisor.common.timeline.fusion_ops_rule_handler import TimelineOpRuleHandler +from msprof_analyze.advisor.utils.log import get_log_level +from msprof_analyze.advisor.utils.utils import get_file_path_by_walk +from msprof_analyze.prof_common.file_manager import FileManager logger = logging.getLogger() logger.setLevel(get_log_level()) diff --git a/profiler/advisor/common/timeline/fusion_ops_rule.py b/profiler/msprof_analyze/advisor/common/timeline/fusion_ops_rule.py similarity index 98% rename from profiler/advisor/common/timeline/fusion_ops_rule.py rename to profiler/msprof_analyze/advisor/common/timeline/fusion_ops_rule.py index deee68edb9..bf8a53207d 100644 --- a/profiler/advisor/common/timeline/fusion_ops_rule.py +++ b/profiler/msprof_analyze/advisor/common/timeline/fusion_ops_rule.py @@ -2,7 +2,7 @@ import copy import logging -from profiler.advisor.utils.log import get_log_level +from msprof_analyze.advisor.utils.log import get_log_level logger = logging.getLogger() logger.setLevel(get_log_level()) diff --git a/profiler/advisor/common/timeline/fusion_ops_rule_handler.py b/profiler/msprof_analyze/advisor/common/timeline/fusion_ops_rule_handler.py similarity index 98% rename from profiler/advisor/common/timeline/fusion_ops_rule_handler.py rename to profiler/msprof_analyze/advisor/common/timeline/fusion_ops_rule_handler.py index 25bbdf100d..808b095f27 100644 --- a/profiler/advisor/common/timeline/fusion_ops_rule_handler.py +++ b/profiler/msprof_analyze/advisor/common/timeline/fusion_ops_rule_handler.py @@ -2,9 +2,9 @@ import copy import logging -from profiler.prof_common.constant import Constant -from profiler.advisor.common.timeline.fusion_ops_rule import OpRule -from profiler.advisor.utils.log import get_log_level +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.advisor.common.timeline.fusion_ops_rule import OpRule +from msprof_analyze.advisor.utils.log import get_log_level logger = logging.getLogger() logger.setLevel(get_log_level()) diff --git a/profiler/advisor/common/version_control.py b/profiler/msprof_analyze/advisor/common/version_control.py similarity index 100% rename from profiler/advisor/common/version_control.py rename to profiler/msprof_analyze/advisor/common/version_control.py diff --git a/profiler/advisor/computation_analysis.ipynb b/profiler/msprof_analyze/advisor/computation_analysis.ipynb similarity index 99% rename from profiler/advisor/computation_analysis.ipynb rename to profiler/msprof_analyze/advisor/computation_analysis.ipynb index 0d4aaadfad..3aa3df9cf2 100644 --- a/profiler/advisor/computation_analysis.ipynb +++ b/profiler/msprof_analyze/advisor/computation_analysis.ipynb @@ -12,7 +12,7 @@ "\n", "from prettytable import PrettyTable, ALL\n", "from textwrap import fill\n", - "from profiler.advisor.interface.interface import Interface" + "from msprof_analyze.advisor.interface.interface import Interface" ] }, { @@ -237,7 +237,7 @@ "source": [ "from prettytable import PrettyTable, ALL\n", "from textwrap import fill\n", - "from profiler.advisor.interface.interface import Interface\n", + "from msprof_analyze.advisor.interface.interface import Interface\n", "\n", "\n", "# 配置profiling采集出来的数据,需要指定到的profiling目录是同一个工具采集的,并且需要采集l0级别以上\n", @@ -475,7 +475,7 @@ "source": [ "from prettytable import PrettyTable, ALL\n", "from textwrap import fill\n", - "from profiler.advisor.interface.interface import Interface\n", + "from msprof_analyze.advisor.interface.interface import Interface\n", "\n", "\n", "# 配置profiling采集出来的数据,需要指定到的profiling目录是同一个工具采集的,并且需要采集l0级别以上\n", diff --git a/profiler/advisor/config/__init__.py b/profiler/msprof_analyze/advisor/config/__init__.py similarity index 100% rename from profiler/advisor/config/__init__.py rename to profiler/msprof_analyze/advisor/config/__init__.py diff --git a/profiler/advisor/config/config.ini b/profiler/msprof_analyze/advisor/config/config.ini similarity index 93% rename from profiler/advisor/config/config.ini rename to profiler/msprof_analyze/advisor/config/config.ini index 108e8c97d0..211d3104e7 100644 --- a/profiler/advisor/config/config.ini +++ b/profiler/msprof_analyze/advisor/config/config.ini @@ -16,7 +16,7 @@ cn-north-9 = cnnorth9-modelarts-sdk cn-southwest-2 = cnsouthwest2-modelarts-sdk cn-north-7 = cnnorth7-modelarts-sdk [URL] -timeline_api_doc_url = https://gitee.com/ascend/mstt/blob/master/profiler/advisor/doc/Samples%20of%20Fused%20Operator%20API%20Replacement.md +timeline_api_doc_url = https://gitee.com/ascend/mstt/blob/master/profiler/msprof_analyze/advisor/doc/Samples%20of%20Fused%20Operator%20API%20Replacement.md timeline_with_stack_doc_url = https://www.hiascend.com/document/detail/zh/canncommercial/80RC2/devaids/auxiliarydevtool/atlasprofiling_16_0038.html pytorch_aoe_operator_tune_url = https://www.hiascend.com/document/detail/zh/canncommercial/80RC2/devaids/auxiliarydevtool/aoe_16_043.html mslite_infer_aoe_operator_tune_url = https://www.mindspore.cn/lite/docs/en/master/mindir/converter_tool_ascend.html#aoe-auto-tuning diff --git a/profiler/advisor/config/config.py b/profiler/msprof_analyze/advisor/config/config.py similarity index 96% rename from profiler/advisor/config/config.py rename to profiler/msprof_analyze/advisor/config/config.py index a7862ea50b..2ab735a286 100644 --- a/profiler/advisor/config/config.py +++ b/profiler/msprof_analyze/advisor/config/config.py @@ -17,9 +17,9 @@ import logging import os -from profiler.advisor.utils.utils import Timer -from profiler.advisor.utils.utils import singleton -from profiler.prof_common.utils import SafeConfigReader +from msprof_analyze.advisor.utils.utils import Timer +from msprof_analyze.advisor.utils.utils import singleton +from msprof_analyze.prof_common.utils import SafeConfigReader logger = logging.getLogger() diff --git a/profiler/advisor/config/enum_parameters.yaml b/profiler/msprof_analyze/advisor/config/enum_parameters.yaml similarity index 94% rename from profiler/advisor/config/enum_parameters.yaml rename to profiler/msprof_analyze/advisor/config/enum_parameters.yaml index 678fe72b43..534859eb9d 100644 --- a/profiler/advisor/config/enum_parameters.yaml +++ b/profiler/msprof_analyze/advisor/config/enum_parameters.yaml @@ -1,58 +1,58 @@ -arguments: - cann_version: - type: str - options: - - 6.3.RC2 - - 7.0.RC1 - - 7.0.0 - - 8.0.RC1 - - 8.0.RC2 - - 8.0.0 - default: 8.0.0 - - torch_version: - type: str - options: - - 1.11.0 - - 2.1.0 - default: 2.1.0 - mindspore_version: - type: str - options: - - 2.3.0 - - 2.4.0 - default: 2.4.0 - analysis_dimensions: - type: list - options: - - [ computation, communication, schedule, memory ] - - [ computation ] - - [ communication ] - - [ schedule ] - - [ memory ] - default: [ computation, communication, schedule, memory ] - - profiling_type: - type: str - options: - - pytorch - - mslite - - msprof - - mindspore - default: pytorch - -envs: - ADVISOR_ANALYZE_PROCESSES: - type: int - options: [ 1, 2, 3, 4, 5, 6, 7, 8 ] - default: 1 - - DISABLE_PROFILING_COMPARISON: - type: boolean - options: [ true, false ] - default: false - - DISABLE_AFFINITY_API: - type: boolean - options: [ true, false ] - default: false +arguments: + cann_version: + type: str + options: + - 6.3.RC2 + - 7.0.RC1 + - 7.0.0 + - 8.0.RC1 + - 8.0.RC2 + - 8.0.0 + default: 8.0.0 + + torch_version: + type: str + options: + - 1.11.0 + - 2.1.0 + default: 2.1.0 + mindspore_version: + type: str + options: + - 2.3.0 + - 2.4.0 + default: 2.4.0 + analysis_dimensions: + type: list + options: + - [ computation, communication, schedule, memory ] + - [ computation ] + - [ communication ] + - [ schedule ] + - [ memory ] + default: [ computation, communication, schedule, memory ] + + profiling_type: + type: str + options: + - pytorch + - mslite + - msprof + - mindspore + default: pytorch + +envs: + ADVISOR_ANALYZE_PROCESSES: + type: int + options: [ 1, 2, 3, 4, 5, 6, 7, 8 ] + default: 1 + + DISABLE_PROFILING_COMPARISON: + type: boolean + options: [ true, false ] + default: false + + DISABLE_AFFINITY_API: + type: boolean + options: [ true, false ] + default: false diff --git a/profiler/advisor/config/profiling_data_version_config.yaml b/profiler/msprof_analyze/advisor/config/profiling_data_version_config.yaml similarity index 100% rename from profiler/advisor/config/profiling_data_version_config.yaml rename to profiler/msprof_analyze/advisor/config/profiling_data_version_config.yaml diff --git a/profiler/advisor/dataset/__init__.py b/profiler/msprof_analyze/advisor/dataset/__init__.py similarity index 100% rename from profiler/advisor/dataset/__init__.py rename to profiler/msprof_analyze/advisor/dataset/__init__.py diff --git a/profiler/advisor/dataset/ai_core_freq/__init__.py b/profiler/msprof_analyze/advisor/dataset/ai_core_freq/__init__.py similarity index 100% rename from profiler/advisor/dataset/ai_core_freq/__init__.py rename to profiler/msprof_analyze/advisor/dataset/ai_core_freq/__init__.py diff --git a/profiler/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py b/profiler/msprof_analyze/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py similarity index 93% rename from profiler/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py rename to profiler/msprof_analyze/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py index d7aab88b93..a7490c2e51 100644 --- a/profiler/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py +++ b/profiler/msprof_analyze/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py @@ -22,11 +22,11 @@ import traceback import ijson from tqdm import tqdm -from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.advisor.utils.utils import get_file_path_from_directory -from profiler.advisor.utils.utils import convert_to_float, parse_json_with_generator -from profiler.advisor.dataset.profiling.device_info import DeviceInfoParser -from profiler.advisor.config.config import Config +from msprof_analyze.advisor.common.timeline.event import TimelineEvent +from msprof_analyze.advisor.utils.utils import get_file_path_from_directory +from msprof_analyze.advisor.utils.utils import convert_to_float, parse_json_with_generator +from msprof_analyze.advisor.dataset.profiling.device_info import DeviceInfoParser +from msprof_analyze.advisor.config.config import Config logger = logging.getLogger() diff --git a/profiler/advisor/dataset/cluster/__init__.py b/profiler/msprof_analyze/advisor/dataset/cluster/__init__.py similarity index 100% rename from profiler/advisor/dataset/cluster/__init__.py rename to profiler/msprof_analyze/advisor/dataset/cluster/__init__.py diff --git a/profiler/advisor/dataset/cluster/cluster_dataset.py b/profiler/msprof_analyze/advisor/dataset/cluster/cluster_dataset.py similarity index 94% rename from profiler/advisor/dataset/cluster/cluster_dataset.py rename to profiler/msprof_analyze/advisor/dataset/cluster/cluster_dataset.py index 4a3b6aa88a..b47f6d4518 100644 --- a/profiler/advisor/dataset/cluster/cluster_dataset.py +++ b/profiler/msprof_analyze/advisor/dataset/cluster/cluster_dataset.py @@ -18,13 +18,13 @@ import os import re from collections import defaultdict -from profiler.advisor.dataset.dataset import Dataset -from profiler.advisor.utils.utils import singleton -from profiler.prof_common.file_manager import FileManager -from profiler.prof_common.constant import Constant -from profiler.cluster_analyse.cluster_analysis import Interface -from profiler.advisor.dataset.cluster.cluster_step_trace_time_bean import ClusterStepTraceTimeBean -from profiler.advisor.dataset.cluster.hccl_collection import HcclInfo +from msprof_analyze.advisor.dataset.dataset import Dataset +from msprof_analyze.advisor.utils.utils import singleton +from msprof_analyze.prof_common.file_manager import FileManager +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.cluster_analyse.cluster_analysis import Interface +from msprof_analyze.advisor.dataset.cluster.cluster_step_trace_time_bean import ClusterStepTraceTimeBean +from msprof_analyze.advisor.dataset.cluster.hccl_collection import HcclInfo logger = logging.getLogger() diff --git a/profiler/advisor/dataset/cluster/cluster_step_trace_time_bean.py b/profiler/msprof_analyze/advisor/dataset/cluster/cluster_step_trace_time_bean.py similarity index 100% rename from profiler/advisor/dataset/cluster/cluster_step_trace_time_bean.py rename to profiler/msprof_analyze/advisor/dataset/cluster/cluster_step_trace_time_bean.py diff --git a/profiler/advisor/dataset/cluster/hccl_collection.py b/profiler/msprof_analyze/advisor/dataset/cluster/hccl_collection.py similarity index 100% rename from profiler/advisor/dataset/cluster/hccl_collection.py rename to profiler/msprof_analyze/advisor/dataset/cluster/hccl_collection.py diff --git a/profiler/advisor/dataset/communication/__init__.py b/profiler/msprof_analyze/advisor/dataset/communication/__init__.py similarity index 100% rename from profiler/advisor/dataset/communication/__init__.py rename to profiler/msprof_analyze/advisor/dataset/communication/__init__.py diff --git a/profiler/advisor/dataset/communication/communication_dataset.py b/profiler/msprof_analyze/advisor/dataset/communication/communication_dataset.py similarity index 92% rename from profiler/advisor/dataset/communication/communication_dataset.py rename to profiler/msprof_analyze/advisor/dataset/communication/communication_dataset.py index f1adec7167..44efddccff 100644 --- a/profiler/advisor/dataset/communication/communication_dataset.py +++ b/profiler/msprof_analyze/advisor/dataset/communication/communication_dataset.py @@ -15,11 +15,11 @@ import logging import os from collections import defaultdict -from profiler.advisor.utils.utils import singleton -from profiler.prof_common.constant import Constant -from profiler.prof_common.file_manager import FileManager -from profiler.advisor.dataset.cluster.hccl_collection import HcclInfo -from profiler.advisor.utils.utils import CheckPathAccess +from msprof_analyze.advisor.utils.utils import singleton +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.prof_common.file_manager import FileManager +from msprof_analyze.advisor.dataset.cluster.hccl_collection import HcclInfo +from msprof_analyze.advisor.utils.utils import CheckPathAccess logger = logging.getLogger() diff --git a/profiler/advisor/dataset/communication/hccl_detail_dataset.py b/profiler/msprof_analyze/advisor/dataset/communication/hccl_detail_dataset.py similarity index 93% rename from profiler/advisor/dataset/communication/hccl_detail_dataset.py rename to profiler/msprof_analyze/advisor/dataset/communication/hccl_detail_dataset.py index 7cdc0820a8..fac5603b99 100644 --- a/profiler/advisor/dataset/communication/hccl_detail_dataset.py +++ b/profiler/msprof_analyze/advisor/dataset/communication/hccl_detail_dataset.py @@ -14,9 +14,9 @@ # limitations under the License. import logging from typing import List -from profiler.advisor.utils.utils import singleton -from profiler.advisor.common.profiling.msprof import Msprof -from profiler.advisor.dataset.profiling.info_collection import TaskInfo, HcclOp, HcclTask +from msprof_analyze.advisor.utils.utils import singleton +from msprof_analyze.advisor.common.profiling.msprof import Msprof +from msprof_analyze.advisor.dataset.profiling.info_collection import TaskInfo, HcclOp, HcclTask logger = logging.getLogger() diff --git a/profiler/advisor/dataset/dataset.py b/profiler/msprof_analyze/advisor/dataset/dataset.py similarity index 96% rename from profiler/advisor/dataset/dataset.py rename to profiler/msprof_analyze/advisor/dataset/dataset.py index d40438dc5b..3cc669480d 100644 --- a/profiler/advisor/dataset/dataset.py +++ b/profiler/msprof_analyze/advisor/dataset/dataset.py @@ -19,7 +19,7 @@ dataset module import logging import os -from profiler.advisor.config.config import Config +from msprof_analyze.advisor.config.config import Config logger = logging.getLogger() diff --git a/profiler/advisor/dataset/environment_variable_dataset.py b/profiler/msprof_analyze/advisor/dataset/environment_variable_dataset.py similarity index 92% rename from profiler/advisor/dataset/environment_variable_dataset.py rename to profiler/msprof_analyze/advisor/dataset/environment_variable_dataset.py index 855b23189e..6fa569ad9f 100644 --- a/profiler/advisor/dataset/environment_variable_dataset.py +++ b/profiler/msprof_analyze/advisor/dataset/environment_variable_dataset.py @@ -15,8 +15,8 @@ import os import logging -from profiler.prof_common.constant import Constant -from profiler.prof_common.file_manager import FileManager +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.prof_common.file_manager import FileManager class EnvironmentVariableDataset: diff --git a/profiler/advisor/dataset/graph_dataset.py b/profiler/msprof_analyze/advisor/dataset/graph_dataset.py similarity index 87% rename from profiler/advisor/dataset/graph_dataset.py rename to profiler/msprof_analyze/advisor/dataset/graph_dataset.py index d02af46b3a..0e68312558 100644 --- a/profiler/advisor/dataset/graph_dataset.py +++ b/profiler/msprof_analyze/advisor/dataset/graph_dataset.py @@ -16,10 +16,10 @@ import logging from typing import List -from profiler.advisor.dataset.dataset import Dataset -from profiler.advisor.common.graph.graph_parser import HostGraphParser -from profiler.advisor.common.graph.graph import Graph -from profiler.advisor.utils.utils import load_parameter, lazy_property, get_file_path_from_directory +from msprof_analyze.advisor.dataset.dataset import Dataset +from msprof_analyze.advisor.common.graph.graph_parser import HostGraphParser +from msprof_analyze.advisor.common.graph.graph import Graph +from msprof_analyze.advisor.utils.utils import load_parameter, lazy_property, get_file_path_from_directory logger = logging.getLogger() diff --git a/profiler/advisor/dataset/profiling/__init__.py b/profiler/msprof_analyze/advisor/dataset/profiling/__init__.py similarity index 100% rename from profiler/advisor/dataset/profiling/__init__.py rename to profiler/msprof_analyze/advisor/dataset/profiling/__init__.py diff --git a/profiler/advisor/dataset/profiling/builder_base.py b/profiler/msprof_analyze/advisor/dataset/profiling/builder_base.py similarity index 91% rename from profiler/advisor/dataset/profiling/builder_base.py rename to profiler/msprof_analyze/advisor/dataset/profiling/builder_base.py index 77bd926f72..5b04399b19 100644 --- a/profiler/advisor/dataset/profiling/builder_base.py +++ b/profiler/msprof_analyze/advisor/dataset/profiling/builder_base.py @@ -19,8 +19,8 @@ profiling base import logging from typing import Dict, List -from profiler.advisor.dataset.profiling.profiling_parser import ProfilingParser -from profiler.advisor.utils.utils import join_prof_path +from msprof_analyze.advisor.dataset.profiling.profiling_parser import ProfilingParser +from msprof_analyze.advisor.utils.utils import join_prof_path logger = logging.getLogger() diff --git a/profiler/advisor/dataset/profiling/db_manager.py b/profiler/msprof_analyze/advisor/dataset/profiling/db_manager.py similarity index 100% rename from profiler/advisor/dataset/profiling/db_manager.py rename to profiler/msprof_analyze/advisor/dataset/profiling/db_manager.py diff --git a/profiler/advisor/dataset/profiling/device_info.py b/profiler/msprof_analyze/advisor/dataset/profiling/device_info.py similarity index 92% rename from profiler/advisor/dataset/profiling/device_info.py rename to profiler/msprof_analyze/advisor/dataset/profiling/device_info.py index 52c811e6dd..b0226ac569 100644 --- a/profiler/advisor/dataset/profiling/device_info.py +++ b/profiler/msprof_analyze/advisor/dataset/profiling/device_info.py @@ -17,9 +17,9 @@ profiling info import json import logging -from profiler.advisor.config.config import Config -from profiler.advisor.utils.utils import get_file_path_from_directory -from profiler.prof_common.file_manager import FileManager +from msprof_analyze.advisor.config.config import Config +from msprof_analyze.advisor.utils.utils import get_file_path_from_directory +from msprof_analyze.prof_common.file_manager import FileManager logger = logging.getLogger() diff --git a/profiler/advisor/dataset/profiling/info_collection.py b/profiler/msprof_analyze/advisor/dataset/profiling/info_collection.py similarity index 99% rename from profiler/advisor/dataset/profiling/info_collection.py rename to profiler/msprof_analyze/advisor/dataset/profiling/info_collection.py index 88333b2d44..8540591c33 100644 --- a/profiler/advisor/dataset/profiling/info_collection.py +++ b/profiler/msprof_analyze/advisor/dataset/profiling/info_collection.py @@ -19,7 +19,7 @@ profiling info import decimal import logging from typing import List -from profiler.advisor.utils.utils import lazy_property +from msprof_analyze.advisor.utils.utils import lazy_property logger = logging.getLogger() diff --git a/profiler/advisor/dataset/profiling/profiling_dataset.py b/profiler/msprof_analyze/advisor/dataset/profiling/profiling_dataset.py similarity index 86% rename from profiler/advisor/dataset/profiling/profiling_dataset.py rename to profiler/msprof_analyze/advisor/dataset/profiling/profiling_dataset.py index 99f11e171a..7981e4140f 100644 --- a/profiler/advisor/dataset/profiling/profiling_dataset.py +++ b/profiler/msprof_analyze/advisor/dataset/profiling/profiling_dataset.py @@ -17,16 +17,16 @@ import logging import os import yaml -from profiler.prof_common.constant import Constant -from profiler.advisor.common.profiling.ge_info import GeInfo -from profiler.advisor.common.profiling.msprof import Msprof -from profiler.advisor.common.profiling.op_summary import OpSummary -from profiler.advisor.common.profiling.tasktime import TaskTime -from profiler.advisor.common.enum_params_parser import EnumParamsParser -from profiler.advisor.dataset.dataset import Dataset -from profiler.advisor.dataset.profiling.device_info import DeviceInfoParser -from profiler.advisor.utils.utils import join_prof_path -from profiler.prof_common.file_manager import FileManager +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.advisor.common.profiling.ge_info import GeInfo +from msprof_analyze.advisor.common.profiling.msprof import Msprof +from msprof_analyze.advisor.common.profiling.op_summary import OpSummary +from msprof_analyze.advisor.common.profiling.tasktime import TaskTime +from msprof_analyze.advisor.common.enum_params_parser import EnumParamsParser +from msprof_analyze.advisor.dataset.dataset import Dataset +from msprof_analyze.advisor.dataset.profiling.device_info import DeviceInfoParser +from msprof_analyze.advisor.utils.utils import join_prof_path +from msprof_analyze.prof_common.file_manager import FileManager logger = logging.getLogger() diff --git a/profiler/advisor/dataset/profiling/profiling_parser.py b/profiler/msprof_analyze/advisor/dataset/profiling/profiling_parser.py similarity index 95% rename from profiler/advisor/dataset/profiling/profiling_parser.py rename to profiler/msprof_analyze/advisor/dataset/profiling/profiling_parser.py index f379fea070..016b914b84 100644 --- a/profiler/advisor/dataset/profiling/profiling_parser.py +++ b/profiler/msprof_analyze/advisor/dataset/profiling/profiling_parser.py @@ -19,9 +19,9 @@ import re from abc import abstractmethod from typing import List, Dict -from profiler.advisor.dataset.profiling.info_collection import logger -from profiler.advisor.utils.utils import get_file_path_from_directory, SafeOpen, format_excel_title -from profiler.prof_common.file_manager import FileManager +from msprof_analyze.advisor.dataset.profiling.info_collection import logger +from msprof_analyze.advisor.utils.utils import get_file_path_from_directory, SafeOpen, format_excel_title +from msprof_analyze.prof_common.file_manager import FileManager class ProfilingParser: diff --git a/profiler/advisor/dataset/timeline_event_dataset.py b/profiler/msprof_analyze/advisor/dataset/timeline_event_dataset.py similarity index 96% rename from profiler/advisor/dataset/timeline_event_dataset.py rename to profiler/msprof_analyze/advisor/dataset/timeline_event_dataset.py index 77d64267c4..b9631445b8 100644 --- a/profiler/advisor/dataset/timeline_event_dataset.py +++ b/profiler/msprof_analyze/advisor/dataset/timeline_event_dataset.py @@ -21,10 +21,10 @@ from collections import OrderedDict import ijson from tqdm import tqdm -from profiler.prof_common.constant import Constant -from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.advisor.utils.utils import get_file_path_from_directory, check_path_valid, singleton, convert_to_float -from profiler.advisor.dataset.timeline_op_collector.timeline_op_collector import ( +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.advisor.common.timeline.event import TimelineEvent +from msprof_analyze.advisor.utils.utils import get_file_path_from_directory, check_path_valid, singleton, convert_to_float +from msprof_analyze.advisor.dataset.timeline_op_collector.timeline_op_collector import ( OpCompileCollector, SynchronizeStreamCollector, MemCollector, diff --git a/profiler/advisor/dataset/timeline_op_collector/__init__.py b/profiler/msprof_analyze/advisor/dataset/timeline_op_collector/__init__.py similarity index 100% rename from profiler/advisor/dataset/timeline_op_collector/__init__.py rename to profiler/msprof_analyze/advisor/dataset/timeline_op_collector/__init__.py diff --git a/profiler/advisor/dataset/timeline_op_collector/timeline_op_collector.py b/profiler/msprof_analyze/advisor/dataset/timeline_op_collector/timeline_op_collector.py similarity index 97% rename from profiler/advisor/dataset/timeline_op_collector/timeline_op_collector.py rename to profiler/msprof_analyze/advisor/dataset/timeline_op_collector/timeline_op_collector.py index a1aa68d208..5def8948b5 100644 --- a/profiler/advisor/dataset/timeline_op_collector/timeline_op_collector.py +++ b/profiler/msprof_analyze/advisor/dataset/timeline_op_collector/timeline_op_collector.py @@ -16,11 +16,11 @@ import math import os from abc import abstractmethod, ABCMeta -from profiler.prof_common.additional_args_manager import AdditionalArgsManager -from profiler.prof_common.constant import Constant -from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.advisor.utils.utils import convert_to_float -from profiler.prof_common.file_manager import FileManager +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.advisor.common.timeline.event import TimelineEvent +from msprof_analyze.advisor.utils.utils import convert_to_float +from msprof_analyze.prof_common.file_manager import FileManager logger = logging.getLogger() diff --git a/profiler/advisor/display/__init__.py b/profiler/msprof_analyze/advisor/display/__init__.py similarity index 100% rename from profiler/advisor/display/__init__.py rename to profiler/msprof_analyze/advisor/display/__init__.py diff --git a/profiler/advisor/display/html/__init__.py b/profiler/msprof_analyze/advisor/display/html/__init__.py similarity index 100% rename from profiler/advisor/display/html/__init__.py rename to profiler/msprof_analyze/advisor/display/html/__init__.py diff --git a/profiler/advisor/display/html/priority_background_color.py b/profiler/msprof_analyze/advisor/display/html/priority_background_color.py similarity index 97% rename from profiler/advisor/display/html/priority_background_color.py rename to profiler/msprof_analyze/advisor/display/html/priority_background_color.py index f5b89b232f..6b03747a81 100644 --- a/profiler/advisor/display/html/priority_background_color.py +++ b/profiler/msprof_analyze/advisor/display/html/priority_background_color.py @@ -1,19 +1,19 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -class PriorityBackgroundColor: - high = "#B5495B" - medium = "#fcaf17" - low = "#65c294" +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class PriorityBackgroundColor: + high = "#B5495B" + medium = "#fcaf17" + low = "#65c294" diff --git a/profiler/advisor/display/html/render.py b/profiler/msprof_analyze/advisor/display/html/render.py similarity index 95% rename from profiler/advisor/display/html/render.py rename to profiler/msprof_analyze/advisor/display/html/render.py index 593531c23d..ffc07509ad 100644 --- a/profiler/advisor/display/html/render.py +++ b/profiler/msprof_analyze/advisor/display/html/render.py @@ -19,10 +19,10 @@ from typing import List, Dict from collections import defaultdict, OrderedDict from jinja2 import Environment, FileSystemLoader -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant -from profiler.advisor.config.config import Config -from profiler.advisor.utils.utils import singleton, safe_write +from msprof_analyze.advisor.config.config import Config +from msprof_analyze.advisor.utils.utils import singleton, safe_write logger = logging.getLogger() diff --git a/profiler/advisor/display/html/templates/__init__.py b/profiler/msprof_analyze/advisor/display/html/templates/__init__.py similarity index 100% rename from profiler/advisor/display/html/templates/__init__.py rename to profiler/msprof_analyze/advisor/display/html/templates/__init__.py diff --git a/profiler/advisor/display/html/templates/affinity_api.html b/profiler/msprof_analyze/advisor/display/html/templates/affinity_api.html similarity index 100% rename from profiler/advisor/display/html/templates/affinity_api.html rename to profiler/msprof_analyze/advisor/display/html/templates/affinity_api.html diff --git a/profiler/advisor/display/html/templates/ai_core_frequency.html b/profiler/msprof_analyze/advisor/display/html/templates/ai_core_frequency.html similarity index 100% rename from profiler/advisor/display/html/templates/ai_core_frequency.html rename to profiler/msprof_analyze/advisor/display/html/templates/ai_core_frequency.html diff --git a/profiler/advisor/display/html/templates/byte_alignment.html b/profiler/msprof_analyze/advisor/display/html/templates/byte_alignment.html similarity index 100% rename from profiler/advisor/display/html/templates/byte_alignment.html rename to profiler/msprof_analyze/advisor/display/html/templates/byte_alignment.html diff --git a/profiler/advisor/display/html/templates/cluster_analysis.html b/profiler/msprof_analyze/advisor/display/html/templates/cluster_analysis.html similarity index 100% rename from profiler/advisor/display/html/templates/cluster_analysis.html rename to profiler/msprof_analyze/advisor/display/html/templates/cluster_analysis.html diff --git a/profiler/advisor/display/html/templates/communication_retransmission_analysis.html b/profiler/msprof_analyze/advisor/display/html/templates/communication_retransmission_analysis.html similarity index 100% rename from profiler/advisor/display/html/templates/communication_retransmission_analysis.html rename to profiler/msprof_analyze/advisor/display/html/templates/communication_retransmission_analysis.html diff --git a/profiler/advisor/display/html/templates/comparison.html b/profiler/msprof_analyze/advisor/display/html/templates/comparison.html similarity index 97% rename from profiler/advisor/display/html/templates/comparison.html rename to profiler/msprof_analyze/advisor/display/html/templates/comparison.html index 5963e75308..b81802d6b0 100644 --- a/profiler/advisor/display/html/templates/comparison.html +++ b/profiler/msprof_analyze/advisor/display/html/templates/comparison.html @@ -1,25 +1,25 @@ -{% if rows|length > 0 %} -
-

{{ sheet_name }}

-
- Issue: {{ desc }} -

- - - {% for header in headers %} - - {% endfor %} - - - {% for row in rows %} - - {% for element in row %} - - {% endfor %} - - {% endfor %} -
{{ header }}
{{ element|safe }}
- -
-
+{% if rows|length > 0 %} +
+

{{ sheet_name }}

+
+ Issue: {{ desc }} +

+ + + {% for header in headers %} + + {% endfor %} + + + {% for row in rows %} + + {% for element in row %} + + {% endfor %} + + {% endfor %} +
{{ header }}
{{ element|safe }}
+ +
+
{% endif %} \ No newline at end of file diff --git a/profiler/advisor/display/html/templates/compute_analysis.html b/profiler/msprof_analyze/advisor/display/html/templates/compute_analysis.html similarity index 100% rename from profiler/advisor/display/html/templates/compute_analysis.html rename to profiler/msprof_analyze/advisor/display/html/templates/compute_analysis.html diff --git a/profiler/advisor/display/html/templates/contention.html b/profiler/msprof_analyze/advisor/display/html/templates/contention.html similarity index 100% rename from profiler/advisor/display/html/templates/contention.html rename to profiler/msprof_analyze/advisor/display/html/templates/contention.html diff --git a/profiler/advisor/display/html/templates/environment_variable.html b/profiler/msprof_analyze/advisor/display/html/templates/environment_variable.html similarity index 100% rename from profiler/advisor/display/html/templates/environment_variable.html rename to profiler/msprof_analyze/advisor/display/html/templates/environment_variable.html diff --git a/profiler/advisor/display/html/templates/fusion.html b/profiler/msprof_analyze/advisor/display/html/templates/fusion.html similarity index 100% rename from profiler/advisor/display/html/templates/fusion.html rename to profiler/msprof_analyze/advisor/display/html/templates/fusion.html diff --git a/profiler/advisor/display/html/templates/gc.html b/profiler/msprof_analyze/advisor/display/html/templates/gc.html similarity index 100% rename from profiler/advisor/display/html/templates/gc.html rename to profiler/msprof_analyze/advisor/display/html/templates/gc.html diff --git a/profiler/advisor/display/html/templates/main.html b/profiler/msprof_analyze/advisor/display/html/templates/main.html similarity index 100% rename from profiler/advisor/display/html/templates/main.html rename to profiler/msprof_analyze/advisor/display/html/templates/main.html diff --git a/profiler/advisor/display/html/templates/memory.html b/profiler/msprof_analyze/advisor/display/html/templates/memory.html similarity index 96% rename from profiler/advisor/display/html/templates/memory.html rename to profiler/msprof_analyze/advisor/display/html/templates/memory.html index a3d75877b6..2bf57f46a1 100644 --- a/profiler/advisor/display/html/templates/memory.html +++ b/profiler/msprof_analyze/advisor/display/html/templates/memory.html @@ -1,21 +1,21 @@ -
-

Memory Operator Issues

-
- {% if rank is not none %} - Analysis of rank {{ rank|safe }}. - {% endif %} - {{ desc }} - - - - - - {% for suggestion in suggestions %} - - - - {% endfor %} -
Suggestions
{{ loop.index }}. {{ suggestion|safe }}
- -
-
+
+

Memory Operator Issues

+
+ {% if rank is not none %} + Analysis of rank {{ rank|safe }}. + {% endif %} + {{ desc }} + + + + + + {% for suggestion in suggestions %} + + + + {% endfor %} +
Suggestions
{{ loop.index }}. {{ suggestion|safe }}
+ +
+
diff --git a/profiler/advisor/display/html/templates/operator_ai_cpu.html b/profiler/msprof_analyze/advisor/display/html/templates/operator_ai_cpu.html similarity index 100% rename from profiler/advisor/display/html/templates/operator_ai_cpu.html rename to profiler/msprof_analyze/advisor/display/html/templates/operator_ai_cpu.html diff --git a/profiler/advisor/display/html/templates/operator_block_dim.html b/profiler/msprof_analyze/advisor/display/html/templates/operator_block_dim.html similarity index 100% rename from profiler/advisor/display/html/templates/operator_block_dim.html rename to profiler/msprof_analyze/advisor/display/html/templates/operator_block_dim.html diff --git a/profiler/advisor/display/html/templates/operator_dispatch.html b/profiler/msprof_analyze/advisor/display/html/templates/operator_dispatch.html similarity index 100% rename from profiler/advisor/display/html/templates/operator_dispatch.html rename to profiler/msprof_analyze/advisor/display/html/templates/operator_dispatch.html diff --git a/profiler/advisor/display/html/templates/operator_dynamic_shape.html b/profiler/msprof_analyze/advisor/display/html/templates/operator_dynamic_shape.html similarity index 100% rename from profiler/advisor/display/html/templates/operator_dynamic_shape.html rename to profiler/msprof_analyze/advisor/display/html/templates/operator_dynamic_shape.html diff --git a/profiler/advisor/display/html/templates/operator_no_bound.html b/profiler/msprof_analyze/advisor/display/html/templates/operator_no_bound.html similarity index 100% rename from profiler/advisor/display/html/templates/operator_no_bound.html rename to profiler/msprof_analyze/advisor/display/html/templates/operator_no_bound.html diff --git a/profiler/advisor/display/html/templates/overall_analysis.html b/profiler/msprof_analyze/advisor/display/html/templates/overall_analysis.html similarity index 100% rename from profiler/advisor/display/html/templates/overall_analysis.html rename to profiler/msprof_analyze/advisor/display/html/templates/overall_analysis.html diff --git a/profiler/advisor/display/html/templates/packet_analysis.html b/profiler/msprof_analyze/advisor/display/html/templates/packet_analysis.html similarity index 100% rename from profiler/advisor/display/html/templates/packet_analysis.html rename to profiler/msprof_analyze/advisor/display/html/templates/packet_analysis.html diff --git a/profiler/advisor/display/html/templates/pp_stage_computation_analysis.html b/profiler/msprof_analyze/advisor/display/html/templates/pp_stage_computation_analysis.html similarity index 97% rename from profiler/advisor/display/html/templates/pp_stage_computation_analysis.html rename to profiler/msprof_analyze/advisor/display/html/templates/pp_stage_computation_analysis.html index 189e6fadf8..6d2792f31a 100644 --- a/profiler/advisor/display/html/templates/pp_stage_computation_analysis.html +++ b/profiler/msprof_analyze/advisor/display/html/templates/pp_stage_computation_analysis.html @@ -1,19 +1,19 @@ -{% if stages_rendered_html|length > 0 %} -
-

Pipeline Parallel Stages Issues

-
- {% for stage_html in stages_rendered_html %} -
-

{{ stage_html['stage']|safe }}

-
- Description: analysis for slow rank {{ stage_html['rank']|safe }} in current stage -

- {% for html in stage_html['html_list'] %} - {{ html|safe }} - {% endfor %} -
-
- {% endfor %} -
-
+{% if stages_rendered_html|length > 0 %} +
+

Pipeline Parallel Stages Issues

+
+ {% for stage_html in stages_rendered_html %} +
+

{{ stage_html['stage']|safe }}

+
+ Description: analysis for slow rank {{ stage_html['rank']|safe }} in current stage +

+ {% for html in stage_html['html_list'] %} + {{ html|safe }} + {% endfor %} +
+
+ {% endfor %} +
+
{% endif %} \ No newline at end of file diff --git a/profiler/advisor/display/html/templates/slow_dataloader.html b/profiler/msprof_analyze/advisor/display/html/templates/slow_dataloader.html similarity index 96% rename from profiler/advisor/display/html/templates/slow_dataloader.html rename to profiler/msprof_analyze/advisor/display/html/templates/slow_dataloader.html index b9ce7a574a..2a3b2c4462 100644 --- a/profiler/advisor/display/html/templates/slow_dataloader.html +++ b/profiler/msprof_analyze/advisor/display/html/templates/slow_dataloader.html @@ -1,21 +1,21 @@ -
-

Slow Dataloader Issues

-
- {% if rank is not none %} - Analysis of rank {{ rank|safe }}. - {% endif %} - {{ desc }} - - - - - - {% for suggestion in suggestions %} - - - - {% endfor %} -
Suggestions
{{ loop.index }}. {{ suggestion|safe }}
- -
-
+
+

Slow Dataloader Issues

+
+ {% if rank is not none %} + Analysis of rank {{ rank|safe }}. + {% endif %} + {{ desc }} + + + + + + {% for suggestion in suggestions %} + + + + {% endfor %} +
Suggestions
{{ loop.index }}. {{ suggestion|safe }}
+ +
+
diff --git a/profiler/advisor/display/html/templates/sync_batchnorm.html b/profiler/msprof_analyze/advisor/display/html/templates/sync_batchnorm.html similarity index 97% rename from profiler/advisor/display/html/templates/sync_batchnorm.html rename to profiler/msprof_analyze/advisor/display/html/templates/sync_batchnorm.html index 402404c8a4..ea32227664 100644 --- a/profiler/advisor/display/html/templates/sync_batchnorm.html +++ b/profiler/msprof_analyze/advisor/display/html/templates/sync_batchnorm.html @@ -1,33 +1,33 @@ - -
-

SyncBatchNorm Issues

-
- {% if rank is not none %} - Analysis of rank {{ rank|safe }}. - {% endif %} - {{ desc }} - - - - - {% for item in solutions %} - {% set rowloop = loop %} - {% for key, value in item.items() %} - - - - {% endfor %} - {% endfor %} -
Suggestions
{{ rowloop.index }}. {{ value.desc }}
- - More efficient code of syncbn forward as follows: - {% for item in solutions %} - {% for key, value in item.items() %} - {% if 'efficient_code' in value %} - 
{{ value.efficient_code|safe }}
- {% endif %} - {% endfor %} - {% endfor %} - -
-
+ +
+

SyncBatchNorm Issues

+
+ {% if rank is not none %} + Analysis of rank {{ rank|safe }}. + {% endif %} + {{ desc }} + + + + + {% for item in solutions %} + {% set rowloop = loop %} + {% for key, value in item.items() %} + + + + {% endfor %} + {% endfor %} +
Suggestions
{{ rowloop.index }}. {{ value.desc }}
+ + More efficient code of syncbn forward as follows: + {% for item in solutions %} + {% for key, value in item.items() %} + {% if 'efficient_code' in value %} + 
{{ value.efficient_code|safe }}
+ {% endif %} + {% endfor %} + {% endfor %} + +
+
diff --git a/profiler/advisor/display/html/templates/synchronize_stream.html b/profiler/msprof_analyze/advisor/display/html/templates/synchronize_stream.html similarity index 97% rename from profiler/advisor/display/html/templates/synchronize_stream.html rename to profiler/msprof_analyze/advisor/display/html/templates/synchronize_stream.html index eb132a6315..8636740275 100644 --- a/profiler/advisor/display/html/templates/synchronize_stream.html +++ b/profiler/msprof_analyze/advisor/display/html/templates/synchronize_stream.html @@ -1,26 +1,26 @@ -
-

Synchronize Stream Issues

-
- {% if rank is not none %} - Analysis of rank {{ rank|safe }}. - {% endif %} - {{ desc }} - - - - - - - {% for item in solutions %} - {% set rowloop = loop %} - {% for key, value in item.items() %} - - - - - {% endfor %} - {% endfor %} -
Suggestions
{{ rowloop.index }}. {{ value.desc }}
- -
-
+
+

Synchronize Stream Issues

+
+ {% if rank is not none %} + Analysis of rank {{ rank|safe }}. + {% endif %} + {{ desc }} + + + + + + + {% for item in solutions %} + {% set rowloop = loop %} + {% for key, value in item.items() %} + + + + + {% endfor %} + {% endfor %} +
Suggestions
{{ rowloop.index }}. {{ value.desc }}
+ +
+
diff --git a/profiler/advisor/display/html/templates/timeline_analysis.html b/profiler/msprof_analyze/advisor/display/html/templates/timeline_analysis.html similarity index 100% rename from profiler/advisor/display/html/templates/timeline_analysis.html rename to profiler/msprof_analyze/advisor/display/html/templates/timeline_analysis.html diff --git a/profiler/advisor/display/prompt/__init__.py b/profiler/msprof_analyze/advisor/display/prompt/__init__.py similarity index 100% rename from profiler/advisor/display/prompt/__init__.py rename to profiler/msprof_analyze/advisor/display/prompt/__init__.py diff --git a/profiler/advisor/display/prompt/base_prompt.py b/profiler/msprof_analyze/advisor/display/prompt/base_prompt.py similarity index 95% rename from profiler/advisor/display/prompt/base_prompt.py rename to profiler/msprof_analyze/advisor/display/prompt/base_prompt.py index 2a538f3dbb..c91a805f50 100644 --- a/profiler/advisor/display/prompt/base_prompt.py +++ b/profiler/msprof_analyze/advisor/display/prompt/base_prompt.py @@ -14,7 +14,7 @@ # limitations under the License. import importlib -from profiler.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager def split_camel_case(word): diff --git a/profiler/advisor/display/prompt/cn/__init__.py b/profiler/msprof_analyze/advisor/display/prompt/cn/__init__.py similarity index 100% rename from profiler/advisor/display/prompt/cn/__init__.py rename to profiler/msprof_analyze/advisor/display/prompt/cn/__init__.py diff --git a/profiler/advisor/display/prompt/cn/ai_core_freq_prompt.py b/profiler/msprof_analyze/advisor/display/prompt/cn/ai_core_freq_prompt.py similarity index 100% rename from profiler/advisor/display/prompt/cn/ai_core_freq_prompt.py rename to profiler/msprof_analyze/advisor/display/prompt/cn/ai_core_freq_prompt.py diff --git a/profiler/advisor/display/prompt/cn/block_dim_prompt.py b/profiler/msprof_analyze/advisor/display/prompt/cn/block_dim_prompt.py similarity index 100% rename from profiler/advisor/display/prompt/cn/block_dim_prompt.py rename to profiler/msprof_analyze/advisor/display/prompt/cn/block_dim_prompt.py diff --git a/profiler/advisor/display/prompt/cn/dynamic_shape_prompt.py b/profiler/msprof_analyze/advisor/display/prompt/cn/dynamic_shape_prompt.py similarity index 100% rename from profiler/advisor/display/prompt/cn/dynamic_shape_prompt.py rename to profiler/msprof_analyze/advisor/display/prompt/cn/dynamic_shape_prompt.py diff --git a/profiler/advisor/display/prompt/cn/environment_variable_prompt.py b/profiler/msprof_analyze/advisor/display/prompt/cn/environment_variable_prompt.py similarity index 100% rename from profiler/advisor/display/prompt/cn/environment_variable_prompt.py rename to profiler/msprof_analyze/advisor/display/prompt/cn/environment_variable_prompt.py diff --git a/profiler/advisor/display/prompt/cn/graph_fusion_prompt.py b/profiler/msprof_analyze/advisor/display/prompt/cn/graph_fusion_prompt.py similarity index 100% rename from profiler/advisor/display/prompt/cn/graph_fusion_prompt.py rename to profiler/msprof_analyze/advisor/display/prompt/cn/graph_fusion_prompt.py diff --git a/profiler/advisor/display/prompt/cn/op_dispatch_prompt.py b/profiler/msprof_analyze/advisor/display/prompt/cn/op_dispatch_prompt.py similarity index 100% rename from profiler/advisor/display/prompt/cn/op_dispatch_prompt.py rename to profiler/msprof_analyze/advisor/display/prompt/cn/op_dispatch_prompt.py diff --git a/profiler/advisor/display/prompt/cn/operator_bound_prompt.py b/profiler/msprof_analyze/advisor/display/prompt/cn/operator_bound_prompt.py similarity index 100% rename from profiler/advisor/display/prompt/cn/operator_bound_prompt.py rename to profiler/msprof_analyze/advisor/display/prompt/cn/operator_bound_prompt.py diff --git a/profiler/advisor/display/prompt/cn/operator_prompt.py b/profiler/msprof_analyze/advisor/display/prompt/cn/operator_prompt.py similarity index 100% rename from profiler/advisor/display/prompt/cn/operator_prompt.py rename to profiler/msprof_analyze/advisor/display/prompt/cn/operator_prompt.py diff --git a/profiler/advisor/display/prompt/cn/overall_summary_advice_prompt.py b/profiler/msprof_analyze/advisor/display/prompt/cn/overall_summary_advice_prompt.py similarity index 100% rename from profiler/advisor/display/prompt/cn/overall_summary_advice_prompt.py rename to profiler/msprof_analyze/advisor/display/prompt/cn/overall_summary_advice_prompt.py diff --git a/profiler/advisor/display/prompt/cn/overall_summary_prompt.py b/profiler/msprof_analyze/advisor/display/prompt/cn/overall_summary_prompt.py similarity index 100% rename from profiler/advisor/display/prompt/cn/overall_summary_prompt.py rename to profiler/msprof_analyze/advisor/display/prompt/cn/overall_summary_prompt.py diff --git a/profiler/advisor/display/prompt/cn/timeline_fusion_ops_prompt.py b/profiler/msprof_analyze/advisor/display/prompt/cn/timeline_fusion_ops_prompt.py similarity index 100% rename from profiler/advisor/display/prompt/cn/timeline_fusion_ops_prompt.py rename to profiler/msprof_analyze/advisor/display/prompt/cn/timeline_fusion_ops_prompt.py diff --git a/profiler/advisor/display/prompt/en/__init__.py b/profiler/msprof_analyze/advisor/display/prompt/en/__init__.py similarity index 100% rename from profiler/advisor/display/prompt/en/__init__.py rename to profiler/msprof_analyze/advisor/display/prompt/en/__init__.py diff --git a/profiler/advisor/display/prompt/en/ai_core_freq_prompt.py b/profiler/msprof_analyze/advisor/display/prompt/en/ai_core_freq_prompt.py similarity index 100% rename from profiler/advisor/display/prompt/en/ai_core_freq_prompt.py rename to profiler/msprof_analyze/advisor/display/prompt/en/ai_core_freq_prompt.py diff --git a/profiler/advisor/display/prompt/en/block_dim_prompt.py b/profiler/msprof_analyze/advisor/display/prompt/en/block_dim_prompt.py similarity index 100% rename from profiler/advisor/display/prompt/en/block_dim_prompt.py rename to profiler/msprof_analyze/advisor/display/prompt/en/block_dim_prompt.py diff --git a/profiler/advisor/display/prompt/en/dynamic_shape_prompt.py b/profiler/msprof_analyze/advisor/display/prompt/en/dynamic_shape_prompt.py similarity index 100% rename from profiler/advisor/display/prompt/en/dynamic_shape_prompt.py rename to profiler/msprof_analyze/advisor/display/prompt/en/dynamic_shape_prompt.py diff --git a/profiler/advisor/display/prompt/en/environment_variable_prompt.py b/profiler/msprof_analyze/advisor/display/prompt/en/environment_variable_prompt.py similarity index 100% rename from profiler/advisor/display/prompt/en/environment_variable_prompt.py rename to profiler/msprof_analyze/advisor/display/prompt/en/environment_variable_prompt.py diff --git a/profiler/advisor/display/prompt/en/graph_fusion_prompt.py b/profiler/msprof_analyze/advisor/display/prompt/en/graph_fusion_prompt.py similarity index 100% rename from profiler/advisor/display/prompt/en/graph_fusion_prompt.py rename to profiler/msprof_analyze/advisor/display/prompt/en/graph_fusion_prompt.py diff --git a/profiler/advisor/display/prompt/en/op_dispatch_prompt.py b/profiler/msprof_analyze/advisor/display/prompt/en/op_dispatch_prompt.py similarity index 100% rename from profiler/advisor/display/prompt/en/op_dispatch_prompt.py rename to profiler/msprof_analyze/advisor/display/prompt/en/op_dispatch_prompt.py diff --git a/profiler/advisor/display/prompt/en/operator_bound_prompt.py b/profiler/msprof_analyze/advisor/display/prompt/en/operator_bound_prompt.py similarity index 100% rename from profiler/advisor/display/prompt/en/operator_bound_prompt.py rename to profiler/msprof_analyze/advisor/display/prompt/en/operator_bound_prompt.py diff --git a/profiler/advisor/display/prompt/en/operator_prompt.py b/profiler/msprof_analyze/advisor/display/prompt/en/operator_prompt.py similarity index 100% rename from profiler/advisor/display/prompt/en/operator_prompt.py rename to profiler/msprof_analyze/advisor/display/prompt/en/operator_prompt.py diff --git a/profiler/advisor/display/prompt/en/overall_summary_advice_prompt.py b/profiler/msprof_analyze/advisor/display/prompt/en/overall_summary_advice_prompt.py similarity index 100% rename from profiler/advisor/display/prompt/en/overall_summary_advice_prompt.py rename to profiler/msprof_analyze/advisor/display/prompt/en/overall_summary_advice_prompt.py diff --git a/profiler/advisor/display/prompt/en/overall_summary_prompt.py b/profiler/msprof_analyze/advisor/display/prompt/en/overall_summary_prompt.py similarity index 100% rename from profiler/advisor/display/prompt/en/overall_summary_prompt.py rename to profiler/msprof_analyze/advisor/display/prompt/en/overall_summary_prompt.py diff --git a/profiler/advisor/display/prompt/en/timeline_fusion_ops_prompt.py b/profiler/msprof_analyze/advisor/display/prompt/en/timeline_fusion_ops_prompt.py similarity index 100% rename from profiler/advisor/display/prompt/en/timeline_fusion_ops_prompt.py rename to profiler/msprof_analyze/advisor/display/prompt/en/timeline_fusion_ops_prompt.py diff --git a/profiler/advisor/doc/Samples of AI CPU Operator Replacement.md b/profiler/msprof_analyze/advisor/doc/Samples of AI CPU Operator Replacement.md similarity index 100% rename from profiler/advisor/doc/Samples of AI CPU Operator Replacement.md rename to profiler/msprof_analyze/advisor/doc/Samples of AI CPU Operator Replacement.md diff --git a/profiler/advisor/doc/Samples of Fused Operator API Replacement.md b/profiler/msprof_analyze/advisor/doc/Samples of Fused Operator API Replacement.md similarity index 100% rename from profiler/advisor/doc/Samples of Fused Operator API Replacement.md rename to profiler/msprof_analyze/advisor/doc/Samples of Fused Operator API Replacement.md diff --git a/profiler/advisor/doc/img/Equal.png b/profiler/msprof_analyze/advisor/doc/img/Equal.png similarity index 100% rename from profiler/advisor/doc/img/Equal.png rename to profiler/msprof_analyze/advisor/doc/img/Equal.png diff --git a/profiler/advisor/doc/img/Mul.png b/profiler/msprof_analyze/advisor/doc/img/Mul.png similarity index 100% rename from profiler/advisor/doc/img/Mul.png rename to profiler/msprof_analyze/advisor/doc/img/Mul.png diff --git a/profiler/advisor/doc/img/Pytorch_dtype.png b/profiler/msprof_analyze/advisor/doc/img/Pytorch_dtype.png similarity index 100% rename from profiler/advisor/doc/img/Pytorch_dtype.png rename to profiler/msprof_analyze/advisor/doc/img/Pytorch_dtype.png diff --git a/profiler/advisor/doc/img/TensorEqual.png b/profiler/msprof_analyze/advisor/doc/img/TensorEqual.png similarity index 100% rename from profiler/advisor/doc/img/TensorEqual.png rename to profiler/msprof_analyze/advisor/doc/img/TensorEqual.png diff --git a/profiler/advisor/doc/img/index by index.png b/profiler/msprof_analyze/advisor/doc/img/index by index.png similarity index 100% rename from profiler/advisor/doc/img/index by index.png rename to profiler/msprof_analyze/advisor/doc/img/index by index.png diff --git a/profiler/advisor/doc/img/index_put_by_mask.png b/profiler/msprof_analyze/advisor/doc/img/index_put_by_mask.png similarity index 100% rename from profiler/advisor/doc/img/index_put_by_mask.png rename to profiler/msprof_analyze/advisor/doc/img/index_put_by_mask.png diff --git a/profiler/advisor/doc/img/single_op_time_CANN63RC2.png b/profiler/msprof_analyze/advisor/doc/img/single_op_time_CANN63RC2.png similarity index 100% rename from profiler/advisor/doc/img/single_op_time_CANN63RC2.png rename to profiler/msprof_analyze/advisor/doc/img/single_op_time_CANN63RC2.png diff --git a/profiler/advisor/doc/img/single_op_time_CANN70RC1.png b/profiler/msprof_analyze/advisor/doc/img/single_op_time_CANN70RC1.png similarity index 100% rename from profiler/advisor/doc/img/single_op_time_CANN70RC1.png rename to profiler/msprof_analyze/advisor/doc/img/single_op_time_CANN70RC1.png diff --git "a/profiler/advisor/doc/img/\346\233\277\346\215\242\345\211\215\350\200\227\346\227\266.png" "b/profiler/msprof_analyze/advisor/doc/img/\346\233\277\346\215\242\345\211\215\350\200\227\346\227\266.png" similarity index 100% rename from "profiler/advisor/doc/img/\346\233\277\346\215\242\345\211\215\350\200\227\346\227\266.png" rename to "profiler/msprof_analyze/advisor/doc/img/\346\233\277\346\215\242\345\211\215\350\200\227\346\227\266.png" diff --git "a/profiler/advisor/doc/img/\346\233\277\346\215\242\345\220\216\350\200\227\346\227\266.png" "b/profiler/msprof_analyze/advisor/doc/img/\346\233\277\346\215\242\345\220\216\350\200\227\346\227\266.png" similarity index 100% rename from "profiler/advisor/doc/img/\346\233\277\346\215\242\345\220\216\350\200\227\346\227\266.png" rename to "profiler/msprof_analyze/advisor/doc/img/\346\233\277\346\215\242\345\220\216\350\200\227\346\227\266.png" diff --git a/profiler/advisor/fusion_operators_api_analysis.ipynb b/profiler/msprof_analyze/advisor/fusion_operators_api_analysis.ipynb similarity index 99% rename from profiler/advisor/fusion_operators_api_analysis.ipynb rename to profiler/msprof_analyze/advisor/fusion_operators_api_analysis.ipynb index ac758f562f..4dcb4f2340 100644 --- a/profiler/advisor/fusion_operators_api_analysis.ipynb +++ b/profiler/msprof_analyze/advisor/fusion_operators_api_analysis.ipynb @@ -18,7 +18,7 @@ "source": [ "from prettytable import PrettyTable, ALL\n", "from textwrap import fill\n", - "from profiler.advisor.interface.interface import Interface" + "from msprof_analyze.advisor.interface.interface import Interface" ] }, { diff --git a/profiler/advisor/img/advisor_result.PNG b/profiler/msprof_analyze/advisor/img/advisor_result.PNG similarity index 100% rename from profiler/advisor/img/advisor_result.PNG rename to profiler/msprof_analyze/advisor/img/advisor_result.PNG diff --git a/profiler/advisor/img/all.png b/profiler/msprof_analyze/advisor/img/all.png similarity index 100% rename from profiler/advisor/img/all.png rename to profiler/msprof_analyze/advisor/img/all.png diff --git a/profiler/advisor/img/bandwidth.png b/profiler/msprof_analyze/advisor/img/bandwidth.png similarity index 100% rename from profiler/advisor/img/bandwidth.png rename to profiler/msprof_analyze/advisor/img/bandwidth.png diff --git a/profiler/advisor/img/block_dim.png b/profiler/msprof_analyze/advisor/img/block_dim.png similarity index 100% rename from profiler/advisor/img/block_dim.png rename to profiler/msprof_analyze/advisor/img/block_dim.png diff --git a/profiler/advisor/img/byte_alignment.png b/profiler/msprof_analyze/advisor/img/byte_alignment.png similarity index 100% rename from profiler/advisor/img/byte_alignment.png rename to profiler/msprof_analyze/advisor/img/byte_alignment.png diff --git a/profiler/advisor/img/cluster.png b/profiler/msprof_analyze/advisor/img/cluster.png similarity index 100% rename from profiler/advisor/img/cluster.png rename to profiler/msprof_analyze/advisor/img/cluster.png diff --git a/profiler/advisor/img/cluster_1.png b/profiler/msprof_analyze/advisor/img/cluster_1.png similarity index 100% rename from profiler/advisor/img/cluster_1.png rename to profiler/msprof_analyze/advisor/img/cluster_1.png diff --git a/profiler/advisor/img/cluster_2.png b/profiler/msprof_analyze/advisor/img/cluster_2.png similarity index 100% rename from profiler/advisor/img/cluster_2.png rename to profiler/msprof_analyze/advisor/img/cluster_2.png diff --git a/profiler/advisor/img/cluster_3.png b/profiler/msprof_analyze/advisor/img/cluster_3.png similarity index 100% rename from profiler/advisor/img/cluster_3.png rename to profiler/msprof_analyze/advisor/img/cluster_3.png diff --git a/profiler/advisor/img/cluster_4.png b/profiler/msprof_analyze/advisor/img/cluster_4.png similarity index 100% rename from profiler/advisor/img/cluster_4.png rename to profiler/msprof_analyze/advisor/img/cluster_4.png diff --git a/profiler/advisor/img/cluster_5.png b/profiler/msprof_analyze/advisor/img/cluster_5.png similarity index 100% rename from profiler/advisor/img/cluster_5.png rename to profiler/msprof_analyze/advisor/img/cluster_5.png diff --git a/profiler/advisor/img/communication.png b/profiler/msprof_analyze/advisor/img/communication.png similarity index 100% rename from profiler/advisor/img/communication.png rename to profiler/msprof_analyze/advisor/img/communication.png diff --git a/profiler/advisor/img/comparison.png b/profiler/msprof_analyze/advisor/img/comparison.png similarity index 100% rename from profiler/advisor/img/comparison.png rename to profiler/msprof_analyze/advisor/img/comparison.png diff --git a/profiler/advisor/img/comparison1.png b/profiler/msprof_analyze/advisor/img/comparison1.png similarity index 100% rename from profiler/advisor/img/comparison1.png rename to profiler/msprof_analyze/advisor/img/comparison1.png diff --git a/profiler/advisor/img/comparison2.png b/profiler/msprof_analyze/advisor/img/comparison2.png similarity index 100% rename from profiler/advisor/img/comparison2.png rename to profiler/msprof_analyze/advisor/img/comparison2.png diff --git a/profiler/advisor/img/comparison3.png b/profiler/msprof_analyze/advisor/img/comparison3.png similarity index 100% rename from profiler/advisor/img/comparison3.png rename to profiler/msprof_analyze/advisor/img/comparison3.png diff --git a/profiler/advisor/img/computation.png b/profiler/msprof_analyze/advisor/img/computation.png similarity index 100% rename from profiler/advisor/img/computation.png rename to profiler/msprof_analyze/advisor/img/computation.png diff --git a/profiler/advisor/img/computation_1.png b/profiler/msprof_analyze/advisor/img/computation_1.png similarity index 100% rename from profiler/advisor/img/computation_1.png rename to profiler/msprof_analyze/advisor/img/computation_1.png diff --git a/profiler/advisor/img/computation_2.png b/profiler/msprof_analyze/advisor/img/computation_2.png similarity index 100% rename from profiler/advisor/img/computation_2.png rename to profiler/msprof_analyze/advisor/img/computation_2.png diff --git a/profiler/advisor/img/dataloader.png b/profiler/msprof_analyze/advisor/img/dataloader.png similarity index 100% rename from profiler/advisor/img/dataloader.png rename to profiler/msprof_analyze/advisor/img/dataloader.png diff --git a/profiler/advisor/img/env_var.png b/profiler/msprof_analyze/advisor/img/env_var.png similarity index 100% rename from profiler/advisor/img/env_var.png rename to profiler/msprof_analyze/advisor/img/env_var.png diff --git a/profiler/advisor/img/gc.png b/profiler/msprof_analyze/advisor/img/gc.png similarity index 100% rename from profiler/advisor/img/gc.png rename to profiler/msprof_analyze/advisor/img/gc.png diff --git a/profiler/advisor/img/jupyter_report.PNG b/profiler/msprof_analyze/advisor/img/jupyter_report.PNG similarity index 100% rename from profiler/advisor/img/jupyter_report.PNG rename to profiler/msprof_analyze/advisor/img/jupyter_report.PNG diff --git a/profiler/advisor/img/memory.png b/profiler/msprof_analyze/advisor/img/memory.png similarity index 100% rename from profiler/advisor/img/memory.png rename to profiler/msprof_analyze/advisor/img/memory.png diff --git a/profiler/advisor/img/op_no_bound.png b/profiler/msprof_analyze/advisor/img/op_no_bound.png similarity index 100% rename from profiler/advisor/img/op_no_bound.png rename to profiler/msprof_analyze/advisor/img/op_no_bound.png diff --git a/profiler/advisor/img/overall.png b/profiler/msprof_analyze/advisor/img/overall.png similarity index 100% rename from profiler/advisor/img/overall.png rename to profiler/msprof_analyze/advisor/img/overall.png diff --git a/profiler/advisor/img/overall_0.png b/profiler/msprof_analyze/advisor/img/overall_0.png similarity index 100% rename from profiler/advisor/img/overall_0.png rename to profiler/msprof_analyze/advisor/img/overall_0.png diff --git a/profiler/advisor/img/schedule.png b/profiler/msprof_analyze/advisor/img/schedule.png similarity index 100% rename from profiler/advisor/img/schedule.png rename to profiler/msprof_analyze/advisor/img/schedule.png diff --git a/profiler/advisor/img/schedule_1.png b/profiler/msprof_analyze/advisor/img/schedule_1.png similarity index 100% rename from profiler/advisor/img/schedule_1.png rename to profiler/msprof_analyze/advisor/img/schedule_1.png diff --git a/profiler/advisor/img/schedule_2.png b/profiler/msprof_analyze/advisor/img/schedule_2.png similarity index 100% rename from profiler/advisor/img/schedule_2.png rename to profiler/msprof_analyze/advisor/img/schedule_2.png diff --git a/profiler/advisor/img/schedule_3.png b/profiler/msprof_analyze/advisor/img/schedule_3.png similarity index 100% rename from profiler/advisor/img/schedule_3.png rename to profiler/msprof_analyze/advisor/img/schedule_3.png diff --git a/profiler/advisor/interface/__init__.py b/profiler/msprof_analyze/advisor/interface/__init__.py similarity index 100% rename from profiler/advisor/interface/__init__.py rename to profiler/msprof_analyze/advisor/interface/__init__.py diff --git a/profiler/advisor/interface/interface.py b/profiler/msprof_analyze/advisor/interface/interface.py similarity index 69% rename from profiler/advisor/interface/interface.py rename to profiler/msprof_analyze/advisor/interface/interface.py index 7b9cb00fdf..4f9b6cba55 100644 --- a/profiler/advisor/interface/interface.py +++ b/profiler/msprof_analyze/advisor/interface/interface.py @@ -22,31 +22,31 @@ sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os. sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))), "compare_tools")) -from profiler.advisor.utils.utils import Timer -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.analyzer.computation.profiling_analyzer import AicpuAnalyzer, BlockDimAnalyzer, \ +from msprof_analyze.advisor.utils.utils import Timer +from msprof_analyze.advisor.result.result import OptimizeResult +from msprof_analyze.advisor.analyzer.computation.profiling_analyzer import AicpuAnalyzer, BlockDimAnalyzer, \ DynamicShapeAnalyzer, OperatorBoundAnalyzer -from profiler.advisor.analyzer.schedule.fusion_ops.fusion_ops_analyzer import TimelineFusionOpsAnalyzer -from profiler.advisor.analyzer.graph_fusion.graph_fusion_analyzer import FusionOPAnalyzer -from profiler.advisor.common.analyzer_scopes import SupportedScopes -from profiler.advisor.analyzer.cluster.slow_rank_analyzer import SlowRankAnalyzer -from profiler.advisor.analyzer.cluster.slow_link_analyzer import SlowLinkAnalyzer -from profiler.advisor.analyzer.communication.retransmission.communication_retransmission_analyzer import \ +from msprof_analyze.advisor.analyzer.schedule.fusion_ops.fusion_ops_analyzer import TimelineFusionOpsAnalyzer +from msprof_analyze.advisor.analyzer.graph_fusion.graph_fusion_analyzer import FusionOPAnalyzer +from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes +from msprof_analyze.advisor.analyzer.cluster.slow_rank_analyzer import SlowRankAnalyzer +from msprof_analyze.advisor.analyzer.cluster.slow_link_analyzer import SlowLinkAnalyzer +from msprof_analyze.advisor.analyzer.communication.retransmission.communication_retransmission_analyzer import \ RDMARetransmissionAnalyzer -from profiler.advisor.analyzer.overall.overall_summary_analyzer import OverallSummaryAnalyzer -from profiler.advisor.analyzer.overall.environment_variable_analyzer import EnvironmentVariableAnalyzer -from profiler.advisor.analyzer.schedule.dispatch.timeline_op_dispatch_analyzer import OpDispatchAnalyzer -from profiler.advisor.analyzer.schedule.syncbn.syncbn_analyzer import SyncBNAnalyzer -from profiler.advisor.analyzer.schedule.synchronize_stream.synchronize_stream_analyzer import SynchronizeStreamAnalyzer -from profiler.advisor.analyzer.dataloader.dataloader_analyzer import DataloaderAnalyzer -from profiler.advisor.analyzer.computation.ai_core_freq.ai_core_freq_analyzer import AICoreFreqAnalyzer -from profiler.advisor.analyzer.memory.memory_analyzer import MemoryAnalyzer -from profiler.advisor.analyzer.communication.packet.packet_analyzer import PacketAnalyzer -from profiler.advisor.analyzer.communication.contention.bandwidth_contention_analyzer import BandwidthContentionAnalyzer -from profiler.advisor.analyzer.communication.alignment.byte_alignment_analyzer import ByteAlignmentAnalyzer -from profiler.advisor.analyzer.schedule.gc.gc_analyzer import GcAnalyzer -from profiler.advisor.analyzer.schedule.conjectured_gc.conjectured_gc_analyzer import ConjecturedGcAnalyzer -from profiler.advisor.analyzer.comparison.comparison_analyzer import ComparisonAnalyzer +from msprof_analyze.advisor.analyzer.overall.overall_summary_analyzer import OverallSummaryAnalyzer +from msprof_analyze.advisor.analyzer.overall.environment_variable_analyzer import EnvironmentVariableAnalyzer +from msprof_analyze.advisor.analyzer.schedule.dispatch.timeline_op_dispatch_analyzer import OpDispatchAnalyzer +from msprof_analyze.advisor.analyzer.schedule.syncbn.syncbn_analyzer import SyncBNAnalyzer +from msprof_analyze.advisor.analyzer.schedule.synchronize_stream.synchronize_stream_analyzer import SynchronizeStreamAnalyzer +from msprof_analyze.advisor.analyzer.dataloader.dataloader_analyzer import DataloaderAnalyzer +from msprof_analyze.advisor.analyzer.computation.ai_core_freq.ai_core_freq_analyzer import AICoreFreqAnalyzer +from msprof_analyze.advisor.analyzer.memory.memory_analyzer import MemoryAnalyzer +from msprof_analyze.advisor.analyzer.communication.packet.packet_analyzer import PacketAnalyzer +from msprof_analyze.advisor.analyzer.communication.contention.bandwidth_contention_analyzer import BandwidthContentionAnalyzer +from msprof_analyze.advisor.analyzer.communication.alignment.byte_alignment_analyzer import ByteAlignmentAnalyzer +from msprof_analyze.advisor.analyzer.schedule.gc.gc_analyzer import GcAnalyzer +from msprof_analyze.advisor.analyzer.schedule.conjectured_gc.conjectured_gc_analyzer import ConjecturedGcAnalyzer +from msprof_analyze.advisor.analyzer.comparison.comparison_analyzer import ComparisonAnalyzer logger = logging.getLogger() diff --git a/profiler/advisor/result/__init__.py b/profiler/msprof_analyze/advisor/result/__init__.py similarity index 100% rename from profiler/advisor/result/__init__.py rename to profiler/msprof_analyze/advisor/result/__init__.py diff --git a/profiler/advisor/result/item.py b/profiler/msprof_analyze/advisor/result/item.py similarity index 100% rename from profiler/advisor/result/item.py rename to profiler/msprof_analyze/advisor/result/item.py diff --git a/profiler/advisor/result/result.py b/profiler/msprof_analyze/advisor/result/result.py similarity index 95% rename from profiler/advisor/result/result.py rename to profiler/msprof_analyze/advisor/result/result.py index 74b9358f62..9645cc5af3 100644 --- a/profiler/advisor/result/result.py +++ b/profiler/msprof_analyze/advisor/result/result.py @@ -22,12 +22,12 @@ import click import xlsxwriter from prettytable import ALL, PrettyTable -from profiler.prof_common.additional_args_manager import AdditionalArgsManager -from profiler.prof_common.constant import Constant -from profiler.advisor.utils.utils import singleton, logger -from profiler.advisor.config.config import Config -from profiler.advisor.utils.file import FdOpen, check_dir_writable -from profiler.prof_common.file_manager import FileManager +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.advisor.utils.utils import singleton, logger +from msprof_analyze.advisor.config.config import Config +from msprof_analyze.advisor.utils.file import FdOpen, check_dir_writable +from msprof_analyze.prof_common.file_manager import FileManager class ResultWriter: diff --git a/profiler/advisor/rules/__init__.py b/profiler/msprof_analyze/advisor/rules/__init__.py similarity index 100% rename from profiler/advisor/rules/__init__.py rename to profiler/msprof_analyze/advisor/rules/__init__.py diff --git a/profiler/advisor/rules/cn/__init__.py b/profiler/msprof_analyze/advisor/rules/cn/__init__.py similarity index 100% rename from profiler/advisor/rules/cn/__init__.py rename to profiler/msprof_analyze/advisor/rules/cn/__init__.py diff --git a/profiler/advisor/rules/cn/aicpu_rules.yaml b/profiler/msprof_analyze/advisor/rules/cn/aicpu_rules.yaml similarity index 97% rename from profiler/advisor/rules/cn/aicpu_rules.yaml rename to profiler/msprof_analyze/advisor/rules/cn/aicpu_rules.yaml index f14fc044d4..a4bcec4078 100644 --- a/profiler/advisor/rules/cn/aicpu_rules.yaml +++ b/profiler/msprof_analyze/advisor/rules/cn/aicpu_rules.yaml @@ -3,7 +3,7 @@ description: "一些算子和任务执行时间超过了{}us，比如：\n" suggestion: "修改代码避免使用aicpu类算子" double_suggestion: "尝试将double类型的算子转换成float，比如{}" DataTypeSuggeation: &DataTypeSuggeation "数据类型{}在{}算子中可能会造成AICpu问题, 如果可以，尝试转换成{}。" -AICPU_DOC_URL: &AICPU_DOC_URL "https://gitee.com/ascend/mstt/blob/master/profiler/advisor/doc/Samples%20of%20AI%20CPU%20Operator%20Replacement.md" +AICPU_DOC_URL: &AICPU_DOC_URL "https://gitee.com/ascend/mstt/blob/master/profiler/msprof_analyze/advisor/doc/Samples%20of%20AI%20CPU%20Operator%20Replacement.md" CommonChecker: - DataTypeChecker: diff --git a/profiler/advisor/rules/cn/bandwidth_contention.yaml b/profiler/msprof_analyze/advisor/rules/cn/bandwidth_contention.yaml similarity index 100% rename from profiler/advisor/rules/cn/bandwidth_contention.yaml rename to profiler/msprof_analyze/advisor/rules/cn/bandwidth_contention.yaml diff --git a/profiler/advisor/rules/cn/byte_alignment.yaml b/profiler/msprof_analyze/advisor/rules/cn/byte_alignment.yaml similarity index 100% rename from profiler/advisor/rules/cn/byte_alignment.yaml rename to profiler/msprof_analyze/advisor/rules/cn/byte_alignment.yaml diff --git a/profiler/advisor/rules/cn/conjectured_gc.yaml b/profiler/msprof_analyze/advisor/rules/cn/conjectured_gc.yaml similarity index 100% rename from profiler/advisor/rules/cn/conjectured_gc.yaml rename to profiler/msprof_analyze/advisor/rules/cn/conjectured_gc.yaml diff --git a/profiler/advisor/rules/cn/dataloader.yaml b/profiler/msprof_analyze/advisor/rules/cn/dataloader.yaml similarity index 100% rename from profiler/advisor/rules/cn/dataloader.yaml rename to profiler/msprof_analyze/advisor/rules/cn/dataloader.yaml diff --git a/profiler/advisor/rules/cn/environment_variable_info.yaml b/profiler/msprof_analyze/advisor/rules/cn/environment_variable_info.yaml similarity index 100% rename from profiler/advisor/rules/cn/environment_variable_info.yaml rename to profiler/msprof_analyze/advisor/rules/cn/environment_variable_info.yaml diff --git a/profiler/advisor/rules/cn/gc.yaml b/profiler/msprof_analyze/advisor/rules/cn/gc.yaml similarity index 100% rename from profiler/advisor/rules/cn/gc.yaml rename to profiler/msprof_analyze/advisor/rules/cn/gc.yaml diff --git a/profiler/advisor/rules/cn/memory.yaml b/profiler/msprof_analyze/advisor/rules/cn/memory.yaml similarity index 100% rename from profiler/advisor/rules/cn/memory.yaml rename to profiler/msprof_analyze/advisor/rules/cn/memory.yaml diff --git a/profiler/advisor/rules/cn/packet.yaml b/profiler/msprof_analyze/advisor/rules/cn/packet.yaml similarity index 100% rename from profiler/advisor/rules/cn/packet.yaml rename to profiler/msprof_analyze/advisor/rules/cn/packet.yaml diff --git a/profiler/advisor/rules/cn/rdma_analysis.yaml b/profiler/msprof_analyze/advisor/rules/cn/rdma_analysis.yaml similarity index 100% rename from profiler/advisor/rules/cn/rdma_analysis.yaml rename to profiler/msprof_analyze/advisor/rules/cn/rdma_analysis.yaml diff --git a/profiler/advisor/rules/cn/sync_batchnorm.yaml b/profiler/msprof_analyze/advisor/rules/cn/sync_batchnorm.yaml similarity index 100% rename from profiler/advisor/rules/cn/sync_batchnorm.yaml rename to profiler/msprof_analyze/advisor/rules/cn/sync_batchnorm.yaml diff --git a/profiler/advisor/rules/cn/synchronize.yaml b/profiler/msprof_analyze/advisor/rules/cn/synchronize.yaml similarity index 100% rename from profiler/advisor/rules/cn/synchronize.yaml rename to profiler/msprof_analyze/advisor/rules/cn/synchronize.yaml diff --git a/profiler/advisor/rules/en/__init__.py b/profiler/msprof_analyze/advisor/rules/en/__init__.py similarity index 100% rename from profiler/advisor/rules/en/__init__.py rename to profiler/msprof_analyze/advisor/rules/en/__init__.py diff --git a/profiler/advisor/rules/en/aicpu_rules.yaml b/profiler/msprof_analyze/advisor/rules/en/aicpu_rules.yaml similarity index 97% rename from profiler/advisor/rules/en/aicpu_rules.yaml rename to profiler/msprof_analyze/advisor/rules/en/aicpu_rules.yaml index d1869a25ef..b414c71a2e 100644 --- a/profiler/advisor/rules/en/aicpu_rules.yaml +++ b/profiler/msprof_analyze/advisor/rules/en/aicpu_rules.yaml @@ -3,7 +3,7 @@ description: "Some operators and task duration exceed {} us, such as :\n" suggestion: "Modify code to avoid aicpu operator" double_suggestion: "Try to convert double type operator to float, such as {}" DataTypeSuggeation: &DataTypeSuggeation "Data type {} in {} operator may cause AICPU issues, Try to convert to {} if possible." -AICPU_DOC_URL: &AICPU_DOC_URL "https://gitee.com/ascend/mstt/blob/master/profiler/advisor/doc/Samples%20of%20AI%20CPU%20Operator%20Replacement.md" +AICPU_DOC_URL: &AICPU_DOC_URL "https://gitee.com/ascend/mstt/blob/master/profiler/msprof_analyze/advisor/doc/Samples%20of%20AI%20CPU%20Operator%20Replacement.md" CommonChecker: - DataTypeChecker: diff --git a/profiler/advisor/rules/en/bandwidth_contention.yaml b/profiler/msprof_analyze/advisor/rules/en/bandwidth_contention.yaml similarity index 100% rename from profiler/advisor/rules/en/bandwidth_contention.yaml rename to profiler/msprof_analyze/advisor/rules/en/bandwidth_contention.yaml diff --git a/profiler/advisor/rules/en/byte_alignment.yaml b/profiler/msprof_analyze/advisor/rules/en/byte_alignment.yaml similarity index 100% rename from profiler/advisor/rules/en/byte_alignment.yaml rename to profiler/msprof_analyze/advisor/rules/en/byte_alignment.yaml diff --git a/profiler/advisor/rules/en/conjectured_gc.yaml b/profiler/msprof_analyze/advisor/rules/en/conjectured_gc.yaml similarity index 100% rename from profiler/advisor/rules/en/conjectured_gc.yaml rename to profiler/msprof_analyze/advisor/rules/en/conjectured_gc.yaml diff --git a/profiler/advisor/rules/en/dataloader.yaml b/profiler/msprof_analyze/advisor/rules/en/dataloader.yaml similarity index 100% rename from profiler/advisor/rules/en/dataloader.yaml rename to profiler/msprof_analyze/advisor/rules/en/dataloader.yaml diff --git a/profiler/advisor/rules/en/environment_variable_info.yaml b/profiler/msprof_analyze/advisor/rules/en/environment_variable_info.yaml similarity index 100% rename from profiler/advisor/rules/en/environment_variable_info.yaml rename to profiler/msprof_analyze/advisor/rules/en/environment_variable_info.yaml diff --git a/profiler/advisor/rules/en/gc.yaml b/profiler/msprof_analyze/advisor/rules/en/gc.yaml similarity index 100% rename from profiler/advisor/rules/en/gc.yaml rename to profiler/msprof_analyze/advisor/rules/en/gc.yaml diff --git a/profiler/advisor/rules/en/memory.yaml b/profiler/msprof_analyze/advisor/rules/en/memory.yaml similarity index 100% rename from profiler/advisor/rules/en/memory.yaml rename to profiler/msprof_analyze/advisor/rules/en/memory.yaml diff --git a/profiler/advisor/rules/en/packet.yaml b/profiler/msprof_analyze/advisor/rules/en/packet.yaml similarity index 100% rename from profiler/advisor/rules/en/packet.yaml rename to profiler/msprof_analyze/advisor/rules/en/packet.yaml diff --git a/profiler/advisor/rules/en/rdma_analysis.yaml b/profiler/msprof_analyze/advisor/rules/en/rdma_analysis.yaml similarity index 100% rename from profiler/advisor/rules/en/rdma_analysis.yaml rename to profiler/msprof_analyze/advisor/rules/en/rdma_analysis.yaml diff --git a/profiler/advisor/rules/en/sync_batchnorm.yaml b/profiler/msprof_analyze/advisor/rules/en/sync_batchnorm.yaml similarity index 100% rename from profiler/advisor/rules/en/sync_batchnorm.yaml rename to profiler/msprof_analyze/advisor/rules/en/sync_batchnorm.yaml diff --git a/profiler/advisor/rules/en/synchronize.yaml b/profiler/msprof_analyze/advisor/rules/en/synchronize.yaml similarity index 100% rename from profiler/advisor/rules/en/synchronize.yaml rename to profiler/msprof_analyze/advisor/rules/en/synchronize.yaml diff --git a/profiler/advisor/rules/op_fusion_pass.yaml b/profiler/msprof_analyze/advisor/rules/op_fusion_pass.yaml similarity index 100% rename from profiler/advisor/rules/op_fusion_pass.yaml rename to profiler/msprof_analyze/advisor/rules/op_fusion_pass.yaml diff --git a/profiler/advisor/rules/timeline_fusion_ops.yaml b/profiler/msprof_analyze/advisor/rules/timeline_fusion_ops.yaml similarity index 100% rename from profiler/advisor/rules/timeline_fusion_ops.yaml rename to profiler/msprof_analyze/advisor/rules/timeline_fusion_ops.yaml diff --git a/profiler/advisor/utils/__init__.py b/profiler/msprof_analyze/advisor/utils/__init__.py similarity index 100% rename from profiler/advisor/utils/__init__.py rename to profiler/msprof_analyze/advisor/utils/__init__.py diff --git a/profiler/advisor/utils/file.py b/profiler/msprof_analyze/advisor/utils/file.py similarity index 93% rename from profiler/advisor/utils/file.py rename to profiler/msprof_analyze/advisor/utils/file.py index db01c38939..516077ee72 100644 --- a/profiler/advisor/utils/file.py +++ b/profiler/msprof_analyze/advisor/utils/file.py @@ -15,9 +15,9 @@ import os import logging -from profiler.prof_common.constant import Constant -from profiler.advisor.utils.utils import check_path_valid -from profiler.advisor.utils.log import get_log_level +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.advisor.utils.utils import check_path_valid +from msprof_analyze.advisor.utils.log import get_log_level logger = logging.getLogger() logger.setLevel(get_log_level()) diff --git a/profiler/advisor/utils/log.py b/profiler/msprof_analyze/advisor/utils/log.py similarity index 97% rename from profiler/advisor/utils/log.py rename to profiler/msprof_analyze/advisor/utils/log.py index f62cf5f6c3..7a7a0ca41b 100644 --- a/profiler/advisor/utils/log.py +++ b/profiler/msprof_analyze/advisor/utils/log.py @@ -16,7 +16,7 @@ log module import logging import os -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant def get_log_level(): diff --git a/profiler/advisor/utils/tools.py b/profiler/msprof_analyze/advisor/utils/tools.py similarity index 100% rename from profiler/advisor/utils/tools.py rename to profiler/msprof_analyze/advisor/utils/tools.py diff --git a/profiler/advisor/utils/utils.py b/profiler/msprof_analyze/advisor/utils/utils.py similarity index 98% rename from profiler/advisor/utils/utils.py rename to profiler/msprof_analyze/advisor/utils/utils.py index dd7b5103f5..4f39da29ff 100644 --- a/profiler/advisor/utils/utils.py +++ b/profiler/msprof_analyze/advisor/utils/utils.py @@ -33,9 +33,9 @@ import requests from requests.adapters import HTTPAdapter from tqdm import tqdm -from profiler.prof_common.constant import Constant -from profiler.advisor.common.version_control import VersionControl -from profiler.advisor.utils.log import init_logger, get_log_level +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.advisor.common.version_control import VersionControl +from msprof_analyze.advisor.utils.log import init_logger, get_log_level logger = logging.getLogger() logger.setLevel(get_log_level()) diff --git a/profiler/advisor/version.py b/profiler/msprof_analyze/advisor/version.py similarity index 100% rename from profiler/advisor/version.py rename to profiler/msprof_analyze/advisor/version.py diff --git a/profiler/affinity_cpu_bind/README.md b/profiler/msprof_analyze/affinity_cpu_bind/README.md similarity index 100% rename from profiler/affinity_cpu_bind/README.md rename to profiler/msprof_analyze/affinity_cpu_bind/README.md diff --git a/profiler/affinity_cpu_bind/bind_core.py b/profiler/msprof_analyze/affinity_cpu_bind/bind_core.py similarity index 99% rename from profiler/affinity_cpu_bind/bind_core.py rename to profiler/msprof_analyze/affinity_cpu_bind/bind_core.py index 6c95cdb9bc..4899681bf2 100644 --- a/profiler/affinity_cpu_bind/bind_core.py +++ b/profiler/msprof_analyze/affinity_cpu_bind/bind_core.py @@ -21,7 +21,7 @@ import logging from datetime import datetime from datetime import timezone -from profiler.prof_common.utils import PrintUtils +from msprof_analyze.prof_common.utils import PrintUtils class PathManager: DATA_FILE_AUTHORITY = 0o640 diff --git a/profiler/cli/__init__.py b/profiler/msprof_analyze/cli/__init__.py similarity index 100% rename from profiler/cli/__init__.py rename to profiler/msprof_analyze/cli/__init__.py diff --git a/profiler/cli/analyze_cli.py b/profiler/msprof_analyze/cli/analyze_cli.py similarity index 93% rename from profiler/cli/analyze_cli.py rename to profiler/msprof_analyze/cli/analyze_cli.py index 31c5ee1531..73cbf6b749 100644 --- a/profiler/cli/analyze_cli.py +++ b/profiler/msprof_analyze/cli/analyze_cli.py @@ -21,13 +21,13 @@ import click sys.path.append(os.path.join(os.path.dirname(os.path.dirname(__file__)), "compare_tools")) sys.path.append(os.path.join(os.path.dirname(os.path.dirname(__file__)), "cluster_analyse")) -from profiler.prof_common.path_manager import PathManager -from profiler.advisor.analyzer.analyzer_controller import AnalyzerController -from profiler.advisor.utils.tools import CONTEXT_SETTINGS, ClickAliasedGroup -from profiler.prof_common.constant import Constant -from profiler.advisor.common.enum_params_parser import EnumParamsParser -from profiler.advisor.utils.utils import debug_option -from profiler.advisor.interface.interface import Interface +from msprof_analyze.prof_common.path_manager import PathManager +from msprof_analyze.advisor.analyzer.analyzer_controller import AnalyzerController +from msprof_analyze.advisor.utils.tools import CONTEXT_SETTINGS, ClickAliasedGroup +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.advisor.common.enum_params_parser import EnumParamsParser +from msprof_analyze.advisor.utils.utils import debug_option +from msprof_analyze.advisor.interface.interface import Interface logger = logging.getLogger() diff --git a/profiler/cli/cluster_cli.py b/profiler/msprof_analyze/cli/cluster_cli.py similarity index 86% rename from profiler/cli/cluster_cli.py rename to profiler/msprof_analyze/cli/cluster_cli.py index 9a707b3619..f6bec28ab8 100644 --- a/profiler/cli/cluster_cli.py +++ b/profiler/msprof_analyze/cli/cluster_cli.py @@ -18,10 +18,10 @@ import click sys.path.append(os.path.dirname(os.path.dirname(__file__))) -from profiler.prof_common.path_manager import PathManager -from profiler.prof_common.constant import Constant -from profiler.cluster_analyse.cluster_analysis import COMM_FEATURE_LIST -from profiler.cluster_analyse.cluster_analysis import cluster_analysis_main +from msprof_analyze.prof_common.path_manager import PathManager +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.cluster_analyse.cluster_analysis import COMM_FEATURE_LIST +from msprof_analyze.cluster_analyse.cluster_analysis import cluster_analysis_main context_settings = dict(Constant.CONTEXT_SETTINGS) diff --git a/profiler/cli/compare_cli.py b/profiler/msprof_analyze/cli/compare_cli.py similarity index 90% rename from profiler/cli/compare_cli.py rename to profiler/msprof_analyze/cli/compare_cli.py index 28470ee7a0..c7af038d0c 100644 --- a/profiler/cli/compare_cli.py +++ b/profiler/msprof_analyze/cli/compare_cli.py @@ -19,11 +19,11 @@ import click sys.path.append(os.path.dirname(os.path.dirname(__file__))) -from profiler.prof_common.path_manager import PathManager -from profiler.prof_common.constant import Constant -from profiler.prof_common.analyze_dict import AnalyzeDict -from profiler.compare_tools.compare_backend.comparison_generator import ComparisonGenerator -from profiler.advisor.utils.utils import debug_option +from msprof_analyze.prof_common.path_manager import PathManager +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.prof_common.analyze_dict import AnalyzeDict +from msprof_analyze.compare_tools.compare_backend.comparison_generator import ComparisonGenerator +from msprof_analyze.advisor.utils.utils import debug_option @click.command(context_settings=Constant.CONTEXT_SETTINGS, name="compare", diff --git a/profiler/cli/complete_cli.py b/profiler/msprof_analyze/cli/complete_cli.py similarity index 96% rename from profiler/cli/complete_cli.py rename to profiler/msprof_analyze/cli/complete_cli.py index 78aea6cf89..1dd4048122 100644 --- a/profiler/cli/complete_cli.py +++ b/profiler/msprof_analyze/cli/complete_cli.py @@ -14,7 +14,7 @@ # limitations under the License. import click -from profiler.advisor.utils.tools import CONTEXT_SETTINGS +from msprof_analyze.advisor.utils.tools import CONTEXT_SETTINGS @click.command(context_settings=CONTEXT_SETTINGS, diff --git a/profiler/cli/entrance.py b/profiler/msprof_analyze/cli/entrance.py similarity index 87% rename from profiler/cli/entrance.py rename to profiler/msprof_analyze/cli/entrance.py index c6d72837b7..aea99e8802 100644 --- a/profiler/cli/entrance.py +++ b/profiler/msprof_analyze/cli/entrance.py @@ -17,11 +17,11 @@ import logging import click -from profiler.cli.analyze_cli import analyze_cli -from profiler.cli.complete_cli import auto_complete_cli -from profiler.cli.compare_cli import compare_cli -from profiler.cli.cluster_cli import cluster_cli -from profiler.advisor.version import print_version_callback, cli_version +from msprof_analyze.cli.analyze_cli import analyze_cli +from msprof_analyze.cli.complete_cli import auto_complete_cli +from msprof_analyze.cli.compare_cli import compare_cli +from msprof_analyze.cli.cluster_cli import cluster_cli +from msprof_analyze.advisor.version import print_version_callback, cli_version logger = logging.getLogger() CONTEXT_SETTINGS = dict(help_option_names=['-H', '-h', '--help'], diff --git a/profiler/cluster_analyse/README.md b/profiler/msprof_analyze/cluster_analyse/README.md similarity index 98% rename from profiler/cluster_analyse/README.md rename to profiler/msprof_analyze/cluster_analyse/README.md index a6df8ad902..f175e99730 100644 --- a/profiler/cluster_analyse/README.md +++ b/profiler/msprof_analyze/cluster_analyse/README.md @@ -1,157 +1,157 @@ -# 集群分析工具 -cluster_analyse（集群分析工具）是在集群场景下，通过此工具来进行集群数据的分析，当前主要对基于通信域的迭代内耗时分析、通信时间分析以及通信矩阵分析为主， 从而定位慢卡、慢节点以及慢链路问题。 - -## 性能数据采集 -当前集群调优工具主要支持PyTorch场景的Ascend PyTorch Profiler采集方式和MindSpore场景的MindSpore Profiler采集方式下的集群数据。 - -此工具只需要NPU的性能数据作为输入。 - -Ascend PyTorch Profiler采集方法请参见《[NPU性能数据采集](https://gitee.com/ascend/mstt/tree/master/profiler)》，MindSpore Profiler采集方法请参见《[性能调试](https://www.mindspore.cn/mindinsight/docs/zh-CN/r2.3/performance_profiling_ascend.html)》。 - -我们要求至少是L1级别的数据。 -```python -experimental_config = torch_npu.profiler._ExperimentalConfig( - profiler_level=torch_npu.profiler.ProfilerLevel.Level1 -) -``` -### 确认数据是否可用 - -打开采集到的某张卡数据(\*ascend_pt、\*ascend_ms结尾的文件夹)，可用的数据应该具备： - -- ./profiler_info_x.json, -- ./ASCEND_PROFILER_OUTPUT/step_trace_time.csv, -- ./ASCEND_PROFILER_OUTPUT/trace_view.json, -- ./ASCEND_PROFILER_OUTPUT/kernel_details.csv, -- ./ASCEND_PROFILER_OUTPUT/communication.json, -- ./ASCEND_PROFILER_OUTPUT/communication_matrix.json - -或者具备： - -- analysis.db -- ascend_pytorch_profiler_{rank_id}.db - -以上csv、json文件与db文件只能存在一类，否则集群分析工具解析异常。MindSpore场景暂不支持以上db文件。 - -确认这几个文件生成后，继续下面的集群分析。 - -## 数据汇聚与解析 - -### 操作步骤 - -1. 参见《[性能工具](../README.md)》完成工具安装。建议安装最新版本。 - -2. 将所有卡的数据拷贝并汇集到一个目录下，运行以下命令，在该目录下即可生成cluster_analysis_output文件夹。 - - ```bash - msprof-analyze cluster -d {cluster profiling data path} [-m mode] [-o output_path] [--data_simplification] [--force] - ``` - - 或 - - ```bash - python3 cluster_analysis.py -d {cluster profiling data path} [-m mode] [-o output_path] [--data_simplification] [--force] - ``` - - 参数说明： - - | 参数名 | 说明 | 是否必选 | - | --------------------- | ------------------------------------------------------------ | -------- | - | --profiling_path或-d | 性能数据汇集目录。未配置-o参数时，运行分析脚本之后会在该目录下自动创建cluster_analysis_output文件夹，保存分析数据。 | 是 | - | --output_path或-o | 自定义输出路径，运行分析脚本之后会在该目录下自动创建cluster_analysis_output文件夹，保存分析数据。 | 否 | - | --mode或-m | 数据解析模式，取值详见“**--mode参数说明**”表。 | 否 | - | --data_simplification | 数据精简模式。对于数据量过大的性能数据db文件，可以通过配置该参数将数据精简，并提高工具分析效率。配置该参数表示开启数据精简，默认未配置表示关闭。 | 否 | - | --force | 强制执行cluster。配置后可强制跳过如下情况：
指定的目录、文件的用户属主不属于当前用户，忽略属主判断直接执行。
csv文件大于5G、json文件大于10G、db文件大于8G，忽略文件过大判断直接执行。
配置该参数表示开启强制执行，默认未配置表示关闭。 | 否 | - - --mode参数说明： - - | 参数名 | 说明 | 是否必选 | - | -------------------- | ------------------------------------------------------------ | -------- | - | communication_matrix | 解析通信矩阵数据。 | 否 | - | communication_time | 解析通信耗时数据。 | 否 | - | all | 同时解析通信矩阵communication_matrix和通信耗时数据communication_time，--mode参数默认值为all。 | 否 | - - - -### 交付件 - -集群分析工具的交付件通过MindStudio Insight工具展示，详见《[MindStudio Insight用户指南](https://www.hiascend.com/document/detail/zh/mindstudio/70RC2/GUI-baseddevelopmenttool/msascendinsightug/AscendInsight_0002.html)》。 - -#### cluster_step_trace_time.csv - -数据解析模式为communication_matrix、communication_time或all时均生成。 - -A列： Step数，是采集性能数据时设置的，一般来说集群性能数据采集一个step足够，如果采集多个step，需要先筛选一下。 - -B列： Type，主要分两种，rank和stage, 和后面的index强相关，可以理解为一个是单卡rank，一个是rank group（pp 并行的stage），如果type为stage，则后面D-K列信息为rank group下的最大值。 - -C列：Index，与type相关，表示卡号。 - -D列：Computing， 此列统计计算时间。 - -E列：Communication(Not Overlapped)，此列统计未被掩盖的通信耗时。 - -F列：Overlapped，统计计算与通信重叠的耗时。 - -G列：Communication，通信时间的全部耗时。 - -H列：Free，空闲时间，指device侧既不在通信也不在计算的耗时，可能在做sdma拷贝或者空等。 - -I列：Stage时间，I、J、K列属于pp并行时有效的数值，stage时间代表除receive算子时间外的时间。 - -J列：Bubble时间，指receive时间的总和。 - -K列：Communication（Not Overlapped and Exclude Receive）指剔除receive算子外的并且不被掩盖的通信时间。 - -L列：Preparing，指迭代开始到首个计算或通信算子运行的时间。 - -M列：DP Index，指集群数据按照并行策略切分后所属DP组的索引， 如果没有采集则不显示。 - -N列：PP Index，指集群数据按照并行策略切分后所属PP组的索引，如果没有采集则不显示。 - -O列：TP Index，指集群数据按照并行策略切分后所属TP组的索引，如果没有采集则不显示。 - -**Tips**：先筛选B列type为stage， 看stage间是否有问题，再筛选B列type为rank，看rank是否有问题，根据以下几点排查。 - -* 根据Computing的时间差异判断是否有慢卡，或者有负载不均衡的现象。 - -* 根据Free统计是否有host bound或者分布不均现象。 - -* 根据Communication（Not Overlapped and Exclude Receive）时间判断是否通信耗时占比过大。 - -* 根据Bubble时间的占比和理论计算公式判断bubble设置是否合理，是否stage间有不均衡现象。 - -以上时间理论上都应该处于持平状态，即最大值小于最小值5%，否则就可能出现慢卡。 - -#### cluster_communication_matrix.json - -数据解析模式为communication_matrix或all时生成。 - -直接打开json（vscode或json查看器）, 搜索"Total", 会有多个搜索结果，一般来说链路带宽信息的结构： - -```bash -{src_rank}-{dst_rank}: { - "Transport Type": "LOCAL", - "Transit Time(ms)": 0.02462, - "Transit Size(MB)": 16.777216, - "Bandwidth(GB/s)": 681.4466 -} -``` -**Tips**：可以根据rank互联的带宽以及链路类型，判断是否有慢链路的问题。 - -- "LOCAL"是片内拷贝，速度最高。 -- “HCCS”或“PCIE”是节点内片间拷贝，速度居中。 -- “RDMA”是节点间拷贝，速度最低。 - -#### cluster_communication.json - -数据解析模式为communication_time或all时生成。 - -主要为通信耗时数据。 - -#### cluster_analysis.db - -解析analysis.db或ascend_pytorch_profiler_{rank_id}.db生成的交付件，根据数据解析模式不同而解析不同的数据，可以使用MindStudio Insight工具展示。 - -#### communication_group.json - -记录通信域信息，解析analysis.db生成的交付件，collective表示集合通信域，P2P表示点对点通信，用户无须关注该文件。 +# 集群分析工具 +cluster_analyse（集群分析工具）是在集群场景下，通过此工具来进行集群数据的分析，当前主要对基于通信域的迭代内耗时分析、通信时间分析以及通信矩阵分析为主， 从而定位慢卡、慢节点以及慢链路问题。 + +## 性能数据采集 +当前集群调优工具主要支持PyTorch场景的Ascend PyTorch Profiler采集方式和MindSpore场景的MindSpore Profiler采集方式下的集群数据。 + +此工具只需要NPU的性能数据作为输入。 + +Ascend PyTorch Profiler采集方法请参见《[NPU性能数据采集](https://gitee.com/ascend/mstt/tree/master/profiler)》，MindSpore Profiler采集方法请参见《[性能调试](https://www.mindspore.cn/mindinsight/docs/zh-CN/r2.3/performance_profiling_ascend.html)》。 + +我们要求至少是L1级别的数据。 +```python +experimental_config = torch_npu.profiler._ExperimentalConfig( + profiler_level=torch_npu.profiler.ProfilerLevel.Level1 +) +``` +### 确认数据是否可用 + +打开采集到的某张卡数据(\*ascend_pt、\*ascend_ms结尾的文件夹)，可用的数据应该具备： + +- ./profiler_info_x.json, +- ./ASCEND_PROFILER_OUTPUT/step_trace_time.csv, +- ./ASCEND_PROFILER_OUTPUT/trace_view.json, +- ./ASCEND_PROFILER_OUTPUT/kernel_details.csv, +- ./ASCEND_PROFILER_OUTPUT/communication.json, +- ./ASCEND_PROFILER_OUTPUT/communication_matrix.json + +或者具备： + +- analysis.db +- ascend_pytorch_profiler_{rank_id}.db + +以上csv、json文件与db文件只能存在一类，否则集群分析工具解析异常。MindSpore场景暂不支持以上db文件。 + +确认这几个文件生成后，继续下面的集群分析。 + +## 数据汇聚与解析 + +### 操作步骤 + +1. 参见《[性能工具](../README.md)》完成工具安装。建议安装最新版本。 + +2. 将所有卡的数据拷贝并汇集到一个目录下，运行以下命令，在该目录下即可生成cluster_analysis_output文件夹。 + + ```bash + msprof-analyze cluster -d {cluster profiling data path} [-m mode] [-o output_path] [--data_simplification] [--force] + ``` + + 或 + + ```bash + python3 cluster_analysis.py -d {cluster profiling data path} [-m mode] [-o output_path] [--data_simplification] [--force] + ``` + + 参数说明： + + | 参数名 | 说明 | 是否必选 | + | --------------------- | ------------------------------------------------------------ | -------- | + | --profiling_path或-d | 性能数据汇集目录。未配置-o参数时，运行分析脚本之后会在该目录下自动创建cluster_analysis_output文件夹，保存分析数据。 | 是 | + | --output_path或-o | 自定义输出路径，运行分析脚本之后会在该目录下自动创建cluster_analysis_output文件夹，保存分析数据。 | 否 | + | --mode或-m | 数据解析模式，取值详见“**--mode参数说明**”表。 | 否 | + | --data_simplification | 数据精简模式。对于数据量过大的性能数据db文件，可以通过配置该参数将数据精简，并提高工具分析效率。配置该参数表示开启数据精简，默认未配置表示关闭。 | 否 | + | --force | 强制执行cluster。配置后可强制跳过如下情况：
指定的目录、文件的用户属主不属于当前用户，忽略属主判断直接执行。
csv文件大于5G、json文件大于10G、db文件大于8G，忽略文件过大判断直接执行。
配置该参数表示开启强制执行，默认未配置表示关闭。 | 否 | + + --mode参数说明： + + | 参数名 | 说明 | 是否必选 | + | -------------------- | ------------------------------------------------------------ | -------- | + | communication_matrix | 解析通信矩阵数据。 | 否 | + | communication_time | 解析通信耗时数据。 | 否 | + | all | 同时解析通信矩阵communication_matrix和通信耗时数据communication_time，--mode参数默认值为all。 | 否 | + + + +### 交付件 + +集群分析工具的交付件通过MindStudio Insight工具展示，详见《[MindStudio Insight用户指南](https://www.hiascend.com/document/detail/zh/mindstudio/70RC2/GUI-baseddevelopmenttool/msascendinsightug/AscendInsight_0002.html)》。 + +#### cluster_step_trace_time.csv + +数据解析模式为communication_matrix、communication_time或all时均生成。 + +A列： Step数，是采集性能数据时设置的，一般来说集群性能数据采集一个step足够，如果采集多个step，需要先筛选一下。 + +B列： Type，主要分两种，rank和stage, 和后面的index强相关，可以理解为一个是单卡rank，一个是rank group（pp 并行的stage），如果type为stage，则后面D-K列信息为rank group下的最大值。 + +C列：Index，与type相关，表示卡号。 + +D列：Computing， 此列统计计算时间。 + +E列：Communication(Not Overlapped)，此列统计未被掩盖的通信耗时。 + +F列：Overlapped，统计计算与通信重叠的耗时。 + +G列：Communication，通信时间的全部耗时。 + +H列：Free，空闲时间，指device侧既不在通信也不在计算的耗时，可能在做sdma拷贝或者空等。 + +I列：Stage时间，I、J、K列属于pp并行时有效的数值，stage时间代表除receive算子时间外的时间。 + +J列：Bubble时间，指receive时间的总和。 + +K列：Communication（Not Overlapped and Exclude Receive）指剔除receive算子外的并且不被掩盖的通信时间。 + +L列：Preparing，指迭代开始到首个计算或通信算子运行的时间。 + +M列：DP Index，指集群数据按照并行策略切分后所属DP组的索引， 如果没有采集则不显示。 + +N列：PP Index，指集群数据按照并行策略切分后所属PP组的索引，如果没有采集则不显示。 + +O列：TP Index，指集群数据按照并行策略切分后所属TP组的索引，如果没有采集则不显示。 + +**Tips**：先筛选B列type为stage， 看stage间是否有问题，再筛选B列type为rank，看rank是否有问题，根据以下几点排查。 + +* 根据Computing的时间差异判断是否有慢卡，或者有负载不均衡的现象。 + +* 根据Free统计是否有host bound或者分布不均现象。 + +* 根据Communication（Not Overlapped and Exclude Receive）时间判断是否通信耗时占比过大。 + +* 根据Bubble时间的占比和理论计算公式判断bubble设置是否合理，是否stage间有不均衡现象。 + +以上时间理论上都应该处于持平状态，即最大值小于最小值5%，否则就可能出现慢卡。 + +#### cluster_communication_matrix.json + +数据解析模式为communication_matrix或all时生成。 + +直接打开json（vscode或json查看器）, 搜索"Total", 会有多个搜索结果，一般来说链路带宽信息的结构： + +```bash +{src_rank}-{dst_rank}: { + "Transport Type": "LOCAL", + "Transit Time(ms)": 0.02462, + "Transit Size(MB)": 16.777216, + "Bandwidth(GB/s)": 681.4466 +} +``` +**Tips**：可以根据rank互联的带宽以及链路类型，判断是否有慢链路的问题。 + +- "LOCAL"是片内拷贝，速度最高。 +- “HCCS”或“PCIE”是节点内片间拷贝，速度居中。 +- “RDMA”是节点间拷贝，速度最低。 + +#### cluster_communication.json + +数据解析模式为communication_time或all时生成。 + +主要为通信耗时数据。 + +#### cluster_analysis.db + +解析analysis.db或ascend_pytorch_profiler_{rank_id}.db生成的交付件，根据数据解析模式不同而解析不同的数据，可以使用MindStudio Insight工具展示。 + +#### communication_group.json + +记录通信域信息，解析analysis.db生成的交付件，collective表示集合通信域，P2P表示点对点通信，用户无须关注该文件。 diff --git a/profiler/cluster_analyse/__init__.py b/profiler/msprof_analyze/cluster_analyse/__init__.py similarity index 100% rename from profiler/cluster_analyse/__init__.py rename to profiler/msprof_analyze/cluster_analyse/__init__.py diff --git a/profiler/cluster_analyse/analysis/__init__.py b/profiler/msprof_analyze/cluster_analyse/analysis/__init__.py similarity index 100% rename from profiler/cluster_analyse/analysis/__init__.py rename to profiler/msprof_analyze/cluster_analyse/analysis/__init__.py diff --git a/profiler/cluster_analyse/analysis/analysis_facade.py b/profiler/msprof_analyze/cluster_analyse/analysis/analysis_facade.py similarity index 97% rename from profiler/cluster_analyse/analysis/analysis_facade.py rename to profiler/msprof_analyze/cluster_analyse/analysis/analysis_facade.py index 48c06fb9eb..f7911fbd43 100644 --- a/profiler/cluster_analyse/analysis/analysis_facade.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/analysis_facade.py @@ -20,7 +20,7 @@ from analysis.comm_matrix_analysis import CommMatrixAnalysis from analysis.comm_matrix_analysis import CommMatrixAnalysisOptimized from analysis.step_trace_time_analysis import StepTraceTimeAnalysis from analysis.host_info_analysis import HostInfoAnalysis -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class AnalysisFacade: diff --git a/profiler/cluster_analyse/analysis/base_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/base_analysis.py similarity index 96% rename from profiler/cluster_analyse/analysis/base_analysis.py rename to profiler/msprof_analyze/cluster_analyse/analysis/base_analysis.py index a20a5e6da5..41086809c7 100644 --- a/profiler/cluster_analyse/analysis/base_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/base_analysis.py @@ -16,8 +16,8 @@ import logging from abc import abstractmethod -from profiler.prof_common.constant import Constant -from profiler.prof_common.file_manager import FileManager +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.prof_common.file_manager import FileManager from cluster_utils.data_transfer_adapter import DataTransferAdapter logger = logging.getLogger() diff --git a/profiler/cluster_analyse/analysis/comm_matrix_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/comm_matrix_analysis.py similarity index 99% rename from profiler/cluster_analyse/analysis/comm_matrix_analysis.py rename to profiler/msprof_analyze/cluster_analyse/analysis/comm_matrix_analysis.py index 522d9db9f2..de4a0b3957 100644 --- a/profiler/cluster_analyse/analysis/comm_matrix_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/comm_matrix_analysis.py @@ -21,7 +21,7 @@ from analysis.base_analysis import BaseAnalysis from common_func.db_manager import DBManager from common_func.utils import increase_shared_value -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant logger = logging.getLogger("cluster") diff --git a/profiler/cluster_analyse/analysis/communication_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/communication_analysis.py similarity index 99% rename from profiler/cluster_analyse/analysis/communication_analysis.py rename to profiler/msprof_analyze/cluster_analyse/analysis/communication_analysis.py index 0cac569511..b9513cb3b4 100644 --- a/profiler/cluster_analyse/analysis/communication_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/communication_analysis.py @@ -23,7 +23,7 @@ from common_func.db_manager import DBManager from common_func.utils import increase_shared_value from prof_bean.communication_bandwidth_bean import CommunicationBandwidthBean from prof_bean.communication_time_bean import CommunicationTimeBean -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant logger = logging.getLogger("cluster") diff --git a/profiler/cluster_analyse/analysis/host_info_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/host_info_analysis.py similarity index 97% rename from profiler/cluster_analyse/analysis/host_info_analysis.py rename to profiler/msprof_analyze/cluster_analyse/analysis/host_info_analysis.py index d0e4dd2eed..7f97394296 100644 --- a/profiler/cluster_analyse/analysis/host_info_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/host_info_analysis.py @@ -20,8 +20,8 @@ from analysis.base_analysis import BaseAnalysis from common_func.db_manager import DBManager from common_func.utils import increase_shared_value -from profiler.prof_common.constant import Constant -from profiler.prof_common.utils import PrintUtils +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.prof_common.utils import PrintUtils logger = logging.getLogger("cluster") diff --git a/profiler/cluster_analyse/analysis/step_trace_time_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/step_trace_time_analysis.py similarity index 98% rename from profiler/cluster_analyse/analysis/step_trace_time_analysis.py rename to profiler/msprof_analyze/cluster_analyse/analysis/step_trace_time_analysis.py index bfff4fb45e..119956a0fb 100644 --- a/profiler/cluster_analyse/analysis/step_trace_time_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/step_trace_time_analysis.py @@ -21,8 +21,8 @@ from common_func.utils import increase_shared_value from cluster_utils.parallel_strategy_calculator import ParallelStrategyCalculator from prof_bean.step_trace_time_bean import StepTraceTimeBean -from profiler.prof_common.constant import Constant -from profiler.prof_common.file_manager import FileManager +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.prof_common.file_manager import FileManager logger = logging.getLogger("cluster") diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/msprof_analyze/cluster_analyse/cluster_analysis.py similarity index 95% rename from profiler/cluster_analyse/cluster_analysis.py rename to profiler/msprof_analyze/cluster_analyse/cluster_analysis.py index 32eb1e00e7..c214e66eb1 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/cluster_analysis.py @@ -17,16 +17,16 @@ import argparse import os import logging import sys -sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) from analysis.analysis_facade import AnalysisFacade from cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor from cluster_data_preprocess.mindspore_data_preprocessor import MindsporeDataPreprocessor from communication_group.communication_group_generator import CommunicationGroupGenerator -from profiler.prof_common.additional_args_manager import AdditionalArgsManager -from profiler.prof_common.constant import Constant -from profiler.prof_common.file_manager import FileManager -from profiler.prof_common.path_manager import PathManager +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.prof_common.file_manager import FileManager +from msprof_analyze.prof_common.path_manager import PathManager COMM_FEATURE_LIST = ['all', 'communication_time', 'communication_matrix'] diff --git a/profiler/cluster_analyse/cluster_data_preprocess/__init__.py b/profiler/msprof_analyze/cluster_analyse/cluster_data_preprocess/__init__.py similarity index 100% rename from profiler/cluster_analyse/cluster_data_preprocess/__init__.py rename to profiler/msprof_analyze/cluster_analyse/cluster_data_preprocess/__init__.py diff --git a/profiler/cluster_analyse/cluster_data_preprocess/data_preprocessor.py b/profiler/msprof_analyze/cluster_analyse/cluster_data_preprocess/data_preprocessor.py similarity index 100% rename from profiler/cluster_analyse/cluster_data_preprocess/data_preprocessor.py rename to profiler/msprof_analyze/cluster_analyse/cluster_data_preprocess/data_preprocessor.py diff --git a/profiler/cluster_analyse/cluster_data_preprocess/mindspore_data_preprocessor.py b/profiler/msprof_analyze/cluster_analyse/cluster_data_preprocess/mindspore_data_preprocessor.py similarity index 100% rename from profiler/cluster_analyse/cluster_data_preprocess/mindspore_data_preprocessor.py rename to profiler/msprof_analyze/cluster_analyse/cluster_data_preprocess/mindspore_data_preprocessor.py diff --git a/profiler/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py b/profiler/msprof_analyze/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py similarity index 94% rename from profiler/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py rename to profiler/msprof_analyze/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py index bc6c8c12a9..28554aa83a 100644 --- a/profiler/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py +++ b/profiler/msprof_analyze/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py @@ -18,8 +18,8 @@ from collections import defaultdict import os import logging from cluster_data_preprocess.data_preprocessor import DataPreprocessor -from profiler.prof_common.constant import Constant -from profiler.prof_common.file_manager import FileManager +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.prof_common.file_manager import FileManager class PytorchDataPreprocessor(DataPreprocessor): diff --git a/profiler/cluster_analyse/cluster_kernels_analysis/README.md b/profiler/msprof_analyze/cluster_analyse/cluster_kernels_analysis/README.md similarity index 100% rename from profiler/cluster_analyse/cluster_kernels_analysis/README.md rename to profiler/msprof_analyze/cluster_analyse/cluster_kernels_analysis/README.md diff --git a/profiler/cluster_analyse/cluster_kernels_analysis/__init__.py b/profiler/msprof_analyze/cluster_analyse/cluster_kernels_analysis/__init__.py similarity index 100% rename from profiler/cluster_analyse/cluster_kernels_analysis/__init__.py rename to profiler/msprof_analyze/cluster_analyse/cluster_kernels_analysis/__init__.py diff --git a/profiler/cluster_analyse/cluster_kernels_analysis/cluster_prof_Info_analysis.py b/profiler/msprof_analyze/cluster_analyse/cluster_kernels_analysis/cluster_prof_Info_analysis.py similarity index 99% rename from profiler/cluster_analyse/cluster_kernels_analysis/cluster_prof_Info_analysis.py rename to profiler/msprof_analyze/cluster_analyse/cluster_kernels_analysis/cluster_prof_Info_analysis.py index 6e164e996e..053f04d158 100644 --- a/profiler/cluster_analyse/cluster_kernels_analysis/cluster_prof_Info_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/cluster_kernels_analysis/cluster_prof_Info_analysis.py @@ -29,8 +29,8 @@ from plotly.offline import plot sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from profiler.prof_common.path_manager import PathManager -from profiler.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.prof_common.path_manager import PathManager +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager MAX_READ_FILE_BYTES = 64 * 1024 * 1024 diff --git a/profiler/cluster_analyse/cluster_utils/__init__.py b/profiler/msprof_analyze/cluster_analyse/cluster_utils/__init__.py similarity index 100% rename from profiler/cluster_analyse/cluster_utils/__init__.py rename to profiler/msprof_analyze/cluster_analyse/cluster_utils/__init__.py diff --git a/profiler/cluster_analyse/cluster_utils/data_transfer_adapter.py b/profiler/msprof_analyze/cluster_analyse/cluster_utils/data_transfer_adapter.py similarity index 99% rename from profiler/cluster_analyse/cluster_utils/data_transfer_adapter.py rename to profiler/msprof_analyze/cluster_analyse/cluster_utils/data_transfer_adapter.py index 40fd4d8fdf..d21fc22446 100644 --- a/profiler/cluster_analyse/cluster_utils/data_transfer_adapter.py +++ b/profiler/msprof_analyze/cluster_analyse/cluster_utils/data_transfer_adapter.py @@ -14,7 +14,7 @@ # limitations under the License. import copy from common_func.table_constant import TableConstant -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class DataTransferAdapter(object): diff --git a/profiler/cluster_analyse/cluster_utils/parallel_algorithm.py b/profiler/msprof_analyze/cluster_analyse/cluster_utils/parallel_algorithm.py similarity index 100% rename from profiler/cluster_analyse/cluster_utils/parallel_algorithm.py rename to profiler/msprof_analyze/cluster_analyse/cluster_utils/parallel_algorithm.py diff --git a/profiler/cluster_analyse/cluster_utils/parallel_strategy_calculator.py b/profiler/msprof_analyze/cluster_analyse/cluster_utils/parallel_strategy_calculator.py similarity index 100% rename from profiler/cluster_analyse/cluster_utils/parallel_strategy_calculator.py rename to profiler/msprof_analyze/cluster_analyse/cluster_utils/parallel_strategy_calculator.py diff --git a/profiler/cluster_analyse/common_func/__init__.py b/profiler/msprof_analyze/cluster_analyse/common_func/__init__.py similarity index 100% rename from profiler/cluster_analyse/common_func/__init__.py rename to profiler/msprof_analyze/cluster_analyse/common_func/__init__.py diff --git a/profiler/cluster_analyse/common_func/db_manager.py b/profiler/msprof_analyze/cluster_analyse/common_func/db_manager.py similarity index 97% rename from profiler/cluster_analyse/common_func/db_manager.py rename to profiler/msprof_analyze/cluster_analyse/common_func/db_manager.py index 5be3ca9a57..e4301d796e 100644 --- a/profiler/cluster_analyse/common_func/db_manager.py +++ b/profiler/msprof_analyze/cluster_analyse/common_func/db_manager.py @@ -20,9 +20,9 @@ import logging from common_func.empty_class import EmptyClass from common_func.tables_config import TablesConfig -from profiler.prof_common.constant import Constant -from profiler.prof_common.file_manager import check_db_path_valid -from profiler.prof_common.utils import PrintUtils +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.prof_common.file_manager import check_db_path_valid +from msprof_analyze.prof_common.utils import PrintUtils class DBManager: """ diff --git a/profiler/cluster_analyse/common_func/empty_class.py b/profiler/msprof_analyze/cluster_analyse/common_func/empty_class.py similarity index 100% rename from profiler/cluster_analyse/common_func/empty_class.py rename to profiler/msprof_analyze/cluster_analyse/common_func/empty_class.py diff --git a/profiler/cluster_analyse/common_func/path_manager.py b/profiler/msprof_analyze/cluster_analyse/common_func/path_manager.py similarity index 98% rename from profiler/cluster_analyse/common_func/path_manager.py rename to profiler/msprof_analyze/cluster_analyse/common_func/path_manager.py index fc2e6bf75f..9cdaba6cca 100644 --- a/profiler/cluster_analyse/common_func/path_manager.py +++ b/profiler/msprof_analyze/cluster_analyse/common_func/path_manager.py @@ -17,7 +17,7 @@ import os import re import shutil import platform -from profiler.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager diff --git a/profiler/cluster_analyse/common_func/table_constant.py b/profiler/msprof_analyze/cluster_analyse/common_func/table_constant.py similarity index 100% rename from profiler/cluster_analyse/common_func/table_constant.py rename to profiler/msprof_analyze/cluster_analyse/common_func/table_constant.py diff --git a/profiler/cluster_analyse/common_func/tables_config.py b/profiler/msprof_analyze/cluster_analyse/common_func/tables_config.py similarity index 100% rename from profiler/cluster_analyse/common_func/tables_config.py rename to profiler/msprof_analyze/cluster_analyse/common_func/tables_config.py diff --git a/profiler/cluster_analyse/common_func/utils.py b/profiler/msprof_analyze/cluster_analyse/common_func/utils.py similarity index 100% rename from profiler/cluster_analyse/common_func/utils.py rename to profiler/msprof_analyze/cluster_analyse/common_func/utils.py diff --git a/profiler/cluster_analyse/communication_group/__init__.py b/profiler/msprof_analyze/cluster_analyse/communication_group/__init__.py similarity index 100% rename from profiler/cluster_analyse/communication_group/__init__.py rename to profiler/msprof_analyze/cluster_analyse/communication_group/__init__.py diff --git a/profiler/cluster_analyse/communication_group/base_communication_group.py b/profiler/msprof_analyze/cluster_analyse/communication_group/base_communication_group.py similarity index 99% rename from profiler/cluster_analyse/communication_group/base_communication_group.py rename to profiler/msprof_analyze/cluster_analyse/communication_group/base_communication_group.py index 527a2bb20d..08156f72f7 100644 --- a/profiler/cluster_analyse/communication_group/base_communication_group.py +++ b/profiler/msprof_analyze/cluster_analyse/communication_group/base_communication_group.py @@ -21,7 +21,7 @@ from multiprocessing import Pool import logging from cluster_utils.data_transfer_adapter import DataTransferAdapter -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant logger = logging.getLogger() diff --git a/profiler/cluster_analyse/communication_group/communication_db_group.py b/profiler/msprof_analyze/cluster_analyse/communication_group/communication_db_group.py similarity index 99% rename from profiler/cluster_analyse/communication_group/communication_db_group.py rename to profiler/msprof_analyze/cluster_analyse/communication_group/communication_db_group.py index 8a4d24b97b..2827e07f7b 100644 --- a/profiler/cluster_analyse/communication_group/communication_db_group.py +++ b/profiler/msprof_analyze/cluster_analyse/communication_group/communication_db_group.py @@ -18,7 +18,7 @@ import logging from common_func.db_manager import DBManager from communication_group.base_communication_group import BaseCommunicationGroup -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant logger = logging.getLogger() diff --git a/profiler/cluster_analyse/communication_group/communication_group_generator.py b/profiler/msprof_analyze/cluster_analyse/communication_group/communication_group_generator.py similarity index 96% rename from profiler/cluster_analyse/communication_group/communication_group_generator.py rename to profiler/msprof_analyze/cluster_analyse/communication_group/communication_group_generator.py index 4d551b0655..00ced8e8bd 100644 --- a/profiler/cluster_analyse/communication_group/communication_group_generator.py +++ b/profiler/msprof_analyze/cluster_analyse/communication_group/communication_group_generator.py @@ -16,7 +16,7 @@ from communication_group.communication_db_group import CommunicationDBGroup from communication_group.communication_db_group import CommunicationDBGroupOptimized from communication_group.communication_json_group import CommunicationJsonGroup -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant SIMPLIFIED = "SIMPLIFIED" diff --git a/profiler/cluster_analyse/communication_group/communication_json_group.py b/profiler/msprof_analyze/cluster_analyse/communication_group/communication_json_group.py similarity index 96% rename from profiler/cluster_analyse/communication_group/communication_json_group.py rename to profiler/msprof_analyze/cluster_analyse/communication_group/communication_json_group.py index 2d450514c5..2d39c8ef7d 100644 --- a/profiler/cluster_analyse/communication_group/communication_json_group.py +++ b/profiler/msprof_analyze/cluster_analyse/communication_group/communication_json_group.py @@ -16,7 +16,7 @@ import os from communication_group.base_communication_group import BaseCommunicationGroup -from profiler.prof_common.file_manager import FileManager +from msprof_analyze.prof_common.file_manager import FileManager class CommunicationJsonGroup(BaseCommunicationGroup): diff --git a/profiler/cluster_analyse/prof_bean/__init__.py b/profiler/msprof_analyze/cluster_analyse/prof_bean/__init__.py similarity index 100% rename from profiler/cluster_analyse/prof_bean/__init__.py rename to profiler/msprof_analyze/cluster_analyse/prof_bean/__init__.py diff --git a/profiler/cluster_analyse/prof_bean/communication_bandwidth_bean.py b/profiler/msprof_analyze/cluster_analyse/prof_bean/communication_bandwidth_bean.py similarity index 100% rename from profiler/cluster_analyse/prof_bean/communication_bandwidth_bean.py rename to profiler/msprof_analyze/cluster_analyse/prof_bean/communication_bandwidth_bean.py diff --git a/profiler/cluster_analyse/prof_bean/communication_time_bean.py b/profiler/msprof_analyze/cluster_analyse/prof_bean/communication_time_bean.py similarity index 100% rename from profiler/cluster_analyse/prof_bean/communication_time_bean.py rename to profiler/msprof_analyze/cluster_analyse/prof_bean/communication_time_bean.py diff --git a/profiler/cluster_analyse/prof_bean/step_trace_time_bean.py b/profiler/msprof_analyze/cluster_analyse/prof_bean/step_trace_time_bean.py similarity index 100% rename from profiler/cluster_analyse/prof_bean/step_trace_time_bean.py rename to profiler/msprof_analyze/cluster_analyse/prof_bean/step_trace_time_bean.py diff --git a/profiler/cluster_analyse/resources/.keep b/profiler/msprof_analyze/cluster_analyse/resources/.keep similarity index 100% rename from profiler/cluster_analyse/resources/.keep rename to profiler/msprof_analyze/cluster_analyse/resources/.keep diff --git a/profiler/compare_tools/README.md b/profiler/msprof_analyze/compare_tools/README.md similarity index 98% rename from profiler/compare_tools/README.md rename to profiler/msprof_analyze/compare_tools/README.md index 4b5f8c3b63..7505c4e1dc 100644 --- a/profiler/compare_tools/README.md +++ b/profiler/msprof_analyze/compare_tools/README.md @@ -131,7 +131,7 @@ MindSpore性能调试工具采集结果数据目录结构如下： ```bash # 进入mstt代码仓目录下的compare_tools目录 -cd mstt/profiler/compare_tools +cd mstt/profiler/msprof_analyze/compare_tools # 执行最简比对命令 python performance_compare.py [基准性能数据文件所在路径] [比对性能数据文件所在路径] --output_path=[比对结果文件存放路径] ``` @@ -193,7 +193,7 @@ MindSpore场景暂不支持。 #### 自定义比对算子 -一般情况下compare功能按照默认配置的算子进行比对，若用户需要对特定算子的性能进行比对和分析，可以通过在[compare_config.ini](https://gitee.com/ascend/mstt/blob/master/profiler/compare_tools/compare_backend/compare_config/compare_config.ini)文件中配置需要比对的算子名的识别关键词，之后再执行比对操作（msprof-analyze compare），比对结果在结果文件performance_comparison_result_{timestamp}.csv中呈现。 +一般情况下compare功能按照默认配置的算子进行比对，若用户需要对特定算子的性能进行比对和分析，可以通过在[compare_config.ini](https://gitee.com/ascend/mstt/blob/master/profiler/msprof_analyze/compare_tools/compare_backend/compare_config/compare_config.ini)文件中配置需要比对的算子名的识别关键词，之后再执行比对操作（msprof-analyze compare），比对结果在结果文件performance_comparison_result_{timestamp}.csv中呈现。 配置算子名的识别关键词为算子名称中的一部分，代表只要算子名称中包含该关键词，那么该算子会进行比对。 diff --git a/profiler/compare_tools/__init__.py b/profiler/msprof_analyze/compare_tools/__init__.py similarity index 100% rename from profiler/compare_tools/__init__.py rename to profiler/msprof_analyze/compare_tools/__init__.py diff --git a/profiler/compare_tools/compare_backend/__init__.py b/profiler/msprof_analyze/compare_tools/compare_backend/__init__.py similarity index 100% rename from profiler/compare_tools/compare_backend/__init__.py rename to profiler/msprof_analyze/compare_tools/compare_backend/__init__.py diff --git a/profiler/compare_tools/compare_backend/comparator/__init__.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/__init__.py similarity index 100% rename from profiler/compare_tools/compare_backend/comparator/__init__.py rename to profiler/msprof_analyze/compare_tools/compare_backend/comparator/__init__.py diff --git a/profiler/compare_tools/compare_backend/comparator/api_compare_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/api_compare_comparator.py similarity index 97% rename from profiler/compare_tools/compare_backend/comparator/api_compare_comparator.py rename to profiler/msprof_analyze/compare_tools/compare_backend/comparator/api_compare_comparator.py index 45cff33512..6dd4ed5fc7 100644 --- a/profiler/compare_tools/compare_backend/comparator/api_compare_comparator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/api_compare_comparator.py @@ -15,7 +15,7 @@ from compare_backend.comparator.base_comparator import BaseComparator from compare_backend.utils.common_func import update_order_id -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class ApiCompareComparator(BaseComparator): diff --git a/profiler/compare_tools/compare_backend/comparator/base_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/base_comparator.py similarity index 100% rename from profiler/compare_tools/compare_backend/comparator/base_comparator.py rename to profiler/msprof_analyze/compare_tools/compare_backend/comparator/base_comparator.py diff --git a/profiler/compare_tools/compare_backend/comparator/communication_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/communication_comparator.py similarity index 96% rename from profiler/compare_tools/compare_backend/comparator/communication_comparator.py rename to profiler/msprof_analyze/compare_tools/compare_backend/comparator/communication_comparator.py index 7306cf3ddd..a9aa221d33 100644 --- a/profiler/compare_tools/compare_backend/comparator/communication_comparator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/communication_comparator.py @@ -16,7 +16,7 @@ from compare_backend.comparator.base_comparator import BaseComparator from compare_backend.compare_bean.communication_bean import CommunicationBean from compare_backend.utils.common_func import update_order_id -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class CommunicationComparator(BaseComparator): diff --git a/profiler/compare_tools/compare_backend/comparator/kernel_compare_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/kernel_compare_comparator.py similarity index 97% rename from profiler/compare_tools/compare_backend/comparator/kernel_compare_comparator.py rename to profiler/msprof_analyze/compare_tools/compare_backend/comparator/kernel_compare_comparator.py index 0aef6f18af..51fce7c156 100644 --- a/profiler/compare_tools/compare_backend/comparator/kernel_compare_comparator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/kernel_compare_comparator.py @@ -15,7 +15,7 @@ from compare_backend.comparator.base_comparator import BaseComparator from compare_backend.utils.common_func import update_order_id -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class KernelCompareComparator(BaseComparator): diff --git a/profiler/compare_tools/compare_backend/comparator/kernel_type_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/kernel_type_comparator.py similarity index 96% rename from profiler/compare_tools/compare_backend/comparator/kernel_type_comparator.py rename to profiler/msprof_analyze/compare_tools/compare_backend/comparator/kernel_type_comparator.py index a939bedfa1..ebca4346fa 100644 --- a/profiler/compare_tools/compare_backend/comparator/kernel_type_comparator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/kernel_type_comparator.py @@ -16,7 +16,7 @@ from compare_backend.comparator.base_comparator import BaseComparator from compare_backend.compare_bean.origin_data_bean.op_stastic_bean import OpStatisticBean from compare_backend.utils.common_func import update_order_id -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class KernelTypeComparator(BaseComparator): diff --git a/profiler/compare_tools/compare_backend/comparator/module_comparetor.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/module_comparetor.py similarity index 95% rename from profiler/compare_tools/compare_backend/comparator/module_comparetor.py rename to profiler/msprof_analyze/compare_tools/compare_backend/comparator/module_comparetor.py index 6c7133bcf9..3b13c24411 100644 --- a/profiler/compare_tools/compare_backend/comparator/module_comparetor.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/module_comparetor.py @@ -15,8 +15,8 @@ from compare_backend.comparator.base_comparator import BaseComparator from compare_backend.utils.common_func import update_order_id -from profiler.prof_common.constant import Constant -from profiler.prof_common.logger import get_logger +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.prof_common.logger import get_logger logger = get_logger() diff --git a/profiler/compare_tools/compare_backend/comparator/module_statistic_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/module_statistic_comparator.py similarity index 100% rename from profiler/compare_tools/compare_backend/comparator/module_statistic_comparator.py rename to profiler/msprof_analyze/compare_tools/compare_backend/comparator/module_statistic_comparator.py diff --git a/profiler/compare_tools/compare_backend/comparator/operator_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/operator_comparator.py similarity index 100% rename from profiler/compare_tools/compare_backend/comparator/operator_comparator.py rename to profiler/msprof_analyze/compare_tools/compare_backend/comparator/operator_comparator.py diff --git a/profiler/compare_tools/compare_backend/comparator/operator_statistic_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/operator_statistic_comparator.py similarity index 100% rename from profiler/compare_tools/compare_backend/comparator/operator_statistic_comparator.py rename to profiler/msprof_analyze/compare_tools/compare_backend/comparator/operator_statistic_comparator.py diff --git a/profiler/compare_tools/compare_backend/comparator/overall_metrics_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/overall_metrics_comparator.py similarity index 96% rename from profiler/compare_tools/compare_backend/comparator/overall_metrics_comparator.py rename to profiler/msprof_analyze/compare_tools/compare_backend/comparator/overall_metrics_comparator.py index 099e1cc71a..ad853f2db8 100644 --- a/profiler/compare_tools/compare_backend/comparator/overall_metrics_comparator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/overall_metrics_comparator.py @@ -17,7 +17,7 @@ from math import isclose from compare_backend.comparator.base_comparator import BaseComparator from compare_backend.utils.excel_config import ExcelConfig -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class OverallMetricsComparator(BaseComparator): diff --git a/profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/overall_performance_comparator.py similarity index 99% rename from profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py rename to profiler/msprof_analyze/compare_tools/compare_backend/comparator/overall_performance_comparator.py index 9f2d485e94..0eee314ee1 100644 --- a/profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/overall_performance_comparator.py @@ -14,7 +14,7 @@ # limitations under the License. from compare_backend.comparator.base_comparator import BaseComparator -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class OverallPerformanceComparator(BaseComparator): diff --git a/profiler/compare_tools/compare_backend/compare_bean/__init__.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/__init__.py similarity index 100% rename from profiler/compare_tools/compare_backend/compare_bean/__init__.py rename to profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/__init__.py diff --git a/profiler/compare_tools/compare_backend/compare_bean/api_compare_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/api_compare_bean.py similarity index 97% rename from profiler/compare_tools/compare_backend/compare_bean/api_compare_bean.py rename to profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/api_compare_bean.py index f0391e0767..5178485700 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/api_compare_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/api_compare_bean.py @@ -15,7 +15,7 @@ from compare_backend.utils.common_func import calculate_diff_ratio from compare_backend.utils.excel_config import ExcelConfig -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class ApiInfo: diff --git a/profiler/compare_tools/compare_backend/compare_bean/communication_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/communication_bean.py similarity index 98% rename from profiler/compare_tools/compare_backend/compare_bean/communication_bean.py rename to profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/communication_bean.py index be87cf7563..f840eb653c 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/communication_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/communication_bean.py @@ -15,7 +15,7 @@ from compare_backend.utils.excel_config import ExcelConfig from compare_backend.utils.common_func import calculate_diff_ratio -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class CommunicationInfo: diff --git a/profiler/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py similarity index 98% rename from profiler/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py rename to profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py index 889c28f3ae..ff10d94dd5 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py @@ -15,7 +15,7 @@ from compare_backend.utils.common_func import calculate_diff_ratio, convert_to_float from compare_backend.utils.excel_config import ExcelConfig -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class KernelCompareInfo: diff --git a/profiler/compare_tools/compare_backend/compare_bean/kernel_type_compare_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/kernel_type_compare_bean.py similarity index 97% rename from profiler/compare_tools/compare_backend/compare_bean/kernel_type_compare_bean.py rename to profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/kernel_type_compare_bean.py index dd5ff81a87..b129facc94 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/kernel_type_compare_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/kernel_type_compare_bean.py @@ -15,7 +15,7 @@ from compare_backend.utils.common_func import calculate_diff_ratio from compare_backend.utils.excel_config import ExcelConfig -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class KernelTypeCompareBean: diff --git a/profiler/compare_tools/compare_backend/compare_bean/memory_compare_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/memory_compare_bean.py similarity index 97% rename from profiler/compare_tools/compare_backend/compare_bean/memory_compare_bean.py rename to profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/memory_compare_bean.py index fc4d79bb67..5e2720a2bb 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/memory_compare_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/memory_compare_bean.py @@ -17,7 +17,7 @@ from compare_backend.utils.excel_config import ExcelConfig from compare_backend.utils.torch_op_node import TorchOpNode from compare_backend.utils.tree_builder import TreeBuilder -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class MemoryCompareBean: diff --git a/profiler/compare_tools/compare_backend/compare_bean/memory_statistic_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/memory_statistic_bean.py similarity index 97% rename from profiler/compare_tools/compare_backend/compare_bean/memory_statistic_bean.py rename to profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/memory_statistic_bean.py index 838cee19aa..18ca502e72 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/memory_statistic_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/memory_statistic_bean.py @@ -16,7 +16,7 @@ from compare_backend.utils.common_func import calculate_diff_ratio from compare_backend.utils.tree_builder import TreeBuilder from compare_backend.utils.excel_config import ExcelConfig -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class MemoryStatisticBean: diff --git a/profiler/compare_tools/compare_backend/compare_bean/module_compare_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/module_compare_bean.py similarity index 98% rename from profiler/compare_tools/compare_backend/compare_bean/module_compare_bean.py rename to profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/module_compare_bean.py index 91e0070da6..9dee49acda 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/module_compare_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/module_compare_bean.py @@ -18,7 +18,7 @@ from compare_backend.utils.module_node import ModuleNode from compare_backend.utils.name_function import NameFunction from compare_backend.utils.torch_op_node import TorchOpNode -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class ModuleCompareBean: diff --git a/profiler/compare_tools/compare_backend/compare_bean/module_statistic_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/module_statistic_bean.py similarity index 99% rename from profiler/compare_tools/compare_backend/compare_bean/module_statistic_bean.py rename to profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/module_statistic_bean.py index 1734434a47..b4448d1eb9 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/module_statistic_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/module_statistic_bean.py @@ -17,7 +17,7 @@ import re from compare_backend.utils.common_func import calculate_diff_ratio from compare_backend.utils.excel_config import ExcelConfig -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class ModuleStatisticBean: diff --git a/profiler/compare_tools/compare_backend/compare_bean/operator_compare_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/operator_compare_bean.py similarity index 97% rename from profiler/compare_tools/compare_backend/compare_bean/operator_compare_bean.py rename to profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/operator_compare_bean.py index 9ad85ca33f..b475a59ef5 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/operator_compare_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/operator_compare_bean.py @@ -17,7 +17,7 @@ from compare_backend.utils.excel_config import ExcelConfig from compare_backend.utils.torch_op_node import TorchOpNode from compare_backend.utils.tree_builder import TreeBuilder -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class OperatorCompareBean: diff --git a/profiler/compare_tools/compare_backend/compare_bean/operator_statistic_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/operator_statistic_bean.py similarity index 97% rename from profiler/compare_tools/compare_backend/compare_bean/operator_statistic_bean.py rename to profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/operator_statistic_bean.py index 769f3afb69..11d555576c 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/operator_statistic_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/operator_statistic_bean.py @@ -16,7 +16,7 @@ from compare_backend.utils.common_func import calculate_diff_ratio from compare_backend.utils.excel_config import ExcelConfig from compare_backend.utils.tree_builder import TreeBuilder -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class OperatorStatisticBean: diff --git a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/__init__.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/__init__.py similarity index 100% rename from profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/__init__.py rename to profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/__init__.py diff --git a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/compare_event.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/compare_event.py similarity index 98% rename from profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/compare_event.py rename to profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/compare_event.py index 9615815ac0..8c9d84075f 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/compare_event.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/compare_event.py @@ -16,7 +16,7 @@ from decimal import Decimal from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class KernelEvent: diff --git a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py similarity index 98% rename from profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py rename to profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py index a6a3066d02..f1ff9f575a 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py @@ -20,7 +20,7 @@ import pandas as pd from compare_backend.utils.common_func import convert_to_float, convert_to_decimal from compare_backend.compare_config.compare_config import CompareConfig -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class KernelDetailsBean: diff --git a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/memory_record_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/memory_record_bean.py similarity index 100% rename from profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/memory_record_bean.py rename to profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/memory_record_bean.py diff --git a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/op_stastic_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/op_stastic_bean.py similarity index 93% rename from profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/op_stastic_bean.py rename to profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/op_stastic_bean.py index 7dde163f16..14d7ff1370 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/op_stastic_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/op_stastic_bean.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from profiler.prof_common.utils import convert_to_float, convert_to_int +from msprof_analyze.prof_common.utils import convert_to_float, convert_to_int class OpStatisticBean: diff --git a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/operator_memory_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/operator_memory_bean.py similarity index 100% rename from profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/operator_memory_bean.py rename to profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/operator_memory_bean.py diff --git a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py similarity index 99% rename from profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py rename to profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py index 9967153287..9afcab5ec0 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py @@ -17,7 +17,7 @@ from decimal import Decimal from compare_backend.utils.common_func import convert_to_float, convert_to_decimal from compare_backend.compare_config.compare_config import CompareConfig -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class TraceEventBean: diff --git a/profiler/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py similarity index 99% rename from profiler/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py rename to profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py index 5fc025b886..c6c8a03ba8 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py @@ -18,7 +18,7 @@ from compare_backend.compare_bean.profiling_info import ProfilingInfo from compare_backend.utils.common_func import calculate_diff_ratio from compare_backend.utils.excel_config import ExcelConfig, CellFormatType -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class OverallMetricsBean: diff --git a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/profiling_info.py similarity index 99% rename from profiler/compare_tools/compare_backend/compare_bean/profiling_info.py rename to profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/profiling_info.py index 75e623f07e..9c803b38eb 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/profiling_info.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class ProfilingInfo: diff --git a/profiler/compare_tools/compare_backend/compare_config/__init__.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_config/__init__.py similarity index 100% rename from profiler/compare_tools/compare_backend/compare_config/__init__.py rename to profiler/msprof_analyze/compare_tools/compare_backend/compare_config/__init__.py diff --git a/profiler/compare_tools/compare_backend/compare_config/compare_config.ini b/profiler/msprof_analyze/compare_tools/compare_backend/compare_config/compare_config.ini similarity index 100% rename from profiler/compare_tools/compare_backend/compare_config/compare_config.ini rename to profiler/msprof_analyze/compare_tools/compare_backend/compare_config/compare_config.ini diff --git a/profiler/compare_tools/compare_backend/compare_config/compare_config.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_config/compare_config.py similarity index 97% rename from profiler/compare_tools/compare_backend/compare_config/compare_config.py rename to profiler/msprof_analyze/compare_tools/compare_backend/compare_config/compare_config.py index 30358df08e..a44c074ea6 100644 --- a/profiler/compare_tools/compare_backend/compare_config/compare_config.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_config/compare_config.py @@ -16,7 +16,7 @@ import os from compare_backend.utils.singleton import Singleton -from profiler.prof_common.utils import SafeConfigReader +from msprof_analyze.prof_common.utils import SafeConfigReader @Singleton diff --git a/profiler/compare_tools/compare_backend/comparison_generator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparison_generator.py similarity index 94% rename from profiler/compare_tools/compare_backend/comparison_generator.py rename to profiler/msprof_analyze/compare_tools/compare_backend/comparison_generator.py index c72bc465dd..4a880822ae 100644 --- a/profiler/compare_tools/compare_backend/comparison_generator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparison_generator.py @@ -19,9 +19,9 @@ from compare_backend.interface.compare_interface import CompareInterface from compare_backend.profiling_parser.gpu_profiling_parser import GPUProfilingParser from compare_backend.profiling_parser.npu_profiling_parser import NPUProfilingParser from compare_backend.utils.args_manager import ArgsManager -from profiler.prof_common.constant import Constant -from profiler.prof_common.additional_args_manager import AdditionalArgsManager -from profiler.prof_common.logger import get_logger +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.prof_common.logger import get_logger logger = get_logger() diff --git a/profiler/compare_tools/compare_backend/data_prepare/__init__.py b/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/__init__.py similarity index 100% rename from profiler/compare_tools/compare_backend/data_prepare/__init__.py rename to profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/__init__.py diff --git a/profiler/compare_tools/compare_backend/data_prepare/module_data_prepare.py b/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/module_data_prepare.py similarity index 98% rename from profiler/compare_tools/compare_backend/data_prepare/module_data_prepare.py rename to profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/module_data_prepare.py index 4ce32b543b..52df18c78b 100644 --- a/profiler/compare_tools/compare_backend/data_prepare/module_data_prepare.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/module_data_prepare.py @@ -20,7 +20,7 @@ from compare_backend.profiling_parser.base_profiling_parser import ProfilingResu from compare_backend.utils.module_node import ModuleNode from compare_backend.utils.tree_builder import TreeBuilder -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class ModuleDataPrepare: diff --git a/profiler/compare_tools/compare_backend/data_prepare/operator_data_prepare.py b/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/operator_data_prepare.py similarity index 96% rename from profiler/compare_tools/compare_backend/data_prepare/operator_data_prepare.py rename to profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/operator_data_prepare.py index 24d4cb90a8..003a023221 100644 --- a/profiler/compare_tools/compare_backend/data_prepare/operator_data_prepare.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/operator_data_prepare.py @@ -15,8 +15,8 @@ from compare_backend.profiling_parser.base_profiling_parser import ProfilingResult from compare_backend.utils.tree_builder import TreeBuilder -from profiler.prof_common.constant import Constant -from profiler.prof_common.logger import get_logger +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.prof_common.logger import get_logger logger = get_logger() diff --git a/profiler/compare_tools/compare_backend/data_prepare/sequence_pre_matching.py b/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/sequence_pre_matching.py similarity index 99% rename from profiler/compare_tools/compare_backend/data_prepare/sequence_pre_matching.py rename to profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/sequence_pre_matching.py index 364e5e531b..b8e674fe98 100644 --- a/profiler/compare_tools/compare_backend/data_prepare/sequence_pre_matching.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/sequence_pre_matching.py @@ -19,7 +19,7 @@ from compare_backend.utils.common_func import longest_common_subsequence_matchin from compare_backend.utils.torch_op_node import TorchOpNode from compare_backend.utils.module_node import ModuleNode -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class SequencePreMatching: diff --git a/profiler/compare_tools/compare_backend/disaggregate/__init__.py b/profiler/msprof_analyze/compare_tools/compare_backend/disaggregate/__init__.py similarity index 100% rename from profiler/compare_tools/compare_backend/disaggregate/__init__.py rename to profiler/msprof_analyze/compare_tools/compare_backend/disaggregate/__init__.py diff --git a/profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py b/profiler/msprof_analyze/compare_tools/compare_backend/disaggregate/overall_perf_interface.py similarity index 96% rename from profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py rename to profiler/msprof_analyze/compare_tools/compare_backend/disaggregate/overall_perf_interface.py index 2a68960df2..c59b35dd66 100644 --- a/profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/disaggregate/overall_perf_interface.py @@ -17,9 +17,9 @@ from compare_backend.profiling_parser.npu_profiling_parser import NPUProfilingPa from compare_backend.utils.args_manager import ArgsManager from compare_backend.utils.compare_args import Args -from profiler.prof_common.constant import Constant -from profiler.prof_common.logger import get_logger -from profiler.prof_common.path_manager import PathManager +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.prof_common.logger import get_logger +from msprof_analyze.prof_common.path_manager import PathManager logger = get_logger() diff --git a/profiler/compare_tools/compare_backend/generator/__init__.py b/profiler/msprof_analyze/compare_tools/compare_backend/generator/__init__.py similarity index 100% rename from profiler/compare_tools/compare_backend/generator/__init__.py rename to profiler/msprof_analyze/compare_tools/compare_backend/generator/__init__.py diff --git a/profiler/compare_tools/compare_backend/generator/base_generator.py b/profiler/msprof_analyze/compare_tools/compare_backend/generator/base_generator.py similarity index 100% rename from profiler/compare_tools/compare_backend/generator/base_generator.py rename to profiler/msprof_analyze/compare_tools/compare_backend/generator/base_generator.py diff --git a/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py b/profiler/msprof_analyze/compare_tools/compare_backend/generator/detail_performance_generator.py similarity index 98% rename from profiler/compare_tools/compare_backend/generator/detail_performance_generator.py rename to profiler/msprof_analyze/compare_tools/compare_backend/generator/detail_performance_generator.py index 0425c3f796..d48353be9c 100644 --- a/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/generator/detail_performance_generator.py @@ -41,8 +41,8 @@ from compare_backend.compare_bean.kernel_type_compare_bean import KernelTypeComp from compare_backend.view.excel_view import ExcelView from compare_backend.data_prepare.sequence_pre_matching import SequencePreMatching -from profiler.prof_common.constant import Constant -from profiler.prof_common.logger import get_logger +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.prof_common.logger import get_logger logger = get_logger() diff --git a/profiler/compare_tools/compare_backend/generator/overall_performance_generator.py b/profiler/msprof_analyze/compare_tools/compare_backend/generator/overall_performance_generator.py similarity index 96% rename from profiler/compare_tools/compare_backend/generator/overall_performance_generator.py rename to profiler/msprof_analyze/compare_tools/compare_backend/generator/overall_performance_generator.py index 76d491eb4e..a162353e89 100644 --- a/profiler/compare_tools/compare_backend/generator/overall_performance_generator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/generator/overall_performance_generator.py @@ -17,7 +17,7 @@ from compare_backend.compare_bean.profiling_info import ProfilingInfo from compare_backend.generator.base_generator import BaseGenerator from compare_backend.view.screen_view import ScreenView -from profiler.prof_common.logger import get_logger +from msprof_analyze.prof_common.logger import get_logger logger = get_logger() diff --git a/profiler/compare_tools/compare_backend/interface/__init__.py b/profiler/msprof_analyze/compare_tools/compare_backend/interface/__init__.py similarity index 100% rename from profiler/compare_tools/compare_backend/interface/__init__.py rename to profiler/msprof_analyze/compare_tools/compare_backend/interface/__init__.py diff --git a/profiler/compare_tools/compare_backend/interface/compare_interface.py b/profiler/msprof_analyze/compare_tools/compare_backend/interface/compare_interface.py similarity index 98% rename from profiler/compare_tools/compare_backend/interface/compare_interface.py rename to profiler/msprof_analyze/compare_tools/compare_backend/interface/compare_interface.py index 98771e9241..58e4a0dbeb 100644 --- a/profiler/compare_tools/compare_backend/interface/compare_interface.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/interface/compare_interface.py @@ -23,7 +23,7 @@ from compare_backend.data_prepare.sequence_pre_matching import SequencePreMatchi from compare_backend.comparator.kernel_type_comparator import KernelTypeComparator from compare_backend.compare_bean.kernel_type_compare_bean import KernelTypeCompareBean -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class CompareInterface: diff --git a/profiler/compare_tools/compare_backend/interface/overall_interface.py b/profiler/msprof_analyze/compare_tools/compare_backend/interface/overall_interface.py similarity index 95% rename from profiler/compare_tools/compare_backend/interface/overall_interface.py rename to profiler/msprof_analyze/compare_tools/compare_backend/interface/overall_interface.py index a4b444c5cd..ab1f2b23d0 100644 --- a/profiler/compare_tools/compare_backend/interface/overall_interface.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/interface/overall_interface.py @@ -14,7 +14,7 @@ # limitations under the License. from compare_backend.comparator.overall_performance_comparator import OverallPerformanceComparator from compare_backend.compare_bean.profiling_info import ProfilingInfo -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class OverallInterface: diff --git a/profiler/compare_tools/compare_backend/profiling_parser/__init__.py b/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/__init__.py similarity index 100% rename from profiler/compare_tools/compare_backend/profiling_parser/__init__.py rename to profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/__init__.py diff --git a/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py b/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py similarity index 98% rename from profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py rename to profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py index 2929219284..c9a31a9269 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py @@ -21,10 +21,10 @@ from compare_backend.compare_bean.origin_data_bean.compare_event import KernelEv from compare_backend.compare_bean.origin_data_bean.kernel_details_bean import KernelDetailsBean from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean from compare_backend.compare_bean.profiling_info import ProfilingInfo -from profiler.prof_common.constant import Constant -from profiler.prof_common.file_manager import FileManager -from profiler.prof_common.logger import get_logger -from profiler.prof_common.path_manager import PathManager +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.prof_common.file_manager import FileManager +from msprof_analyze.prof_common.logger import get_logger +from msprof_analyze.prof_common.path_manager import PathManager logger = get_logger() diff --git a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py b/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py similarity index 98% rename from profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py rename to profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py index 6cbe624235..b69950deed 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py @@ -17,8 +17,8 @@ from collections import defaultdict, Counter from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean from compare_backend.profiling_parser.base_profiling_parser import BaseProfilingParser -from profiler.prof_common.constant import Constant -from profiler.prof_common.logger import get_logger +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.prof_common.logger import get_logger logger = get_logger() diff --git a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py b/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py similarity index 98% rename from profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py rename to profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py index 094c1b3d97..941acedf80 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py @@ -22,10 +22,10 @@ from compare_backend.compare_bean.origin_data_bean.operator_memory_bean import O from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean from compare_backend.profiling_parser.base_profiling_parser import BaseProfilingParser -from profiler.compare_tools.compare_backend.compare_bean.origin_data_bean.op_stastic_bean import OpStatisticBean -from profiler.prof_common.constant import Constant -from profiler.prof_common.file_manager import FileManager -from profiler.prof_common.logger import get_logger +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.op_stastic_bean import OpStatisticBean +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.prof_common.file_manager import FileManager +from msprof_analyze.prof_common.logger import get_logger logger = get_logger() diff --git a/profiler/compare_tools/compare_backend/utils/__init__.py b/profiler/msprof_analyze/compare_tools/compare_backend/utils/__init__.py similarity index 100% rename from profiler/compare_tools/compare_backend/utils/__init__.py rename to profiler/msprof_analyze/compare_tools/compare_backend/utils/__init__.py diff --git a/profiler/compare_tools/compare_backend/utils/args_manager.py b/profiler/msprof_analyze/compare_tools/compare_backend/utils/args_manager.py similarity index 97% rename from profiler/compare_tools/compare_backend/utils/args_manager.py rename to profiler/msprof_analyze/compare_tools/compare_backend/utils/args_manager.py index ab35e5b5a3..3e01918363 100644 --- a/profiler/compare_tools/compare_backend/utils/args_manager.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/utils/args_manager.py @@ -16,10 +16,10 @@ import os.path import re from compare_backend.utils.singleton import Singleton -from profiler.prof_common.constant import Constant -from profiler.prof_common.file_manager import FileManager -from profiler.prof_common.logger import get_logger -from profiler.prof_common.path_manager import PathManager +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.prof_common.file_manager import FileManager +from msprof_analyze.prof_common.logger import get_logger +from msprof_analyze.prof_common.path_manager import PathManager logger = get_logger() diff --git a/profiler/compare_tools/compare_backend/utils/common_func.py b/profiler/msprof_analyze/compare_tools/compare_backend/utils/common_func.py similarity index 98% rename from profiler/compare_tools/compare_backend/utils/common_func.py rename to profiler/msprof_analyze/compare_tools/compare_backend/utils/common_func.py index c5c2ce748e..ac9b4726ea 100644 --- a/profiler/compare_tools/compare_backend/utils/common_func.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/utils/common_func.py @@ -16,7 +16,7 @@ from decimal import Decimal import numpy as np -from profiler.prof_common.logger import get_logger +from msprof_analyze.prof_common.logger import get_logger logger = get_logger() diff --git a/profiler/compare_tools/compare_backend/utils/compare_args.py b/profiler/msprof_analyze/compare_tools/compare_backend/utils/compare_args.py similarity index 100% rename from profiler/compare_tools/compare_backend/utils/compare_args.py rename to profiler/msprof_analyze/compare_tools/compare_backend/utils/compare_args.py diff --git a/profiler/compare_tools/compare_backend/utils/excel_config.py b/profiler/msprof_analyze/compare_tools/compare_backend/utils/excel_config.py similarity index 99% rename from profiler/compare_tools/compare_backend/utils/excel_config.py rename to profiler/msprof_analyze/compare_tools/compare_backend/utils/excel_config.py index 1977106fd5..75b1c64b9e 100644 --- a/profiler/compare_tools/compare_backend/utils/excel_config.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/utils/excel_config.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class CellFormatType: diff --git a/profiler/compare_tools/compare_backend/utils/module_node.py b/profiler/msprof_analyze/compare_tools/compare_backend/utils/module_node.py similarity index 100% rename from profiler/compare_tools/compare_backend/utils/module_node.py rename to profiler/msprof_analyze/compare_tools/compare_backend/utils/module_node.py diff --git a/profiler/compare_tools/compare_backend/utils/name_function.py b/profiler/msprof_analyze/compare_tools/compare_backend/utils/name_function.py similarity index 100% rename from profiler/compare_tools/compare_backend/utils/name_function.py rename to profiler/msprof_analyze/compare_tools/compare_backend/utils/name_function.py diff --git a/profiler/compare_tools/compare_backend/utils/singleton.py b/profiler/msprof_analyze/compare_tools/compare_backend/utils/singleton.py similarity index 100% rename from profiler/compare_tools/compare_backend/utils/singleton.py rename to profiler/msprof_analyze/compare_tools/compare_backend/utils/singleton.py diff --git a/profiler/compare_tools/compare_backend/utils/torch_op_node.py b/profiler/msprof_analyze/compare_tools/compare_backend/utils/torch_op_node.py similarity index 98% rename from profiler/compare_tools/compare_backend/utils/torch_op_node.py rename to profiler/msprof_analyze/compare_tools/compare_backend/utils/torch_op_node.py index bd49c65df6..9692241a96 100644 --- a/profiler/compare_tools/compare_backend/utils/torch_op_node.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/utils/torch_op_node.py @@ -15,7 +15,7 @@ from compare_backend.compare_bean.origin_data_bean.compare_event import MemoryEvent from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class TorchOpNode: diff --git a/profiler/compare_tools/compare_backend/utils/tree_builder.py b/profiler/msprof_analyze/compare_tools/compare_backend/utils/tree_builder.py similarity index 100% rename from profiler/compare_tools/compare_backend/utils/tree_builder.py rename to profiler/msprof_analyze/compare_tools/compare_backend/utils/tree_builder.py diff --git a/profiler/compare_tools/compare_backend/view/__init__.py b/profiler/msprof_analyze/compare_tools/compare_backend/view/__init__.py similarity index 100% rename from profiler/compare_tools/compare_backend/view/__init__.py rename to profiler/msprof_analyze/compare_tools/compare_backend/view/__init__.py diff --git a/profiler/compare_tools/compare_backend/view/base_view.py b/profiler/msprof_analyze/compare_tools/compare_backend/view/base_view.py similarity index 100% rename from profiler/compare_tools/compare_backend/view/base_view.py rename to profiler/msprof_analyze/compare_tools/compare_backend/view/base_view.py diff --git a/profiler/compare_tools/compare_backend/view/excel_view.py b/profiler/msprof_analyze/compare_tools/compare_backend/view/excel_view.py similarity index 95% rename from profiler/compare_tools/compare_backend/view/excel_view.py rename to profiler/msprof_analyze/compare_tools/compare_backend/view/excel_view.py index 7074f68050..7cf3a8554c 100644 --- a/profiler/compare_tools/compare_backend/view/excel_view.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/view/excel_view.py @@ -18,7 +18,7 @@ from xlsxwriter import Workbook from compare_backend.view.base_view import BaseView from compare_backend.view.work_sheet_creator import WorkSheetCreator -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class ExcelView(BaseView): diff --git a/profiler/compare_tools/compare_backend/view/screen_view.py b/profiler/msprof_analyze/compare_tools/compare_backend/view/screen_view.py similarity index 100% rename from profiler/compare_tools/compare_backend/view/screen_view.py rename to profiler/msprof_analyze/compare_tools/compare_backend/view/screen_view.py diff --git a/profiler/compare_tools/compare_backend/view/work_sheet_creator.py b/profiler/msprof_analyze/compare_tools/compare_backend/view/work_sheet_creator.py similarity index 100% rename from profiler/compare_tools/compare_backend/view/work_sheet_creator.py rename to profiler/msprof_analyze/compare_tools/compare_backend/view/work_sheet_creator.py diff --git a/profiler/compare_tools/compare_interface/__init__.py b/profiler/msprof_analyze/compare_tools/compare_interface/__init__.py similarity index 100% rename from profiler/compare_tools/compare_interface/__init__.py rename to profiler/msprof_analyze/compare_tools/compare_interface/__init__.py diff --git a/profiler/compare_tools/compare_interface/comparison_interface.py b/profiler/msprof_analyze/compare_tools/compare_interface/comparison_interface.py similarity index 92% rename from profiler/compare_tools/compare_interface/comparison_interface.py rename to profiler/msprof_analyze/compare_tools/compare_interface/comparison_interface.py index d657a5d1fe..bb3a18eb4b 100644 --- a/profiler/compare_tools/compare_interface/comparison_interface.py +++ b/profiler/msprof_analyze/compare_tools/compare_interface/comparison_interface.py @@ -22,9 +22,9 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from compare_backend.comparison_generator import ComparisonGenerator from compare_backend.disaggregate.overall_perf_interface import OverallPerfInterface from compare_backend.utils.compare_args import Args -from profiler.prof_common.constant import Constant -from profiler.prof_common.analyze_dict import AnalyzeDict -from profiler.prof_common.logger import get_logger +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.prof_common.analyze_dict import AnalyzeDict +from msprof_analyze.prof_common.logger import get_logger logger = get_logger() diff --git a/profiler/compare_tools/img/OverallMetrics.png b/profiler/msprof_analyze/compare_tools/img/OverallMetrics.png similarity index 100% rename from profiler/compare_tools/img/OverallMetrics.png rename to profiler/msprof_analyze/compare_tools/img/OverallMetrics.png diff --git a/profiler/compare_tools/img/config.PNG b/profiler/msprof_analyze/compare_tools/img/config.PNG similarity index 100% rename from profiler/compare_tools/img/config.PNG rename to profiler/msprof_analyze/compare_tools/img/config.PNG diff --git a/profiler/compare_tools/performance_compare.py b/profiler/msprof_analyze/compare_tools/performance_compare.py similarity index 94% rename from profiler/compare_tools/performance_compare.py rename to profiler/msprof_analyze/compare_tools/performance_compare.py index ee1033de65..c934d7ec8c 100644 --- a/profiler/compare_tools/performance_compare.py +++ b/profiler/msprof_analyze/compare_tools/performance_compare.py @@ -20,12 +20,12 @@ import sys sys.path.append( os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "cluster_analyse")) -sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) from compare_backend.comparison_generator import ComparisonGenerator -from profiler.prof_common.analyze_dict import AnalyzeDict -from profiler.prof_common.logger import get_logger -from profiler.prof_common.path_manager import PathManager +from msprof_analyze.prof_common.analyze_dict import AnalyzeDict +from msprof_analyze.prof_common.logger import get_logger +from msprof_analyze.prof_common.path_manager import PathManager logger = get_logger() diff --git a/profiler/config/config.ini b/profiler/msprof_analyze/config/config.ini similarity index 100% rename from profiler/config/config.ini rename to profiler/msprof_analyze/config/config.ini diff --git a/profiler/example/__init__.py b/profiler/msprof_analyze/example/__init__.py similarity index 100% rename from profiler/example/__init__.py rename to profiler/msprof_analyze/example/__init__.py diff --git a/profiler/example/mstx_torch_plugin/README.md b/profiler/msprof_analyze/example/mstx_torch_plugin/README.md similarity index 100% rename from profiler/example/mstx_torch_plugin/README.md rename to profiler/msprof_analyze/example/mstx_torch_plugin/README.md diff --git a/profiler/example/mstx_torch_plugin/__init__.py b/profiler/msprof_analyze/example/mstx_torch_plugin/__init__.py similarity index 100% rename from profiler/example/mstx_torch_plugin/__init__.py rename to profiler/msprof_analyze/example/mstx_torch_plugin/__init__.py diff --git a/profiler/example/mstx_torch_plugin/img/result.png b/profiler/msprof_analyze/example/mstx_torch_plugin/img/result.png similarity index 100% rename from profiler/example/mstx_torch_plugin/img/result.png rename to profiler/msprof_analyze/example/mstx_torch_plugin/img/result.png diff --git a/profiler/example/mstx_torch_plugin/mstx_torch_plugin.py b/profiler/msprof_analyze/example/mstx_torch_plugin/mstx_torch_plugin.py similarity index 100% rename from profiler/example/mstx_torch_plugin/mstx_torch_plugin.py rename to profiler/msprof_analyze/example/mstx_torch_plugin/mstx_torch_plugin.py diff --git a/profiler/example/setup.py b/profiler/msprof_analyze/example/setup.py similarity index 100% rename from profiler/example/setup.py rename to profiler/msprof_analyze/example/setup.py diff --git a/profiler/prof_common/__init__.py b/profiler/msprof_analyze/prof_common/__init__.py similarity index 93% rename from profiler/prof_common/__init__.py rename to profiler/msprof_analyze/prof_common/__init__.py index 8b7e7544bb..6110927cff 100644 --- a/profiler/prof_common/__init__.py +++ b/profiler/msprof_analyze/prof_common/__init__.py @@ -14,4 +14,4 @@ # limitations under the License. import os import sys -sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) \ No newline at end of file +sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) \ No newline at end of file diff --git a/profiler/prof_common/additional_args_manager.py b/profiler/msprof_analyze/prof_common/additional_args_manager.py similarity index 96% rename from profiler/prof_common/additional_args_manager.py rename to profiler/msprof_analyze/prof_common/additional_args_manager.py index bed0f2bf91..9136b049bc 100644 --- a/profiler/prof_common/additional_args_manager.py +++ b/profiler/msprof_analyze/prof_common/additional_args_manager.py @@ -29,7 +29,7 @@ # limitations under the License. from typing import Dict -from profiler.advisor.utils.utils import singleton +from msprof_analyze.advisor.utils.utils import singleton @singleton diff --git a/profiler/prof_common/analyze_dict.py b/profiler/msprof_analyze/prof_common/analyze_dict.py similarity index 100% rename from profiler/prof_common/analyze_dict.py rename to profiler/msprof_analyze/prof_common/analyze_dict.py diff --git a/profiler/prof_common/constant.py b/profiler/msprof_analyze/prof_common/constant.py similarity index 100% rename from profiler/prof_common/constant.py rename to profiler/msprof_analyze/prof_common/constant.py diff --git a/profiler/prof_common/file_manager.py b/profiler/msprof_analyze/prof_common/file_manager.py similarity index 95% rename from profiler/prof_common/file_manager.py rename to profiler/msprof_analyze/prof_common/file_manager.py index ac7d4e5992..948938b089 100644 --- a/profiler/prof_common/file_manager.py +++ b/profiler/msprof_analyze/prof_common/file_manager.py @@ -18,11 +18,11 @@ import csv import json import yaml -from profiler.prof_common.constant import Constant -from profiler.prof_common.logger import get_logger -from profiler.prof_common.path_manager import PathManager -from profiler.prof_common.additional_args_manager import AdditionalArgsManager -from profiler.prof_common.utils import PrintUtils +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.prof_common.logger import get_logger +from msprof_analyze.prof_common.path_manager import PathManager +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.prof_common.utils import PrintUtils logger = get_logger() diff --git a/profiler/prof_common/logger.py b/profiler/msprof_analyze/prof_common/logger.py similarity index 96% rename from profiler/prof_common/logger.py rename to profiler/msprof_analyze/prof_common/logger.py index 0b90c53a43..d409727b4e 100644 --- a/profiler/prof_common/logger.py +++ b/profiler/msprof_analyze/prof_common/logger.py @@ -13,7 +13,7 @@ import logging import os -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant def get_log_level(): diff --git a/profiler/prof_common/path_manager.py b/profiler/msprof_analyze/prof_common/path_manager.py similarity index 98% rename from profiler/prof_common/path_manager.py rename to profiler/msprof_analyze/prof_common/path_manager.py index cd758d1cbf..ed0a0554eb 100644 --- a/profiler/prof_common/path_manager.py +++ b/profiler/msprof_analyze/prof_common/path_manager.py @@ -17,8 +17,8 @@ import re import shutil import platform -from profiler.prof_common.constant import Constant -from profiler.prof_common.additional_args_manager import AdditionalArgsManager +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager class PathManager: diff --git a/profiler/prof_common/utils.py b/profiler/msprof_analyze/prof_common/utils.py similarity index 100% rename from profiler/prof_common/utils.py rename to profiler/msprof_analyze/prof_common/utils.py diff --git a/profiler/requirements.txt b/profiler/msprof_analyze/requirements.txt similarity index 100% rename from profiler/requirements.txt rename to profiler/msprof_analyze/requirements.txt diff --git a/profiler/requirements/build.txt b/profiler/msprof_analyze/requirements/build.txt similarity index 100% rename from profiler/requirements/build.txt rename to profiler/msprof_analyze/requirements/build.txt diff --git a/profiler/requirements/tests.txt b/profiler/msprof_analyze/requirements/tests.txt similarity index 100% rename from profiler/requirements/tests.txt rename to profiler/msprof_analyze/requirements/tests.txt diff --git a/profiler/setup.cfg b/profiler/msprof_analyze/setup.cfg similarity index 100% rename from profiler/setup.cfg rename to profiler/msprof_analyze/setup.cfg diff --git a/profiler/setup.py b/profiler/msprof_analyze/setup.py similarity index 84% rename from profiler/setup.py rename to profiler/msprof_analyze/setup.py index 966483a222..35435ff5b5 100644 --- a/profiler/setup.py +++ b/profiler/msprof_analyze/setup.py @@ -19,13 +19,13 @@ import os import sys -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) from setuptools import find_packages, setup # type: ignore -from profiler.prof_common.path_manager import PathManager -from profiler.prof_common.file_manager import FileManager -from profiler.prof_common.utils import SafeConfigReader +from msprof_analyze.prof_common.path_manager import PathManager +from msprof_analyze.prof_common.file_manager import FileManager +from msprof_analyze.prof_common.utils import SafeConfigReader extras = { "test": [ @@ -63,7 +63,7 @@ try: except Exception as e: raise RuntimeError("The configuration file is incomplete and not configured ms_email information.") from e -root_path = os.path.dirname(os.path.abspath(os.path.dirname(__file__))) +root_path = os.path.dirname(os.path.dirname(os.path.abspath(os.path.dirname(__file__)))) setup( name="msprof-analyze", version=version, @@ -74,7 +74,9 @@ setup( url=url, author="MindStudio", author_email=author_email, - package_dir={"": root_path}, + package_dir={ + "": root_path, + }, packages=find_packages(root_path, exclude=["example"]), include_package_data=False, python_requires='>=3.7', @@ -84,7 +86,7 @@ setup( license='Apache License 2.0', entry_points=""" [console_scripts] - msprof-analyze=profiler.cli.entrance:msprof_analyze_cli + msprof-analyze=msprof_analyze.cli.entrance:msprof_analyze_cli """ ) diff --git a/profiler/test/__init__.py b/profiler/msprof_analyze/test/__init__.py similarity index 100% rename from profiler/test/__init__.py rename to profiler/msprof_analyze/test/__init__.py diff --git a/profiler/test/resource/advisor/cluster_analysis_output/cluster_communication.json b/profiler/msprof_analyze/test/resource/advisor/cluster_analysis_output/cluster_communication.json similarity index 100% rename from profiler/test/resource/advisor/cluster_analysis_output/cluster_communication.json rename to profiler/msprof_analyze/test/resource/advisor/cluster_analysis_output/cluster_communication.json diff --git a/profiler/test/resource/advisor/cluster_analysis_output/cluster_step_trace_time.csv b/profiler/msprof_analyze/test/resource/advisor/cluster_analysis_output/cluster_step_trace_time.csv similarity index 100% rename from profiler/test/resource/advisor/cluster_analysis_output/cluster_step_trace_time.csv rename to profiler/msprof_analyze/test/resource/advisor/cluster_analysis_output/cluster_step_trace_time.csv diff --git a/profiler/test/resource/event_list.json b/profiler/msprof_analyze/test/resource/event_list.json similarity index 100% rename from profiler/test/resource/event_list.json rename to profiler/msprof_analyze/test/resource/event_list.json diff --git a/profiler/test/resource/pipeline_view.png b/profiler/msprof_analyze/test/resource/pipeline_view.png similarity index 100% rename from profiler/test/resource/pipeline_view.png rename to profiler/msprof_analyze/test/resource/pipeline_view.png diff --git a/profiler/test/resource/test.csv b/profiler/msprof_analyze/test/resource/test.csv similarity index 100% rename from profiler/test/resource/test.csv rename to profiler/msprof_analyze/test/resource/test.csv diff --git a/profiler/test/run_st.py b/profiler/msprof_analyze/test/run_st.py similarity index 100% rename from profiler/test/run_st.py rename to profiler/msprof_analyze/test/run_st.py diff --git a/profiler/test/run_ut.py b/profiler/msprof_analyze/test/run_ut.py similarity index 97% rename from profiler/test/run_ut.py rename to profiler/msprof_analyze/test/run_ut.py index e75af167ff..26b3d6b564 100644 --- a/profiler/test/run_ut.py +++ b/profiler/msprof_analyze/test/run_ut.py @@ -12,7 +12,7 @@ def set_python_path(): os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "advisor") advisor_backend_root = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "advisor", "advisor_backend") - profiler_parent_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + profiler_parent_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) # Update PYTHONPATH python_path = os.environ.get("PYTHONPATH", "") if not python_path: diff --git a/profiler/test/st/__init__.py b/profiler/msprof_analyze/test/st/__init__.py similarity index 100% rename from profiler/test/st/__init__.py rename to profiler/msprof_analyze/test/st/__init__.py diff --git a/profiler/test/st/advisor/__init__.py b/profiler/msprof_analyze/test/st/advisor/__init__.py similarity index 100% rename from profiler/test/st/advisor/__init__.py rename to profiler/msprof_analyze/test/st/advisor/__init__.py diff --git a/profiler/test/st/advisor/test_advisor_cmd_single_ascend_pt_compare.py b/profiler/msprof_analyze/test/st/advisor/test_advisor_cmd_single_ascend_pt_compare.py similarity index 99% rename from profiler/test/st/advisor/test_advisor_cmd_single_ascend_pt_compare.py rename to profiler/msprof_analyze/test/st/advisor/test_advisor_cmd_single_ascend_pt_compare.py index 671903b7dc..3ec56be2f1 100644 --- a/profiler/test/st/advisor/test_advisor_cmd_single_ascend_pt_compare.py +++ b/profiler/msprof_analyze/test/st/advisor/test_advisor_cmd_single_ascend_pt_compare.py @@ -7,7 +7,7 @@ import math import pandas as pd from bs4 import BeautifulSoup -from profiler.prof_common.path_manager import PathManager +from msprof_analyze.prof_common.path_manager import PathManager from .utils import get_files, execute_cmd diff --git a/profiler/test/st/advisor/utils.py b/profiler/msprof_analyze/test/st/advisor/utils.py similarity index 100% rename from profiler/test/st/advisor/utils.py rename to profiler/msprof_analyze/test/st/advisor/utils.py diff --git a/profiler/test/st/cluster_analyse/__init__.py b/profiler/msprof_analyze/test/st/cluster_analyse/__init__.py similarity index 100% rename from profiler/test/st/cluster_analyse/__init__.py rename to profiler/msprof_analyze/test/st/cluster_analyse/__init__.py diff --git a/profiler/test/st/cluster_analyse/cluster_communication_analyzer_bandwidth_db.py b/profiler/msprof_analyze/test/st/cluster_analyse/cluster_communication_analyzer_bandwidth_db.py similarity index 100% rename from profiler/test/st/cluster_analyse/cluster_communication_analyzer_bandwidth_db.py rename to profiler/msprof_analyze/test/st/cluster_analyse/cluster_communication_analyzer_bandwidth_db.py diff --git a/profiler/test/st/cluster_analyse/cluster_communication_analyzer_matrix_db.py b/profiler/msprof_analyze/test/st/cluster_analyse/cluster_communication_analyzer_matrix_db.py similarity index 100% rename from profiler/test/st/cluster_analyse/cluster_communication_analyzer_matrix_db.py rename to profiler/msprof_analyze/test/st/cluster_analyse/cluster_communication_analyzer_matrix_db.py diff --git a/profiler/test/st/cluster_analyse/cluster_communication_analyzer_time_db.py b/profiler/msprof_analyze/test/st/cluster_analyse/cluster_communication_analyzer_time_db.py similarity index 100% rename from profiler/test/st/cluster_analyse/cluster_communication_analyzer_time_db.py rename to profiler/msprof_analyze/test/st/cluster_analyse/cluster_communication_analyzer_time_db.py diff --git a/profiler/test/st/cluster_analyse/cluster_step_trace_time_db.py b/profiler/msprof_analyze/test/st/cluster_analyse/cluster_step_trace_time_db.py similarity index 100% rename from profiler/test/st/cluster_analyse/cluster_step_trace_time_db.py rename to profiler/msprof_analyze/test/st/cluster_analyse/cluster_step_trace_time_db.py diff --git a/profiler/test/st/cluster_analyse/test_cluster_analyse_pytorch_db.py b/profiler/msprof_analyze/test/st/cluster_analyse/test_cluster_analyse_pytorch_db.py similarity index 98% rename from profiler/test/st/cluster_analyse/test_cluster_analyse_pytorch_db.py rename to profiler/msprof_analyze/test/st/cluster_analyse/test_cluster_analyse_pytorch_db.py index cbadbd84e4..ba9cd68062 100644 --- a/profiler/test/st/cluster_analyse/test_cluster_analyse_pytorch_db.py +++ b/profiler/msprof_analyze/test/st/cluster_analyse/test_cluster_analyse_pytorch_db.py @@ -19,9 +19,9 @@ from unittest import TestCase import pandas as pd -from profiler.test.st.utils import execute_cmd, select_count, select_by_query -from profiler.prof_common.file_manager import FileManager -from profiler.prof_common.path_manager import PathManager +from msprof_analyze.test.st.utils import execute_cmd, select_count, select_by_query +from msprof_analyze.prof_common.file_manager import FileManager +from msprof_analyze.prof_common.path_manager import PathManager from .cluster_communication_analyzer_bandwidth_db import ClusterCommunicationAnalyzerBandwidthDb from .cluster_communication_analyzer_matrix_db import ClusterCommunicationAnalyzerMatrixDb from .cluster_communication_analyzer_time_db import ClusterCommunicationAnalyzerTime diff --git a/profiler/test/st/cluster_analyse/test_cluster_analyse_pytorch_text.py b/profiler/msprof_analyze/test/st/cluster_analyse/test_cluster_analyse_pytorch_text.py similarity index 99% rename from profiler/test/st/cluster_analyse/test_cluster_analyse_pytorch_text.py rename to profiler/msprof_analyze/test/st/cluster_analyse/test_cluster_analyse_pytorch_text.py index 2ff8e87bc7..f1a6aa31a4 100644 --- a/profiler/test/st/cluster_analyse/test_cluster_analyse_pytorch_text.py +++ b/profiler/msprof_analyze/test/st/cluster_analyse/test_cluster_analyse_pytorch_text.py @@ -4,7 +4,7 @@ import logging import subprocess import pandas as pd from unittest import TestCase -from profiler.prof_common.path_manager import PathManager +from msprof_analyze.prof_common.path_manager import PathManager class TestClusterAnalyseCmdPytorchText(TestCase): diff --git a/profiler/test/st/compare_tools/__init__.py b/profiler/msprof_analyze/test/st/compare_tools/__init__.py similarity index 100% rename from profiler/test/st/compare_tools/__init__.py rename to profiler/msprof_analyze/test/st/compare_tools/__init__.py diff --git a/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_api_compare.py b/profiler/msprof_analyze/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_api_compare.py similarity index 94% rename from profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_api_compare.py rename to profiler/msprof_analyze/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_api_compare.py index 8b0039c21a..9f6f9034a0 100644 --- a/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_api_compare.py +++ b/profiler/msprof_analyze/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_api_compare.py @@ -3,8 +3,8 @@ from unittest import TestCase import pandas as pd -from profiler.prof_common.path_manager import PathManager -from profiler.test.st.utils import execute_cmd, check_result_file +from msprof_analyze.prof_common.path_manager import PathManager +from msprof_analyze.test.st.utils import execute_cmd, check_result_file class TestCompareToolsCmdPytorchNpuVsNpuEnableApiCompare(TestCase): diff --git a/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_communication_compare.py b/profiler/msprof_analyze/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_communication_compare.py similarity index 95% rename from profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_communication_compare.py rename to profiler/msprof_analyze/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_communication_compare.py index 4bec194a94..7161725de1 100644 --- a/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_communication_compare.py +++ b/profiler/msprof_analyze/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_communication_compare.py @@ -4,8 +4,8 @@ from unittest import TestCase import pandas as pd -from profiler.prof_common.path_manager import PathManager -from profiler.test.st.utils import execute_cmd, check_result_file +from msprof_analyze.prof_common.path_manager import PathManager +from msprof_analyze.test.st.utils import execute_cmd, check_result_file class TestCompareToolsCmdPytorchNpuVsNpuEnableCommunicationCompare(TestCase): diff --git a/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_kernel_compare.py b/profiler/msprof_analyze/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_kernel_compare.py similarity index 94% rename from profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_kernel_compare.py rename to profiler/msprof_analyze/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_kernel_compare.py index ff3f89da72..0afb2fc82a 100644 --- a/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_kernel_compare.py +++ b/profiler/msprof_analyze/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_kernel_compare.py @@ -3,8 +3,8 @@ from unittest import TestCase import pandas as pd -from profiler.prof_common.path_manager import PathManager -from profiler.test.st.utils import execute_cmd, check_result_file +from msprof_analyze.prof_common.path_manager import PathManager +from msprof_analyze.test.st.utils import execute_cmd, check_result_file class TestCompareToolsCmdPytorchNpuVsNpuEnableKernelCompare(TestCase): diff --git a/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_memory_compare.py b/profiler/msprof_analyze/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_memory_compare.py similarity index 94% rename from profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_memory_compare.py rename to profiler/msprof_analyze/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_memory_compare.py index 450e19acdb..69f8b72cb2 100644 --- a/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_memory_compare.py +++ b/profiler/msprof_analyze/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_memory_compare.py @@ -3,8 +3,8 @@ from unittest import TestCase import pandas as pd -from profiler.prof_common.path_manager import PathManager -from profiler.test.st.utils import execute_cmd, check_result_file +from msprof_analyze.prof_common.path_manager import PathManager +from msprof_analyze.test.st.utils import execute_cmd, check_result_file class TestCompareToolsCmdPytorchNpuVsNpuEnableMemoryCompare(TestCase): diff --git a/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_operator_compare.py b/profiler/msprof_analyze/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_operator_compare.py similarity index 98% rename from profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_operator_compare.py rename to profiler/msprof_analyze/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_operator_compare.py index e66354192d..514e3f4da2 100644 --- a/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_operator_compare.py +++ b/profiler/msprof_analyze/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_operator_compare.py @@ -4,8 +4,8 @@ from unittest import TestCase import pandas as pd -from profiler.prof_common.path_manager import PathManager -from profiler.test.st.utils import execute_cmd, check_result_file +from msprof_analyze.prof_common.path_manager import PathManager +from msprof_analyze.test.st.utils import execute_cmd, check_result_file class TestCompareToolsCmdPytorchNpuVsNpuEnableOperatorCompare(TestCase): diff --git a/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_profiling.py b/profiler/msprof_analyze/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_profiling.py similarity index 95% rename from profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_profiling.py rename to profiler/msprof_analyze/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_profiling.py index 03f4c9aabd..1a3b626b1e 100644 --- a/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_profiling.py +++ b/profiler/msprof_analyze/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_profiling.py @@ -4,8 +4,8 @@ from unittest import TestCase import pandas as pd -from profiler.prof_common.path_manager import PathManager -from profiler.test.st.utils import execute_cmd, check_result_file +from msprof_analyze.prof_common.path_manager import PathManager +from msprof_analyze.test.st.utils import execute_cmd, check_result_file class TestCompareToolsCmdPytorchNpuVsNpuEnableProfiling(TestCase): diff --git a/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_vs_npu.py b/profiler/msprof_analyze/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_vs_npu.py similarity index 96% rename from profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_vs_npu.py rename to profiler/msprof_analyze/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_vs_npu.py index d3b6837106..c3384f08a2 100644 --- a/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_vs_npu.py +++ b/profiler/msprof_analyze/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_vs_npu.py @@ -3,8 +3,8 @@ from unittest import TestCase import pandas as pd -from profiler.prof_common.path_manager import PathManager -from profiler.test.st.utils import execute_cmd, check_result_file +from msprof_analyze.prof_common.path_manager import PathManager +from msprof_analyze.test.st.utils import execute_cmd, check_result_file class TestCompareToolsCmdPytorchNpuVsNpu(TestCase): diff --git a/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_vs_npu_step.py b/profiler/msprof_analyze/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_vs_npu_step.py similarity index 97% rename from profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_vs_npu_step.py rename to profiler/msprof_analyze/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_vs_npu_step.py index 39cbd63015..e980e70d6d 100644 --- a/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_vs_npu_step.py +++ b/profiler/msprof_analyze/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_vs_npu_step.py @@ -4,8 +4,8 @@ from unittest import TestCase import pandas as pd -from profiler.prof_common.path_manager import PathManager -from profiler.test.st.utils import execute_cmd, check_result_file +from msprof_analyze.prof_common.path_manager import PathManager +from msprof_analyze.test.st.utils import execute_cmd, check_result_file class TestCompareToolsCmdPytorchNpuVsNpu(TestCase): diff --git a/profiler/test/st/dev/test_advisor_cmd_cluster_ascend_pt_compare.py b/profiler/msprof_analyze/test/st/dev/test_advisor_cmd_cluster_ascend_pt_compare.py similarity index 99% rename from profiler/test/st/dev/test_advisor_cmd_cluster_ascend_pt_compare.py rename to profiler/msprof_analyze/test/st/dev/test_advisor_cmd_cluster_ascend_pt_compare.py index b461619647..98c93195a6 100644 --- a/profiler/test/st/dev/test_advisor_cmd_cluster_ascend_pt_compare.py +++ b/profiler/msprof_analyze/test/st/dev/test_advisor_cmd_cluster_ascend_pt_compare.py @@ -6,8 +6,8 @@ import math import pandas as pd from bs4 import BeautifulSoup -from profiler.prof_common.path_manager import PathManager -from profiler.test.st.advisor.utils import get_files, execute_cmd +from msprof_analyze.prof_common.path_manager import PathManager +from msprof_analyze.test.st.advisor.utils import get_files, execute_cmd class TestAdvisorCmdClusterAscendPtCompare(TestCase): diff --git a/profiler/test/st/dev/test_advisor_cmd_cluster_ascend_pt_no_compare.py b/profiler/msprof_analyze/test/st/dev/test_advisor_cmd_cluster_ascend_pt_no_compare.py similarity index 99% rename from profiler/test/st/dev/test_advisor_cmd_cluster_ascend_pt_no_compare.py rename to profiler/msprof_analyze/test/st/dev/test_advisor_cmd_cluster_ascend_pt_no_compare.py index 861a987269..e21963e1c7 100644 --- a/profiler/test/st/dev/test_advisor_cmd_cluster_ascend_pt_no_compare.py +++ b/profiler/msprof_analyze/test/st/dev/test_advisor_cmd_cluster_ascend_pt_no_compare.py @@ -6,8 +6,8 @@ import math import pandas as pd from bs4 import BeautifulSoup -from profiler.prof_common.path_manager import PathManager -from profiler.test.st.advisor.utils import get_files, execute_cmd +from msprof_analyze.prof_common.path_manager import PathManager +from msprof_analyze.test.st.advisor.utils import get_files, execute_cmd class TestAdvisorCmdClusterAscendPtNoCompare(TestCase): diff --git a/profiler/test/st/dev/test_advisor_cmd_single_ascend_pt_no_compare.py b/profiler/msprof_analyze/test/st/dev/test_advisor_cmd_single_ascend_pt_no_compare.py similarity index 99% rename from profiler/test/st/dev/test_advisor_cmd_single_ascend_pt_no_compare.py rename to profiler/msprof_analyze/test/st/dev/test_advisor_cmd_single_ascend_pt_no_compare.py index a04fa32a44..11366ccd11 100644 --- a/profiler/test/st/dev/test_advisor_cmd_single_ascend_pt_no_compare.py +++ b/profiler/msprof_analyze/test/st/dev/test_advisor_cmd_single_ascend_pt_no_compare.py @@ -7,8 +7,8 @@ import math import pandas as pd from bs4 import BeautifulSoup -from profiler.prof_common.path_manager import PathManager -from profiler.test.st.advisor.utils import get_files, execute_cmd +from msprof_analyze.prof_common.path_manager import PathManager +from msprof_analyze.test.st.advisor.utils import get_files, execute_cmd class TestAdvisorCmdSingleAscendPtNoCompare(TestCase): diff --git a/profiler/test/st/utils.py b/profiler/msprof_analyze/test/st/utils.py similarity index 100% rename from profiler/test/st/utils.py rename to profiler/msprof_analyze/test/st/utils.py diff --git a/profiler/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_advice_base.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_advice_base.py similarity index 100% rename from profiler/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_advice_base.py rename to profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_advice_base.py diff --git a/profiler/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_pipeline_advice.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_pipeline_advice.py similarity index 100% rename from profiler/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_pipeline_advice.py rename to profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_pipeline_advice.py diff --git a/profiler/test/ut/advisor/advisor_backend/cluster_advice/test_kernel_cluster_advice.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_kernel_cluster_advice.py similarity index 98% rename from profiler/test/ut/advisor/advisor_backend/cluster_advice/test_kernel_cluster_advice.py rename to profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_kernel_cluster_advice.py index e7f0cb85bc..62dd532b22 100644 --- a/profiler/test/ut/advisor/advisor_backend/cluster_advice/test_kernel_cluster_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_kernel_cluster_advice.py @@ -5,7 +5,7 @@ import unittest from unittest import mock from unittest.mock import MagicMock -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant from advisor_backend.cluster_advice.kernel_cluster_advice import KernelClusterAdvice diff --git a/profiler/test/ut/advisor/advisor_backend/cluster_advice/test_slow_link_advice.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_slow_link_advice.py similarity index 100% rename from profiler/test/ut/advisor/advisor_backend/cluster_advice/test_slow_link_advice.py rename to profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_slow_link_advice.py diff --git a/profiler/test/ut/advisor/advisor_backend/cluster_advice/test_slow_rank_advice.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_slow_rank_advice.py similarity index 100% rename from profiler/test/ut/advisor/advisor_backend/cluster_advice/test_slow_rank_advice.py rename to profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_slow_rank_advice.py diff --git a/profiler/test/ut/advisor/advisor_backend/compute_advice/kernel_details.csv b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/compute_advice/kernel_details.csv similarity index 100% rename from profiler/test/ut/advisor/advisor_backend/compute_advice/kernel_details.csv rename to profiler/msprof_analyze/test/ut/advisor/advisor_backend/compute_advice/kernel_details.csv diff --git a/profiler/test/ut/advisor/advisor_backend/compute_advice/test_npu_slow_advice.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/compute_advice/test_npu_slow_advice.py similarity index 100% rename from profiler/test/ut/advisor/advisor_backend/compute_advice/test_npu_slow_advice.py rename to profiler/msprof_analyze/test/ut/advisor/advisor_backend/compute_advice/test_npu_slow_advice.py diff --git a/profiler/test/ut/advisor/advisor_backend/compute_advice/test_npufused_advice.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/compute_advice/test_npufused_advice.py similarity index 100% rename from profiler/test/ut/advisor/advisor_backend/compute_advice/test_npufused_advice.py rename to profiler/msprof_analyze/test/ut/advisor/advisor_backend/compute_advice/test_npufused_advice.py diff --git a/profiler/test/ut/advisor/advisor_backend/prof_bean_advisor/test_cluster_step_trace_time_bean.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/prof_bean_advisor/test_cluster_step_trace_time_bean.py similarity index 100% rename from profiler/test/ut/advisor/advisor_backend/prof_bean_advisor/test_cluster_step_trace_time_bean.py rename to profiler/msprof_analyze/test/ut/advisor/advisor_backend/prof_bean_advisor/test_cluster_step_trace_time_bean.py diff --git a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_opsche_advice.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/timeline_advice/test_opsche_advice.py similarity index 100% rename from profiler/test/ut/advisor/advisor_backend/timeline_advice/test_opsche_advice.py rename to profiler/msprof_analyze/test/ut/advisor/advisor_backend/timeline_advice/test_opsche_advice.py diff --git a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_optimizer_advice.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/timeline_advice/test_optimizer_advice.py similarity index 100% rename from profiler/test/ut/advisor/advisor_backend/timeline_advice/test_optimizer_advice.py rename to profiler/msprof_analyze/test/ut/advisor/advisor_backend/timeline_advice/test_optimizer_advice.py diff --git a/profiler/test/ut/advisor/advisor_backend/timeline_advice/trace_view.json b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/timeline_advice/trace_view.json similarity index 100% rename from profiler/test/ut/advisor/advisor_backend/timeline_advice/trace_view.json rename to profiler/msprof_analyze/test/ut/advisor/advisor_backend/timeline_advice/trace_view.json diff --git a/profiler/test/ut/advisor/advisor_backend/tools/__init__.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/tools/__init__.py similarity index 100% rename from profiler/test/ut/advisor/advisor_backend/tools/__init__.py rename to profiler/msprof_analyze/test/ut/advisor/advisor_backend/tools/__init__.py diff --git a/profiler/test/ut/advisor/advisor_backend/tools/tool.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/tools/tool.py similarity index 100% rename from profiler/test/ut/advisor/advisor_backend/tools/tool.py rename to profiler/msprof_analyze/test/ut/advisor/advisor_backend/tools/tool.py diff --git a/profiler/test/ut/advisor/common/__init__.py b/profiler/msprof_analyze/test/ut/advisor/common/__init__.py similarity index 100% rename from profiler/test/ut/advisor/common/__init__.py rename to profiler/msprof_analyze/test/ut/advisor/common/__init__.py diff --git a/profiler/test/ut/advisor/common/test_enum_params_parser.py b/profiler/msprof_analyze/test/ut/advisor/common/test_enum_params_parser.py similarity index 84% rename from profiler/test/ut/advisor/common/test_enum_params_parser.py rename to profiler/msprof_analyze/test/ut/advisor/common/test_enum_params_parser.py index c27a632f64..0f8984d1f6 100644 --- a/profiler/test/ut/advisor/common/test_enum_params_parser.py +++ b/profiler/msprof_analyze/test/ut/advisor/common/test_enum_params_parser.py @@ -1,55 +1,55 @@ -import unittest -import sys -import os - -profiler_root_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))) -sys.path.append(os.path.join(profiler_root_dir, "compare_tools")) -sys.path.append(os.path.join(profiler_root_dir, "cluster_analyse")) - -from profiler.advisor.common.enum_params_parser import EnumParamsParser -from profiler.test.ut.advisor.advisor_backend.tools.tool import recover_env - - -class TestEnumParamsParser(unittest.TestCase): - @classmethod - def tearDownClass(cls) -> None: - recover_env() - - def setUp(self) -> None: - self.enum_params_parser = EnumParamsParser() - self.argument_keys = sorted(["cann_version", "torch_version", "analysis_dimensions", "profiling_type", "mindspore_version"]) - self.env_keys = ["ADVISOR_ANALYZE_PROCESSES", "DISABLE_PROFILING_COMPARISON", "DISABLE_AFFINITY_API"] - - def test_get_keys(self): - total_keys = sorted(self.argument_keys + self.env_keys) - keys = sorted(self.enum_params_parser.get_keys()) - self.assertTrue(isinstance(keys, list)) - self.assertEqual(keys, total_keys) - - def test_get_argument_keys(self): - argument_keys = sorted(self.enum_params_parser.get_arguments_keys()) - self.assertTrue(isinstance(argument_keys, list)) - self.assertEqual(argument_keys, self.argument_keys) - - def test_get_env_keys(self): - env_keys = sorted(self.enum_params_parser.get_envs_keys()) - self.assertTrue(isinstance(env_keys, list)) - self.assertEqual(env_keys, sorted(self.env_keys)) - - def test_get_default(self): - self.assertTrue(self.enum_params_parser.get_default("cann_version"), "8.0.RC1") - self.assertTrue(self.enum_params_parser.get_default("torch_version"), "2.1.0") - self.assertTrue(self.enum_params_parser.get_default("analysis_dimensions"), - ["computation", "communication", "schedule", "memory"]) - self.assertTrue(self.enum_params_parser.get_default("profiling_type"), "ascend_pytorch_profiler") - self.assertTrue(self.enum_params_parser.get_default("ADVISOR_ANALYZE_PROCESSES"), 1) - - def test_get_options(self): - self.assertTrue(self.enum_params_parser.get_options("cann_version"), ["6.3.RC2", "7.0.RC1", "7.0.0", "8.0.RC1"]) - self.assertTrue(self.enum_params_parser.get_options("torch_version"), ["1.11.0", "2.1.0"]) - self.assertTrue(self.enum_params_parser.get_options("analysis_dimensions"), - [["computation", "communication", "schedule", "memory"], ["communication"], ["schedule"], - ["computation"], ["memory"]]) - self.assertTrue(self.enum_params_parser.get_options("profiling_type"), - ["ascend_pytorch_profiler", "mslite", "msprof"]) - self.assertTrue(self.enum_params_parser.get_options("ADVISOR_ANALYZE_PROCESSES"), list(range(1, 9))) +import unittest +import sys +import os + +msprof_analyze_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))) +sys.path.append(os.path.join(msprof_analyze_dir, "compare_tools")) +sys.path.append(os.path.join(msprof_analyze_dir, "cluster_analyse")) + +from msprof_analyze.advisor.common.enum_params_parser import EnumParamsParser +from msprof_analyze.test.ut.advisor.advisor_backend.tools.tool import recover_env + + +class TestEnumParamsParser(unittest.TestCase): + @classmethod + def tearDownClass(cls) -> None: + recover_env() + + def setUp(self) -> None: + self.enum_params_parser = EnumParamsParser() + self.argument_keys = sorted(["cann_version", "torch_version", "analysis_dimensions", "profiling_type", "mindspore_version"]) + self.env_keys = ["ADVISOR_ANALYZE_PROCESSES", "DISABLE_PROFILING_COMPARISON", "DISABLE_AFFINITY_API"] + + def test_get_keys(self): + total_keys = sorted(self.argument_keys + self.env_keys) + keys = sorted(self.enum_params_parser.get_keys()) + self.assertTrue(isinstance(keys, list)) + self.assertEqual(keys, total_keys) + + def test_get_argument_keys(self): + argument_keys = sorted(self.enum_params_parser.get_arguments_keys()) + self.assertTrue(isinstance(argument_keys, list)) + self.assertEqual(argument_keys, self.argument_keys) + + def test_get_env_keys(self): + env_keys = sorted(self.enum_params_parser.get_envs_keys()) + self.assertTrue(isinstance(env_keys, list)) + self.assertEqual(env_keys, sorted(self.env_keys)) + + def test_get_default(self): + self.assertTrue(self.enum_params_parser.get_default("cann_version"), "8.0.RC1") + self.assertTrue(self.enum_params_parser.get_default("torch_version"), "2.1.0") + self.assertTrue(self.enum_params_parser.get_default("analysis_dimensions"), + ["computation", "communication", "schedule", "memory"]) + self.assertTrue(self.enum_params_parser.get_default("profiling_type"), "ascend_pytorch_profiler") + self.assertTrue(self.enum_params_parser.get_default("ADVISOR_ANALYZE_PROCESSES"), 1) + + def test_get_options(self): + self.assertTrue(self.enum_params_parser.get_options("cann_version"), ["6.3.RC2", "7.0.RC1", "7.0.0", "8.0.RC1"]) + self.assertTrue(self.enum_params_parser.get_options("torch_version"), ["1.11.0", "2.1.0"]) + self.assertTrue(self.enum_params_parser.get_options("analysis_dimensions"), + [["computation", "communication", "schedule", "memory"], ["communication"], ["schedule"], + ["computation"], ["memory"]]) + self.assertTrue(self.enum_params_parser.get_options("profiling_type"), + ["ascend_pytorch_profiler", "mslite", "msprof"]) + self.assertTrue(self.enum_params_parser.get_options("ADVISOR_ANALYZE_PROCESSES"), list(range(1, 9))) diff --git a/profiler/test/ut/advisor/communication_advice/test_bandwidth_contention_advice.py b/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_bandwidth_contention_advice.py similarity index 98% rename from profiler/test/ut/advisor/communication_advice/test_bandwidth_contention_advice.py rename to profiler/msprof_analyze/test/ut/advisor/communication_advice/test_bandwidth_contention_advice.py index 6cef67dbcf..3e6bc74658 100644 --- a/profiler/test/ut/advisor/communication_advice/test_bandwidth_contention_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_bandwidth_contention_advice.py @@ -4,8 +4,8 @@ import stat import json import csv import unittest -from profiler.advisor.interface.interface import Interface -from profiler.advisor.common.analyzer_scopes import SupportedScopes +from msprof_analyze.advisor.interface.interface import Interface +from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes class TestBandwidthContentionAdvice(unittest.TestCase): diff --git a/profiler/test/ut/advisor/communication_advice/test_byte_alignment_analyzer.py b/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_byte_alignment_analyzer.py similarity index 97% rename from profiler/test/ut/advisor/communication_advice/test_byte_alignment_analyzer.py rename to profiler/msprof_analyze/test/ut/advisor/communication_advice/test_byte_alignment_analyzer.py index 9516435366..8f8a084a85 100644 --- a/profiler/test/ut/advisor/communication_advice/test_byte_alignment_analyzer.py +++ b/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_byte_alignment_analyzer.py @@ -3,8 +3,8 @@ import shutil import stat import json import unittest -from profiler.advisor.interface.interface import Interface -from profiler.advisor.common.analyzer_scopes import SupportedScopes +from msprof_analyze.advisor.interface.interface import Interface +from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes class TestByteAlignmentAnalyzer(unittest.TestCase): diff --git a/profiler/test/ut/advisor/communication_advice/test_packet_advice.py b/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_packet_advice.py similarity index 98% rename from profiler/test/ut/advisor/communication_advice/test_packet_advice.py rename to profiler/msprof_analyze/test/ut/advisor/communication_advice/test_packet_advice.py index dc30a8743a..ef26392d8a 100644 --- a/profiler/test/ut/advisor/communication_advice/test_packet_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_packet_advice.py @@ -4,8 +4,8 @@ import stat import json import unittest -from profiler.advisor.interface.interface import Interface -from profiler.advisor.common.analyzer_scopes import SupportedScopes +from msprof_analyze.advisor.interface.interface import Interface +from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes class TestPacketAdvice(unittest.TestCase): diff --git a/profiler/test/ut/advisor/communication_advice/test_rdma_retransmission_advice.py b/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_rdma_retransmission_advice.py similarity index 98% rename from profiler/test/ut/advisor/communication_advice/test_rdma_retransmission_advice.py rename to profiler/msprof_analyze/test/ut/advisor/communication_advice/test_rdma_retransmission_advice.py index 5ffb569d43..a0aa23f92d 100644 --- a/profiler/test/ut/advisor/communication_advice/test_rdma_retransmission_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_rdma_retransmission_advice.py @@ -4,8 +4,8 @@ import stat import json import unittest -from profiler.advisor.interface.interface import Interface -from profiler.advisor.common.analyzer_scopes import SupportedScopes +from msprof_analyze.advisor.interface.interface import Interface +from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes class TestRdmaAdvice(unittest.TestCase): diff --git a/profiler/test/ut/advisor/compute_advice/test_frequency_advice.py b/profiler/msprof_analyze/test/ut/advisor/compute_advice/test_frequency_advice.py similarity index 96% rename from profiler/test/ut/advisor/compute_advice/test_frequency_advice.py rename to profiler/msprof_analyze/test/ut/advisor/compute_advice/test_frequency_advice.py index d3bcae7a7d..049d080eed 100644 --- a/profiler/test/ut/advisor/compute_advice/test_frequency_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/compute_advice/test_frequency_advice.py @@ -4,9 +4,9 @@ import stat import json import unittest -from profiler.advisor.interface.interface import Interface -from profiler.advisor.common.analyzer_scopes import SupportedScopes -from profiler.advisor.dataset.timeline_event_dataset import ComputationAnalysisDataset +from msprof_analyze.advisor.interface.interface import Interface +from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes +from msprof_analyze.advisor.dataset.timeline_event_dataset import ComputationAnalysisDataset class TestFrequencyAdvice(unittest.TestCase): diff --git a/profiler/test/ut/advisor/compute_advice/test_pp_stage_computation_analyzer.py b/profiler/msprof_analyze/test/ut/advisor/compute_advice/test_pp_stage_computation_analyzer.py similarity index 90% rename from profiler/test/ut/advisor/compute_advice/test_pp_stage_computation_analyzer.py rename to profiler/msprof_analyze/test/ut/advisor/compute_advice/test_pp_stage_computation_analyzer.py index 781dc3dcc3..93f99ad60d 100644 --- a/profiler/test/ut/advisor/compute_advice/test_pp_stage_computation_analyzer.py +++ b/profiler/msprof_analyze/test/ut/advisor/compute_advice/test_pp_stage_computation_analyzer.py @@ -2,8 +2,8 @@ import unittest import copy import os -from profiler.advisor.analyzer.computation.pp_stage_computation_analyzer import PPStageComputationAnalyzer -from profiler.test.ut.advisor.advisor_backend.tools.tool import recover_env +from msprof_analyze.advisor.analyzer.computation.pp_stage_computation_analyzer import PPStageComputationAnalyzer +from msprof_analyze.test.ut.advisor.advisor_backend.tools.tool import recover_env mock_profiling_path = os.path.realpath(__file__) diff --git a/profiler/test/ut/advisor/timeline_advice/test_conjectured_gc_advice.py b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_conjectured_gc_advice.py similarity index 98% rename from profiler/test/ut/advisor/timeline_advice/test_conjectured_gc_advice.py rename to profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_conjectured_gc_advice.py index bd8051869a..07dc8a36b5 100644 --- a/profiler/test/ut/advisor/timeline_advice/test_conjectured_gc_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_conjectured_gc_advice.py @@ -4,8 +4,8 @@ import stat import json import multiprocessing import unittest -from profiler.advisor.interface.interface import Interface -from profiler.advisor.common.analyzer_scopes import SupportedScopes +from msprof_analyze.advisor.interface.interface import Interface +from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes class TestCompatibleGcAdvice(unittest.TestCase): diff --git a/profiler/test/ut/advisor/timeline_advice/test_dataloader_checker.py b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_dataloader_checker.py similarity index 90% rename from profiler/test/ut/advisor/timeline_advice/test_dataloader_checker.py rename to profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_dataloader_checker.py index 0ca468800a..6737197d26 100644 --- a/profiler/test/ut/advisor/timeline_advice/test_dataloader_checker.py +++ b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_dataloader_checker.py @@ -3,9 +3,9 @@ import os import sys import yaml -from profiler.advisor.analyzer.dataloader.dataloader_checker import DataloaderChecker -from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.test.ut.advisor.advisor_backend.tools.tool import recover_env +from msprof_analyze.advisor.analyzer.dataloader.dataloader_checker import DataloaderChecker +from msprof_analyze.advisor.common.timeline.event import TimelineEvent +from msprof_analyze.test.ut.advisor.advisor_backend.tools.tool import recover_env class TestDataloaderChecker(unittest.TestCase): diff --git a/profiler/test/ut/advisor/timeline_advice/test_gc_checker.py b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_gc_checker.py similarity index 97% rename from profiler/test/ut/advisor/timeline_advice/test_gc_checker.py rename to profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_gc_checker.py index 6cea5081ee..77f0814df3 100644 --- a/profiler/test/ut/advisor/timeline_advice/test_gc_checker.py +++ b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_gc_checker.py @@ -4,8 +4,8 @@ import stat import json import multiprocessing import unittest -from profiler.advisor.interface.interface import Interface -from profiler.advisor.common.analyzer_scopes import SupportedScopes +from msprof_analyze.advisor.interface.interface import Interface +from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes class TestGcAdvice(unittest.TestCase): diff --git a/profiler/test/ut/advisor/timeline_advice/test_memory_op_checker.py b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_memory_op_checker.py similarity index 88% rename from profiler/test/ut/advisor/timeline_advice/test_memory_op_checker.py rename to profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_memory_op_checker.py index e90fd86d43..8c14417771 100644 --- a/profiler/test/ut/advisor/timeline_advice/test_memory_op_checker.py +++ b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_memory_op_checker.py @@ -3,9 +3,9 @@ import os import sys import yaml -from profiler.advisor.analyzer.memory.memory_checker import MemoryOpsChecker -from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.test.ut.advisor.advisor_backend.tools.tool import recover_env +from msprof_analyze.advisor.analyzer.memory.memory_checker import MemoryOpsChecker +from msprof_analyze.advisor.common.timeline.event import TimelineEvent +from msprof_analyze.test.ut.advisor.advisor_backend.tools.tool import recover_env class TestMemOpChecker(unittest.TestCase): diff --git a/profiler/test/ut/advisor/timeline_advice/test_syncbn_checker.py b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_syncbn_checker.py similarity index 88% rename from profiler/test/ut/advisor/timeline_advice/test_syncbn_checker.py rename to profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_syncbn_checker.py index 2aaad53353..aae3dfe1bb 100644 --- a/profiler/test/ut/advisor/timeline_advice/test_syncbn_checker.py +++ b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_syncbn_checker.py @@ -3,9 +3,9 @@ import os import sys import yaml -from profiler.advisor.analyzer.schedule.syncbn.syncbn_checker import SyncBNChecker -from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.test.ut.advisor.advisor_backend.tools.tool import recover_env +from msprof_analyze.advisor.analyzer.schedule.syncbn.syncbn_checker import SyncBNChecker +from msprof_analyze.advisor.common.timeline.event import TimelineEvent +from msprof_analyze.test.ut.advisor.advisor_backend.tools.tool import recover_env class TestSyncBNChecker(unittest.TestCase): diff --git a/profiler/test/ut/advisor/timeline_advice/test_synchronize_stream.py b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_synchronize_stream.py similarity index 85% rename from profiler/test/ut/advisor/timeline_advice/test_synchronize_stream.py rename to profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_synchronize_stream.py index e942339b82..6363fffb18 100644 --- a/profiler/test/ut/advisor/timeline_advice/test_synchronize_stream.py +++ b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_synchronize_stream.py @@ -3,11 +3,11 @@ import os import sys import yaml -from profiler.advisor.analyzer.schedule.synchronize_stream.synchronize_stream_checker import SynchronizeStreamChecker -from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.prof_common.constant import Constant -from profiler.advisor.utils.utils import safe_division -from profiler.test.ut.advisor.advisor_backend.tools.tool import recover_env +from msprof_analyze.advisor.analyzer.schedule.synchronize_stream.synchronize_stream_checker import SynchronizeStreamChecker +from msprof_analyze.advisor.common.timeline.event import TimelineEvent +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.advisor.utils.utils import safe_division +from msprof_analyze.test.ut.advisor.advisor_backend.tools.tool import recover_env class TestSynchronizeChecker(unittest.TestCase): diff --git a/profiler/test/ut/advisor/timeline_advice/test_timeline_op_collector.py b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_timeline_op_collector.py similarity index 94% rename from profiler/test/ut/advisor/timeline_advice/test_timeline_op_collector.py rename to profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_timeline_op_collector.py index cb9121d4cf..e67dd398ee 100644 --- a/profiler/test/ut/advisor/timeline_advice/test_timeline_op_collector.py +++ b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_timeline_op_collector.py @@ -1,141 +1,141 @@ -import unittest -import os -import sys -import yaml - -from profiler.advisor.dataset.timeline_op_collector.timeline_op_collector import ( - OpCompileCollector, - SynchronizeStreamCollector, - MemCollector, - DataloaderCollector, - SyncBNCollector, - AtenCollector, - OptimizerCollector, - FrequencyCollector, - SpecificTaskTypeOpCollector, - TorchToNpuCollector, - AclToNpuCollector, - OpStackCollector, - StepCollector -) -from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.test.ut.advisor.advisor_backend.tools.tool import recover_env - - -class TestTimelineOpCollector(unittest.TestCase): - @classmethod - def tearDownClass(cls) -> None: - recover_env() - - def setUp(self) -> None: - self.mock_step_event = TimelineEvent(dict(name="ProfilerStep#1", ts=1, dur=1000)) - self.mock_op_compile_event = TimelineEvent(dict(name="AscendCL@aclopCompileAndExecute", ts=2, dur=1)) - self.mock_sync_stream_event = TimelineEvent(dict(name="AscendCL@aclrtSynchronizeStream", dur=1000000000)) - self.mock_mem_op_event = TimelineEvent(dict(name="AscendCL@aclMallocMemInner", dur=10)) - self.mock_dataloader_event = TimelineEvent(dict(name="dataloader")) - self.mock_sync_bn_event = TimelineEvent(dict(name="syncbatchnorm")) - self.mock_aten_event = TimelineEvent(dict(name="aten::conv3d")) - self.mock_optimizer_event = TimelineEvent(dict(name="Optimizer.step#")) - self.mock_AI_CPU_event = TimelineEvent( - {"name": "index", "args": TimelineEvent({"Task Type": "AI_CPU"}), "ts": 1}) - self.mock_torch_to_npu_event = TimelineEvent(dict(name="torch_to_npu", tid=1, ts=1, ph=1, id=1)) - self.mock_acl_to_npu_event = TimelineEvent(dict(name="acl_to_npu", ts=1)) - self.mock_op_stack_event = TimelineEvent( - {"name": "aten::conv3d", "dataset_index": 1, "ts": 1, "args": TimelineEvent({"Call stack": "mock_stack"})}) - - def test_step_collector(self): - step_collector = StepCollector() - step_collector.add_op(self.mock_step_event) - step_collector.post_process() - self.assertEqual(step_collector.attribute_to_dataset.get("profiler_step"), [self.mock_step_event]) - - def test_op_compile_collector(self): - op_compile_collector = OpCompileCollector() - op_compile_collector.add_op(self.mock_op_compile_event) - op_compile_collector.post_process(op_compile_collector.op_list) - self.assertEqual(op_compile_collector.attribute_to_dataset.get("ops_compile"), op_compile_collector) - self.assertEqual(op_compile_collector.total_time, 1) - self.assertEqual(op_compile_collector.total_count, 1) - - def test_sync_stream_collector(self): - sync_stream_collector = SynchronizeStreamCollector() - sync_stream_collector.post_process() - self.assertEqual(sync_stream_collector.attribute_to_dataset.get("synchronize_stream"), []) - - def test_mem_op_collector(self): - mem_op_collector = MemCollector() - mem_op_collector.add_op(self.mock_mem_op_event) - mem_op_collector.post_process(mem_op_collector.op_list) - self.assertEqual(mem_op_collector.attribute_to_dataset.get("memory_ops"), mem_op_collector) - self.assertEqual(mem_op_collector.mem_op_info.get("AscendCL@aclMallocMemInner"), {"count": 1, "total_dur": 10}) - - def test_dataloader_collector(self): - dataloader_collector = DataloaderCollector() - dataloader_collector.add_op(self.mock_dataloader_event) - dataloader_collector.post_process() - self.assertEqual(len(dataloader_collector.attribute_to_dataset.get("dataloader")), 1) - - def test_sync_bn_collector(self): - sync_bn_collector = SyncBNCollector() - sync_bn_collector.add_op(self.mock_sync_bn_event) - sync_bn_collector.post_process(sync_bn_collector.op_list) - self.assertEqual(len(sync_bn_collector.attribute_to_dataset.get("sync_batchnorm")), 1) - - def test_aten_collector(self): - aten_collector = AtenCollector() - aten_collector.add_op(self.mock_aten_event) - aten_collector.add_op(self.mock_sync_stream_event) - aten_collector.post_process(aten_collector.op_list) - self.assertEqual(len(aten_collector.attribute_to_dataset.get("aten")), 2) - - def test_optimizer_collector(self): - optimizer_collector = OptimizerCollector() - optimizer_collector.add_op(self.mock_optimizer_event) - optimizer_collector.post_process(optimizer_collector.op_list) - self.assertEqual(len(optimizer_collector.attribute_to_dataset.get("optimizer")), 1) - - def test_specific_task_type_op_collector(self): - specific_task_type_op_collector = SpecificTaskTypeOpCollector() - specific_task_type_op_collector.add_op(self.mock_AI_CPU_event) - specific_task_type_op_collector.post_process(specific_task_type_op_collector.op_list) - key = f"{self.mock_AI_CPU_event.name}-{self.mock_AI_CPU_event.ts}" - self.assertTrue( - specific_task_type_op_collector.attribute_to_dataset.get("ops_with_task_type", {}).get(key)) - self.assertTrue(specific_task_type_op_collector.attribute_to_dataset.get("task_op_names"), [key]) - - def test_torch_to_npu_collector(self): - torch_to_npu_collector = TorchToNpuCollector() - torch_to_npu_collector.add_op(self.mock_torch_to_npu_event) - torch_to_npu_collector.post_process(torch_to_npu_collector.op_list) - key = f"{self.mock_torch_to_npu_event.ph}-{self.mock_torch_to_npu_event.id}" - self.assertTrue("1-1" in torch_to_npu_collector.attribute_to_dataset.get("torch_to_npu")) - - def test_acl_to_npu_collector(self): - acl_to_npu_collector = AclToNpuCollector() - acl_to_npu_collector.add_op(self.mock_acl_to_npu_event) - acl_to_npu_collector.post_process(acl_to_npu_collector.op_list) - self.assertEqual(acl_to_npu_collector.attribute_to_dataset.get("acl_to_npu"), - set([str(self.mock_acl_to_npu_event.ts)])) - - def test_op_stack_collector(self): - op_stack_collector = OpStackCollector() - op_stack_collector.add_op(self.mock_op_stack_event) - op_stack_collector.post_process(op_stack_collector.op_list) - self.assertTrue( - str(self.mock_op_stack_event.ts) in op_stack_collector.attribute_to_dataset.get("ops_with_stack")) - - -if __name__ == '__main__': - tester = TestTimelineOpCollector() - tester.test_step_collector() - tester.test_op_compile_collector() - tester.test_sync_stream_collector() - tester.test_mem_op_collector() - tester.test_dataloader_collector() - tester.test_sync_bn_collector() - tester.test_aten_collector() - tester.test_optimizer_collector() - tester.test_specific_task_type_op_collector() - tester.test_torch_to_npu_collector() - tester.test_acl_to_npu_collector() - tester.test_op_stack_collector() +import unittest +import os +import sys +import yaml + +from msprof_analyze.advisor.dataset.timeline_op_collector.timeline_op_collector import ( + OpCompileCollector, + SynchronizeStreamCollector, + MemCollector, + DataloaderCollector, + SyncBNCollector, + AtenCollector, + OptimizerCollector, + FrequencyCollector, + SpecificTaskTypeOpCollector, + TorchToNpuCollector, + AclToNpuCollector, + OpStackCollector, + StepCollector +) +from msprof_analyze.advisor.common.timeline.event import TimelineEvent +from msprof_analyze.test.ut.advisor.advisor_backend.tools.tool import recover_env + + +class TestTimelineOpCollector(unittest.TestCase): + @classmethod + def tearDownClass(cls) -> None: + recover_env() + + def setUp(self) -> None: + self.mock_step_event = TimelineEvent(dict(name="ProfilerStep#1", ts=1, dur=1000)) + self.mock_op_compile_event = TimelineEvent(dict(name="AscendCL@aclopCompileAndExecute", ts=2, dur=1)) + self.mock_sync_stream_event = TimelineEvent(dict(name="AscendCL@aclrtSynchronizeStream", dur=1000000000)) + self.mock_mem_op_event = TimelineEvent(dict(name="AscendCL@aclMallocMemInner", dur=10)) + self.mock_dataloader_event = TimelineEvent(dict(name="dataloader")) + self.mock_sync_bn_event = TimelineEvent(dict(name="syncbatchnorm")) + self.mock_aten_event = TimelineEvent(dict(name="aten::conv3d")) + self.mock_optimizer_event = TimelineEvent(dict(name="Optimizer.step#")) + self.mock_AI_CPU_event = TimelineEvent( + {"name": "index", "args": TimelineEvent({"Task Type": "AI_CPU"}), "ts": 1}) + self.mock_torch_to_npu_event = TimelineEvent(dict(name="torch_to_npu", tid=1, ts=1, ph=1, id=1)) + self.mock_acl_to_npu_event = TimelineEvent(dict(name="acl_to_npu", ts=1)) + self.mock_op_stack_event = TimelineEvent( + {"name": "aten::conv3d", "dataset_index": 1, "ts": 1, "args": TimelineEvent({"Call stack": "mock_stack"})}) + + def test_step_collector(self): + step_collector = StepCollector() + step_collector.add_op(self.mock_step_event) + step_collector.post_process() + self.assertEqual(step_collector.attribute_to_dataset.get("profiler_step"), [self.mock_step_event]) + + def test_op_compile_collector(self): + op_compile_collector = OpCompileCollector() + op_compile_collector.add_op(self.mock_op_compile_event) + op_compile_collector.post_process(op_compile_collector.op_list) + self.assertEqual(op_compile_collector.attribute_to_dataset.get("ops_compile"), op_compile_collector) + self.assertEqual(op_compile_collector.total_time, 1) + self.assertEqual(op_compile_collector.total_count, 1) + + def test_sync_stream_collector(self): + sync_stream_collector = SynchronizeStreamCollector() + sync_stream_collector.post_process() + self.assertEqual(sync_stream_collector.attribute_to_dataset.get("synchronize_stream"), []) + + def test_mem_op_collector(self): + mem_op_collector = MemCollector() + mem_op_collector.add_op(self.mock_mem_op_event) + mem_op_collector.post_process(mem_op_collector.op_list) + self.assertEqual(mem_op_collector.attribute_to_dataset.get("memory_ops"), mem_op_collector) + self.assertEqual(mem_op_collector.mem_op_info.get("AscendCL@aclMallocMemInner"), {"count": 1, "total_dur": 10}) + + def test_dataloader_collector(self): + dataloader_collector = DataloaderCollector() + dataloader_collector.add_op(self.mock_dataloader_event) + dataloader_collector.post_process() + self.assertEqual(len(dataloader_collector.attribute_to_dataset.get("dataloader")), 1) + + def test_sync_bn_collector(self): + sync_bn_collector = SyncBNCollector() + sync_bn_collector.add_op(self.mock_sync_bn_event) + sync_bn_collector.post_process(sync_bn_collector.op_list) + self.assertEqual(len(sync_bn_collector.attribute_to_dataset.get("sync_batchnorm")), 1) + + def test_aten_collector(self): + aten_collector = AtenCollector() + aten_collector.add_op(self.mock_aten_event) + aten_collector.add_op(self.mock_sync_stream_event) + aten_collector.post_process(aten_collector.op_list) + self.assertEqual(len(aten_collector.attribute_to_dataset.get("aten")), 2) + + def test_optimizer_collector(self): + optimizer_collector = OptimizerCollector() + optimizer_collector.add_op(self.mock_optimizer_event) + optimizer_collector.post_process(optimizer_collector.op_list) + self.assertEqual(len(optimizer_collector.attribute_to_dataset.get("optimizer")), 1) + + def test_specific_task_type_op_collector(self): + specific_task_type_op_collector = SpecificTaskTypeOpCollector() + specific_task_type_op_collector.add_op(self.mock_AI_CPU_event) + specific_task_type_op_collector.post_process(specific_task_type_op_collector.op_list) + key = f"{self.mock_AI_CPU_event.name}-{self.mock_AI_CPU_event.ts}" + self.assertTrue( + specific_task_type_op_collector.attribute_to_dataset.get("ops_with_task_type", {}).get(key)) + self.assertTrue(specific_task_type_op_collector.attribute_to_dataset.get("task_op_names"), [key]) + + def test_torch_to_npu_collector(self): + torch_to_npu_collector = TorchToNpuCollector() + torch_to_npu_collector.add_op(self.mock_torch_to_npu_event) + torch_to_npu_collector.post_process(torch_to_npu_collector.op_list) + key = f"{self.mock_torch_to_npu_event.ph}-{self.mock_torch_to_npu_event.id}" + self.assertTrue("1-1" in torch_to_npu_collector.attribute_to_dataset.get("torch_to_npu")) + + def test_acl_to_npu_collector(self): + acl_to_npu_collector = AclToNpuCollector() + acl_to_npu_collector.add_op(self.mock_acl_to_npu_event) + acl_to_npu_collector.post_process(acl_to_npu_collector.op_list) + self.assertEqual(acl_to_npu_collector.attribute_to_dataset.get("acl_to_npu"), + set([str(self.mock_acl_to_npu_event.ts)])) + + def test_op_stack_collector(self): + op_stack_collector = OpStackCollector() + op_stack_collector.add_op(self.mock_op_stack_event) + op_stack_collector.post_process(op_stack_collector.op_list) + self.assertTrue( + str(self.mock_op_stack_event.ts) in op_stack_collector.attribute_to_dataset.get("ops_with_stack")) + + +if __name__ == '__main__': + tester = TestTimelineOpCollector() + tester.test_step_collector() + tester.test_op_compile_collector() + tester.test_sync_stream_collector() + tester.test_mem_op_collector() + tester.test_dataloader_collector() + tester.test_sync_bn_collector() + tester.test_aten_collector() + tester.test_optimizer_collector() + tester.test_specific_task_type_op_collector() + tester.test_torch_to_npu_collector() + tester.test_acl_to_npu_collector() + tester.test_op_stack_collector() diff --git a/profiler/test/ut/advisor/timeline_advice/test_timeline_op_compile_checker.py b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_timeline_op_compile_checker.py similarity index 74% rename from profiler/test/ut/advisor/timeline_advice/test_timeline_op_compile_checker.py rename to profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_timeline_op_compile_checker.py index 9cf1c4862d..5546859eca 100644 --- a/profiler/test/ut/advisor/timeline_advice/test_timeline_op_compile_checker.py +++ b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_timeline_op_compile_checker.py @@ -2,15 +2,15 @@ import unittest import os import sys -work_path = os.path.dirname(os.path.dirname( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))))) +work_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))))))) sys.path.insert(0, work_path) from unittest.mock import patch -from profiler.advisor.analyzer.schedule import dispatch -from profiler.advisor.analyzer.schedule.dispatch.timeline_op_dispatch_analyzer import OpDispatchAnalyzer -from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset -from profiler.advisor.display.html.render import HTMLRender -from profiler.test.ut.advisor.advisor_backend.tools.tool import recover_env +from msprof_analyze.advisor.analyzer.schedule import dispatch +from msprof_analyze.advisor.analyzer.schedule.dispatch.timeline_op_dispatch_analyzer import OpDispatchAnalyzer +from msprof_analyze.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset +from msprof_analyze.advisor.display.html.render import HTMLRender +from msprof_analyze.test.ut.advisor.advisor_backend.tools.tool import recover_env class TestOperatorDispatchAnalyzer(unittest.TestCase): diff --git a/profiler/test/ut/advisor/timeline_advice/trace_view.json b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/trace_view.json similarity index 100% rename from profiler/test/ut/advisor/timeline_advice/trace_view.json rename to profiler/msprof_analyze/test/ut/advisor/timeline_advice/trace_view.json diff --git a/profiler/test/ut/cluster_analyse/cluster_data_preprocess/test_pytorch_data_preprocessor.py b/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_data_preprocess/test_pytorch_data_preprocessor.py similarity index 100% rename from profiler/test/ut/cluster_analyse/cluster_data_preprocess/test_pytorch_data_preprocessor.py rename to profiler/msprof_analyze/test/ut/cluster_analyse/cluster_data_preprocess/test_pytorch_data_preprocessor.py diff --git a/profiler/test/ut/cluster_analyse/cluster_data_preprocess/test_step_trace_time_analysis.py b/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_data_preprocess/test_step_trace_time_analysis.py similarity index 98% rename from profiler/test/ut/cluster_analyse/cluster_data_preprocess/test_step_trace_time_analysis.py rename to profiler/msprof_analyze/test/ut/cluster_analyse/cluster_data_preprocess/test_step_trace_time_analysis.py index 34befa1104..9913faee51 100644 --- a/profiler/test/ut/cluster_analyse/cluster_data_preprocess/test_step_trace_time_analysis.py +++ b/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_data_preprocess/test_step_trace_time_analysis.py @@ -20,7 +20,7 @@ from unittest import mock from analysis.step_trace_time_analysis import StepTraceTimeAnalysis from prof_bean.step_trace_time_bean import StepTraceTimeBean -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class TestStepTraceTimeAnalysis(unittest.TestCase): diff --git a/profiler/test/ut/cluster_analyse/cluster_utils/test_parallel_strategy_calculator.py b/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_utils/test_parallel_strategy_calculator.py similarity index 100% rename from profiler/test/ut/cluster_analyse/cluster_utils/test_parallel_strategy_calculator.py rename to profiler/msprof_analyze/test/ut/cluster_analyse/cluster_utils/test_parallel_strategy_calculator.py diff --git a/profiler/test/ut/cluster_analyse/common_func/test_file_manager.py b/profiler/msprof_analyze/test/ut/cluster_analyse/common_func/test_file_manager.py similarity index 97% rename from profiler/test/ut/cluster_analyse/common_func/test_file_manager.py rename to profiler/msprof_analyze/test/ut/cluster_analyse/common_func/test_file_manager.py index 071d11e7ff..9dec7532fa 100644 --- a/profiler/test/ut/cluster_analyse/common_func/test_file_manager.py +++ b/profiler/msprof_analyze/test/ut/cluster_analyse/common_func/test_file_manager.py @@ -5,7 +5,7 @@ import json import unittest import pytest -from profiler.prof_common.file_manager import FileManager +from msprof_analyze.prof_common.file_manager import FileManager from prof_bean.step_trace_time_bean import StepTraceTimeBean diff --git a/profiler/test/ut/cluster_analyse/common_func/test_path_manager.py b/profiler/msprof_analyze/test/ut/cluster_analyse/common_func/test_path_manager.py similarity index 98% rename from profiler/test/ut/cluster_analyse/common_func/test_path_manager.py rename to profiler/msprof_analyze/test/ut/cluster_analyse/common_func/test_path_manager.py index 93b5f3e1f3..7515c1dba1 100644 --- a/profiler/test/ut/cluster_analyse/common_func/test_path_manager.py +++ b/profiler/msprof_analyze/test/ut/cluster_analyse/common_func/test_path_manager.py @@ -3,7 +3,7 @@ import os import time import pytest -from profiler.prof_common.path_manager import PathManager +from msprof_analyze.prof_common.path_manager import PathManager PATH_DIR = "resource" diff --git a/profiler/test/ut/cluster_analyse/communication_group/test_communication_group_generator.py b/profiler/msprof_analyze/test/ut/cluster_analyse/communication_group/test_communication_group_generator.py similarity index 98% rename from profiler/test/ut/cluster_analyse/communication_group/test_communication_group_generator.py rename to profiler/msprof_analyze/test/ut/cluster_analyse/communication_group/test_communication_group_generator.py index 1d00c4f6fd..e65015d774 100644 --- a/profiler/test/ut/cluster_analyse/communication_group/test_communication_group_generator.py +++ b/profiler/msprof_analyze/test/ut/cluster_analyse/communication_group/test_communication_group_generator.py @@ -19,7 +19,7 @@ import unittest from unittest import mock from communication_group.communication_group_generator import CommunicationGroupGenerator -from profiler.prof_common.constant import Constant +from msprof_analyze.prof_common.constant import Constant class TestCommunicationGroupGenerator(unittest.TestCase): diff --git a/profiler/test/ut/cluster_analyse/prof_bean/test_step_trace_time_bean.py b/profiler/msprof_analyze/test/ut/cluster_analyse/prof_bean/test_step_trace_time_bean.py similarity index 100% rename from profiler/test/ut/cluster_analyse/prof_bean/test_step_trace_time_bean.py rename to profiler/msprof_analyze/test/ut/cluster_analyse/prof_bean/test_step_trace_time_bean.py diff --git a/profiler/test/ut/compare_tools/__init__.py b/profiler/msprof_analyze/test/ut/compare_tools/__init__.py similarity index 100% rename from profiler/test/ut/compare_tools/__init__.py rename to profiler/msprof_analyze/test/ut/compare_tools/__init__.py diff --git a/profiler/test/ut/compare_tools/comparator/test_communication_comparator.py b/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_communication_comparator.py similarity index 100% rename from profiler/test/ut/compare_tools/comparator/test_communication_comparator.py rename to profiler/msprof_analyze/test/ut/compare_tools/comparator/test_communication_comparator.py diff --git a/profiler/test/ut/compare_tools/comparator/test_operator_comparator.py b/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_comparator.py similarity index 100% rename from profiler/test/ut/compare_tools/comparator/test_operator_comparator.py rename to profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_comparator.py diff --git a/profiler/test/ut/compare_tools/comparator/test_operator_statistic_comparator.py b/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_statistic_comparator.py similarity index 100% rename from profiler/test/ut/compare_tools/comparator/test_operator_statistic_comparator.py rename to profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_statistic_comparator.py diff --git a/profiler/test/ut/compare_tools/compare_bean/__init__.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/__init__.py similarity index 100% rename from profiler/test/ut/compare_tools/compare_bean/__init__.py rename to profiler/msprof_analyze/test/ut/compare_tools/compare_bean/__init__.py diff --git a/profiler/test/ut/compare_tools/compare_bean/origin_data_bean/__init__.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/__init__.py similarity index 100% rename from profiler/test/ut/compare_tools/compare_bean/origin_data_bean/__init__.py rename to profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/__init__.py diff --git a/profiler/test/ut/compare_tools/compare_bean/origin_data_bean/test_compare_event.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_compare_event.py similarity index 100% rename from profiler/test/ut/compare_tools/compare_bean/origin_data_bean/test_compare_event.py rename to profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_compare_event.py diff --git a/profiler/test/ut/compare_tools/compare_bean/origin_data_bean/test_kernel_details_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_kernel_details_bean.py similarity index 100% rename from profiler/test/ut/compare_tools/compare_bean/origin_data_bean/test_kernel_details_bean.py rename to profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_kernel_details_bean.py diff --git a/profiler/test/ut/compare_tools/compare_bean/origin_data_bean/test_memory_record_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_memory_record_bean.py similarity index 100% rename from profiler/test/ut/compare_tools/compare_bean/origin_data_bean/test_memory_record_bean.py rename to profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_memory_record_bean.py diff --git a/profiler/test/ut/compare_tools/compare_bean/origin_data_bean/test_operator_memory_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_operator_memory_bean.py similarity index 100% rename from profiler/test/ut/compare_tools/compare_bean/origin_data_bean/test_operator_memory_bean.py rename to profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_operator_memory_bean.py diff --git a/profiler/test/ut/compare_tools/compare_bean/origin_data_bean/test_trace_event_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_trace_event_bean.py similarity index 100% rename from profiler/test/ut/compare_tools/compare_bean/origin_data_bean/test_trace_event_bean.py rename to profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_trace_event_bean.py diff --git a/profiler/test/ut/compare_tools/compare_bean/test_communication_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_communication_bean.py similarity index 100% rename from profiler/test/ut/compare_tools/compare_bean/test_communication_bean.py rename to profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_communication_bean.py diff --git a/profiler/test/ut/compare_tools/compare_bean/test_memory_compare_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_compare_bean.py similarity index 100% rename from profiler/test/ut/compare_tools/compare_bean/test_memory_compare_bean.py rename to profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_compare_bean.py diff --git a/profiler/test/ut/compare_tools/compare_bean/test_memory_statistic_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_statistic_bean.py similarity index 100% rename from profiler/test/ut/compare_tools/compare_bean/test_memory_statistic_bean.py rename to profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_statistic_bean.py diff --git a/profiler/test/ut/compare_tools/compare_bean/test_operator_compare_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_compare_bean.py similarity index 100% rename from profiler/test/ut/compare_tools/compare_bean/test_operator_compare_bean.py rename to profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_compare_bean.py diff --git a/profiler/test/ut/compare_tools/compare_bean/test_operator_statistic_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_statistic_bean.py similarity index 100% rename from profiler/test/ut/compare_tools/compare_bean/test_operator_statistic_bean.py rename to profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_statistic_bean.py diff --git a/profiler/test/ut/compare_tools/compare_bean/test_profiling_info.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_profiling_info.py similarity index 100% rename from profiler/test/ut/compare_tools/compare_bean/test_profiling_info.py rename to profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_profiling_info.py diff --git a/profiler/test/ut/compare_tools/profiling_parser/__init__.py b/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/__init__.py similarity index 100% rename from profiler/test/ut/compare_tools/profiling_parser/__init__.py rename to profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/__init__.py diff --git a/profiler/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py b/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py similarity index 100% rename from profiler/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py rename to profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py diff --git a/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py b/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py similarity index 100% rename from profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py rename to profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py diff --git a/profiler/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py b/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py similarity index 100% rename from profiler/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py rename to profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py diff --git a/profiler/test/ut/compare_tools/utils/test_name_function.py b/profiler/msprof_analyze/test/ut/compare_tools/utils/test_name_function.py similarity index 100% rename from profiler/test/ut/compare_tools/utils/test_name_function.py rename to profiler/msprof_analyze/test/ut/compare_tools/utils/test_name_function.py diff --git a/profiler/test/ut/compare_tools/utils/test_tree_builder.py b/profiler/msprof_analyze/test/ut/compare_tools/utils/test_tree_builder.py similarity index 100% rename from profiler/test/ut/compare_tools/utils/test_tree_builder.py rename to profiler/msprof_analyze/test/ut/compare_tools/utils/test_tree_builder.py diff --git a/profiler/test/ut/compare_tools/view/__init__.py b/profiler/msprof_analyze/test/ut/compare_tools/view/__init__.py similarity index 100% rename from profiler/test/ut/compare_tools/view/__init__.py rename to profiler/msprof_analyze/test/ut/compare_tools/view/__init__.py diff --git a/profiler/test/ut/compare_tools/view/test_excel_view.py b/profiler/msprof_analyze/test/ut/compare_tools/view/test_excel_view.py similarity index 100% rename from profiler/test/ut/compare_tools/view/test_excel_view.py rename to profiler/msprof_analyze/test/ut/compare_tools/view/test_excel_view.py diff --git a/profiler/test/ut/compare_tools/view/test_screen_view.py b/profiler/msprof_analyze/test/ut/compare_tools/view/test_screen_view.py similarity index 100% rename from profiler/test/ut/compare_tools/view/test_screen_view.py rename to profiler/msprof_analyze/test/ut/compare_tools/view/test_screen_view.py diff --git a/profiler/test/ut/compare_tools/view/test_worker_sheet_creator.py b/profiler/msprof_analyze/test/ut/compare_tools/view/test_worker_sheet_creator.py similarity index 100% rename from profiler/test/ut/compare_tools/view/test_worker_sheet_creator.py rename to profiler/msprof_analyze/test/ut/compare_tools/view/test_worker_sheet_creator.py diff --git a/profiler/version.txt b/profiler/msprof_analyze/version.txt similarity index 100% rename from profiler/version.txt rename to profiler/msprof_analyze/version.txt diff --git "a/\345\205\254\347\275\221URL\350\257\264\346\230\216.md" "b/\345\205\254\347\275\221URL\350\257\264\346\230\216.md" index 67064cd045..e0ed58a7c1 100644 --- "a/\345\205\254\347\275\221URL\350\257\264\346\230\216.md" +++ "b/\345\205\254\347\275\221URL\350\257\264\346\230\216.md" @@ -2,13 +2,13 @@ | 软件类型 | 软件名 | 路径 | 类型 | 内容 | 用途说明 | |------|----------------------------------------------------|------------------------------------------|------|------------------------------------------------------------------------------------------------------------|--------------------| -| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/config/config.ini | 公网地址 | https://gitee.com/ascend/mstt/blob/master/profiler/advisor/doc/Samples%20of%20Fused%20Operator%20API%20Replacement.md" | Advisor优化手段参考示例 | -| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/config/config.ini | 公网地址 | https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/modeldevpt/ptmigr/AImpug_0067.html | Advisor优化手段参考示例 | -| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/config/config.ini | 公网地址 | https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/devtools/auxiliarydevtool/aoe_16_045.html | Advisor优化手段参考示例 | -| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/config/config.ini | 公网地址 | https://www.mindspore.cn/lite/docs/en/master/use/cloud_infer/converter_tool_ascend.html#aoe-auto-tuning | Advisor优化手段参考示例 | -| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/config/config.ini | 公网地址 | https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/modeldevpt/ptmigr/AImpug_0059.html | Advisor优化手段参考示例 | -| 开源软件 | MindStudio Training Tools - msprof-analyze | /profiler/config/config.ini | 公网地址 | https://gitee.com/ascend/mstt/tree/master/profiler | msprof-analyze工具地址 | -| 开源软件 | MindStudio Training Tools - msprof-analyze | /profiler/LICENSE | 公网地址 | http://www.apache.org/licenses/LICENSE-2.0 | 开源软件协议地址 | -| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/rules/aicpu_rules.ymal | 公网地址 | https://gitee.com/ascend/mstt/blob/master/profiler/advisor/doc/Samples%20of%20AI%20CPU%20Operator%20Replacement.md | AI CPU 算子替换样例 | -| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/rules/environment_variable_info.yaml | 公网地址 | https://support.huawei.com/enterprise/zh/doc/EDOC1100371278/5eeeed85?idPath=23710424 | 组网指南 | -| 开源软件 | MindStudio Training Tools - msprof-analyze | /profiler/config/config.ini | 公网地址 | pmail_mindstudio@huawei.com | 公网邮箱 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/msprof_analyze/advisor/config/config.ini | 公网地址 | https://gitee.com/ascend/mstt/blob/master/profiler/advisor/doc/Samples%20of%20Fused%20Operator%20API%20Replacement.md" | Advisor优化手段参考示例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/msprof_analyze/advisor/config/config.ini | 公网地址 | https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/modeldevpt/ptmigr/AImpug_0067.html | Advisor优化手段参考示例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/msprof_analyze/advisor/config/config.ini | 公网地址 | https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/devtools/auxiliarydevtool/aoe_16_045.html | Advisor优化手段参考示例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/msprof_analyze/advisor/config/config.ini | 公网地址 | https://www.mindspore.cn/lite/docs/en/master/use/cloud_infer/converter_tool_ascend.html#aoe-auto-tuning | Advisor优化手段参考示例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/msprof_analyze/advisor/config/config.ini | 公网地址 | https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/modeldevpt/ptmigr/AImpug_0059.html | Advisor优化手段参考示例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze | /profiler/msprof_analyze/config/config.ini | 公网地址 | https://gitee.com/ascend/mstt/tree/master/profiler | msprof-analyze工具地址 | +| 开源软件 | MindStudio Training Tools - msprof-analyze | /profiler/msprof_analyze/LICENSE | 公网地址 | http://www.apache.org/licenses/LICENSE-2.0 | 开源软件协议地址 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/msprof_analyze/advisor/rules/aicpu_rules.ymal | 公网地址 | https://gitee.com/ascend/mstt/blob/master/profiler/msprof_analyze/advisor/doc/Samples%20of%20AI%20CPU%20Operator%20Replacement.md | AI CPU 算子替换样例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/msprof_analyze/advisor/rules/environment_variable_info.yaml | 公网地址 | https://support.huawei.com/enterprise/zh/doc/EDOC1100371278/5eeeed85?idPath=23710424 | 组网指南 | +| 开源软件 | MindStudio Training Tools - msprof-analyze | /profiler/msprof_analyze/config/config.ini | 公网地址 | pmail_mindstudio@huawei.com | 公网邮箱 | -- Gitee