From ea9c869bafba78a312576139364e8eff46ca68a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=A4=A9?= Date: Wed, 20 Dec 2023 06:39:02 +0000 Subject: [PATCH 1/5] update profiler/cluster_analyse/cluster_analysis.py. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 李天 --- profiler/cluster_analyse/cluster_analysis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py index 50eeedd73..031dbdfab 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/cluster_analyse/cluster_analysis.py @@ -68,6 +68,7 @@ class Interface: Constant.COMM_DATA_DICT: comm_data_dict } AnalysisFacade(params).cluster_analyze() + print(f"[INFO] Cluster analysis success! Result is saved to {self.collection_path}.") if __name__ == "__main__": -- Gitee From b9cffef4275f06752d7466bf75ef793ed1d316bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=A4=A9?= Date: Wed, 20 Dec 2023 07:41:55 +0000 Subject: [PATCH 2/5] update profiler/cluster_analyse/cluster_analysis.py. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 李天 --- profiler/cluster_analyse/cluster_analysis.py | 1 - 1 file changed, 1 deletion(-) diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py index 031dbdfab..50eeedd73 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/cluster_analyse/cluster_analysis.py @@ -68,7 +68,6 @@ class Interface: Constant.COMM_DATA_DICT: comm_data_dict } AnalysisFacade(params).cluster_analyze() - print(f"[INFO] Cluster analysis success! Result is saved to {self.collection_path}.") if __name__ == "__main__": -- Gitee From c25a4addbdab20367d316835e6c0c174084f708d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=A4=A9?= Date: Wed, 20 Dec 2023 07:44:57 +0000 Subject: [PATCH 3/5] update profiler/cluster_analyse/analysis/analysis_facade.py. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 李天 --- .../analysis/analysis_facade.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/profiler/cluster_analyse/analysis/analysis_facade.py b/profiler/cluster_analyse/analysis/analysis_facade.py index 34228f97a..67f557d37 100644 --- a/profiler/cluster_analyse/analysis/analysis_facade.py +++ b/profiler/cluster_analyse/analysis/analysis_facade.py @@ -13,10 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from multiprocessing import Process +from multiprocessing import Pool from analysis.communication_analysis import CommunicationAnalysis from analysis.step_trace_time_analysis import StepTraceTimeAnalysis from analysis.communication_analysis import CommMatrixAnalysis +from common_func.constant import Constant class AnalysisFacade: @@ -27,11 +28,11 @@ class AnalysisFacade: def cluster_analyze(self): # 多个profiler用多进程处理 - process_list = [] - for analysis in self.analysis_module: - process = Process(target=analysis(self.param).run) - process.start() - process_list.append(process) - - for process in process_list: - process.join() + with Pool(processes=len(self.analysis_module)) as pool: + results = [pool.apply_async(analysis(self.param).run) for analysis in self.analysis_module] + outputs = [result.get() for result in results] + for i in outputs: + if i != 1: + return + print(f"[INFO] Cluster analysis success! Result is saved to {self.param.get(Constant.COLLECTION_PATH)}") + \ No newline at end of file -- Gitee From 05bfad324d0c0f39412a86365db970372b5a2715 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=A4=A9?= Date: Wed, 20 Dec 2023 07:45:24 +0000 Subject: [PATCH 4/5] update profiler/cluster_analyse/analysis/communication_analysis.py. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 李天 --- profiler/cluster_analyse/analysis/communication_analysis.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/profiler/cluster_analyse/analysis/communication_analysis.py b/profiler/cluster_analyse/analysis/communication_analysis.py index a3c51d46a..fcb2149e5 100644 --- a/profiler/cluster_analyse/analysis/communication_analysis.py +++ b/profiler/cluster_analyse/analysis/communication_analysis.py @@ -85,6 +85,7 @@ class CommunicationAnalysis(BaseCommAnalysis): self.split_op_by_group() self.combine_ops_total_info() self.dump_data() + return 1 def compute_total_info(self, comm_ops: dict): if not comm_ops: @@ -161,6 +162,7 @@ class CommMatrixAnalysis(BaseCommAnalysis): self.split_op_by_group() self.combine_ops_total_info() self.dump_data() + return 1 def compute_total_info(self, step_dict: dict): self.merge_same_links(step_dict) -- Gitee From 466e9ad204d169688cc095ce247e287e8de22eee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=A4=A9?= Date: Wed, 20 Dec 2023 07:45:45 +0000 Subject: [PATCH 5/5] update profiler/cluster_analyse/analysis/step_trace_time_analysis.py. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 李天 --- profiler/cluster_analyse/analysis/step_trace_time_analysis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/profiler/cluster_analyse/analysis/step_trace_time_analysis.py b/profiler/cluster_analyse/analysis/step_trace_time_analysis.py index d24a7f1fe..ffac66c59 100644 --- a/profiler/cluster_analyse/analysis/step_trace_time_analysis.py +++ b/profiler/cluster_analyse/analysis/step_trace_time_analysis.py @@ -47,6 +47,7 @@ class StepTraceTimeAnalysis: self.load_step_trace_time_data() self.analyze_step_time() self.dump_data() + return 1 def dump_data(self): if not self.step_data_list: -- Gitee