diff --git a/failslow/failslow/dataloader/marker_data_reader.py b/failslow/failslow/dataloader/marker_data_reader.py index 7ab8e31b5ff6bcdd8c4ccc54c8f457cd42a0a9eb..6bbff8c640d6157a0a858769e4d35ac383478385 100644 --- a/failslow/failslow/dataloader/marker_data_reader.py +++ b/failslow/failslow/dataloader/marker_data_reader.py @@ -4,7 +4,6 @@ import datetime import multiprocessing import numpy as np import pandas as pd -import matplotlib.pyplot as plt from collections import Counter from typing import List, Dict, Tuple @@ -412,32 +411,6 @@ class MarkerDataloader: return selected_indices, comm_ops - def get_broadcast_ops(self, broadcast_ops="HcclBroadcast"): - ''' Use broadcast time estimate step time ''' - # for csv_file in self.csv_files: - # csv_path = os.path.join(self._root_path, csv_file) - # data_df = self.read_csv(csv_path) - # data_df['start_stamp'] = data_df['start'].apply(self.convert_timestamp2datetime) - # data_df['end_stamp'] = data_df['end'].apply(self.convert_timestamp2datetime) - # data_df.to_csv(f"new_{csv_file}", index=False) - - csv_path = os.path.join(self._root_path, self.csv_files[0]) - data_df = self.read_csv(csv_path) - data_df['start_stamp'] = data_df[TableItem.ex_start_ts].apply(self.convert_timestamp2datetime) - # data_df.to_csv("./broadcast_df.csv", index=False) - # n = len(data_df) - # quarter = n // 4 - # data_df = data_df[quarter:] - mask = data_df[TableItem.name] == broadcast_ops - broadcast_df = data_df[mask] - logger.info(f"broadcast df length: {len(broadcast_df)}") - broadcast_df['start_stamp'] = broadcast_df[TableItem.ex_start_ts].apply(self.convert_timestamp2datetime) - # broadcast_df.to_csv("./broadcast_df.csv", index=False) - broadcast_df = self._filter_conti_index(broadcast_df) - step_time = np.array(broadcast_df[TableItem.ex_start_ts].diff() / 1e6)[2:] - logger.info(f"estimate length: {len(broadcast_df)}") - - self.plot_step_time(step_time) def convert_timestamp2datetime(self, data): @@ -446,21 +419,6 @@ class MarkerDataloader: date_time = dt_object.strftime('%Y-%m-%d %H:%M:%S') return date_time - def plot_step_time(self, data): - step_time = extract_step_time_from_log(self._root_path) - mean_value = np.mean(data) - plt.figure(figsize=(10, 6)) - plt.plot(step_time[10:110], label='GT', marker='^') - plt.plot(data[10:110], label='extimate_time', marker='o') - # 绘制均值线 - # plt.axhline(y=mean_value, color='r', linestyle='--', label='Mean') - # 设置图表标题和坐标轴标签 - plt.title('Estimate step time by dataloader.') - plt.xlabel('step index') - plt.ylabel('latency(ms)') - plt.legend() - plt.grid(True) - plt.show() @staticmethod def _filter_deviation_data(data): @@ -503,7 +461,6 @@ if __name__ == "__main__": convert_jsons2csv(_root_path) dataloader = MarkerDataloader(_root_path) - # all_comm_groups = dataloader.get_broadcast_ops() logger.info(f"{dataloader.csv_files}") from restore_comm import RestoreComm diff --git a/failslow/setup.py b/failslow/setup.py index 618f37722267314a85cae082f63158dad5be2127..a881cd6131eddea02ca55b9f17e5aa47eb29c63f 100644 --- a/failslow/setup.py +++ b/failslow/setup.py @@ -43,10 +43,9 @@ setup( install_requires=[ "numpy", "pandas", - "matplotlib", "joblib", "scikit_learn", - "protobuf==3.20.3" + "protobuf" ], entry_points={ "console_scripts": [ diff --git a/raw_mcp_py39.zip b/raw_mcp_py39.zip new file mode 100644 index 0000000000000000000000000000000000000000..effba50c7cae2fd4dca3f8da9b46511e80a0a243 Binary files /dev/null and b/raw_mcp_py39.zip differ diff --git a/test.sh b/test.sh new file mode 100644 index 0000000000000000000000000000000000000000..1144ea2aa7319ef1da0c6c8493fa259a4a7cc861 --- /dev/null +++ b/test.sh @@ -0,0 +1,109 @@ +#!/bin/bash +# 颜色定义 +COLOR_INFO='\033[34m' # 蓝色信息 +COLOR_SUCCESS='\033[32m' # 绿色成功 +COLOR_ERROR='\033[31m' # 红色错误 +COLOR_WARNING='\033[33m' # 黄色警告 +COLOR_RESET='\033[0m' # 重置颜色 + + +#管理员权限启动 +if [[ $(id -u) -ne 0 ]]; then + echo -e "${COLOR_ERROR}[Error] 请以root权限运行该脚本!${COLOR_RESET}" + return 1 +fi +#添加代理 +export http_proxy=http://peulerosci:EulerOS_123@proxy.huawei.com:8080 +export https_proxy=http://peulerosci:EulerOS_123@proxy.huawei.com:8080 +export no_proxy=127.0.0.1 + +#换源 +echo "更新yum源" +echo "[SHELL] +name=SHELL +baseurl=https://eulermaker.compass-ci.openeuler.openatom.cn/api/ems1/repositories/framework-agent/openEuler%3A24.03-LTS-SP2/x86_64/ +enabled=1 +gpgcheck=0 +sslverify=0 +gpgkey=http://repo.openeuler.org/openEuler-24.03-LTS-SP2/OS//RPM-GPG-KEY-openEuler">>/etc/yum.repos.d/shell.repo + +yum clean all +yum makecache + +#启动perf +git config --global http.sslVerify false +cd /home +git clone https://gitee.com/zxstty/perf_mcp +dnf install -y python3-devel python3-pip perf +pip install pyyaml psutil mcp==1.6.0 -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host pypi.tuna.tsinghua.edu.cn +cd perf_mcp +firewall-cmd --permanent --add-port=12141/tcp +firewall-cmd --reload +nohup python3 src/server.py >perf_mcp.log 2>&1 & + +#安装rpm +dnf install -y openeuler-intelligence-cli openeuler-intelligence-installer + +#修改配置文件 +cd /usr/lib/openeuler-intelligence/scripts/5-resource +sed -i.bak \ + -e '/^\[llm\]$/,/^\[.*\]$/ s|^endpoint = .*|endpoint = "https://api.deepseek.com"|' \ + -e '/^\[llm\]$/,/^\[.*\]$/ s|^key = .*|key = "sk-440510c3614a488283f389834085e173"|' \ + -e '/^\[llm\]$/,/^\[.*\]$/ s|^model = .*|model = "deepseek-chat"|' \ + -e '/^\[function_call\]$/,/^\[.*\]$/ s|^backend = .*|backend = "openai"|' \ + -e '/^\[function_call\]$/,/^\[.*\]$/ s|^endpoint = .*|endpoint = "https://api.deepseek.com"|' \ + -e '/^\[function_call\]$/,/^\[.*\]$/ s|^model = .*|model = "deepseek-chat"|' \ + -e '/^\[function_call\]$/,/^\[.*\]$/ s|^api_key = .*|api_key = "sk-440510c3614a488283f389834085e173"|' \ + config.toml + +FRAMEWORK_FILE="framework.service" + +if [ ! -f "$FRAMEWORK_FILE" ]; then + echo "错误:文件 $FRAMEWORK_FILE 不存在!" + exit 1 +fi +echo '[Unit] +After=network.target nss-lookup.target +[Service] +User=root +WorkingDirectory=/usr/lib/euler-copilot-framework +Environment="PYTHONPATH=/usr/lib/euler-copilot-framework" +Environment="CONFIG=/etc/euler-copilot-framework/config.toml" +Environment="http_proxy=http://peulerosci:EulerOS_123@proxy.huawei.com:8080" +Environment="https_proxy=http://peulerosci:EulerOS_123@proxy.huawei.com:8080" +Environment="no_proxy=127.0.0.1" +ExecStart=/usr/bin/python3 apps/main.py +ExecReload=/bin/kill -HUP $MAINPID +Restart=on-failure +RestartSec=10 +LimitNOFILE=infinity +[Install] +WantedBy=multi-user.target' > "$FRAMEWORK_FILE" + +CLIENT_FILE="/usr/lib/python3.11/site-packages/openai/_base_client.py" + +if [ ! -f "$CLIENT_FILE" ]; then + echo "openai未安装 安装中" + dnf install -y python3-openai +fi +sed -i.bak -E '/(Sync|Async)HttpxClientWrapper\($/,/timeout=cast\(Timeout, timeout\),/ { + /timeout=cast\(Timeout, timeout\),/ a\ verify=False +}' "$CLIENT_FILE" + +#启动 +openeuler-intelligence-installer +echo "开始初始化智能助手" +sleep 5 +JSON_FILE="/usr/lib/openeuler-intelligence/scripts/5-resource/mcp_config/perf_mcp/config.json" + +if [ ! -f "$JSON_FILE" ]; then + echo "错误:文件 $JSON_FILE 不存在!" + exit 1 +fi + +sed -i.bak 's/"perf 分析工具"/"OS智能助手"/g' "$JSON_FILE" + +openeuler-intelligence-installer --a init /usr/lib/openeuler-intelligence/scripts/5-resource/mcp_config/perf_mcp/config.json +openeuler-intelligence-installer --a create /usr/lib/openeuler-intelligence/scripts/5-resource/mcp_config/perf_mcp/config.json + +echo "请在命令行输入 oi 进入 shell端使用界面 " \ No newline at end of file