From d2a501318a08d35c32a0e0cd819c5ff2a1a0c18b Mon Sep 17 00:00:00 2001 From: yzy_dev Date: Thu, 28 Nov 2024 15:00:41 +0800 Subject: [PATCH 1/2] AI log_analysis plugin --- llmops/app_factory.py | 2 + llmops/config/config.py | 17 ++-- llmops/controller/log_analysis_controller.py | 13 +++ llmops/llm_tools/log_analysis/agentfactory.py | 15 ++++ .../log_analysis/log_analysis_workflow.py | 88 +++++++++++++++++++ llmops/router/log_analysis_router.py | 15 ++++ llmops/service/log_analysis_service.py | 7 ++ 7 files changed, 152 insertions(+), 5 deletions(-) create mode 100644 llmops/controller/log_analysis_controller.py create mode 100644 llmops/llm_tools/log_analysis/agentfactory.py create mode 100644 llmops/llm_tools/log_analysis/log_analysis_workflow.py create mode 100644 llmops/router/log_analysis_router.py create mode 100644 llmops/service/log_analysis_service.py diff --git a/llmops/app_factory.py b/llmops/app_factory.py index 9c207c38..2e4f6d42 100644 --- a/llmops/app_factory.py +++ b/llmops/app_factory.py @@ -1,5 +1,6 @@ from flask import Flask from config.config import init_config +from llmops.router.log_analysis_router import log_analysis_router from utils.logger import setup_logger from router.task import task_blueprint @@ -19,6 +20,7 @@ def create_app() -> Flask: # 注册蓝图 app.register_blueprint(task_blueprint, url_prefix="/task") + app.register_blueprint(log_analysis_router) return app diff --git a/llmops/config/config.py b/llmops/config/config.py index 7b8e00c5..d772b467 100644 --- a/llmops/config/config.py +++ b/llmops/config/config.py @@ -9,16 +9,22 @@ class AppConf: port: str debug: bool +@dataclass +class LlmConf: + model: str + apikey: str + baseurl: str + current_model: str + class Config: - filename = "../llm-ops.yaml" + filename = "llm-ops.yaml" def __init__(self): self.app_conf: AppConf = None - + self.llm_conf: LlmConf = None def load_config(self): - current_dir = os.path.dirname(os.path.abspath(__file__)) - config_path = os.path.join(current_dir, self.filename) + config_path = os.path.join(os.path.dirname(__file__), Config.filename) try: with open(config_path, "r") as f: @@ -27,9 +33,10 @@ class Config: raise ValueError("Load llm-ops config file failed") from e self.app_conf = AppConf(**conf["app"]) - + self.llm_conf = LlmConf(**conf["llm"]) def init_config() -> Config: + """initialize llm-ops config""" conf = Config() conf.load_config() return conf diff --git a/llmops/controller/log_analysis_controller.py b/llmops/controller/log_analysis_controller.py new file mode 100644 index 00000000..69c0b7d7 --- /dev/null +++ b/llmops/controller/log_analysis_controller.py @@ -0,0 +1,13 @@ +from flask import jsonify + +from llmops.service.log_analysis_service import LogAnalysisService +from llmops.utils.response import success + + +class LogAnalysisController: + def __init__(self): + self.logAnalysis = LogAnalysisService() + + def log_analysis(self,user_input): + log_analysis = self.logAnalysis.loganalysis(user_input) + return success(log_analysis,"返回成功") \ No newline at end of file diff --git a/llmops/llm_tools/log_analysis/agentfactory.py b/llmops/llm_tools/log_analysis/agentfactory.py new file mode 100644 index 00000000..47214a52 --- /dev/null +++ b/llmops/llm_tools/log_analysis/agentfactory.py @@ -0,0 +1,15 @@ +import Agently + +from llmops.config.config import Config + + +def agentfactory(config: Config): + agent_factory = ( + Agently.AgentFactory() + .set_settings("current_model", config.llm_conf.current_model) + # .set_settings("model.OAIClient.auth.api_key", config.llm_conf.apikey) + .set_settings("model.OAIClient.options.model", config.llm_conf.model) + .set_settings("model.OAIClient.url", config.llm_conf.baseurl) + ) + return agent_factory + diff --git a/llmops/llm_tools/log_analysis/log_analysis_workflow.py b/llmops/llm_tools/log_analysis/log_analysis_workflow.py new file mode 100644 index 00000000..3d076d7b --- /dev/null +++ b/llmops/llm_tools/log_analysis/log_analysis_workflow.py @@ -0,0 +1,88 @@ +# 日志分析 +import os + +import Agently + +from llmops.config.config import init_config +from llmops.llm_tools.log_analysis.agentfactory import agentfactory + + +def logworkflow(user_input): + conf = init_config() # 初始化配置 + factory = agentfactory(conf) # 初始化agentfactory工厂类 + agent = factory.create_agent(); # 初始化agent + main_workflow = Agently.Workflow() + + # @main_workflow.chunk("user_input") + # def user_input(inputs, storage): + # storage.set( + # "user_input", + # # input("[请输入你的问题]: ") + # user_input + # ) + + # 返回shell脚本 + @main_workflow.chunk("assistant_reply") + def assistant_shell_reply(inputs, storage): + print("分析中.....") + # assistant_reason_reply = ( + # agent + # .info("回答尽量精简") + # .user_info("用户是服务器运维人员，操作系统是macOs") + # .input({ + # "user_input": storage.get("user_input"), + # }) + # .instruct([ + # "replay中输出语言为中文", + # f"请根据：{user_input}找出日志的问题", + # "用简洁的方式说明问题" + # "操作系统是maoOs" + # ]) + # .output({ + # "replay": ("str", "对{user_input}进行解释"), + # "improvement": ([("str", "根据用户输入{input}输出相应脚本处理用户的要求"), "找出改进方案"]) + # }) + # .start_async() + # ) + # storage.set("replay", assistant_reason_reply.get("replay")) + # print("*" * 50) + # print("一.大模型分析：") + # print(assistant_reason_reply.get("replay")) + # print("*" * 50) + # print("二.生成的脚本内容:") + # storage.set("shell", assistant_reason_reply.get("improvement")) + # print(assistant_reason_reply.get("improvement")) + # print("*" * 50) + # return assistant_reason_reply.get("improvement") + assistant_reason_reply = ( + agent + .input(user_input) + .info("用中文对系统日志进行解释，结果用字典形式例如：{reason:'解释'}") + .instruct("目标语言：中文") + .output({ + "info_list": [ + { + "知识对象": ("str", "回答{input}问题时，需要了解相关知识的具体对象"), + "关键知识点": ("str", "回答{input}问题时，需要了解的关键知识") + } + ], + "analyse": ("str", "根据{info_list}出现的问题进行解释和分析"), + "reasons": [("str", "{info_list}根据找出报错的原因并按列表格式输出，添加编号")] + }) + .start() + ) + return assistant_reason_reply + + ( + main_workflow + # .connect_to("user_input") + .connect_to("assistant_reply") + .connect_to("END") + ) + print(main_workflow.draw()) + result = main_workflow.start() + return result + + +# r = logworkflow("Can not connect to MySQL server. Too many connections -mysql 1040错误") +# print(r['default']) diff --git a/llmops/router/log_analysis_router.py b/llmops/router/log_analysis_router.py new file mode 100644 index 00000000..7ff94808 --- /dev/null +++ b/llmops/router/log_analysis_router.py @@ -0,0 +1,15 @@ +from flask import Blueprint,request + +from controller.log_analysis_controller import LogAnalysisController + + +log_analysis_router = Blueprint("log_analysis", __name__) +logAnalysisController = LogAnalysisController() + + +@log_analysis_router.route("/log_analysis", methods=["POST"]) +def log_analysis(): + print("diaoyog") + user_input = request.get_json("user_input") + print(user_input) + return logAnalysisController.log_analysis(user_input) diff --git a/llmops/service/log_analysis_service.py b/llmops/service/log_analysis_service.py new file mode 100644 index 00000000..81dad9d0 --- /dev/null +++ b/llmops/service/log_analysis_service.py @@ -0,0 +1,7 @@ +from llm_tools.log_analysis import log_analysis_workflow + + +class LogAnalysisService: + def loganalysis(self, userinput: str): + result = log_analysis_workflow.logworkflow(userinput) + return result -- Gitee From 2c83729678c409690be8ed6ba0f35115ab530275 Mon Sep 17 00:00:00 2001 From: yzy_dev Date: Thu, 28 Nov 2024 15:02:31 +0800 Subject: [PATCH 2/2] add copyright information --- llmops/app_factory.py | 5 +++++ llmops/config/config.py | 5 +++++ llmops/controller/log_analysis_controller.py | 5 +++++ llmops/controller/task.py | 5 +++++ llmops/llm_tools/log_analysis/agentfactory.py | 5 +++++ llmops/llm_tools/log_analysis/la_test.py | 5 +++++ llmops/llm_tools/log_analysis/log_analysis_workflow.py | 5 +++++ llmops/llm_tools/nlp/nlp_test.py | 5 +++++ llmops/llm_tools/rca/rca_test.py | 5 +++++ llmops/main.py | 5 +++++ llmops/router/log_analysis_router.py | 5 +++++ llmops/router/task.py | 5 +++++ llmops/service/log_analysis_service.py | 5 +++++ llmops/service/task.py | 5 +++++ llmops/utils/command.py | 5 +++++ llmops/utils/logger.py | 5 +++++ llmops/utils/response.py | 5 +++++ 17 files changed, 85 insertions(+) diff --git a/llmops/app_factory.py b/llmops/app_factory.py index 2e4f6d42..4c953ed0 100644 --- a/llmops/app_factory.py +++ b/llmops/app_factory.py @@ -1,3 +1,8 @@ +# * Copyright (c) KylinSoft Co., Ltd. 2024.All rights reserved. +# * PilotGo-plugins licensed under the Mulan Permissive Software License, Version 2. +# * See LICENSE file for more details. +# * Author: zhanghan2021 +# * Date: Thu Nov 7 15:01:23 2024 +0800 from flask import Flask from config.config import init_config from llmops.router.log_analysis_router import log_analysis_router diff --git a/llmops/config/config.py b/llmops/config/config.py index d772b467..20db29de 100644 --- a/llmops/config/config.py +++ b/llmops/config/config.py @@ -1,3 +1,8 @@ +# * Copyright (c) KylinSoft Co., Ltd. 2024.All rights reserved. +# * PilotGo-plugins licensed under the Mulan Permissive Software License, Version 2. +# * See LICENSE file for more details. +# * Author: zhanghan2021 +# * Date: Thu Nov 7 15:01:23 2024 +0800 import os import yaml from dataclasses import dataclass diff --git a/llmops/controller/log_analysis_controller.py b/llmops/controller/log_analysis_controller.py index 69c0b7d7..d27cbea5 100644 --- a/llmops/controller/log_analysis_controller.py +++ b/llmops/controller/log_analysis_controller.py @@ -1,3 +1,8 @@ +# * Copyright (c) KylinSoft Co., Ltd. 2024.All rights reserved. +# * PilotGo-plugins licensed under the Mulan Permissive Software License, Version 2. +# * See LICENSE file for more details. +# * Author: yzy_dev +# * Date: Thu Nov 28 15:00:41 2024 +0800 from flask import jsonify from llmops.service.log_analysis_service import LogAnalysisService diff --git a/llmops/controller/task.py b/llmops/controller/task.py index 0f37724e..90ca54eb 100644 --- a/llmops/controller/task.py +++ b/llmops/controller/task.py @@ -1,3 +1,8 @@ +# * Copyright (c) KylinSoft Co., Ltd. 2024.All rights reserved. +# * PilotGo-plugins licensed under the Mulan Permissive Software License, Version 2. +# * See LICENSE file for more details. +# * Author: zhanghan2021 +# * Date: Thu Nov 7 16:30:24 2024 +0800 from flask import request from utils.response import success, fail, not_found from service.task import create_new_task, get_task_status, confirm_task_option diff --git a/llmops/llm_tools/log_analysis/agentfactory.py b/llmops/llm_tools/log_analysis/agentfactory.py index 47214a52..460f8a7a 100644 --- a/llmops/llm_tools/log_analysis/agentfactory.py +++ b/llmops/llm_tools/log_analysis/agentfactory.py @@ -1,3 +1,8 @@ +# * Copyright (c) KylinSoft Co., Ltd. 2024.All rights reserved. +# * PilotGo-plugins licensed under the Mulan Permissive Software License, Version 2. +# * See LICENSE file for more details. +# * Author: yzy_dev +# * Date: Thu Nov 28 15:00:41 2024 +0800 import Agently from llmops.config.config import Config diff --git a/llmops/llm_tools/log_analysis/la_test.py b/llmops/llm_tools/log_analysis/la_test.py index 6e85df85..8bfc2e71 100644 --- a/llmops/llm_tools/log_analysis/la_test.py +++ b/llmops/llm_tools/log_analysis/la_test.py @@ -1 +1,6 @@ +# * Copyright (c) KylinSoft Co., Ltd. 2024.All rights reserved. +# * PilotGo-plugins licensed under the Mulan Permissive Software License, Version 2. +# * See LICENSE file for more details. +# * Author: zhanghan2021 +# * Date: Fri Nov 8 11:39:53 2024 +0800 # 日志分析 diff --git a/llmops/llm_tools/log_analysis/log_analysis_workflow.py b/llmops/llm_tools/log_analysis/log_analysis_workflow.py index 3d076d7b..43402886 100644 --- a/llmops/llm_tools/log_analysis/log_analysis_workflow.py +++ b/llmops/llm_tools/log_analysis/log_analysis_workflow.py @@ -1,3 +1,8 @@ +# * Copyright (c) KylinSoft Co., Ltd. 2024.All rights reserved. +# * PilotGo-plugins licensed under the Mulan Permissive Software License, Version 2. +# * See LICENSE file for more details. +# * Author: yzy_dev +# * Date: Thu Nov 28 15:00:41 2024 +0800 # 日志分析 import os diff --git a/llmops/llm_tools/nlp/nlp_test.py b/llmops/llm_tools/nlp/nlp_test.py index 9dd4dd94..4e6d5e9d 100644 --- a/llmops/llm_tools/nlp/nlp_test.py +++ b/llmops/llm_tools/nlp/nlp_test.py @@ -1 +1,6 @@ +# * Copyright (c) KylinSoft Co., Ltd. 2024.All rights reserved. +# * PilotGo-plugins licensed under the Mulan Permissive Software License, Version 2. +# * See LICENSE file for more details. +# * Author: zhanghan2021 +# * Date: Fri Nov 8 11:39:53 2024 +0800 # 自然语言交互 diff --git a/llmops/llm_tools/rca/rca_test.py b/llmops/llm_tools/rca/rca_test.py index 327cce23..3da68030 100644 --- a/llmops/llm_tools/rca/rca_test.py +++ b/llmops/llm_tools/rca/rca_test.py @@ -1 +1,6 @@ +# * Copyright (c) KylinSoft Co., Ltd. 2024.All rights reserved. +# * PilotGo-plugins licensed under the Mulan Permissive Software License, Version 2. +# * See LICENSE file for more details. +# * Author: zhanghan2021 +# * Date: Fri Nov 8 11:39:53 2024 +0800 # 故障定位 diff --git a/llmops/main.py b/llmops/main.py index f2a0ca04..6227462f 100644 --- a/llmops/main.py +++ b/llmops/main.py @@ -1,3 +1,8 @@ +# * Copyright (c) KylinSoft Co., Ltd. 2024.All rights reserved. +# * PilotGo-plugins licensed under the Mulan Permissive Software License, Version 2. +# * See LICENSE file for more details. +# * Author: zhanghan2021 +# * Date: Thu Nov 7 15:01:23 2024 +0800 from app_factory import run_app if __name__ == "__main__": diff --git a/llmops/router/log_analysis_router.py b/llmops/router/log_analysis_router.py index 7ff94808..1012d05c 100644 --- a/llmops/router/log_analysis_router.py +++ b/llmops/router/log_analysis_router.py @@ -1,3 +1,8 @@ +# * Copyright (c) KylinSoft Co., Ltd. 2024.All rights reserved. +# * PilotGo-plugins licensed under the Mulan Permissive Software License, Version 2. +# * See LICENSE file for more details. +# * Author: yzy_dev +# * Date: Thu Nov 28 15:00:41 2024 +0800 from flask import Blueprint,request from controller.log_analysis_controller import LogAnalysisController diff --git a/llmops/router/task.py b/llmops/router/task.py index 5aadea70..8b47cff3 100644 --- a/llmops/router/task.py +++ b/llmops/router/task.py @@ -1,3 +1,8 @@ +# * Copyright (c) KylinSoft Co., Ltd. 2024.All rights reserved. +# * PilotGo-plugins licensed under the Mulan Permissive Software License, Version 2. +# * See LICENSE file for more details. +# * Author: zhanghan2021 +# * Date: Thu Nov 7 16:30:24 2024 +0800 from flask import Blueprint from controller.task import create_task, task_status, confirm_option diff --git a/llmops/service/log_analysis_service.py b/llmops/service/log_analysis_service.py index 81dad9d0..03aeccca 100644 --- a/llmops/service/log_analysis_service.py +++ b/llmops/service/log_analysis_service.py @@ -1,3 +1,8 @@ +# * Copyright (c) KylinSoft Co., Ltd. 2024.All rights reserved. +# * PilotGo-plugins licensed under the Mulan Permissive Software License, Version 2. +# * See LICENSE file for more details. +# * Author: yzy_dev +# * Date: Thu Nov 28 15:00:41 2024 +0800 from llm_tools.log_analysis import log_analysis_workflow diff --git a/llmops/service/task.py b/llmops/service/task.py index cdcf63d5..840c56c9 100644 --- a/llmops/service/task.py +++ b/llmops/service/task.py @@ -1,3 +1,8 @@ +# * Copyright (c) KylinSoft Co., Ltd. 2024.All rights reserved. +# * PilotGo-plugins licensed under the Mulan Permissive Software License, Version 2. +# * See LICENSE file for more details. +# * Author: zhanghan2021 +# * Date: Thu Nov 7 16:30:24 2024 +0800 import uuid import time from threading import Thread diff --git a/llmops/utils/command.py b/llmops/utils/command.py index 43d07e56..eacdbdab 100644 --- a/llmops/utils/command.py +++ b/llmops/utils/command.py @@ -1,3 +1,8 @@ +# * Copyright (c) KylinSoft Co., Ltd. 2024.All rights reserved. +# * PilotGo-plugins licensed under the Mulan Permissive Software License, Version 2. +# * See LICENSE file for more details. +# * Author: zhanghan2021 +# * Date: Thu Nov 7 15:53:47 2024 +0800 import subprocess diff --git a/llmops/utils/logger.py b/llmops/utils/logger.py index a9a5fb72..c63861f3 100644 --- a/llmops/utils/logger.py +++ b/llmops/utils/logger.py @@ -1,3 +1,8 @@ +# * Copyright (c) KylinSoft Co., Ltd. 2024.All rights reserved. +# * PilotGo-plugins licensed under the Mulan Permissive Software License, Version 2. +# * See LICENSE file for more details. +# * Author: zhanghan2021 +# * Date: Thu Nov 7 15:53:47 2024 +0800 import logging diff --git a/llmops/utils/response.py b/llmops/utils/response.py index 8daf6241..12ed02e6 100644 --- a/llmops/utils/response.py +++ b/llmops/utils/response.py @@ -1,3 +1,8 @@ +# * Copyright (c) KylinSoft Co., Ltd. 2024.All rights reserved. +# * PilotGo-plugins licensed under the Mulan Permissive Software License, Version 2. +# * See LICENSE file for more details. +# * Author: zhanghan2021 +# * Date: Thu Nov 7 15:53:47 2024 +0800 from flask import jsonify -- Gitee