From 8973d311bafc9bfe5c9d389b9c0080478d882fec Mon Sep 17 00:00:00 2001 From: wangdong_cmcc Date: Thu, 10 Oct 2024 01:50:39 +0800 Subject: [PATCH] add vmcore full text matching --- environment/0_env/install.sh | 2 +- .../sysom_vmcore/apps/vmcore/views.py | 20 ++++++++++++++++++- sysom_web/src/locales/en-US/pages.js | 3 ++- sysom_web/src/locales/zh-CN/pages.js | 7 ++++--- sysom_web/src/pages/vmcore/Match/index.jsx | 6 ++++++ 5 files changed, 32 insertions(+), 6 deletions(-) diff --git a/environment/0_env/install.sh b/environment/0_env/install.sh index 1833a42d..b9b342a5 100644 --- a/environment/0_env/install.sh +++ b/environment/0_env/install.sh @@ -46,7 +46,7 @@ check_requirements() { touch "$requirements_log" || exit ### atomic-0.7.3 need cffi, we show install cffi first### pip install --upgrade pip - pip install cffi + pip install cffi scikit-learn pip install -r requirements.txt | tee -a "${requirements_log}" || exit 1 local pip_res=$? if [ $pip_res -ne 0 ]; then diff --git a/sysom_server/sysom_vmcore/apps/vmcore/views.py b/sysom_server/sysom_vmcore/apps/vmcore/views.py index b4ff9e9f..c76bd870 100644 --- a/sysom_server/sysom_vmcore/apps/vmcore/views.py +++ b/sysom_server/sysom_vmcore/apps/vmcore/views.py @@ -4,6 +4,8 @@ from rest_framework.viewsets import GenericViewSet from rest_framework import mixins from rest_framework.views import APIView from django_filters.rest_framework import DjangoFilterBackend +from sklearn.feature_extraction.text import TfidfVectorizer #将文本转换为TF-IDF向量。 +from sklearn.metrics.pairwise import cosine_similarity #用于计算余弦相似度 # from apps.accounts.permissions import IsAdminPermission # from apps.accounts.authentication import Authentication @@ -217,6 +219,12 @@ class VmcoreViewSet(GenericViewSet, i = i + 1 return -1 + # 定义函数，计算文件文本的相似度 + def calculate_similarity(self, text1, text2): + vectorizer = TfidfVectorizer() + tfidf = vectorizer.fit_transform([text1, text2]) + similarity = cosine_similarity(tfidf[0], tfidf[1])[0][0] + return similarity def parse_calltrace(self,dmesg): if len(dmesg) <= 0: @@ -281,6 +289,11 @@ class VmcoreViewSet(GenericViewSet, if fcnt > 1: data = serializer.PanicListSerializer(last_name.vmcore).data data['rate'] = round(fcnt*100/s2,2) + panic_objects = models.Panic.objects.filter(name=last_name.name) + for panic_obj in panic_objects: + dmesg_content = panic_obj.dmesg + data['similarity'] = round((self.calculate_similarity(str(dmesg_content), str(dmesg)))*100) + result.append(data) #result[last_name]=(fcnt*100/s2)+fpow last_idx=0 @@ -301,6 +314,11 @@ class VmcoreViewSet(GenericViewSet, if fcnt > 1: data = serializer.PanicListSerializer(calltrace_set.vmcore).data data['rate'] = round(fcnt*100/s2,2) + panic_objects = models.Panic.objects.filter(name=last_name.name) + for panic_obj in panic_objects: + dmesg_content = panic_obj.dmesg + data['similarity'] = round((self.calculate_similarity(str(dmesg_content), str(dmesg)))*100) + result.append(data) if len(result) > 0: break @@ -462,4 +480,4 @@ class VmcoreConfigTest(APIView): class HealthViewset(GenericViewSet): def health_check(self, request, *args, **kwargs): - return success(result={}) \ No newline at end of file + return success(result={}) diff --git a/sysom_web/src/locales/en-US/pages.js b/sysom_web/src/locales/en-US/pages.js index cadb45aa..e3518b6f 100644 --- a/sysom_web/src/locales/en-US/pages.js +++ b/sysom_web/src/locales/en-US/pages.js @@ -385,6 +385,7 @@ export default { 'pages.vmcore.view': 'View', 'pages.vmcore.similarstack': 'Similar stack', 'pages.vmcore.similarity': 'Similarity', + 'pages.vmcore.textsimilarity': 'TextSimilarity', 'pages.vmcore.configuration': 'Current configuration:', 'pages.vmcore.warehousename': 'Warehouse name', 'pages.vmcore.warehousevmcore': 'Name of the warehouse where the vmcore is faulty', @@ -512,4 +513,4 @@ export default { 'pages.monitor.components.podName': "POD Name", 'pages.monitor.components.namespace': "Namespace", 'pages.monitor.components.detail': "Detail", -}; \ No newline at end of file +}; diff --git a/sysom_web/src/locales/zh-CN/pages.js b/sysom_web/src/locales/zh-CN/pages.js index 19874ad7..1cdc3d5e 100644 --- a/sysom_web/src/locales/zh-CN/pages.js +++ b/sysom_web/src/locales/zh-CN/pages.js @@ -429,8 +429,9 @@ export default { 'pages.vmcore.solution': '解决方案', 'pages.vmcore.outagedetails': '宕机详情', 'pages.vmcore.view': '查看', - 'pages.vmcore.similarstack': '相似调用栈', - 'pages.vmcore.similarity': '相似度', + 'pages.vmcore.similarstack': '相似调用栈或全文日志', + 'pages.vmcore.similarity': '调用栈相似度', + 'pages.vmcore.textsimilarity': '全文日志相似度', 'pages.vmcore.configuration': '当前配置：', 'pages.vmcore.warehousename': '仓库名', 'pages.vmcore.warehousevmcore': '存储宕机vmcore的仓库名称', @@ -619,4 +620,4 @@ export default { 'pages.diagnose.net.pcap.pcap-lost-segment': "丢包数(\\ms)", 'pages.diagnose.net.pcap.pcap-node': '节点', -}; \ No newline at end of file +}; diff --git a/sysom_web/src/pages/vmcore/Match/index.jsx b/sysom_web/src/pages/vmcore/Match/index.jsx index dd08fc8f..5dee0ba8 100644 --- a/sysom_web/src/pages/vmcore/Match/index.jsx +++ b/sysom_web/src/pages/vmcore/Match/index.jsx @@ -84,6 +84,12 @@ const VmcoreMatch = () => { hideInSearch: true, valueType: "textarea", }, + { + title: , + dataIndex: "similarity", + hideInSearch: true, + valueType: "textarea", + }, { title: , dataIndex: "option", -- Gitee