From 3c890593acb8cbe650e76a9e1fe05ac6977a2c31 Mon Sep 17 00:00:00 2001 From: jinsaihang Date: Tue, 14 Jan 2025 10:36:15 +0800 Subject: [PATCH] adapt to the 6.6 kernel Signed-off-by: jinsaihang --- ...that-function-cpu_report_result-is-c.patch | 35 - Remove-ANSI-escape-sequences.patch | 32 - ...i-threshold-slow-io-detection-plugin.patch | 1201 --------- add-boundary-check-for-settings.patch | 39 - add-collect-module-to-sysSentry.patch | 1165 --------- add-deleted-code-to-plugin-rasdaemon.patch | 31 - add-detail-time.patch | 32 - add-get_disk_type-and-fix-some-bugs.patch | 176 -- add-hbm-online-repair.patch | 2194 ----------------- add-log-for-improving-maintainability.patch | 251 -- ...rm-when-sending-msg-and-clean-invali.patch | 24 - add-log-level-and-change-log-format.patch | 522 ---- ...me_range-alarm_id-and-alarm_clear_ti.patch | 104 - ...-pySentryNotify-add-multi-users-supp.patch | 678 ----- add-root-cause-analysis.patch | 1253 ---------- ...get_alarm-module_name-s-time_range-d.patch | 438 ---- ...eanup-invalid-server-socket-peroidly.patch | 90 - ai_block_io-adapt-alarm-module.patch | 221 -- ai_block_io-fix-some-bugs.patch | 235 -- ...fix-some-config-parameters-parse-bug.patch | 626 ----- ai_block_io-lack-section-exit.patch | 98 - ...pport-absolute-threshold-lower-limit.patch | 728 ------ ai_block_io-support-iodump.patch | 200 -- ai_block_io-support-stage-and-iotype.patch | 906 ------- avg_block_io-send-alarm-to-xalarmd.patch | 73 - bugfix-typo.patch | 34 - change-alarm-length.patch | 56 - change-avg_block_io-config.patch | 55 - ...tus-of-period-task-and-sort-mod-file.patch | 36 - ...ty-and-cpu_patrol-must-be-an-integer.patch | 41 - diff-disk-type-use-diff-config.patch | 430 ---- enrich-alert-info-about-kernel-stack.patch | 29 - feature-add-avg_block_io-plugin.patch | 572 ----- fix-ai_block_io-root-cause-bug.patch | 33 - fix-ai_block_io-some-issues.patch | 832 ------- fix-alarm_info-newline-break-error.patch | 48 - ...bout-collect-module-and-avg-block-io.patch | 323 --- fix-config-relative-some-issues.patch | 243 -- ...onfigparser.InterpolationSyntaxError.patch | 37 - fix-error-handling.patch | 25 - fix-excessive-CPU-usage.patch | 41 - fix-frequency-param-check-bug.patch | 70 - fix-get_alarm-error.patch | 36 - ...-online-repair-notice-and-efi-create.patch | 508 ---- fix-io_dump-for-collect-module.patch | 25 - ...ython-3.7-not-support-list-bool-type.patch | 53 - fix-result-when-process-output-is-None.patch | 36 - ...bout-collect-module-and-avg-block-io.patch | 226 -- ...ils-to-be-started-when-cpu_sentry-is.patch | 56 - fix-test_ai_block_io-fail.patch | 91 - ...g-and-change-isolation-default-value.patch | 61 - fix-version-in-setup.py.patch | 25 - fix-word-error.patch | 53 - fix-write-file-return-code-bug.patch | 69 - fix-xalarm-non-uniform-log-formatting.patch | 60 - ...t-function-not-refuse-alarm-msg-exce.patch | 76 - ...de-not-return-val-and-fail-when-thre.patch | 71 - get_alarm-d-abnomal-display.patch | 26 - ...ed-wont-stop-avg_block_io-and-del-di.patch | 168 -- hbm_online_repair-add-unload-driver.patch | 107 - ...of-collect-module-exits-occasionally.patch | 104 - make-debug-msg-clear.patch | 69 - ...stack-when-the-disk-field-is-not-con.patch | 28 - modify-logrotate-rule.patch | 27 - optimize-log-printing.patch | 125 - ...g-of-cat-cli-error-msg-in-cpu_sentry.patch | 77 - ...-should-be-warn-level-log-in-cat-cli.patch | 25 - param-must-be-integer.patch | 23 - precise-alarm-query-time.patch | 91 - ...py-and-bugfix-uncorrect-slow-io-repo.patch | 566 ----- set-logrotate.patch | 92 - setting-parameters-must-be-integer.patch | 38 - split-cpu_sentry-and-syssentry.patch | 155 -- sysSentry-1.0.2.tar.gz | Bin 317440 -> 0 bytes sysSentry-1.0.3.tar.gz | Bin 0 -> 1354119 bytes sysSentry.spec | 619 +---- ...avg_block_io-log-and-ai_block_io-log.patch | 63 - update-collect-log.patch | 25 - update-collect-plugin-period-max.patch | 44 - update-log-when-it-is-not-lock-collect.patch | 35 - update-nvme-config.patch | 51 - xalarm-add-alarm-msg-length-to-8192.patch | 112 - 82 files changed, 20 insertions(+), 18353 deletions(-) delete mode 100644 Fix-the-problem-that-function-cpu_report_result-is-c.patch delete mode 100644 Remove-ANSI-escape-sequences.patch delete mode 100644 add-ai-threshold-slow-io-detection-plugin.patch delete mode 100644 add-boundary-check-for-settings.patch delete mode 100644 add-collect-module-to-sysSentry.patch delete mode 100644 add-deleted-code-to-plugin-rasdaemon.patch delete mode 100644 add-detail-time.patch delete mode 100644 add-get_disk_type-and-fix-some-bugs.patch delete mode 100644 add-hbm-online-repair.patch delete mode 100644 add-log-for-improving-maintainability.patch delete mode 100644 add-log-for-xalarm-when-sending-msg-and-clean-invali.patch delete mode 100644 add-log-level-and-change-log-format.patch delete mode 100644 add-parameter-time_range-alarm_id-and-alarm_clear_ti.patch delete mode 100644 add-pyxalarm-and-pySentryNotify-add-multi-users-supp.patch delete mode 100644 add-root-cause-analysis.patch delete mode 100644 add-sentryctl-get_alarm-module_name-s-time_range-d.patch delete mode 100644 add-xalarm-cleanup-invalid-server-socket-peroidly.patch delete mode 100644 ai_block_io-adapt-alarm-module.patch delete mode 100644 ai_block_io-fix-some-bugs.patch delete mode 100644 ai_block_io-fix-some-config-parameters-parse-bug.patch delete mode 100644 ai_block_io-lack-section-exit.patch delete mode 100644 ai_block_io-support-absolute-threshold-lower-limit.patch delete mode 100644 ai_block_io-support-iodump.patch delete mode 100644 ai_block_io-support-stage-and-iotype.patch delete mode 100644 avg_block_io-send-alarm-to-xalarmd.patch delete mode 100644 bugfix-typo.patch delete mode 100644 change-alarm-length.patch delete mode 100644 change-avg_block_io-config.patch delete mode 100644 change-status-of-period-task-and-sort-mod-file.patch delete mode 100644 cpu_utility-and-cpu_patrol-must-be-an-integer.patch delete mode 100644 diff-disk-type-use-diff-config.patch delete mode 100644 enrich-alert-info-about-kernel-stack.patch delete mode 100644 feature-add-avg_block_io-plugin.patch delete mode 100644 fix-ai_block_io-root-cause-bug.patch delete mode 100644 fix-ai_block_io-some-issues.patch delete mode 100644 fix-alarm_info-newline-break-error.patch delete mode 100644 fix-bug-step-2-about-collect-module-and-avg-block-io.patch delete mode 100644 fix-config-relative-some-issues.patch delete mode 100644 fix-configparser.InterpolationSyntaxError.patch delete mode 100644 fix-error-handling.patch delete mode 100644 fix-excessive-CPU-usage.patch delete mode 100644 fix-frequency-param-check-bug.patch delete mode 100644 fix-get_alarm-error.patch delete mode 100644 fix-hbm-online-repair-notice-and-efi-create.patch delete mode 100644 fix-io_dump-for-collect-module.patch delete mode 100644 fix-python-3.7-not-support-list-bool-type.patch delete mode 100644 fix-result-when-process-output-is-None.patch delete mode 100644 fix-some-about-collect-module-and-avg-block-io.patch delete mode 100644 fix-syssentry-fails-to-be-started-when-cpu_sentry-is.patch delete mode 100644 fix-test_ai_block_io-fail.patch delete mode 100644 fix-uint8-bug-and-change-isolation-default-value.patch delete mode 100644 fix-version-in-setup.py.patch delete mode 100644 fix-word-error.patch delete mode 100644 fix-write-file-return-code-bug.patch delete mode 100644 fix-xalarm-non-uniform-log-formatting.patch delete mode 100644 fix-xalarm_Report-function-not-refuse-alarm-msg-exce.patch delete mode 100644 fix-xalarm_upgrade-not-return-val-and-fail-when-thre.patch delete mode 100644 get_alarm-d-abnomal-display.patch delete mode 100644 get_io_data-failed-wont-stop-avg_block_io-and-del-di.patch delete mode 100644 hbm_online_repair-add-unload-driver.patch delete mode 100644 listen-thread-of-collect-module-exits-occasionally.patch delete mode 100644 make-debug-msg-clear.patch delete mode 100644 modify-abnormal-stack-when-the-disk-field-is-not-con.patch delete mode 100644 modify-logrotate-rule.patch delete mode 100644 optimize-log-printing.patch delete mode 100644 optimize-the-handing-of-cat-cli-error-msg-in-cpu_sentry.patch delete mode 100644 over-threshold-should-be-warn-level-log-in-cat-cli.patch delete mode 100644 param-must-be-integer.patch delete mode 100644 precise-alarm-query-time.patch delete mode 100644 refactor-config.py-and-bugfix-uncorrect-slow-io-repo.patch delete mode 100644 set-logrotate.patch delete mode 100644 setting-parameters-must-be-integer.patch delete mode 100644 split-cpu_sentry-and-syssentry.patch delete mode 100644 sysSentry-1.0.2.tar.gz create mode 100644 sysSentry-1.0.3.tar.gz delete mode 100644 uniform-avg_block_io-log-and-ai_block_io-log.patch delete mode 100644 update-collect-log.patch delete mode 100644 update-collect-plugin-period-max.patch delete mode 100644 update-log-when-it-is-not-lock-collect.patch delete mode 100644 update-nvme-config.patch delete mode 100644 xalarm-add-alarm-msg-length-to-8192.patch diff --git a/Fix-the-problem-that-function-cpu_report_result-is-c.patch b/Fix-the-problem-that-function-cpu_report_result-is-c.patch deleted file mode 100644 index 7a46590..0000000 --- a/Fix-the-problem-that-function-cpu_report_result-is-c.patch +++ /dev/null @@ -1,35 +0,0 @@ -From 3e2721852ad1f8047ad219a5ab6c68fd4c9d6f5c Mon Sep 17 00:00:00 2001 -From: shixuantong -Date: Wed, 24 Jul 2024 16:17:54 +0800 -Subject: [PATCH] Fix the problem that function cpu_report_result() is called - more than once - -when task is running, user to exec "sentryctl stop cpu_sentry", cpu_report_result() will be called twice. This will cause the log to be printed twice ---- - src/python/syssentry/cpu_sentry.py | 5 +++-- - 1 file changed, 3 insertions(+), 2 deletions(-) - -diff --git a/src/python/syssentry/cpu_sentry.py b/src/python/syssentry/cpu_sentry.py -index 7e77654..3c4d58d 100644 ---- a/src/python/syssentry/cpu_sentry.py -+++ b/src/python/syssentry/cpu_sentry.py -@@ -133,6 +133,7 @@ class CpuSentry: - - result_level = self.send_result.get("result", ResultLevel.FAIL) - report_result(task_name, result_level, details) -+ self.init_send_result() - - def kill_process(signum, _f, cpu_sentry_obj): - """kill process by 'pkill -9'""" -@@ -179,6 +180,6 @@ def main(): - cpu_sentry_task.send_result["result"] = ResultLevel.FAIL - cpu_sentry_task.send_result["details"]["code"] = 1004 - cpu_sentry_task.send_result["details"]["msg"] = "run cmd [%s] raise Error" % cpu_sentry_task_cmd -- finally: - cpu_sentry_task.cpu_report_result() -- cpu_sentry_task.init_send_result() -+ else: -+ cpu_sentry_task.cpu_report_result() --- -2.27.0 - diff --git a/Remove-ANSI-escape-sequences.patch b/Remove-ANSI-escape-sequences.patch deleted file mode 100644 index 713bfc0..0000000 --- a/Remove-ANSI-escape-sequences.patch +++ /dev/null @@ -1,32 +0,0 @@ -From 91aa47999030503fda4935d4cc238b82d6842238 Mon Sep 17 00:00:00 2001 -From: shixuantong -Date: Sun, 11 Aug 2024 18:36:23 +0800 -Subject: [PATCH] Remove ANSI escape sequences - ---- - src/python/syssentry/cpu_sentry.py | 9 ++++++++- - 1 file changed, 8 insertions(+), 1 deletion(-) - -diff --git a/src/python/syssentry/cpu_sentry.py b/src/python/syssentry/cpu_sentry.py -index 9287e2f..99af127 100644 ---- a/src/python/syssentry/cpu_sentry.py -+++ b/src/python/syssentry/cpu_sentry.py -@@ -97,7 +97,14 @@ class CpuSentry: - if "ERROR" in stdout: - self.send_result["result"] = ResultLevel.FAIL - self.send_result["details"]["code"] = 1004 -- self.send_result["details"]["msg"] = stdout.split("\n")[0] -+ -+ # Remove ANSI escape sequences -+ error_info = stdout.split("\n")[0] -+ if error_info.startswith("\u001b"): -+ ansi_escape = r'\x1b\[([0-9]+)(;[0-9]+)*([A-Za-z])' -+ error_info = re.sub(ansi_escape, '', error_info) -+ -+ self.send_result["details"]["msg"] = error_info - return - - out_split = stdout.split("\n") --- -2.33.0 - diff --git a/add-ai-threshold-slow-io-detection-plugin.patch b/add-ai-threshold-slow-io-detection-plugin.patch deleted file mode 100644 index 6f707a4..0000000 --- a/add-ai-threshold-slow-io-detection-plugin.patch +++ /dev/null @@ -1,1201 +0,0 @@ -From 3d72fa7f517e6e99af1205e965c3775dc23461f4 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?=E8=B4=BA=E6=9C=89=E5=BF=97?= <1037617413@qq.com> -Date: Mon, 23 Sep 2024 11:03:26 +0800 -Subject: [PATCH] add ai threshold slow io detection to sysSentry - ---- - .../ai_threshold_slow_io_detection.ini | 16 ++ - .../tasks/ai_threshold_slow_io_detection.mod | 5 + - .../test_ai_threshold_slow_io_detection.py | 165 ++++++++++++++++++ - .../ai_threshold_slow_io_detection/README.md | 2 + - .../__init__.py | 0 - .../alarm_report.py | 49 ++++++ - .../config_parser.py | 141 +++++++++++++++ - .../data_access.py | 91 ++++++++++ - .../detector.py | 48 +++++ - .../ai_threshold_slow_io_detection/io_data.py | 74 ++++++++ - .../sliding_window.py | 113 ++++++++++++ - .../slow_io_detection.py | 133 ++++++++++++++ - .../threshold.py | 160 +++++++++++++++++ - .../ai_threshold_slow_io_detection/utils.py | 67 +++++++ - src/python/setup.py | 3 +- - 15 files changed, 1066 insertions(+), 1 deletion(-) - create mode 100644 config/plugins/ai_threshold_slow_io_detection.ini - create mode 100644 config/tasks/ai_threshold_slow_io_detection.mod - create mode 100644 selftest/test/test_ai_threshold_slow_io_detection.py - create mode 100644 src/python/sentryPlugins/ai_threshold_slow_io_detection/README.md - create mode 100644 src/python/sentryPlugins/ai_threshold_slow_io_detection/__init__.py - create mode 100644 src/python/sentryPlugins/ai_threshold_slow_io_detection/alarm_report.py - create mode 100644 src/python/sentryPlugins/ai_threshold_slow_io_detection/config_parser.py - create mode 100644 src/python/sentryPlugins/ai_threshold_slow_io_detection/data_access.py - create mode 100644 src/python/sentryPlugins/ai_threshold_slow_io_detection/detector.py - create mode 100644 src/python/sentryPlugins/ai_threshold_slow_io_detection/io_data.py - create mode 100644 src/python/sentryPlugins/ai_threshold_slow_io_detection/sliding_window.py - create mode 100644 src/python/sentryPlugins/ai_threshold_slow_io_detection/slow_io_detection.py - create mode 100644 src/python/sentryPlugins/ai_threshold_slow_io_detection/threshold.py - create mode 100644 src/python/sentryPlugins/ai_threshold_slow_io_detection/utils.py - -diff --git a/config/plugins/ai_threshold_slow_io_detection.ini b/config/plugins/ai_threshold_slow_io_detection.ini -new file mode 100644 -index 0000000..44eb928 ---- /dev/null -+++ b/config/plugins/ai_threshold_slow_io_detection.ini -@@ -0,0 +1,16 @@ -+[common] -+absolute_threshold=40 -+slow_io_detect_frequency=1 -+log_level=info -+ -+[algorithm] -+train_data_duration=0.1 -+train_update_duration=0.02 -+algorithm_type=n_sigma -+boxplot_parameter=1.5 -+n_sigma_parameter=3 -+ -+[sliding_window] -+sliding_window_type=not_continuous -+window_size=30 -+window_minimum_threshold=6 -\ No newline at end of file -diff --git a/config/tasks/ai_threshold_slow_io_detection.mod b/config/tasks/ai_threshold_slow_io_detection.mod -new file mode 100644 -index 0000000..2729f72 ---- /dev/null -+++ b/config/tasks/ai_threshold_slow_io_detection.mod -@@ -0,0 +1,5 @@ -+[common] -+enabled=yes -+task_start=/usr/bin/python3 /usr/bin/ai_threshold_slow_io_detection -+task_stop=pkill -f /usr/bin/ai_threshold_slow_io_detection -+type=oneshot -\ No newline at end of file -diff --git a/selftest/test/test_ai_threshold_slow_io_detection.py b/selftest/test/test_ai_threshold_slow_io_detection.py -new file mode 100644 -index 0000000..c36fef5 ---- /dev/null -+++ b/selftest/test/test_ai_threshold_slow_io_detection.py -@@ -0,0 +1,165 @@ -+# coding: utf-8 -+# Copyright (c) 2024 Huawei Technologies Co., Ltd. -+# sysSentry is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+ -+import unittest -+import numpy as np -+ -+from sentryPlugins.ai_threshold_slow_io_detection.threshold import AbsoluteThreshold, BoxplotThreshold, NSigmaThreshold -+from sentryPlugins.ai_threshold_slow_io_detection.sliding_window import (NotContinuousSlidingWindow, -+ ContinuousSlidingWindow, MedianSlidingWindow) -+ -+ -+def _get_boxplot_threshold(data_list: list, parameter): -+ q1 = np.percentile(data_list, 25) -+ q3 = np.percentile(data_list, 75) -+ iqr = q3 - q1 -+ return q3 + parameter * iqr -+ -+ -+def _get_n_sigma_threshold(data_list: list, parameter): -+ mean = np.mean(data_list) -+ std = np.std(data_list) -+ return mean + parameter * std -+ -+ -+class Test(unittest.TestCase): -+ @classmethod -+ def setUpClass(cls): -+ print("UnitTest Begin...") -+ -+ @classmethod -+ def tearDownClass(cls): -+ print("UnitTest End...") -+ -+ def setUp(self): -+ print("Begin...") -+ -+ def tearDown(self): -+ print("End...") -+ -+ def test_absolute_threshold(self): -+ absolute = AbsoluteThreshold() -+ self.assertEqual(None, absolute.get_threshold()) -+ self.assertFalse(absolute.is_abnormal(5000)) -+ absolute.set_threshold(40) -+ self.assertEqual(40, absolute.get_threshold()) -+ self.assertTrue(absolute.is_abnormal(50)) -+ -+ def test_boxplot_threshold(self): -+ boxplot = BoxplotThreshold(1.5, 5, 1) -+ # 阶段1：尚未初始化 -+ self.assertEqual(None, boxplot.get_threshold()) -+ self.assertFalse(boxplot.is_abnormal(5000)) -+ # 往boxplot中插入5个元素后，会生成阈值 -+ data_list = [20, 20, 20, 30, 10] -+ for data in data_list: -+ boxplot.push_latest_data_to_queue(data) -+ # 阶段2：初始化 -+ boxplot_threshold = boxplot.get_threshold() -+ self.assertEqual(_get_boxplot_threshold(data_list, 1.5), boxplot_threshold) -+ self.assertTrue(boxplot.is_abnormal(5000)) -+ data_list.pop(0) -+ data_list.append(100) -+ boxplot.push_latest_data_to_queue(100) -+ # 阶段3：更新阈值 -+ boxplot_threshold = boxplot.get_threshold() -+ self.assertEqual(_get_boxplot_threshold(data_list, 1.5), boxplot_threshold) -+ -+ def test_n_sigma_threshold(self): -+ n_sigma = NSigmaThreshold(3, 5, 1) -+ self.assertEqual(None, n_sigma.get_threshold()) -+ self.assertFalse(n_sigma.is_abnormal(5000)) -+ data_list = [20, 20, 20, 30, 10] -+ for data in data_list: -+ n_sigma.push_latest_data_to_queue(data) -+ n_sigma_threshold = n_sigma.get_threshold() -+ self.assertEqual(_get_n_sigma_threshold(data_list, 3), n_sigma_threshold) -+ self.assertTrue(n_sigma.is_abnormal(5000)) -+ data_list.pop(0) -+ data_list.append(100) -+ n_sigma.push_latest_data_to_queue(100) -+ # 阶段3：更新阈值 -+ n_sigma_threshold = n_sigma.get_threshold() -+ self.assertEqual(_get_n_sigma_threshold(data_list, 3), n_sigma_threshold) -+ -+ def test_not_continuous_sliding_window(self): -+ not_continuous = NotContinuousSlidingWindow(5, 3) -+ boxplot_threshold = BoxplotThreshold(1.5, 10, 8) -+ boxplot_threshold.attach_observer(not_continuous) -+ data_list1 = [19, 20, 20, 20, 20, 20, 22, 24, 23, 20] -+ for data in data_list1: -+ boxplot_threshold.push_latest_data_to_queue(data) -+ result = not_continuous.is_slow_io_event(data) -+ self.assertFalse(result[0]) -+ self.assertEqual(23.75, boxplot_threshold.get_threshold()) -+ boxplot_threshold.push_latest_data_to_queue(24) -+ result = not_continuous.is_slow_io_event(24) -+ self.assertFalse(result[0]) -+ boxplot_threshold.push_latest_data_to_queue(25) -+ result = not_continuous.is_slow_io_event(25) -+ self.assertTrue(result[0]) -+ data_list2 = [20, 20, 20, 20, 20, 20] -+ for data in data_list2: -+ boxplot_threshold.push_latest_data_to_queue(data) -+ result = not_continuous.is_slow_io_event(data) -+ self.assertFalse(result[0]) -+ self.assertEqual(25.625, boxplot_threshold.get_threshold()) -+ -+ def test_continuous_sliding_window(self): -+ continuous = ContinuousSlidingWindow(5, 3) -+ boxplot_threshold = BoxplotThreshold(1.5, 10, 8) -+ boxplot_threshold.attach_observer(continuous) -+ data_list = [19, 20, 20, 20, 20, 20, 22, 24, 23, 20] -+ for data in data_list: -+ boxplot_threshold.push_latest_data_to_queue(data) -+ result = continuous.is_slow_io_event(data) -+ self.assertFalse(result[0]) -+ self.assertEqual(23.75, boxplot_threshold.get_threshold()) -+ # 没有三个异常点 -+ self.assertFalse(continuous.is_slow_io_event(25)[0]) -+ # 不连续的三个异常点 -+ self.assertFalse(continuous.is_slow_io_event(25)[0]) -+ # 连续的三个异常点 -+ self.assertTrue(continuous.is_slow_io_event(25)[0]) -+ -+ def test_median_sliding_window(self): -+ median = MedianSlidingWindow(5, 3) -+ absolute_threshold = AbsoluteThreshold(10, 8) -+ absolute_threshold.attach_observer(median) -+ absolute_threshold.set_threshold(24.5) -+ data_list = [24, 24, 24, 25, 25] -+ for data in data_list: -+ self.assertFalse(median.is_slow_io_event(data)[0]) -+ self.assertTrue(median.is_slow_io_event(25)[0]) -+ -+ def test_parse_collect_data(self): -+ collect = { -+ "read": [1.0, 2.0, 3.0, 4.0], -+ "write": [5.0, 6.0, 7.0, 8.0], -+ "flush": [9.0, 10.0, 11.0, 12.0], -+ "discard": [13.0, 14.0, 15.0, 16.0], -+ } -+ from io_data import BaseData -+ from data_access import _get_io_stage_data -+ -+ io_data = _get_io_stage_data(collect) -+ self.assertEqual( -+ io_data.read, BaseData(latency=1.0, io_dump=2.0, io_length=3.0, iops=4.0) -+ ) -+ self.assertEqual( -+ io_data.write, BaseData(latency=5.0, io_dump=6.0, io_length=7.0, iops=8.0) -+ ) -+ self.assertEqual( -+ io_data.flush, BaseData(latency=9.0, io_dump=10.0, io_length=11.0, iops=12.0) -+ ) -+ self.assertEqual( -+ io_data.discard, BaseData(latency=13.0, io_dump=14.0, io_length=15.0, iops=16.0) -+ ) -diff --git a/src/python/sentryPlugins/ai_threshold_slow_io_detection/README.md b/src/python/sentryPlugins/ai_threshold_slow_io_detection/README.md -new file mode 100644 -index 0000000..f9b8388 ---- /dev/null -+++ b/src/python/sentryPlugins/ai_threshold_slow_io_detection/README.md -@@ -0,0 +1,2 @@ -+# slow_io_detection -+ -diff --git a/src/python/sentryPlugins/ai_threshold_slow_io_detection/__init__.py b/src/python/sentryPlugins/ai_threshold_slow_io_detection/__init__.py -new file mode 100644 -index 0000000..e69de29 -diff --git a/src/python/sentryPlugins/ai_threshold_slow_io_detection/alarm_report.py b/src/python/sentryPlugins/ai_threshold_slow_io_detection/alarm_report.py -new file mode 100644 -index 0000000..3f4f34e ---- /dev/null -+++ b/src/python/sentryPlugins/ai_threshold_slow_io_detection/alarm_report.py -@@ -0,0 +1,49 @@ -+# coding: utf-8 -+# Copyright (c) 2024 Huawei Technologies Co., Ltd. -+# sysSentry is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+ -+from syssentry.result import ResultLevel, report_result -+import logging -+import json -+ -+ -+class AlarmReport: -+ TASK_NAME = "SLOW_IO_DETECTION" -+ -+ @staticmethod -+ def report_pass(info: str): -+ report_result(AlarmReport.TASK_NAME, ResultLevel.PASS, json.dumps({"msg": info})) -+ logging.info(f'Report {AlarmReport.TASK_NAME} PASS: {info}') -+ -+ @staticmethod -+ def report_fail(info: str): -+ report_result(AlarmReport.TASK_NAME, ResultLevel.FAIL, json.dumps({"msg": info})) -+ logging.info(f'Report {AlarmReport.TASK_NAME} FAIL: {info}') -+ -+ @staticmethod -+ def report_skip(info: str): -+ report_result(AlarmReport.TASK_NAME, ResultLevel.SKIP, json.dumps({"msg": info})) -+ logging.info(f'Report {AlarmReport.TASK_NAME} SKIP: {info}') -+ -+ @staticmethod -+ def report_minor_alm(info: str): -+ report_result(AlarmReport.TASK_NAME, ResultLevel.MINOR_ALM, json.dumps({"msg": info})) -+ logging.info(f'Report {AlarmReport.TASK_NAME} MINOR_ALM: {info}') -+ -+ @staticmethod -+ def report_major_alm(info: str): -+ report_result(AlarmReport.TASK_NAME, ResultLevel.MAJOR_ALM, json.dumps({"msg": info})) -+ logging.info(f'Report {AlarmReport.TASK_NAME} MAJOR_ALM: {info}') -+ -+ @staticmethod -+ def report_critical_alm(info: str): -+ report_result(AlarmReport.TASK_NAME, ResultLevel.CRITICAL_ALM, json.dumps({"msg": info})) -+ logging.info(f'Report {AlarmReport.TASK_NAME} CRITICAL_ALM: {info}') -+ -diff --git a/src/python/sentryPlugins/ai_threshold_slow_io_detection/config_parser.py b/src/python/sentryPlugins/ai_threshold_slow_io_detection/config_parser.py -new file mode 100644 -index 0000000..cd4e6f1 ---- /dev/null -+++ b/src/python/sentryPlugins/ai_threshold_slow_io_detection/config_parser.py -@@ -0,0 +1,141 @@ -+# coding: utf-8 -+# Copyright (c) 2024 Huawei Technologies Co., Ltd. -+# sysSentry is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+ -+import configparser -+import logging -+ -+ -+class ConfigParser: -+ -+ DEFAULT_ABSOLUTE_THRESHOLD = 40 -+ DEFAULT_SLOW_IO_DETECTION_FREQUENCY = 1 -+ DEFAULT_LOG_LEVEL = 'info' -+ DEFAULT_TRAIN_DATA_DURATION = 24 -+ DEFAULT_TRAIN_UPDATE_DURATION = 2 -+ DEFAULT_ALGORITHM_TYPE = 'boxplot' -+ DEFAULT_N_SIGMA_PARAMETER = 3 -+ DEFAULT_BOXPLOT_PARAMETER = 1.5 -+ DEFAULT_SLIDING_WINDOW_TYPE = 'not_continuous' -+ DEFAULT_WINDOW_SIZE = 30 -+ DEFAULT_WINDOW_MINIMUM_THRESHOLD = 6 -+ -+ def __init__(self, config_file_name): -+ self.__boxplot_parameter = None -+ self.__window_minimum_threshold = None -+ self.__window_size = None -+ self.__sliding_window_type = None -+ self.__n_sigma_parameter = None -+ self.__algorithm_type = None -+ self.__train_update_duration = None -+ self.__log_level = None -+ self.__slow_io_detect_frequency = None -+ self.__absolute_threshold = None -+ self.__train_data_duration = None -+ self.__config_file_name = config_file_name -+ -+ def read_config_from_file(self): -+ -+ con = configparser.ConfigParser() -+ con.read(self.__config_file_name, encoding='utf-8') -+ -+ items_common = dict(con.items('common')) -+ items_algorithm = dict(con.items('algorithm')) -+ items_sliding_window = dict(con.items('sliding_window')) -+ -+ try: -+ self.__absolute_threshold = int(items_common.get('absolute_threshold', -+ ConfigParser.DEFAULT_ABSOLUTE_THRESHOLD)) -+ except ValueError: -+ self.__absolute_threshold = ConfigParser.DEFAULT_ABSOLUTE_THRESHOLD -+ logging.warning('absolute threshold type conversion has error, use default value.') -+ -+ try: -+ self.__slow_io_detect_frequency = int(items_common.get('slow_io_detect_frequency', -+ ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY)) -+ except ValueError: -+ self.__slow_io_detect_frequency = ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY -+ logging.warning('slow_io_detect_frequency type conversion has error, use default value.') -+ -+ self.__log_level = items_common.get('log_level', ConfigParser.DEFAULT_LOG_LEVEL) -+ -+ try: -+ self.__train_data_duration = float(items_algorithm.get('train_data_duration', -+ ConfigParser.DEFAULT_TRAIN_DATA_DURATION)) -+ except ValueError: -+ self.__train_data_duration = ConfigParser.DEFAULT_TRAIN_DATA_DURATION -+ logging.warning('train_data_duration type conversion has error, use default value.') -+ -+ try: -+ self.__train_update_duration = float(items_algorithm.get('train_update_duration', -+ ConfigParser.DEFAULT_TRAIN_UPDATE_DURATION)) -+ except ValueError: -+ self.__train_update_duration = ConfigParser.DEFAULT_TRAIN_UPDATE_DURATION -+ logging.warning('train_update_duration type conversion has error, use default value.') -+ -+ try: -+ self.__algorithm_type = items_algorithm.get('algorithm_type', ConfigParser.DEFAULT_ALGORITHM_TYPE) -+ except ValueError: -+ self.__algorithm_type = ConfigParser.DEFAULT_ALGORITHM_TYPE -+ logging.warning('algorithmType type conversion has error, use default value.') -+ -+ if self.__algorithm_type == 'n_sigma': -+ try: -+ self.__n_sigma_parameter = float(items_algorithm.get('n_sigma_parameter', -+ ConfigParser.DEFAULT_N_SIGMA_PARAMETER)) -+ except ValueError: -+ self.__n_sigma_parameter = ConfigParser.DEFAULT_N_SIGMA_PARAMETER -+ logging.warning('n_sigma_parameter type conversion has error, use default value.') -+ elif self.__algorithm_type == 'boxplot': -+ try: -+ self.__boxplot_parameter = float(items_algorithm.get('boxplot_parameter', -+ ConfigParser.DEFAULT_BOXPLOT_PARAMETER)) -+ except ValueError: -+ self.__boxplot_parameter = ConfigParser.DEFAULT_BOXPLOT_PARAMETER -+ logging.warning('boxplot_parameter type conversion has error, use default value.') -+ -+ self.__sliding_window_type = items_sliding_window.get('sliding_window_type', -+ ConfigParser.DEFAULT_SLIDING_WINDOW_TYPE) -+ -+ try: -+ self.__window_size = int(items_sliding_window.get('window_size', -+ ConfigParser.DEFAULT_WINDOW_SIZE)) -+ except ValueError: -+ self.__window_size = ConfigParser.DEFAULT_WINDOW_SIZE -+ logging.warning('window_size type conversion has error, use default value.') -+ -+ try: -+ self.__window_minimum_threshold = ( -+ int(items_sliding_window.get('window_minimum_threshold', -+ ConfigParser.DEFAULT_WINDOW_MINIMUM_THRESHOLD))) -+ except ValueError: -+ self.__window_minimum_threshold = ConfigParser.DEFAULT_WINDOW_MINIMUM_THRESHOLD -+ logging.warning('window_minimum_threshold type conversion has error, use default value.') -+ -+ def get_slow_io_detect_frequency(self): -+ return self.__slow_io_detect_frequency -+ -+ def get_algorithm_type(self): -+ return self.__algorithm_type -+ -+ def get_sliding_window_type(self): -+ return self.__sliding_window_type -+ -+ def get_train_data_duration_and_train_update_duration(self): -+ return self.__train_data_duration, self.__train_update_duration -+ -+ def get_window_size_and_window_minimum_threshold(self): -+ return self.__window_size, self.__window_minimum_threshold -+ -+ def get_absolute_threshold(self): -+ return self.__absolute_threshold -+ -+ def get_log_level(self): -+ return self.__log_level -diff --git a/src/python/sentryPlugins/ai_threshold_slow_io_detection/data_access.py b/src/python/sentryPlugins/ai_threshold_slow_io_detection/data_access.py -new file mode 100644 -index 0000000..d9f3460 ---- /dev/null -+++ b/src/python/sentryPlugins/ai_threshold_slow_io_detection/data_access.py -@@ -0,0 +1,91 @@ -+# coding: utf-8 -+# Copyright (c) 2024 Huawei Technologies Co., Ltd. -+# sysSentry is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+ -+import json -+import logging -+ -+from sentryCollector.collect_plugin import ( -+ Result_Messages, -+ get_io_data, -+ is_iocollect_valid, -+) -+from .io_data import IOStageData, IOData -+ -+COLLECT_STAGES = [ -+ "throtl", -+ "wbt", -+ "gettag", -+ "plug", -+ "bfq", -+ "hctx", -+ "requeue", -+ "rq_driver", -+ "bio", -+ "iocost", -+] -+ -+def check_collect_valid(period): -+ data_raw = is_iocollect_valid(period) -+ if data_raw["ret"] == 0: -+ try: -+ data = json.loads(data_raw["message"]) -+ except Exception as e: -+ logging.warning(f"get io data failed, {e}") -+ return [] -+ return [k for k in data.keys()] -+ else: -+ return [] -+ -+ -+def _get_raw_data(period, disk_list): -+ return get_io_data( -+ period, -+ disk_list, -+ COLLECT_STAGES, -+ ["read", "write", "flush", "discard"], -+ ) -+ -+ -+def _get_io_stage_data(data): -+ io_stage_data = IOStageData() -+ for data_type in ('read', 'write', 'flush', 'discard'): -+ if data_type in data: -+ getattr(io_stage_data, data_type).latency = data[data_type][0] -+ getattr(io_stage_data, data_type).io_dump = data[data_type][1] -+ getattr(io_stage_data, data_type).io_length = data[data_type][2] -+ getattr(io_stage_data, data_type).iops = data[data_type][3] -+ return io_stage_data -+ -+ -+def get_io_data_from_collect_plug(period, disk_list): -+ data_raw = _get_raw_data(period, disk_list) -+ if data_raw["ret"] == 0: -+ ret = {} -+ try: -+ data = json.loads(data_raw["message"]) -+ except json.decoder.JSONDecodeError as e: -+ logging.warning(f"get io data failed, {e}") -+ return None -+ -+ for disk in data: -+ disk_data = data[disk] -+ disk_ret = IOData() -+ for k, v in disk_data.items(): -+ try: -+ getattr(disk_ret, k) -+ setattr(disk_ret, k, _get_io_stage_data(v)) -+ except AttributeError: -+ logging.debug(f'no attr {k}') -+ continue -+ ret[disk] = disk_ret -+ return ret -+ logging.warning(f'get io data failed with message: {data_raw["message"]}') -+ return None -diff --git a/src/python/sentryPlugins/ai_threshold_slow_io_detection/detector.py b/src/python/sentryPlugins/ai_threshold_slow_io_detection/detector.py -new file mode 100644 -index 0000000..eda9825 ---- /dev/null -+++ b/src/python/sentryPlugins/ai_threshold_slow_io_detection/detector.py -@@ -0,0 +1,48 @@ -+# coding: utf-8 -+# Copyright (c) 2024 Huawei Technologies Co., Ltd. -+# sysSentry is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+import logging -+ -+from .io_data import MetricName -+from .threshold import Threshold -+from .sliding_window import SlidingWindow -+from .utils import get_metric_value_from_io_data_dict_by_metric_name -+ -+ -+class Detector: -+ _metric_name: MetricName = None -+ _threshold: Threshold = None -+ _slidingWindow: SlidingWindow = None -+ -+ def __init__(self, metric_name: MetricName, threshold: Threshold, sliding_window: SlidingWindow): -+ self._metric_name = metric_name -+ self._threshold = threshold -+ self._slidingWindow = sliding_window -+ self._threshold.attach_observer(self._slidingWindow) -+ -+ def get_metric_name(self): -+ return self._metric_name -+ -+ def is_slow_io_event(self, io_data_dict_with_disk_name: dict): -+ logging.debug(f'Enter Detector: {self}') -+ metric_value = get_metric_value_from_io_data_dict_by_metric_name(io_data_dict_with_disk_name, self._metric_name) -+ if metric_value > 1e-6: -+ logging.debug(f'Input metric value: {str(metric_value)}') -+ self._threshold.push_latest_data_to_queue(metric_value) -+ detection_result = self._slidingWindow.is_slow_io_event(metric_value) -+ logging.debug(f'Detection result: {str(detection_result)}') -+ logging.debug(f'Exit Detector: {self}') -+ return detection_result -+ -+ def __repr__(self): -+ return (f'disk_name: {self._metric_name.get_disk_name()}, stage_name: {self._metric_name.get_stage_name()},' -+ f' access_type_name: {self._metric_name.get_io_access_type_name()},' -+ f' metric_name: {self._metric_name.get_metric_name()}, threshold_type: {self._threshold},' -+ f' sliding_window_type: {self._slidingWindow}') -diff --git a/src/python/sentryPlugins/ai_threshold_slow_io_detection/io_data.py b/src/python/sentryPlugins/ai_threshold_slow_io_detection/io_data.py -new file mode 100644 -index 0000000..0e17051 ---- /dev/null -+++ b/src/python/sentryPlugins/ai_threshold_slow_io_detection/io_data.py -@@ -0,0 +1,74 @@ -+# coding: utf-8 -+# Copyright (c) 2024 Huawei Technologies Co., Ltd. -+# sysSentry is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+ -+from dataclasses import dataclass, field -+from datetime import datetime -+from typing import Optional -+ -+ -+@dataclass -+class BaseData: -+ latency: Optional[float] = field(default_factory=lambda: None) -+ io_dump: Optional[int] = field(default_factory=lambda: None) -+ io_length: Optional[int] = field(default_factory=lambda: None) -+ iops: Optional[int] = field(default_factory=lambda: None) -+ -+ -+@dataclass -+class IOStageData: -+ read: BaseData = field(default_factory=lambda: BaseData()) -+ write: BaseData = field(default_factory=lambda: BaseData()) -+ flush: BaseData = field(default_factory=lambda: BaseData()) -+ discard: BaseData = field(default_factory=lambda: BaseData()) -+ -+ -+@dataclass -+class IOData: -+ throtl: IOStageData = field(default_factory=lambda: IOStageData()) -+ wbt: IOStageData = field(default_factory=lambda: IOStageData()) -+ gettag: IOStageData = field(default_factory=lambda: IOStageData()) -+ iocost: IOStageData = field(default_factory=lambda: IOStageData()) -+ plug: IOStageData = field(default_factory=lambda: IOStageData()) -+ bfq: IOStageData = field(default_factory=lambda: IOStageData()) -+ hctx: IOStageData = field(default_factory=lambda: IOStageData()) -+ requeue: IOStageData = field(default_factory=lambda: IOStageData()) -+ rq_driver: IOStageData = field(default_factory=lambda: IOStageData()) -+ bio: IOStageData = field(default_factory=lambda: IOStageData()) -+ time_stamp: float = field(default_factory=lambda: datetime.now().timestamp()) -+ -+ -+class MetricName: -+ _disk_name: str = None -+ _stage_name: str = None -+ _io_access_type_name: str = None -+ _metric_name: str = None -+ -+ def __init__(self, disk_name: str, stage_name: str, io_access_type_name: str, metric_name: str): -+ self._disk_name = disk_name -+ self._stage_name = stage_name -+ self._io_access_type_name = io_access_type_name -+ self._metric_name = metric_name -+ -+ def get_disk_name(self): -+ return self._disk_name -+ -+ def get_stage_name(self): -+ return self._stage_name -+ -+ def get_io_access_type_name(self): -+ return self._io_access_type_name -+ -+ def get_metric_name(self): -+ return self._metric_name -+ -+ def __repr__(self): -+ return (f'disk: {self._disk_name}, stage: {self._stage_name}, io_access_type: {self._io_access_type_name},' -+ f'metric: {self._metric_name}') -diff --git a/src/python/sentryPlugins/ai_threshold_slow_io_detection/sliding_window.py b/src/python/sentryPlugins/ai_threshold_slow_io_detection/sliding_window.py -new file mode 100644 -index 0000000..d395d48 ---- /dev/null -+++ b/src/python/sentryPlugins/ai_threshold_slow_io_detection/sliding_window.py -@@ -0,0 +1,113 @@ -+# coding: utf-8 -+# Copyright (c) 2024 Huawei Technologies Co., Ltd. -+# sysSentry is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+ -+from enum import Enum, unique -+import numpy as np -+ -+ -+@unique -+class SlidingWindowType(Enum): -+ NotContinuousSlidingWindow = 0 -+ ContinuousSlidingWindow = 1 -+ MedianSlidingWindow = 2 -+ -+ -+class SlidingWindow: -+ _ai_threshold = None -+ _queue_length = None -+ _queue_threshold = None -+ _io_data_queue: list = None -+ _io_data_queue_abnormal_tag: list = None -+ -+ def __init__(self, queue_length: int, threshold: int): -+ self._queue_length = queue_length -+ self._queue_threshold = threshold -+ self._io_data_queue = [] -+ self._io_data_queue_abnormal_tag = [] -+ -+ def push(self, data: float): -+ if len(self._io_data_queue) == self._queue_length: -+ self._io_data_queue.pop(0) -+ self._io_data_queue_abnormal_tag.pop(0) -+ self._io_data_queue.append(data) -+ self._io_data_queue_abnormal_tag.append(data >= self._ai_threshold if self._ai_threshold is not None else False) -+ -+ def update(self, threshold): -+ if self._ai_threshold == threshold: -+ return -+ self._ai_threshold = threshold -+ self._io_data_queue_abnormal_tag.clear() -+ for data in self._io_data_queue: -+ self._io_data_queue_abnormal_tag.append(data >= self._ai_threshold) -+ -+ def is_slow_io_event(self, data): -+ return False, None, None -+ -+ def __repr__(self): -+ return "SlidingWindow" -+ -+ -+class NotContinuousSlidingWindow(SlidingWindow): -+ def is_slow_io_event(self, data): -+ super().push(data) -+ if len(self._io_data_queue) < self._queue_length or self._ai_threshold is None: -+ return False, self._io_data_queue, self._ai_threshold -+ if self._io_data_queue_abnormal_tag.count(True) >= self._queue_threshold: -+ return True, self._io_data_queue, self._ai_threshold -+ return False, self._io_data_queue, self._ai_threshold -+ -+ def __repr__(self): -+ return "NotContinuousSlidingWindow" -+ -+ -+class ContinuousSlidingWindow(SlidingWindow): -+ def is_slow_io_event(self, data): -+ super().push(data) -+ if len(self._io_data_queue) < self._queue_length or self._ai_threshold is None: -+ return False, self._io_data_queue, self._ai_threshold -+ consecutive_count = 0 -+ for tag in self._io_data_queue_abnormal_tag: -+ if tag: -+ consecutive_count += 1 -+ if consecutive_count >= self._queue_threshold: -+ return True, self._io_data_queue, self._ai_threshold -+ else: -+ consecutive_count = 0 -+ return False, self._io_data_queue, self._ai_threshold -+ -+ def __repr__(self): -+ return "ContinuousSlidingWindow" -+ -+ -+class MedianSlidingWindow(SlidingWindow): -+ def is_slow_io_event(self, data): -+ super().push(data) -+ if len(self._io_data_queue) < self._queue_length or self._ai_threshold is None: -+ return False, self._io_data_queue, self._ai_threshold -+ median = np.median(self._io_data_queue) -+ if median >= self._ai_threshold: -+ return True, self._io_data_queue, self._ai_threshold -+ return False, self._io_data_queue, self._ai_threshold -+ -+ def __repr__(self): -+ return "MedianSlidingWindow" -+ -+ -+class SlidingWindowFactory: -+ def get_sliding_window(self, sliding_window_type: SlidingWindowType, *args, **kwargs): -+ if sliding_window_type == SlidingWindowType.NotContinuousSlidingWindow: -+ return NotContinuousSlidingWindow(*args, **kwargs) -+ elif sliding_window_type == SlidingWindowType.ContinuousSlidingWindow: -+ return ContinuousSlidingWindow(*args, **kwargs) -+ elif sliding_window_type == SlidingWindowType.MedianSlidingWindow: -+ return MedianSlidingWindow(*args, **kwargs) -+ else: -+ return NotContinuousSlidingWindow(*args, **kwargs) -diff --git a/src/python/sentryPlugins/ai_threshold_slow_io_detection/slow_io_detection.py b/src/python/sentryPlugins/ai_threshold_slow_io_detection/slow_io_detection.py -new file mode 100644 -index 0000000..43cf770 ---- /dev/null -+++ b/src/python/sentryPlugins/ai_threshold_slow_io_detection/slow_io_detection.py -@@ -0,0 +1,133 @@ -+# coding: utf-8 -+# Copyright (c) 2024 Huawei Technologies Co., Ltd. -+# sysSentry is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+ -+import time -+import signal -+import logging -+ -+from .detector import Detector -+from .threshold import ThresholdFactory, AbsoluteThreshold -+from .sliding_window import SlidingWindowFactory -+from .utils import (get_threshold_type_enum, get_sliding_window_type_enum, get_data_queue_size_and_update_size, -+ get_log_level) -+from .config_parser import ConfigParser -+from .data_access import get_io_data_from_collect_plug, check_collect_valid -+from .io_data import MetricName -+from .alarm_report import AlarmReport -+ -+CONFIG_FILE = "/etc/sysSentry/plugins/ai_threshold_slow_io_detection.ini" -+ -+ -+def sig_handler(signum, frame): -+ logging.info("receive signal: %d", signum) -+ AlarmReport().report_fail(f"receive signal: {signum}") -+ exit(signum) -+ -+ -+class SlowIODetection: -+ _config_parser = None -+ _disk_list = None -+ _detector_name_list = [] -+ _detectors = {} -+ -+ def __init__(self, config_parser: ConfigParser): -+ self._config_parser = config_parser -+ self.__set_log_format() -+ self.__init_detector_name_list() -+ self.__init_detector() -+ -+ def __set_log_format(self): -+ log_format = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s" -+ log_level = get_log_level(self._config_parser.get_log_level()) -+ logging.basicConfig(level=log_level, format=log_format) -+ -+ def __init_detector_name_list(self): -+ self._disk_list = check_collect_valid(self._config_parser.get_slow_io_detect_frequency()) -+ for disk in self._disk_list: -+ self._detector_name_list.append(MetricName(disk, "bio", "read", "latency")) -+ self._detector_name_list.append(MetricName(disk, "bio", "write", "latency")) -+ -+ def __init_detector(self): -+ train_data_duration, train_update_duration = (self._config_parser. -+ get_train_data_duration_and_train_update_duration()) -+ slow_io_detection_frequency = self._config_parser.get_slow_io_detect_frequency() -+ threshold_type = get_threshold_type_enum(self._config_parser.get_algorithm_type()) -+ data_queue_size, update_size = get_data_queue_size_and_update_size(train_data_duration, -+ train_update_duration, -+ slow_io_detection_frequency) -+ sliding_window_type = get_sliding_window_type_enum(self._config_parser.get_sliding_window_type()) -+ window_size, window_threshold = self._config_parser.get_window_size_and_window_minimum_threshold() -+ -+ for detector_name in self._detector_name_list: -+ threshold = ThresholdFactory().get_threshold(threshold_type, data_queue_size=data_queue_size, -+ data_queue_update_size=update_size) -+ sliding_window = SlidingWindowFactory().get_sliding_window(sliding_window_type, queue_length=window_size, -+ threshold=window_threshold) -+ detector = Detector(detector_name, threshold, sliding_window) -+ # 绝对阈值的阈值初始化 -+ if isinstance(threshold, AbsoluteThreshold): -+ threshold.set_threshold(self._config_parser.get_absolute_threshold()) -+ self._detectors[detector_name] = detector -+ logging.info(f"add detector: {detector}") -+ -+ def launch(self): -+ while True: -+ logging.debug('step0. AI threshold slow io event detection is looping.') -+ -+ # Step1：获取IO数据 -+ io_data_dict_with_disk_name = get_io_data_from_collect_plug( -+ self._config_parser.get_slow_io_detect_frequency(), self._disk_list -+ ) -+ logging.debug(f'step1. Get io data: {str(io_data_dict_with_disk_name)}') -+ if io_data_dict_with_disk_name is None: -+ continue -+ # Step2：慢IO检测 -+ logging.debug('step2. Start to detection slow io event.') -+ slow_io_event_list = [] -+ for metric_name, detector in self._detectors.items(): -+ result = detector.is_slow_io_event(io_data_dict_with_disk_name) -+ if result[0]: -+ slow_io_event_list.append((detector.get_metric_name(), result)) -+ logging.debug('step2. End to detection slow io event.') -+ -+ # Step3：慢IO事件上报 -+ logging.debug('step3. Report slow io event to sysSentry.') -+ for slow_io_event in slow_io_event_list: -+ metric_name: MetricName = slow_io_event[0] -+ result = slow_io_event[1] -+ AlarmReport.report_major_alm(f"disk {metric_name.get_disk_name()} has slow io event." -+ f"stage: {metric_name.get_metric_name()}," -+ f"type: {metric_name.get_io_access_type_name()}," -+ f"metric: {metric_name.get_metric_name()}," -+ f"current window: {result[1]}," -+ f"threshold: {result[2]}") -+ logging.error(f"slow io event happen: {str(slow_io_event)}") -+ -+ # Step4：等待检测时间 -+ logging.debug('step4. Wait to start next slow io event detection loop.') -+ time.sleep(self._config_parser.get_slow_io_detect_frequency()) -+ -+ -+def main(): -+ # Step1：注册消息处理函数 -+ signal.signal(signal.SIGINT, sig_handler) -+ signal.signal(signal.SIGTERM, sig_handler) -+ # Step2：断点恢复 -+ # todo: -+ -+ # Step3：读取配置 -+ config_file_name = CONFIG_FILE -+ config = ConfigParser(config_file_name) -+ config.read_config_from_file() -+ -+ # Step4：启动慢IO检测 -+ slow_io_detection = SlowIODetection(config) -+ slow_io_detection.launch() -diff --git a/src/python/sentryPlugins/ai_threshold_slow_io_detection/threshold.py b/src/python/sentryPlugins/ai_threshold_slow_io_detection/threshold.py -new file mode 100644 -index 0000000..9e1ca7b ---- /dev/null -+++ b/src/python/sentryPlugins/ai_threshold_slow_io_detection/threshold.py -@@ -0,0 +1,160 @@ -+# coding: utf-8 -+# Copyright (c) 2024 Huawei Technologies Co., Ltd. -+# sysSentry is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+import logging -+from enum import Enum -+import queue -+import numpy as np -+import math -+ -+from .sliding_window import SlidingWindow -+ -+ -+class ThresholdState(Enum): -+ INIT = 0 -+ START = 1 -+ -+ -+class Threshold: -+ threshold = None -+ data_queue: queue.Queue = None -+ data_queue_update_size: int = None -+ new_data_size: int = None -+ threshold_state: ThresholdState = None -+ -+ def __init__(self, data_queue_size: int = 10000, data_queue_update_size: int = 1000): -+ self._observer = None -+ self.data_queue = queue.Queue(data_queue_size) -+ self.data_queue_update_size = data_queue_update_size -+ self.new_data_size = 0 -+ self.threshold_state = ThresholdState.INIT -+ self.threshold = math.inf -+ -+ def set_threshold(self, threshold): -+ self.threshold = threshold -+ self.threshold_state = ThresholdState.START -+ self.notify_observer() -+ -+ def get_threshold(self): -+ if self.threshold_state == ThresholdState.INIT: -+ return None -+ return self.threshold -+ -+ def is_abnormal(self, data): -+ if self.threshold_state == ThresholdState.INIT: -+ return False -+ return data >= self.threshold -+ -+ # 使用观察者模式，当阈值更新时，自动同步刷新滑窗中的阈值 -+ def attach_observer(self, observer: SlidingWindow): -+ self._observer = observer -+ -+ def notify_observer(self): -+ if self._observer is not None: -+ self._observer.update(self.threshold) -+ -+ def push_latest_data_to_queue(self, data): -+ pass -+ -+ def __repr__(self): -+ return "Threshold" -+ -+ -+class AbsoluteThreshold(Threshold): -+ def __init__(self, data_queue_size: int = 10000, data_queue_update_size: int = 1000): -+ super().__init__(data_queue_size, data_queue_update_size) -+ -+ def push_latest_data_to_queue(self, data): -+ pass -+ -+ def __repr__(self): -+ return "AbsoluteThreshold" -+ -+ -+class BoxplotThreshold(Threshold): -+ def __init__(self, parameter: float = 1.5, data_queue_size: int = 10000, data_queue_update_size: int = 1000): -+ super().__init__(data_queue_size, data_queue_update_size) -+ self.parameter = parameter -+ -+ def _update_threshold(self): -+ data = list(self.data_queue.queue) -+ q1 = np.percentile(data, 25) -+ q3 = np.percentile(data, 75) -+ iqr = q3 - q1 -+ self.threshold = q3 + self.parameter * iqr -+ if self.threshold_state == ThresholdState.INIT: -+ self.threshold_state = ThresholdState.START -+ self.notify_observer() -+ -+ def push_latest_data_to_queue(self, data): -+ try: -+ self.data_queue.put(data, block=False) -+ except queue.Full: -+ self.data_queue.get() -+ self.data_queue.put(data) -+ self.new_data_size += 1 -+ if (self.data_queue.full() and (self.threshold_state == ThresholdState.INIT or -+ (self.threshold_state == ThresholdState.START and -+ self.new_data_size >= self.data_queue_update_size))): -+ self._update_threshold() -+ self.new_data_size = 0 -+ -+ def __repr__(self): -+ return "BoxplotThreshold" -+ -+ -+class NSigmaThreshold(Threshold): -+ def __init__(self, parameter: float = 2.0, data_queue_size: int = 10000, data_queue_update_size: int = 1000): -+ super().__init__(data_queue_size, data_queue_update_size) -+ self.parameter = parameter -+ -+ def _update_threshold(self): -+ data = list(self.data_queue.queue) -+ mean = np.mean(data) -+ std = np.std(data) -+ self.threshold = mean + self.parameter * std -+ if self.threshold_state == ThresholdState.INIT: -+ self.threshold_state = ThresholdState.START -+ self.notify_observer() -+ -+ def push_latest_data_to_queue(self, data): -+ try: -+ self.data_queue.put(data, block=False) -+ except queue.Full: -+ self.data_queue.get() -+ self.data_queue.put(data) -+ self.new_data_size += 1 -+ if (self.data_queue.full() and (self.threshold_state == ThresholdState.INIT or -+ (self.threshold_state == ThresholdState.START and -+ self.new_data_size >= self.data_queue_update_size))): -+ self._update_threshold() -+ self.new_data_size = 0 -+ -+ def __repr__(self): -+ return "NSigmaThreshold" -+ -+ -+class ThresholdType(Enum): -+ AbsoluteThreshold = 0 -+ BoxplotThreshold = 1 -+ NSigmaThreshold = 2 -+ -+ -+class ThresholdFactory: -+ def get_threshold(self, threshold_type: ThresholdType, *args, **kwargs): -+ if threshold_type == ThresholdType.AbsoluteThreshold: -+ return AbsoluteThreshold(*args, **kwargs) -+ elif threshold_type == ThresholdType.BoxplotThreshold: -+ return BoxplotThreshold(*args, **kwargs) -+ elif threshold_type == ThresholdType.NSigmaThreshold: -+ return NSigmaThreshold(*args, **kwargs) -+ else: -+ raise ValueError(f"Invalid threshold type: {threshold_type}") -+ -diff --git a/src/python/sentryPlugins/ai_threshold_slow_io_detection/utils.py b/src/python/sentryPlugins/ai_threshold_slow_io_detection/utils.py -new file mode 100644 -index 0000000..f66e5ed ---- /dev/null -+++ b/src/python/sentryPlugins/ai_threshold_slow_io_detection/utils.py -@@ -0,0 +1,67 @@ -+# coding: utf-8 -+# Copyright (c) 2024 Huawei Technologies Co., Ltd. -+# sysSentry is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+import logging -+from dataclasses import asdict -+ -+from .threshold import ThresholdType -+from .sliding_window import SlidingWindowType -+from .io_data import MetricName, IOData -+ -+def get_threshold_type_enum(algorithm_type: str): -+ if algorithm_type.lower() == 'absolute': -+ return ThresholdType.AbsoluteThreshold -+ if algorithm_type.lower() == 'boxplot': -+ return ThresholdType.BoxplotThreshold -+ if algorithm_type.lower() == 'n_sigma': -+ return ThresholdType.NSigmaThreshold -+ logging.info('not found correct algorithm type, use default: boxplot.') -+ return ThresholdType.BoxplotThreshold -+ -+ -+def get_sliding_window_type_enum(sliding_window_type: str): -+ if sliding_window_type.lower() == 'not_continuous': -+ return SlidingWindowType.NotContinuousSlidingWindow -+ if sliding_window_type.lower() == 'continuous': -+ return SlidingWindowType.ContinuousSlidingWindow -+ if sliding_window_type.lower() == 'median': -+ return SlidingWindowType.MedianSlidingWindow -+ logging.info('not found correct sliding window type, use default: not_continuous.') -+ return SlidingWindowType.NotContinuousSlidingWindow -+ -+ -+def get_metric_value_from_io_data_dict_by_metric_name(io_data_dict: dict, metric_name: MetricName): -+ try: -+ io_data: IOData = io_data_dict[metric_name.get_disk_name()] -+ io_stage_data = asdict(io_data)[metric_name.get_stage_name()] -+ base_data = io_stage_data[metric_name.get_io_access_type_name()] -+ metric_value = base_data[metric_name.get_metric_name()] -+ return metric_value -+ except KeyError: -+ return None -+ -+ -+def get_data_queue_size_and_update_size(training_data_duration: float, train_update_duration: float, -+ slow_io_detect_frequency: int): -+ data_queue_size = int(training_data_duration * 60 * 60 / slow_io_detect_frequency) -+ update_size = int(train_update_duration * 60 * 60 / slow_io_detect_frequency) -+ return data_queue_size, update_size -+ -+ -+def get_log_level(log_level: str): -+ if log_level.lower() == 'debug': -+ return logging.DEBUG -+ elif log_level.lower() == 'info': -+ return logging.INFO -+ elif log_level.lower() == 'warning': -+ return logging.WARNING -+ elif log_level.lower() == 'fatal': -+ return logging.FATAL -+ return None -diff --git a/src/python/setup.py b/src/python/setup.py -index c28c691..dac6481 100644 ---- a/src/python/setup.py -+++ b/src/python/setup.py -@@ -33,7 +33,8 @@ setup( - 'syssentry=syssentry.syssentry:main', - 'xalarmd=xalarm.xalarm_daemon:alarm_process_create', - 'sentryCollector=sentryCollector.collectd:main', -- 'avg_block_io=sentryPlugins.avg_block_io.avg_block_io:main' -+ 'avg_block_io=sentryPlugins.avg_block_io.avg_block_io:main', -+ 'ai_threshold_slow_io_detection=sentryPlugins.ai_threshold_slow_io_detection.slow_io_detection:main' - ] - }, - ) --- -2.23.0 - diff --git a/add-boundary-check-for-settings.patch b/add-boundary-check-for-settings.patch deleted file mode 100644 index 05184c0..0000000 --- a/add-boundary-check-for-settings.patch +++ /dev/null @@ -1,39 +0,0 @@ -From abf36bf0351efde388c089245aed9f6d8d2e6d3b Mon Sep 17 00:00:00 2001 -From: luckky -Date: Wed, 6 Nov 2024 11:42:53 +0800 -Subject: [PATCH] add boundary check for settings -1. add two boundary checks for page_isolation_threshold and hbm_online_repair_log_level -(0 <= page_isolation_threshold) -(0(LOG_DEBUG) <= hbm_online_repair_log_level <= 3(LOG_ERROR)) - ---- - src/c/hbm_online_repair/hbm_online_repair.c | 6 ++++++ - 1 file changed, 6 insertions(+) - -diff --git a/src/c/hbm_online_repair/hbm_online_repair.c b/src/c/hbm_online_repair/hbm_online_repair.c -index 943f201..00c9c0b 100644 ---- a/src/c/hbm_online_repair/hbm_online_repair.c -+++ b/src/c/hbm_online_repair/hbm_online_repair.c -@@ -89,6 +89,9 @@ void hbm_param_init(void) - if (ret < 0) { - global_level_setting = DEFAULT_LOG_LEVEL; - log(LOG_WARNING, "Get log level from config failed, set the default value %d\n", DEFAULT_LOG_LEVEL); -+ } else if (global_level_setting < LOG_DEBUG || global_level_setting > LOG_ERROR) { -+ log(LOG_WARNING, "The log level value %d in config is out of range, set the default value %d\n", global_level_setting, DEFAULT_LOG_LEVEL); -+ global_level_setting = DEFAULT_LOG_LEVEL; - } else { - log(LOG_INFO, "log level: %d\n", global_level_setting); - } -@@ -98,6 +101,9 @@ void hbm_param_init(void) - if (ret < 0) { - page_isolation_threshold = DEFAULT_PAGE_ISOLATION_THRESHOLD; - log(LOG_WARNING, "Get page_isolation_threshold from config failed, set the default value %d\n", DEFAULT_PAGE_ISOLATION_THRESHOLD); -+ } else if (page_isolation_threshold < 0) { -+ log(LOG_WARNING, "The page_isolation_threshold %d in config is out of range, set the default value %d\n", page_isolation_threshold, DEFAULT_PAGE_ISOLATION_THRESHOLD); -+ page_isolation_threshold = DEFAULT_PAGE_ISOLATION_THRESHOLD; - } else { - log(LOG_INFO, "page_isolation_threshold: %d\n", page_isolation_threshold); - } --- -2.43.0 - diff --git a/add-collect-module-to-sysSentry.patch b/add-collect-module-to-sysSentry.patch deleted file mode 100644 index f8fa667..0000000 --- a/add-collect-module-to-sysSentry.patch +++ /dev/null @@ -1,1165 +0,0 @@ -From bd32dc01000126d593c188d47404cfdbe1df343e Mon Sep 17 00:00:00 2001 -From: zhuofeng -Date: Thu, 12 Sep 2024 11:29:01 +0800 -Subject: [PATCH 1/2] add collect module to sysSentry - ---- - config/collector.conf | 7 + - service/sentryCollector.service | 12 + - service/sysSentry.service | 2 +- - src/python/sentryCollector/__init__.py | 0 - src/python/sentryCollector/__main__.py | 17 ++ - src/python/sentryCollector/collect_config.py | 118 ++++++++ - src/python/sentryCollector/collect_io.py | 239 ++++++++++++++++ - src/python/sentryCollector/collect_plugin.py | 276 ++++++++++++++++++ - src/python/sentryCollector/collect_server.py | 285 +++++++++++++++++++ - src/python/sentryCollector/collectd.py | 99 +++++++ - src/python/setup.py | 4 +- - 11 files changed, 1057 insertions(+), 2 deletions(-) - create mode 100644 config/collector.conf - create mode 100644 service/sentryCollector.service - create mode 100644 src/python/sentryCollector/__init__.py - create mode 100644 src/python/sentryCollector/__main__.py - create mode 100644 src/python/sentryCollector/collect_config.py - create mode 100644 src/python/sentryCollector/collect_io.py - create mode 100644 src/python/sentryCollector/collect_plugin.py - create mode 100644 src/python/sentryCollector/collect_server.py - create mode 100644 src/python/sentryCollector/collectd.py - -diff --git a/config/collector.conf b/config/collector.conf -new file mode 100644 -index 0000000..9baa086 ---- /dev/null -+++ b/config/collector.conf -@@ -0,0 +1,7 @@ -+[common] -+modules=io -+ -+[io] -+period_time=1 -+max_save=10 -+disk=default -\ No newline at end of file -diff --git a/service/sentryCollector.service b/service/sentryCollector.service -new file mode 100644 -index 0000000..4ee07d5 ---- /dev/null -+++ b/service/sentryCollector.service -@@ -0,0 +1,12 @@ -+[Unit] -+Description = Collection module added for sysSentry and kernel lock-free collection -+ -+[Service] -+ExecStart=/usr/bin/python3 /usr/bin/sentryCollector -+ExecStop=/bin/kill $MAINPID -+KillMode=process -+Restart=on-failure -+RestartSec=10s -+ -+[Install] -+WantedBy = multi-user.target -diff --git a/service/sysSentry.service b/service/sysSentry.service -index 4d85a6c..1d8338f 100644 ---- a/service/sysSentry.service -+++ b/service/sysSentry.service -@@ -2,7 +2,7 @@ - Description=EulerOS System Inspection Frame - - [Service] --ExecStart=/usr/bin/syssentry -+ExecStart=/usr/bin/python3 /usr/bin/syssentry - ExecStop=/bin/kill $MAINPID - KillMode=process - Restart=on-failure -diff --git a/src/python/sentryCollector/__init__.py b/src/python/sentryCollector/__init__.py -new file mode 100644 -index 0000000..e69de29 -diff --git a/src/python/sentryCollector/__main__.py b/src/python/sentryCollector/__main__.py -new file mode 100644 -index 0000000..9c2ae50 ---- /dev/null -+++ b/src/python/sentryCollector/__main__.py -@@ -0,0 +1,17 @@ -+# coding: utf-8 -+# Copyright (c) 2023 Huawei Technologies Co., Ltd. -+# sysSentry is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+ -+""" -+main -+""" -+from collectd import collectd -+ -+collectd.main() -diff --git a/src/python/sentryCollector/collect_config.py b/src/python/sentryCollector/collect_config.py -new file mode 100644 -index 0000000..b6cc75c ---- /dev/null -+++ b/src/python/sentryCollector/collect_config.py -@@ -0,0 +1,118 @@ -+# coding: utf-8 -+# Copyright (c) 2024 Huawei Technologies Co., Ltd. -+# sysSentry is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+ -+""" -+Read and save collector.conf value. -+""" -+import configparser -+import logging -+import os -+import re -+ -+ -+COLLECT_CONF_PATH = "/etc/sysSentry/collector.conf" -+ -+CONF_COMMON = 'common' -+CONF_MODULES = 'modules' -+ -+# io -+CONF_IO = 'io' -+CONF_IO_PERIOD_TIME = 'period_time' -+CONF_IO_MAX_SAVE = 'max_save' -+CONF_IO_DISK = 'disk' -+CONF_IO_PERIOD_TIME_DEFAULT = 1 -+CONF_IO_MAX_SAVE_DEFAULT = 10 -+CONF_IO_DISK_DEFAULT = "default" -+ -+class CollectConfig: -+ def __init__(self, filename=COLLECT_CONF_PATH): -+ -+ self.filename = filename -+ self.modules = [] -+ self.module_count = 0 -+ self.load_config() -+ -+ def load_config(self): -+ if not os.path.exists(self.filename): -+ logging.error("%s is not exists", self.filename) -+ return -+ -+ try: -+ self.config = configparser.ConfigParser() -+ self.config.read(self.filename) -+ except configparser.Error: -+ logging.error("collectd configure file read failed") -+ return -+ -+ try: -+ common_config = self.config[CONF_COMMON] -+ modules_str = common_config[CONF_MODULES] -+ # remove space -+ modules_list = modules_str.replace(" ", "").split(',') -+ except KeyError as e: -+ logging.error("read config data failed, %s", e) -+ return -+ -+ pattern = r'^[a-zA-Z0-9-_]+$' -+ for module_name in modules_list: -+ if not re.match(pattern, module_name): -+ logging.warning("module_name: %s is invalid", module_name) -+ continue -+ if not self.config.has_section(module_name): -+ logging.warning("module_name: %s config is incorrect", module_name) -+ continue -+ self.modules.append(module_name) -+ -+ def load_module_config(self, module_name): -+ module_name = module_name.strip().lower() -+ if module_name in self.modules and self.config.has_section(module_name): -+ return {key.lower(): value for key, value in self.config[module_name].items()} -+ else: -+ raise ValueError(f"Module '{module_name}' not found in configuration") -+ -+ def get_io_config(self): -+ result_io_config = {} -+ io_map_value = self.load_module_config(CONF_IO) -+ # period_time -+ period_time = io_map_value.get(CONF_IO_PERIOD_TIME) -+ if period_time and period_time.isdigit() and int(period_time) >= 1 and int(period_time) <= 300: -+ result_io_config[CONF_IO_PERIOD_TIME] = int(period_time) -+ else: -+ logging.warning("module_name = %s section, field = %s is incorrect, use default %d", -+ CONF_IO, CONF_IO_PERIOD_TIME, CONF_IO_PERIOD_TIME_DEFAULT) -+ result_io_config[CONF_IO_PERIOD_TIME] = CONF_IO_PERIOD_TIME_DEFAULT -+ # max_save -+ max_save = io_map_value.get(CONF_IO_MAX_SAVE) -+ if max_save and max_save.isdigit() and int(max_save) >= 1 and int(max_save) <= 300: -+ result_io_config[CONF_IO_MAX_SAVE] = int(max_save) -+ else: -+ logging.warning("module_name = %s section, field = %s is incorrect, use default %d", -+ CONF_IO, CONF_IO_MAX_SAVE, CONF_IO_MAX_SAVE_DEFAULT) -+ result_io_config[CONF_IO_MAX_SAVE] = CONF_IO_MAX_SAVE_DEFAULT -+ # disk -+ disk = io_map_value.get(CONF_IO_DISK) -+ if disk: -+ disk_str = disk.replace(" ", "") -+ pattern = r'^[a-zA-Z0-9-_,]+$' -+ if not re.match(pattern, disk_str): -+ logging.warning("module_name = %s section, field = %s is incorrect, use default %s", -+ CONF_IO, CONF_IO_DISK, CONF_IO_DISK_DEFAULT) -+ disk_str = CONF_IO_DISK_DEFAULT -+ result_io_config[CONF_IO_DISK] = disk_str -+ else: -+ logging.warning("module_name = %s section, field = %s is incorrect, use default %s", -+ CONF_IO, CONF_IO_DISK, CONF_IO_DISK_DEFAULT) -+ result_io_config[CONF_IO_DISK] = CONF_IO_DISK_DEFAULT -+ logging.info("config get_io_config: %s", result_io_config) -+ return result_io_config -+ -+ def get_common_config(self): -+ return {key.lower(): value for key, value in self.config['common'].items()} -diff --git a/src/python/sentryCollector/collect_io.py b/src/python/sentryCollector/collect_io.py -new file mode 100644 -index 0000000..b826dc4 ---- /dev/null -+++ b/src/python/sentryCollector/collect_io.py -@@ -0,0 +1,239 @@ -+# coding: utf-8 -+# Copyright (c) 2024 Huawei Technologies Co., Ltd. -+# sysSentry is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+ -+""" -+collect module -+""" -+import os -+import time -+import logging -+import threading -+ -+from .collect_config import CollectConfig -+ -+Io_Category = ["read", "write", "flush", "discard"] -+IO_GLOBAL_DATA = {} -+IO_CONFIG_DATA = [] -+ -+class IoStatus(): -+ TOTAL = 0 -+ FINISH = 1 -+ LATENCY = 2 -+ -+class CollectIo(): -+ -+ def __init__(self, module_config): -+ -+ io_config = module_config.get_io_config() -+ -+ self.period_time = io_config['period_time'] -+ self.max_save = io_config['max_save'] -+ disk_str = io_config['disk'] -+ -+ self.disk_map_stage = {} -+ self.window_value = {} -+ -+ self.loop_all = False -+ -+ if disk_str == "default": -+ self.loop_all = True -+ else: -+ self.disk_list = disk_str.strip().split(',') -+ -+ self.stop_event = threading.Event() -+ -+ IO_CONFIG_DATA.append(self.period_time) -+ IO_CONFIG_DATA.append(self.max_save) -+ -+ def get_blk_io_hierarchy(self, disk_name, stage_list): -+ stats_file = '/sys/kernel/debug/block/{}/blk_io_hierarchy/stats'.format(disk_name) -+ try: -+ with open(stats_file, 'r') as file: -+ lines = file.read() -+ except FileNotFoundError: -+ logging.error("The file %s does not exist", stats_file) -+ return -1 -+ except Exception as e: -+ logging.error("An error occurred3: %s", e) -+ return -1 -+ -+ curr_value = lines.strip().split('\n') -+ -+ for stage_val in curr_value: -+ stage = stage_val.split(' ')[0] -+ if (len(self.window_value[disk_name][stage])) >= 2: -+ self.window_value[disk_name][stage].pop(0) -+ -+ curr_stage_value = stage_val.split(' ')[1:-1] -+ self.window_value[disk_name][stage].append(curr_stage_value) -+ return 0 -+ -+ def append_period_lat(self, disk_name, stage_list): -+ for stage in stage_list: -+ if len(self.window_value[disk_name][stage]) < 2: -+ return -+ curr_stage_value = self.window_value[disk_name][stage][-1] -+ last_stage_value = self.window_value[disk_name][stage][-2] -+ -+ for index in range(len(Io_Category)): -+ # read=0, write=1, flush=2, discard=3 -+ if (len(IO_GLOBAL_DATA[disk_name][stage][Io_Category[index]])) >= self.max_save: -+ IO_GLOBAL_DATA[disk_name][stage][Io_Category[index]].pop() -+ -+ curr_lat = self.get_latency_value(curr_stage_value, last_stage_value, index) -+ curr_iops = self.get_iops(curr_stage_value, last_stage_value, index) -+ curr_io_length = self.get_io_length(curr_stage_value, last_stage_value, index) -+ curr_io_dump = self.get_io_dump(disk_name, stage, index) -+ -+ IO_GLOBAL_DATA[disk_name][stage][Io_Category[index]].insert(0, [curr_lat, curr_io_dump, curr_io_length, curr_iops]) -+ -+ def get_iops(self, curr_stage_value, last_stage_value, category): -+ try: -+ finish = int(curr_stage_value[category * 3 + IoStatus.FINISH]) - int(last_stage_value[category * 3 + IoStatus.FINISH]) -+ except ValueError as e: -+ logging.error("get_iops convert to int failed, %s", e) -+ return 0 -+ value = finish / self.period_time -+ if value.is_integer(): -+ return int(value) -+ else: -+ return round(value, 1) -+ -+ def get_latency_value(self, curr_stage_value, last_stage_value, category): -+ try: -+ finish = int(curr_stage_value[category * 3 + IoStatus.FINISH]) - int(last_stage_value[category * 3 + IoStatus.FINISH]) -+ lat_time = (int(curr_stage_value[category * 3 + IoStatus.LATENCY]) - int(last_stage_value[category * 3 + IoStatus.LATENCY])) -+ except ValueError as e: -+ logging.error("get_latency_value convert to int failed, %s", e) -+ return 0 -+ if finish <= 0 or lat_time <= 0: -+ return 0 -+ value = lat_time / finish / 1000 / 1000 -+ if value.is_integer(): -+ return int(value) -+ else: -+ return round(value, 1) -+ -+ def get_io_length(self, curr_stage_value, last_stage_value, category): -+ try: -+ finish = int(curr_stage_value[category * 3 + IoStatus.FINISH]) - int(last_stage_value[category * 3 + IoStatus.FINISH]) -+ except ValueError as e: -+ logging.error("get_io_length convert to int failed, %s", e) -+ return 0 -+ value = finish / self.period_time / 1000 / 1000 -+ if value.is_integer(): -+ return int(value) -+ else: -+ return round(value, 1) -+ -+ def get_io_dump(self, disk_name, stage, category): -+ io_dump_file = '/sys/kernel/debug/block/{}/blk_io_hierarchy/{}/io_dump'.format(disk_name, stage) -+ count = 0 -+ try: -+ with open(io_dump_file, 'r') as file: -+ for line in file: -+ count += line.count('.op=' + Io_Category[category]) -+ except FileNotFoundError: -+ logging.error("The file %s does not exist.", io_dump_file) -+ return count -+ except Exception as e: -+ logging.error("An error occurred1: %s", e) -+ return count -+ return count -+ -+ def extract_first_column(self, file_path): -+ column_names = [] -+ try: -+ with open(file_path, 'r') as file: -+ for line in file: -+ parts = line.strip().split() -+ if parts: -+ column_names.append(parts[0]) -+ except FileNotFoundError: -+ logging.error("The file %s does not exist.", file_path) -+ except Exception as e: -+ logging.error("An error occurred2: %s", e) -+ return column_names -+ -+ def task_loop(self): -+ if self.stop_event.is_set(): -+ logging.info("collect io thread exit") -+ return -+ -+ for disk_name, stage_list in self.disk_map_stage.items(): -+ if self.get_blk_io_hierarchy(disk_name, stage_list) < 0: -+ continue -+ self.append_period_lat(disk_name, stage_list) -+ -+ threading.Timer(self.period_time, self.task_loop).start() -+ -+ def main_loop(self): -+ logging.info("collect io thread start") -+ base_path = '/sys/kernel/debug/block' -+ for disk_name in os.listdir(base_path): -+ if not self.loop_all and disk_name not in self.disk_list: -+ continue -+ -+ disk_path = os.path.join(base_path, disk_name) -+ blk_io_hierarchy_path = os.path.join(disk_path, 'blk_io_hierarchy') -+ -+ if not os.path.exists(blk_io_hierarchy_path): -+ logging.error("no blk_io_hierarchy directory found in %s, skipping.", disk_name) -+ continue -+ -+ for file_name in os.listdir(blk_io_hierarchy_path): -+ file_path = os.path.join(blk_io_hierarchy_path, file_name) -+ -+ if file_name == 'stats': -+ stage_list = self.extract_first_column(file_path) -+ self.disk_map_stage[disk_name] = stage_list -+ self.window_value[disk_name] = {} -+ IO_GLOBAL_DATA[disk_name] = {} -+ -+ if len(self.disk_map_stage) == 0: -+ logging.warning("no disks meet the requirements. the thread exits") -+ return -+ -+ for disk_name, stage_list in self.disk_map_stage.items(): -+ for stage in stage_list: -+ self.window_value[disk_name][stage] = [] -+ IO_GLOBAL_DATA[disk_name][stage] = {} -+ for category in Io_Category: -+ IO_GLOBAL_DATA[disk_name][stage][category] = [] -+ -+ while True: -+ start_time = time.time() -+ -+ if self.stop_event.is_set(): -+ logging.info("collect io thread exit") -+ return -+ -+ for disk_name, stage_list in self.disk_map_stage.items(): -+ if self.get_blk_io_hierarchy(disk_name, stage_list) < 0: -+ continue -+ self.append_period_lat(disk_name, stage_list) -+ -+ elapsed_time = time.time() - start_time -+ sleep_time = self.period_time - elapsed_time -+ if sleep_time < 0: -+ continue -+ while sleep_time > 1: -+ if self.stop_event.is_set(): -+ logging.info("collect io thread exit") -+ return -+ time.sleep(1) -+ sleep_time -= 1 -+ time.sleep(sleep_time) -+ -+ # set stop event, notify thread exit -+ def stop_thread(self): -+ logging.info("collect io thread is preparing to exit") -+ self.stop_event.set() -diff --git a/src/python/sentryCollector/collect_plugin.py b/src/python/sentryCollector/collect_plugin.py -new file mode 100644 -index 0000000..49ce0a8 ---- /dev/null -+++ b/src/python/sentryCollector/collect_plugin.py -@@ -0,0 +1,276 @@ -+# coding: utf-8 -+# Copyright (c) 2024 Huawei Technologies Co., Ltd. -+# sysSentry is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+ -+""" -+collcet plugin -+""" -+import json -+import socket -+import logging -+import re -+ -+COLLECT_SOCKET_PATH = "/var/run/sysSentry/collector.sock" -+ -+# data length param -+CLT_MSG_HEAD_LEN = 9 #3+2+4 -+CLT_MSG_PRO_LEN = 2 -+CLT_MSG_MAGIC_LEN = 3 -+CLT_MSG_LEN_LEN = 4 -+ -+CLT_MAGIC = "CLT" -+RES_MAGIC = "RES" -+ -+# disk limit -+LIMIT_DISK_CHAR_LEN = 32 -+LIMIT_DISK_LIST_LEN = 10 -+ -+# stage limit -+LIMIT_STAGE_CHAR_LEN = 20 -+LIMIT_STAGE_LIST_LEN = 15 -+ -+#iotype limit -+LIMIT_IOTYPE_CHAR_LEN = 7 -+LIMIT_IOTYPE_LIST_LEN = 4 -+ -+#period limit -+LIMIT_PERIOD_MIN_LEN = 1 -+LIMIT_PERIOD_MAX_LEN = 300 -+ -+# interface protocol -+class ClientProtocol(): -+ IS_IOCOLLECT_VALID = 0 -+ GET_IO_DATA = 1 -+ PRO_END = 3 -+ -+class ResultMessage(): -+ RESULT_SUCCEED = 0 -+ RESULT_UNKNOWN = 1 # unknown error -+ RESULT_NOT_PARAM = 2 # the parameter does not exist or the type does not match. -+ RESULT_INVALID_LENGTH = 3 # invalid parameter length. -+ RESULT_EXCEED_LIMIT = 4 # the parameter length exceeds the limit. -+ RESULT_PARSE_FAILED = 5 # parse failed -+ RESULT_INVALID_CHAR = 6 # invalid char -+ -+Result_Messages = { -+ ResultMessage.RESULT_SUCCEED: "Succeed", -+ ResultMessage.RESULT_UNKNOWN: "Unknown error", -+ ResultMessage.RESULT_NOT_PARAM: "The parameter does not exist or the type does not match", -+ ResultMessage.RESULT_INVALID_LENGTH: "Invalid parameter length", -+ ResultMessage.RESULT_EXCEED_LIMIT: "The parameter length exceeds the limit", -+ ResultMessage.RESULT_PARSE_FAILED: "Parse failed", -+ ResultMessage.RESULT_INVALID_CHAR: "Invalid char" -+} -+ -+ -+def client_send_and_recv(request_data, data_str_len, protocol): -+ """client socket send and recv message""" -+ try: -+ client_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) -+ except socket.error: -+ print("collect_plugin: client creat socket error") -+ return None -+ -+ try: -+ client_socket.connect(COLLECT_SOCKET_PATH) -+ except OSError: -+ client_socket.close() -+ print("collect_plugin: client connect error") -+ return None -+ -+ req_data_len = len(request_data) -+ request_msg = CLT_MAGIC + str(protocol).zfill(CLT_MSG_PRO_LEN) + str(req_data_len).zfill(CLT_MSG_LEN_LEN) + request_data -+ -+ try: -+ client_socket.send(request_msg.encode()) -+ res_data = client_socket.recv(len(RES_MAGIC) + CLT_MSG_PRO_LEN + data_str_len) -+ res_data = res_data.decode() -+ except (OSError, UnicodeError): -+ client_socket.close() -+ print("collect_plugin: client communicate error") -+ return None -+ -+ res_magic = res_data[:CLT_MSG_MAGIC_LEN] -+ if res_magic != "RES": -+ print("res msg format error") -+ return None -+ -+ protocol_str = res_data[CLT_MSG_MAGIC_LEN:CLT_MSG_MAGIC_LEN+CLT_MSG_PRO_LEN] -+ try: -+ protocol_id = int(protocol_str) -+ except ValueError: -+ print("recv msg protocol id is invalid %s", protocol_str) -+ return None -+ -+ if protocol_id >= ClientProtocol.PRO_END: -+ print("protocol id is invalid") -+ return None -+ -+ try: -+ res_data_len = int(res_data[CLT_MSG_MAGIC_LEN+CLT_MSG_PRO_LEN:]) -+ res_msg_data = client_socket.recv(res_data_len) -+ res_msg_data = res_msg_data.decode() -+ return res_msg_data -+ except (OSError, ValueError, UnicodeError): -+ print("collect_plugin: client recv res msg error") -+ finally: -+ client_socket.close() -+ -+ return None -+ -+def validate_parameters(param, len_limit, char_limit): -+ ret = ResultMessage.RESULT_SUCCEED -+ if not param: -+ print("parm is invalid") -+ ret = ResultMessage.RESULT_NOT_PARAM -+ return [False, ret] -+ -+ if not isinstance(param, list): -+ print(f"{param} is not list type.") -+ ret = ResultMessage.RESULT_NOT_PARAM -+ return [False, ret] -+ -+ if len(param) <= 0: -+ print(f"{param} length is 0.") -+ ret = ResultMessage.RESULT_INVALID_LENGTH -+ return [False, ret] -+ -+ if len(param) > len_limit: -+ print(f"{param} length more than {len_limit}") -+ ret = ResultMessage.RESULT_EXCEED_LIMIT -+ return [False, ret] -+ -+ pattern = r'^[a-zA-Z0-9_-]+$' -+ for info in param: -+ if len(info) > char_limit: -+ print(f"{info} length more than {char_limit}") -+ ret = ResultMessage.RESULT_EXCEED_LIMIT -+ return [False, ret] -+ if not re.match(pattern, info): -+ print(f"{info} is invalid char") -+ ret = ResultMessage.RESULT_INVALID_CHAR -+ return [False, ret] -+ -+ return [True, ret] -+ -+def is_iocollect_valid(period, disk_list=None, stage=None): -+ result = inter_is_iocollect_valid(period, disk_list, stage) -+ error_code = result['ret'] -+ if error_code != ResultMessage.RESULT_SUCCEED: -+ result['message'] = Result_Messages[error_code] -+ return result -+ -+def inter_is_iocollect_valid(period, disk_list=None, stage=None): -+ result = {} -+ result['ret'] = ResultMessage.RESULT_UNKNOWN -+ result['message'] = "" -+ -+ if not period or not isinstance(period, int): -+ result['ret'] = ResultMessage.RESULT_NOT_PARAM -+ return result -+ if period < LIMIT_PERIOD_MIN_LEN or period > LIMIT_PERIOD_MAX_LEN: -+ result['ret'] = ResultMessage.RESULT_INVALID_LENGTH -+ return result -+ -+ if not disk_list: -+ disk_list = [] -+ else: -+ res = validate_parameters(disk_list, LIMIT_DISK_LIST_LEN, LIMIT_DISK_CHAR_LEN) -+ if not res[0]: -+ result['ret'] = res[1] -+ return result -+ -+ if not stage: -+ stage = [] -+ else: -+ res = validate_parameters(stage, LIMIT_STAGE_LIST_LEN, LIMIT_STAGE_CHAR_LEN) -+ if not res[0]: -+ result['ret'] = res[1] -+ return result -+ -+ req_msg_struct = { -+ 'disk_list': json.dumps(disk_list), -+ 'period': period, -+ 'stage': json.dumps(stage) -+ } -+ request_message = json.dumps(req_msg_struct) -+ result_message = client_send_and_recv(request_message, CLT_MSG_LEN_LEN, ClientProtocol.IS_IOCOLLECT_VALID) -+ if not result_message: -+ print("collect_plugin: client_send_and_recv failed") -+ return result -+ -+ try: -+ json.loads(result_message) -+ except json.JSONDecodeError: -+ print("is_iocollect_valid: json decode error") -+ result['ret'] = ResultMessage.RESULT_PARSE_FAILED -+ return result -+ -+ result['ret'] = ResultMessage.RESULT_SUCCEED -+ result['message'] = result_message -+ return result -+ -+def get_io_data(period, disk_list, stage, iotype): -+ result = inter_get_io_data(period, disk_list, stage, iotype) -+ error_code = result['ret'] -+ if error_code != ResultMessage.RESULT_SUCCEED: -+ result['message'] = Result_Messages[error_code] -+ return result -+ -+def inter_get_io_data(period, disk_list, stage, iotype): -+ result = {} -+ result['ret'] = ResultMessage.RESULT_UNKNOWN -+ result['message'] = "" -+ -+ if not isinstance(period, int): -+ result['ret'] = ResultMessage.RESULT_NOT_PARAM -+ return result -+ if period < LIMIT_PERIOD_MIN_LEN or period > LIMIT_PERIOD_MAX_LEN: -+ result['ret'] = ResultMessage.RESULT_INVALID_LENGTH -+ return result -+ -+ res = validate_parameters(disk_list, LIMIT_DISK_LIST_LEN, LIMIT_DISK_CHAR_LEN) -+ if not res[0]: -+ result['ret'] = res[1] -+ return result -+ -+ res = validate_parameters(stage, LIMIT_STAGE_LIST_LEN, LIMIT_STAGE_CHAR_LEN) -+ if not res[0]: -+ result['ret'] = res[1] -+ return result -+ -+ res = validate_parameters(iotype, LIMIT_IOTYPE_LIST_LEN, LIMIT_IOTYPE_CHAR_LEN) -+ if not res[0]: -+ result['ret'] = res[1] -+ return result -+ -+ req_msg_struct = { -+ 'disk_list': json.dumps(disk_list), -+ 'period': period, -+ 'stage': json.dumps(stage), -+ 'iotype' : json.dumps(iotype) -+ } -+ -+ request_message = json.dumps(req_msg_struct) -+ result_message = client_send_and_recv(request_message, CLT_MSG_LEN_LEN, ClientProtocol.GET_IO_DATA) -+ if not result_message: -+ print("collect_plugin: client_send_and_recv failed") -+ return result -+ try: -+ json.loads(result_message) -+ except json.JSONDecodeError: -+ print("get_io_data: json decode error") -+ result['ret'] = ResultMessage.RESULT_PARSE_FAILED -+ return result -+ -+ result['ret'] = ResultMessage.RESULT_SUCCEED -+ result['message'] = result_message -+ return result -+ -diff --git a/src/python/sentryCollector/collect_server.py b/src/python/sentryCollector/collect_server.py -new file mode 100644 -index 0000000..fa49781 ---- /dev/null -+++ b/src/python/sentryCollector/collect_server.py -@@ -0,0 +1,285 @@ -+# coding: utf-8 -+# Copyright (c) 2024 Huawei Technologies Co., Ltd. -+# sysSentry is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+ -+""" -+listen module -+""" -+import sys -+import signal -+import traceback -+import socket -+import os -+import json -+import logging -+import fcntl -+import select -+import threading -+import time -+ -+from .collect_io import IO_GLOBAL_DATA, IO_CONFIG_DATA -+from .collect_config import CollectConfig -+ -+SENTRY_RUN_DIR = "/var/run/sysSentry" -+COLLECT_SOCKET_PATH = "/var/run/sysSentry/collector.sock" -+ -+# socket param -+CLT_LISTEN_QUEUE_LEN = 5 -+SERVER_EPOLL_TIMEOUT = 0.3 -+ -+# data length param -+CLT_MSG_HEAD_LEN = 9 #3+2+4 -+CLT_MSG_PRO_LEN = 2 -+CLT_MSG_MAGIC_LEN = 3 -+CLT_MSG_LEN_LEN = 4 -+ -+# data flag param -+CLT_MAGIC = "CLT" -+RES_MAGIC = "RES" -+ -+# interface protocol -+class ServerProtocol(): -+ IS_IOCOLLECT_VALID = 0 -+ GET_IO_DATA = 1 -+ PRO_END = 3 -+ -+class CollectServer(): -+ -+ def __init__(self): -+ -+ self.io_global_data = {} -+ -+ self.stop_event = threading.Event() -+ -+ def is_iocollect_valid(self, data_struct): -+ -+ result_rev = {} -+ self.io_global_data = IO_GLOBAL_DATA -+ -+ if len(IO_CONFIG_DATA) == 0: -+ logging.error("the collect thread is not started, the data is invalid. ") -+ return json.dumps(result_rev) -+ -+ period_time = IO_CONFIG_DATA[0] -+ max_save = IO_CONFIG_DATA[1] -+ -+ disk_list = json.loads(data_struct['disk_list']) -+ period = int(data_struct['period']) -+ stage_list = json.loads(data_struct['stage']) -+ -+ if (period < period_time) or (period > period_time * max_save) or (period % period_time): -+ logging.error("is_iocollect_valid: period time: %d is invalid", period) -+ return json.dumps(result_rev) -+ -+ for disk_name, stage_info in self.io_global_data.items(): -+ if len(disk_list) > 0 and disk_name not in disk_list: -+ continue -+ result_rev[disk_name] = [] -+ if len(stage_list) == 0: -+ result_rev[disk_name] = list(stage_info.keys()) -+ continue -+ for stage_name, stage_data in stage_info.items(): -+ if stage_name in stage_list: -+ result_rev[disk_name].append(stage_name) -+ -+ return json.dumps(result_rev) -+ -+ def get_io_data(self, data_struct): -+ result_rev = {} -+ self.io_global_data = IO_GLOBAL_DATA -+ -+ if len(IO_CONFIG_DATA) == 0: -+ logging.error("the collect thread is not started, the data is invalid. ") -+ return json.dumps(result_rev) -+ period_time = IO_CONFIG_DATA[0] -+ max_save = IO_CONFIG_DATA[1] -+ -+ period = int(data_struct['period']) -+ disk_list = json.loads(data_struct['disk_list']) -+ stage_list = json.loads(data_struct['stage']) -+ iotype_list = json.loads(data_struct['iotype']) -+ -+ if (period < period_time) or (period > period_time * max_save) or (period % period_time): -+ logging.error("get_io_data: period time: %d is invalid", period) -+ return json.dumps(result_rev) -+ -+ collect_index = period // period_time - 1 -+ logging.debug("period: %d, collect_index: %d", period, collect_index) -+ -+ for disk_name, stage_info in self.io_global_data.items(): -+ if disk_name not in disk_list: -+ continue -+ result_rev[disk_name] = {} -+ for stage_name, iotype_info in stage_info.items(): -+ if len(stage_list) > 0 and stage_name not in stage_list: -+ continue -+ result_rev[disk_name][stage_name] = {} -+ for iotype_name, iotype_info in iotype_info.items(): -+ if iotype_name not in iotype_list: -+ continue -+ if len(iotype_info) < collect_index: -+ continue -+ result_rev[disk_name][stage_name][iotype_name] = iotype_info[collect_index] -+ -+ return json.dumps(result_rev) -+ -+ def msg_data_process(self, msg_data, protocal_id): -+ """message data process""" -+ logging.debug("msg_data %s", msg_data) -+ protocol_name = msg_data[0] -+ try: -+ data_struct = json.loads(msg_data) -+ except json.JSONDecodeError: -+ logging.error("msg data process: json decode error") -+ return "Request message decode failed" -+ -+ if protocal_id == ServerProtocol.IS_IOCOLLECT_VALID: -+ res_msg = self.is_iocollect_valid(data_struct) -+ elif protocal_id == ServerProtocol.GET_IO_DATA: -+ res_msg = self.get_io_data(data_struct) -+ -+ return res_msg -+ -+ def msg_head_process(self, msg_head): -+ """message head process""" -+ ctl_magic = msg_head[:CLT_MSG_MAGIC_LEN] -+ if ctl_magic != CLT_MAGIC: -+ logging.error("recv msg head magic invalid") -+ return None -+ -+ protocol_str = msg_head[CLT_MSG_MAGIC_LEN:CLT_MSG_MAGIC_LEN+CLT_MSG_PRO_LEN] -+ try: -+ protocol_id = int(protocol_str) -+ except ValueError: -+ logging.error("recv msg protocol id is invalid") -+ return None -+ -+ data_len_str = msg_head[CLT_MSG_MAGIC_LEN+CLT_MSG_PRO_LEN:CLT_MSG_HEAD_LEN] -+ try: -+ data_len = int(data_len_str) -+ except ValueError: -+ logging.error("recv msg data len is invalid %s", data_len_str) -+ return None -+ -+ return [protocol_id, data_len] -+ -+ def server_recv(self, server_socket: socket.socket): -+ """server receive""" -+ try: -+ client_socket, _ = server_socket.accept() -+ logging.debug("server_fd listen ok") -+ except socket.error: -+ logging.error("server accept failed, %s", socket.error) -+ return -+ -+ try: -+ msg_head = client_socket.recv(CLT_MSG_HEAD_LEN) -+ logging.debug("recv msg head: %s", msg_head.decode()) -+ head_info = self.msg_head_process(msg_head.decode()) -+ except (OSError, UnicodeError): -+ client_socket.close() -+ logging.error("server recv HEAD failed") -+ return -+ -+ protocol_id = head_info[0] -+ data_len = head_info[1] -+ logging.debug("msg protocol id: %d, data length: %d", protocol_id, data_len) -+ if protocol_id >= ServerProtocol.PRO_END: -+ client_socket.close() -+ logging.error("protocol id is invalid") -+ return -+ -+ if data_len < 0: -+ client_socket.close() -+ logging.error("msg head parse failed") -+ return -+ -+ try: -+ msg_data = client_socket.recv(data_len) -+ msg_data_decode = msg_data.decode() -+ logging.debug("msg data %s", msg_data_decode) -+ except (OSError, UnicodeError): -+ client_socket.close() -+ logging.error("server recv MSG failed") -+ return -+ -+ res_data = self.msg_data_process(msg_data_decode, protocol_id) -+ logging.debug("res data %s", res_data) -+ -+ # server send -+ res_head = RES_MAGIC -+ res_head += str(protocol_id).zfill(CLT_MSG_PRO_LEN) -+ res_data_len = str(len(res_data)).zfill(CLT_MSG_LEN_LEN) -+ res_head += res_data_len -+ logging.debug("res head %s", res_head) -+ -+ res_msg = res_head + res_data -+ logging.debug("res msg %s", res_msg) -+ -+ try: -+ client_socket.send(res_msg.encode()) -+ except OSError: -+ logging.error("server recv failed") -+ finally: -+ client_socket.close() -+ return -+ -+ def server_fd_create(self): -+ """create server fd""" -+ if not os.path.exists(SENTRY_RUN_DIR): -+ logging.error("%s not exist, failed", SENTRY_RUN_DIR) -+ return None -+ -+ try: -+ server_fd = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) -+ server_fd.setblocking(False) -+ if os.path.exists(COLLECT_SOCKET_PATH): -+ os.remove(COLLECT_SOCKET_PATH) -+ -+ server_fd.bind(COLLECT_SOCKET_PATH) -+ os.chmod(COLLECT_SOCKET_PATH, 0o600) -+ server_fd.listen(CLT_LISTEN_QUEUE_LEN) -+ logging.debug("%s bind and listen", COLLECT_SOCKET_PATH) -+ except socket.error: -+ logging.error("server fd create failed") -+ server_fd = None -+ -+ return server_fd -+ -+ -+ def server_loop(self): -+ """main loop""" -+ logging.info("collect server thread start") -+ server_fd = self.server_fd_create() -+ if not server_fd: -+ return -+ -+ epoll_fd = select.epoll() -+ epoll_fd.register(server_fd.fileno(), select.EPOLLIN) -+ -+ logging.debug("start server_loop loop") -+ while True: -+ if self.stop_event.is_set(): -+ logging.info("collect server thread exit") -+ server_fd = None -+ return -+ try: -+ events_list = epoll_fd.poll(SERVER_EPOLL_TIMEOUT) -+ for event_fd, _ in events_list: -+ if event_fd == server_fd.fileno(): -+ self.server_recv(server_fd) -+ else: -+ continue -+ except socket.error: -+ pass -+ -+ def stop_thread(self): -+ logging.info("collect server thread is preparing to exit") -+ self.stop_event.set() -diff --git a/src/python/sentryCollector/collectd.py b/src/python/sentryCollector/collectd.py -new file mode 100644 -index 0000000..b77c642 ---- /dev/null -+++ b/src/python/sentryCollector/collectd.py -@@ -0,0 +1,99 @@ -+# coding: utf-8 -+# Copyright (c) 2024 Huawei Technologies Co., Ltd. -+# sysSentry is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+ -+""" -+main loop for collect. -+""" -+import sys -+import signal -+import traceback -+import socket -+import os -+import json -+import logging -+import fcntl -+import select -+ -+import threading -+ -+from .collect_io import CollectIo -+from .collect_server import CollectServer -+from .collect_config import CollectConfig -+ -+SENTRY_RUN_DIR = "/var/run/sysSentry" -+COLLECT_SOCKET_PATH = "/var/run/sysSentry/collector.sock" -+SENTRY_RUN_DIR_PERM = 0o750 -+ -+COLLECT_LOG_FILE = "/var/log/sysSentry/collector.log" -+Thread_List = [] -+Module_Map_Class = {"io" : CollectIo} -+ -+def remove_sock_file(): -+ try: -+ os.unlink(COLLECT_SOCKET_PATH) -+ except FileNotFoundError: -+ pass -+ -+def sig_handler(signum, _f): -+ if signum not in (signal.SIGINT, signal.SIGTERM): -+ return -+ for i in range(len(Thread_List)): -+ Thread_List[i][0].stop_thread() -+ -+ remove_sock_file() -+ sys.exit(0) -+ -+def main(): -+ """main -+ """ -+ if not os.path.exists(SENTRY_RUN_DIR): -+ os.mkdir(SENTRY_RUN_DIR) -+ os.chmod(SENTRY_RUN_DIR, mode=SENTRY_RUN_DIR_PERM) -+ -+ logging.basicConfig(filename=COLLECT_LOG_FILE, level=logging.INFO) -+ os.chmod(COLLECT_LOG_FILE, 0o600) -+ -+ try: -+ signal.signal(signal.SIGINT, sig_handler) -+ signal.signal(signal.SIGTERM, sig_handler) -+ signal.signal(signal.SIGHUP, sig_handler) -+ -+ logging.info("finish main parse_args") -+ -+ module_config = CollectConfig() -+ module_list = module_config.modules -+ -+ # listen thread -+ cs = CollectServer() -+ listen_thread = threading.Thread(target=cs.server_loop) -+ listen_thread.start() -+ Thread_List.append([cs, listen_thread]) -+ -+ # collect thread -+ for info in module_list: -+ class_name = Module_Map_Class.get(info) -+ if not class_name: -+ logging.info("%s correspond to class is not exists", info) -+ continue -+ cn = class_name(module_config) -+ collect_thread = threading.Thread(target=cn.main_loop) -+ collect_thread.start() -+ Thread_List.append([cn, collect_thread]) -+ -+ for i in range(len(Thread_List)): -+ Thread_List[i][1].join() -+ -+ except Exception: -+ logging.error('%s', traceback.format_exc()) -+ finally: -+ pass -+ -+ logging.info("All threads have finished. Main thread is exiting.") -\ No newline at end of file -diff --git a/src/python/setup.py b/src/python/setup.py -index f96a96e..c28c691 100644 ---- a/src/python/setup.py -+++ b/src/python/setup.py -@@ -31,7 +31,9 @@ setup( - 'console_scripts': [ - 'cpu_sentry=syssentry.cpu_sentry:main', - 'syssentry=syssentry.syssentry:main', -- 'xalarmd=xalarm.xalarm_daemon:alarm_process_create' -+ 'xalarmd=xalarm.xalarm_daemon:alarm_process_create', -+ 'sentryCollector=sentryCollector.collectd:main', -+ 'avg_block_io=sentryPlugins.avg_block_io.avg_block_io:main' - ] - }, - ) --- -2.33.0 - diff --git a/add-deleted-code-to-plugin-rasdaemon.patch b/add-deleted-code-to-plugin-rasdaemon.patch deleted file mode 100644 index 89d1cc4..0000000 --- a/add-deleted-code-to-plugin-rasdaemon.patch +++ /dev/null @@ -1,31 +0,0 @@ -From eca8c542875aef5cfbf947d697c4b644490d1c05 Mon Sep 17 00:00:00 2001 -From: zhuofeng -Date: Fri, 30 Aug 2024 19:58:41 +0800 -Subject: [PATCH] add deleted code to plugin rasdaemon - ---- - src/python/syssentry/syssentry.py | 8 ++++++++ - 1 file changed, 8 insertions(+) - -diff --git a/src/python/syssentry/syssentry.py b/src/python/syssentry/syssentry.py -index 32b81e3..3d5cb8d 100644 ---- a/src/python/syssentry/syssentry.py -+++ b/src/python/syssentry/syssentry.py -@@ -462,6 +462,14 @@ def main_loop(): - epoll_fd.register(cpu_alarm_fd.fileno(), select.EPOLLIN) - - logging.debug("start main loop") -+ # onstart_tasks_handle() -+ for task_type in TasksMap.tasks_dict: -+ for task_name in TasksMap.tasks_dict.get(task_type): -+ task = TasksMap.tasks_dict.get(task_type).get(task_name) -+ if not task: -+ continue -+ task.onstart_handle() -+ - while True: - try: - events_list = epoll_fd.poll(SERVER_EPOLL_TIMEOUT) --- -2.33.0 - diff --git a/add-detail-time.patch b/add-detail-time.patch deleted file mode 100644 index 8e23452..0000000 --- a/add-detail-time.patch +++ /dev/null @@ -1,32 +0,0 @@ -From 9ecd4c2c9c9f9578f5ec4780360dc67b182b384a Mon Sep 17 00:00:00 2001 -From: jinsaihang -Date: Wed, 9 Oct 2024 08:09:04 +0000 -Subject: [PATCH 2/2] add detail time - -Signed-off-by: jinsaihang ---- - src/python/syssentry/alarm.py | 4 +++- - 1 file changed, 3 insertions(+), 1 deletion(-) - -diff --git a/src/python/syssentry/alarm.py b/src/python/syssentry/alarm.py -index 74a2716..d5337d3 100644 ---- a/src/python/syssentry/alarm.py -+++ b/src/python/syssentry/alarm.py -@@ -118,11 +118,13 @@ def get_alarm_result(task_name: str, time_range: int, detailed: bool) -> List[Di - logging.debug(f"get_alarm_result: final alarm_list of {alarm_id} has {len(alarm_list)} elements") - - def xalarm_to_dict(alarm_info: Xalarm) -> dict: -+ timestamp = alarm_info.timetamp.tv_sec + alarm_info.timetamp.tv_usec / 1000000 -+ dt_object = datetime.fromtimestamp(int(timestamp)) - return { - 'alarm_id': xalarm_getid(alarm_info), - 'alarm_type': xalarm_gettype(alarm_info), - 'alarm_level': xalarm_getlevel(alarm_info), -- 'timetamp': xalarm_gettime(alarm_info), -+ 'timestamp': dt_object.strftime("%Y-%m-%d %H:%M:%S"), - 'msg1': xalarm_getdesc(alarm_info) - } - --- -2.27.0 - diff --git a/add-get_disk_type-and-fix-some-bugs.patch b/add-get_disk_type-and-fix-some-bugs.patch deleted file mode 100644 index b5e59e8..0000000 --- a/add-get_disk_type-and-fix-some-bugs.patch +++ /dev/null @@ -1,176 +0,0 @@ -From c2ffc679eddda5d78362612d89a9319d268da7e3 Mon Sep 17 00:00:00 2001 -From: zhuofeng -Date: Thu, 10 Oct 2024 20:17:34 +0800 -Subject: [PATCH] add get_disk_type and fix some bugs - ---- - service/sentryCollector.service | 2 +- - src/python/sentryCollector/collect_io.py | 16 ++++- - src/python/sentryCollector/collect_plugin.py | 68 +++++++++++++++++++- - 3 files changed, 81 insertions(+), 5 deletions(-) - -diff --git a/service/sentryCollector.service b/service/sentryCollector.service -index 4ee07d5..e09ddb3 100644 ---- a/service/sentryCollector.service -+++ b/service/sentryCollector.service -@@ -1,5 +1,5 @@ - [Unit] --Description = Collection module added for sysSentry and kernel lock-free collection -+Description = Collection module added for sysSentry - - [Service] - ExecStart=/usr/bin/python3 /usr/bin/sentryCollector -diff --git a/src/python/sentryCollector/collect_io.py b/src/python/sentryCollector/collect_io.py -index 8780648..6699a90 100644 ---- a/src/python/sentryCollector/collect_io.py -+++ b/src/python/sentryCollector/collect_io.py -@@ -116,7 +116,7 @@ class CollectIo(): - return 0 - if finish <= 0 or lat_time <= 0: - return 0 -- value = lat_time / finish / 1000 / 1000 -+ value = lat_time / finish / 1000 - if value.is_integer(): - return int(value) - else: -@@ -124,11 +124,17 @@ class CollectIo(): - - def get_io_length(self, curr_stage_value, last_stage_value, category): - try: -- finish = int(curr_stage_value[category * 3 + IoStatus.FINISH]) - int(last_stage_value[category * 3 + IoStatus.FINISH]) -+ lat_time = (int(curr_stage_value[category * 3 + IoStatus.LATENCY]) - int(last_stage_value[category * 3 + IoStatus.LATENCY])) - except ValueError as e: - logging.error("get_io_length convert to int failed, %s", e) - return 0 -- value = finish / self.period_time / 1000 / 1000 -+ if lat_time <= 0: -+ return 0 -+ # ns convert us -+ lat_time = lat_time / 1000 -+ # s convert us -+ period_time = self.period_time * 1000 * 1000 -+ value = lat_time / period_time - if value.is_integer(): - return int(value) - else: -@@ -141,6 +147,8 @@ class CollectIo(): - with open(io_dump_file, 'r') as file: - for line in file: - count += line.count('.op=' + Io_Category[category]) -+ if count > 0: -+ logging.info(f"io_dump info : {disk_name}, {stage}, {category}, {count}") - except FileNotFoundError: - logging.error("The file %s does not exist.", io_dump_file) - return count -@@ -223,6 +231,8 @@ class CollectIo(): - if self.get_blk_io_hierarchy(disk_name, stage_list) < 0: - continue - self.append_period_lat(disk_name, stage_list) -+ -+ logging.debug(f"no-lock collect data : {IO_GLOBAL_DATA}") - - elapsed_time = time.time() - start_time - sleep_time = self.period_time - elapsed_time -diff --git a/src/python/sentryCollector/collect_plugin.py b/src/python/sentryCollector/collect_plugin.py -index 3e2cf4c..31bf11b 100644 ---- a/src/python/sentryCollector/collect_plugin.py -+++ b/src/python/sentryCollector/collect_plugin.py -@@ -16,6 +16,7 @@ import json - import socket - import logging - import re -+import os - - COLLECT_SOCKET_PATH = "/var/run/sysSentry/collector.sock" - -@@ -58,6 +59,8 @@ class ResultMessage(): - RESULT_EXCEED_LIMIT = 4 # the parameter length exceeds the limit. - RESULT_PARSE_FAILED = 5 # parse failed - RESULT_INVALID_CHAR = 6 # invalid char -+ RESULT_DISK_NOEXIST = 7 # disk is not exist -+ RESULT_DISK_TYPE_MISMATCH= 8 # disk type mismatch - - Result_Messages = { - ResultMessage.RESULT_SUCCEED: "Succeed", -@@ -66,9 +69,15 @@ Result_Messages = { - ResultMessage.RESULT_INVALID_LENGTH: "Invalid parameter length", - ResultMessage.RESULT_EXCEED_LIMIT: "The parameter length exceeds the limit", - ResultMessage.RESULT_PARSE_FAILED: "Parse failed", -- ResultMessage.RESULT_INVALID_CHAR: "Invalid char" -+ ResultMessage.RESULT_INVALID_CHAR: "Invalid char", -+ ResultMessage.RESULT_DISK_NOEXIST: "Disk is not exist", -+ ResultMessage.RESULT_DISK_TYPE_MISMATCH: "Disk type mismatch" - } - -+class DiskType(): -+ TYPE_NVME_SSD = 0 -+ TYPE_SATA_SSD = 1 -+ TYPE_SATA_HDD = 2 - - def client_send_and_recv(request_data, data_str_len, protocol): - """client socket send and recv message""" -@@ -273,3 +282,60 @@ def inter_get_io_data(period, disk_list, stage, iotype): - result['message'] = result_message - return result - -+def get_disk_type(disk): -+ result = {} -+ result['ret'] = ResultMessage.RESULT_UNKNOWN -+ result['message'] = "" -+ if not disk: -+ logging.error("param is invalid") -+ result['ret'] = ResultMessage.RESULT_NOT_PARAM -+ return result -+ if len(disk) <= 0 or len(disk) > LIMIT_DISK_CHAR_LEN: -+ logging.error("invalid disk length") -+ result['ret'] = ResultMessage.RESULT_INVALID_LENGTH -+ return result -+ pattern = r'^[a-zA-Z0-9_-]+$' -+ if not re.match(pattern, disk): -+ logging.error("%s is invalid char", disk) -+ result['ret'] = ResultMessage.RESULT_INVALID_CHAR -+ return result -+ -+ base_path = '/sys/block' -+ all_disk = [] -+ for disk_name in os.listdir(base_path): -+ all_disk.append(disk_name) -+ -+ if disk not in all_disk: -+ logging.error("disk %s is not exist", disk) -+ result['ret'] = ResultMessage.RESULT_DISK_NOEXIST -+ return result -+ -+ if disk[0:4] == "nvme": -+ result['message'] = str(DiskType.TYPE_NVME_SSD) -+ elif disk[0:2] == "sd": -+ disk_file = '/sys/block/{}/queue/rotational'.format(disk) -+ try: -+ with open(disk_file, 'r') as file: -+ num = int(file.read()) -+ if num == 1: -+ result['message'] = str(DiskType.TYPE_SATA_SSD) -+ elif num == 0: -+ result['message'] = str(DiskType.TYPE_SATA_HDD) -+ else: -+ logging.error("disk %s is not support, num = %d", disk, num) -+ result['ret'] = ResultMessage.RESULT_DISK_TYPE_MISMATCH -+ return result -+ except FileNotFoundError: -+ logging.error("The disk_file [%s] does not exist", disk_file) -+ result['ret'] = ResultMessage.RESULT_DISK_NOEXIST -+ return result -+ except Exception as e: -+ logging.error("open disk_file %s happen an error: %s", disk_file, e) -+ return result -+ else: -+ logging.error("disk %s is not support", disk) -+ result['ret'] = ResultMessage.RESULT_DISK_TYPE_MISMATCH -+ return result -+ -+ result['ret'] = ResultMessage.RESULT_SUCCEED -+ return result -\ No newline at end of file --- -2.33.0 - diff --git a/add-hbm-online-repair.patch b/add-hbm-online-repair.patch deleted file mode 100644 index c6906ff..0000000 --- a/add-hbm-online-repair.patch +++ /dev/null @@ -1,2194 +0,0 @@ -From abdeacfa6ae54b503714cb98f3262a39d883972e Mon Sep 17 00:00:00 2001 -From: luckky -Date: Fri, 11 Oct 2024 09:49:40 +0000 -Subject: [PATCH] add hbm online repair - ---- - config/tasks/hbm_online_repair.mod | 9 + - src/c/hbm_online_repair/.gitignore | 6 + - src/c/hbm_online_repair/Makefile | 25 + - src/c/hbm_online_repair/hbm_online_repair.c | 144 ++++ - src/c/hbm_online_repair/hbm_online_repair.env | 2 + - src/c/hbm_online_repair/logger.h | 31 + - .../non-standard-hbm-repair.c | 799 ++++++++++++++++++ - .../non-standard-hbm-repair.h | 89 ++ - src/c/hbm_online_repair/ras-events.c | 534 ++++++++++++ - src/c/hbm_online_repair/ras-events.h | 28 + - .../ras-non-standard-handler.c | 81 ++ - .../ras-non-standard-handler.h | 25 + - src/python/.gitignore | 1 + - src/python/syssentry/bmc_alarm.py | 159 ++++ - src/python/syssentry/syssentry.py | 78 +- - 15 files changed, 2001 insertions(+), 10 deletions(-) - create mode 100644 config/tasks/hbm_online_repair.mod - create mode 100644 src/c/hbm_online_repair/.gitignore - create mode 100644 src/c/hbm_online_repair/Makefile - create mode 100644 src/c/hbm_online_repair/hbm_online_repair.c - create mode 100644 src/c/hbm_online_repair/hbm_online_repair.env - create mode 100644 src/c/hbm_online_repair/logger.h - create mode 100644 src/c/hbm_online_repair/non-standard-hbm-repair.c - create mode 100644 src/c/hbm_online_repair/non-standard-hbm-repair.h - create mode 100644 src/c/hbm_online_repair/ras-events.c - create mode 100644 src/c/hbm_online_repair/ras-events.h - create mode 100644 src/c/hbm_online_repair/ras-non-standard-handler.c - create mode 100644 src/c/hbm_online_repair/ras-non-standard-handler.h - create mode 100644 src/python/.gitignore - create mode 100644 src/python/syssentry/bmc_alarm.py - -diff --git a/config/tasks/hbm_online_repair.mod b/config/tasks/hbm_online_repair.mod -new file mode 100644 -index 0000000..77dd73e ---- /dev/null -+++ b/config/tasks/hbm_online_repair.mod -@@ -0,0 +1,9 @@ -+[common] -+enabled=yes -+task_start=/usr/bin/hbm_online_repair -+task_stop=kill $pid -+type=period -+interval=180 -+onstart=yes -+env_file=/etc/sysconfig/hbm_online_repair.env -+conflict=up -\ No newline at end of file -diff --git a/src/c/hbm_online_repair/.gitignore b/src/c/hbm_online_repair/.gitignore -new file mode 100644 -index 0000000..a577882 ---- /dev/null -+++ b/src/c/hbm_online_repair/.gitignore -@@ -0,0 +1,6 @@ -+*.o -+*.c~ -+*.h~ -+hbm_online_repair -+ -+.vscode/ -diff --git a/src/c/hbm_online_repair/Makefile b/src/c/hbm_online_repair/Makefile -new file mode 100644 -index 0000000..16ebcd8 ---- /dev/null -+++ b/src/c/hbm_online_repair/Makefile -@@ -0,0 +1,25 @@ -+CC = gcc -+ -+CFLAGS = -Wall -o3 -+ -+LDFLAGS = -ltraceevent -+ -+SRC = $(wildcard *.c) -+HDR = $(wildcard *.h) -+ -+OBJ = $(SRC:.c=.o) -+ -+TARGET = hbm_online_repair -+ -+all: $(TARGET) -+ -+$(TARGET): $(OBJ) -+ $(CC) $(OBJ) -o $@ $(LDFLAGS) -+ -+%.o: %.c $(HDR) -+ $(CC) $(CFLAGS) -c $< -o $@ -+ -+clean: -+ rm -f $(OBJ) $(TARGET) -+ -+.PHONY: all clean -diff --git a/src/c/hbm_online_repair/hbm_online_repair.c b/src/c/hbm_online_repair/hbm_online_repair.c -new file mode 100644 -index 0000000..3ace206 ---- /dev/null -+++ b/src/c/hbm_online_repair/hbm_online_repair.c -@@ -0,0 +1,144 @@ -+#include -+#include -+#include -+#include -+#include -+ -+#include "logger.h" -+#include "ras-events.h" -+#include "non-standard-hbm-repair.h" -+ -+#define DEFAULT_LOG_LEVEL LOG_INFO -+#define DEFAULT_PAGE_ISOLATION_THRESHOLD 128 -+ -+int global_level_setting; -+int page_isolation_threshold; -+ -+int string2int(const char* str, int* value) -+{ -+ if (!str) { -+ return -1; -+ } -+ char *endptr; -+ errno = 0; -+ long val = strtol(str, &endptr, 10); -+ if (errno != 0 || *endptr != '\0') { -+ return -1; -+ } -+ *value = (int)val; -+ if (val != (long)*value) { -+ return -1; -+ } -+ return 0; -+} -+ -+int execute_command(const char *command) -+{ -+ FILE *fp; -+ char buffer[128] = {0}; -+ int ret; -+ fp = popen(command, "r"); -+ if (!fp) { -+ log(LOG_ERROR, "popen failed\n"); -+ return -1; -+ } -+ -+ fgets(buffer, sizeof(buffer), fp); -+ log(LOG_DEBUG, "output of command is: %s\n", buffer); -+ -+ ret = pclose(fp); -+ if (ret < 0) { -+ log(LOG_ERROR, "pclose failed\n"); -+ return -1; -+ } -+ -+ if (!WIFEXITED(ret)) { -+ log(LOG_ERROR, "command did not terminate normally\n"); -+ return -1; -+ } -+ -+ ret = WEXITSTATUS(ret); -+ log(LOG_DEBUG, "command exited with status: %d\n", ret); -+ return ret; -+} -+ -+int load_required_driver(void) -+{ -+ int ret; -+ ret = execute_command("modprobe hisi_mem_ras 2>&1"); -+ if (ret < 0) { -+ log(LOG_ERROR, "load repair driver failed\n"); -+ return ret; -+ } -+ ret = execute_command("modprobe page_eject 2>&1"); -+ if (ret < 0) { -+ log(LOG_ERROR, "load page driver failed\n"); -+ return ret; -+ } -+ log(LOG_INFO, "load required driver success\n"); -+ return ret; -+} -+ -+void hbm_param_init(void) -+{ -+ int ret; -+ char *env; -+ -+ env = getenv("HBM_ONLINE_REPAIR_LOG_LEVEL"); -+ ret = string2int(env, &global_level_setting); -+ if (ret < 0) { -+ global_level_setting = DEFAULT_LOG_LEVEL; -+ log(LOG_WARNING, "Get log level from config failed, set the default value %d\n", DEFAULT_LOG_LEVEL); -+ } else { -+ log(LOG_INFO, "log level: %d\n", global_level_setting); -+ } -+ -+ env = getenv("PAGE_ISOLATION_THRESHOLD"); -+ ret = string2int(env, &page_isolation_threshold); -+ if (ret < 0) { -+ page_isolation_threshold = DEFAULT_PAGE_ISOLATION_THRESHOLD; -+ log(LOG_WARNING, "Get page_isolation_threshold from config failed, set the default value %d\n", DEFAULT_PAGE_ISOLATION_THRESHOLD); -+ } else { -+ log(LOG_INFO, "page_isolation_threshold: %d\n", page_isolation_threshold); -+ } -+} -+ -+ -+int main(int argc, char *argv[]) -+{ -+ int ret; -+ -+ hbm_param_init(); -+ -+ ret = load_required_driver(); -+ if (ret < 0) { -+ log(LOG_DEBUG, "load required driver failed\n"); -+ return ret; -+ } -+ -+ struct ras_events *ras = init_trace_instance(); -+ if (!ras) -+ return -1; -+ -+ ret = toggle_ras_event(ras->tracing, "ras", "non_standard_event", 1); -+ if (ret < 0) { -+ log(LOG_WARNING, "unable to enable ras non_standard_event.\n"); -+ free(ras); -+ return -1; -+ } -+ -+ ret = init_all_flash(); -+ if (ret < 0) { -+ log(LOG_ERROR, "flash writer init failed\n"); -+ } -+ -+ handle_ras_events(ras); -+ -+ ret = toggle_ras_event(ras->tracing, "ras", "non_standard_event", 0); -+ if (ret < 0) { -+ log(LOG_WARNING, "unable to disable ras non_standard_event.\n"); -+ } -+ -+ free(ras); -+ return ret; -+} -diff --git a/src/c/hbm_online_repair/hbm_online_repair.env b/src/c/hbm_online_repair/hbm_online_repair.env -new file mode 100644 -index 0000000..de56079 ---- /dev/null -+++ b/src/c/hbm_online_repair/hbm_online_repair.env -@@ -0,0 +1,2 @@ -+HBM_ONLINE_REPAIR_LOG_LEVEL=1 -+PAGE_ISOLATION_THRESHOLD=128 -diff --git a/src/c/hbm_online_repair/logger.h b/src/c/hbm_online_repair/logger.h -new file mode 100644 -index 0000000..ddfa932 ---- /dev/null -+++ b/src/c/hbm_online_repair/logger.h -@@ -0,0 +1,31 @@ -+#ifndef __LOGGER_H -+#define __LOGGER_H -+ -+#define TOOL_NAME "hbm_online_repair" -+ -+#define LOG_DEBUG 0 -+#define LOG_INFO 1 -+#define LOG_WARNING 2 -+#define LOG_ERROR 3 -+ -+extern int global_level_setting; -+ -+#define log_prefix(level) \ -+ (level == LOG_DEBUG ? "DEBUG" : \ -+ level == LOG_INFO ? "INFO" : \ -+ level == LOG_WARNING ? "WARNING" : \ -+ level == LOG_ERROR ? "ERROR" : \ -+ "UNKNOWN_LEVEL") -+ -+#define log_fd(level) \ -+ (level == LOG_ERROR ? stderr : stdout) -+ -+#define log(level, fmt, args...) do {\ -+ if (level >= global_level_setting) {\ -+ fprintf(log_fd(level), "[%s] %s: ", log_prefix(level), TOOL_NAME);\ -+ fprintf(log_fd(level), fmt, ##args);\ -+ fflush(log_fd(level));\ -+ }\ -+} while (0) -+ -+#endif -diff --git a/src/c/hbm_online_repair/non-standard-hbm-repair.c b/src/c/hbm_online_repair/non-standard-hbm-repair.c -new file mode 100644 -index 0000000..b175e14 ---- /dev/null -+++ b/src/c/hbm_online_repair/non-standard-hbm-repair.c -@@ -0,0 +1,799 @@ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "logger.h" -+#include "non-standard-hbm-repair.h" -+ -+extern int page_isolation_threshold; -+size_t total_size = 0; -+struct hisi_common_error_section { -+ uint32_t val_bits; -+ uint8_t version; -+ uint8_t soc_id; -+ uint8_t socket_id; -+ uint8_t totem_id; -+ uint8_t nimbus_id; -+ uint8_t subsystem_id; -+ uint8_t module_id; -+ uint8_t submodule_id; -+ uint8_t core_id; -+ uint8_t port_id; -+ uint16_t err_type; -+ struct { -+ uint8_t function; -+ uint8_t device; -+ uint16_t segment; -+ uint8_t bus; -+ uint8_t reserved[3]; -+ } pcie_info; -+ uint8_t err_severity; -+ uint8_t reserved[3]; -+ uint32_t reg_array_size; -+ uint32_t reg_array[]; -+}; -+ -+struct fault_addr_info { -+ uint32_t processer_id; -+ uint32_t die_id; -+ uint32_t stack_id; -+ uint32_t sid; -+ uint32_t channel_id; -+ uint32_t bankgroup_id; -+ uint32_t bank_id; -+ uint32_t row_id; -+ uint32_t column_id; -+ uint32_t error_type; -+ uint32_t repair_type; -+ uint32_t reserved; -+ uint32_t crc8; -+}; -+ -+typedef struct { -+ const char *VariableName; -+ const char *VendorGuid; -+ uint32_t DataSize; -+ uint8_t *Data; -+ uint32_t Attributes; -+} efi_variable_t; -+ -+char* flash_names[FLASH_ENTRY_NUM] = { -+ "repair0000", -+ "repair0001", -+ "repair0100", -+ "repair0101", -+ "repair0200", -+ "repair0201", -+ "repair0300", -+ "repair0301", -+}; -+char *flash_guids[FLASH_ENTRY_NUM] = { -+ "CD2FF4D9-D937-4e1d-B810-A1A568C37C01", -+ "DD92CC91-43E6-4c69-A42A-B08F72FCB157", -+ "4A8E0D1E-4CFA-47b2-9359-DA3A0006878B", -+ "733F9979-4ED4-478d-BD6A-E4D0F0390FDB", -+ "9BFBBA1F-5A93-4d36-AD47-D3C2D714D914", -+ "A0920D6F-78B8-4c09-9F61-7CEC845F116C", -+ "0049CE5E-8C18-414c-BDC1-A87E60CEEFD7", -+ "6AED17B4-50C7-4a40-A5A7-48AF55DD8EAC" -+}; -+ -+static int get_guid_index(uint32_t socket_id, uint32_t error_type) { -+ if (2 * socket_id + error_type >= FLASH_ENTRY_NUM) -+ return -1; -+ return 2 * socket_id + error_type; -+} -+ -+static void parse_fault_addr_info(struct fault_addr_info* info_struct, unsigned long long fault_addr) -+{ -+ info_struct->processer_id = fault_addr & FAULT_ADDR_PROCESSOR_ID_MASK; -+ fault_addr >>= FAULT_ADDR_PROCESSOR_ID_LEN; -+ info_struct->die_id = fault_addr & FAULT_ADDR_DIE_ID_MASK; -+ fault_addr >>= FAULT_ADDR_DIE_ID_LEN; -+ info_struct->stack_id = fault_addr & FAULT_ADDR_STACK_ID_MASK; -+ fault_addr >>= FAULT_ADDR_STACK_ID_LEN; -+ info_struct->sid = fault_addr & FAULT_ADDR_SID_MASK; -+ fault_addr >>= FAULT_ADDR_SID_LEN; -+ info_struct->channel_id = fault_addr & FAULT_ADDR_CHANNEL_ID_MASK; -+ fault_addr >>= FAULT_ADDR_CHANNEL_ID_LEN; -+ info_struct->bankgroup_id = fault_addr & FAULT_ADDR_BANKGROUP_ID_MASK; -+ fault_addr >>= FAULT_ADDR_BANKGROUP_ID_LEN; -+ info_struct->bank_id = fault_addr & FAULT_ADDR_BANK_ID_MASK; -+ fault_addr >>= FAULT_ADDR_BANK_ID_LEN; -+ info_struct->row_id = fault_addr & FAULT_ADDR_ROW_ID_MASK; -+ fault_addr >>= FAULT_ADDR_ROW_ID_LEN; -+ info_struct->column_id = fault_addr & FAULT_ADDR_COLUMN_ID_MASK; -+ fault_addr >>= FAULT_ADDR_CHANNEL_ID_LEN; -+ info_struct->error_type = fault_addr & FAULT_ADDR_ERROR_TYPE_MASK; -+ fault_addr >>= FAULT_ADDR_ERROR_TYPE_LEN; -+ info_struct->repair_type = fault_addr & FAULT_ADDR_REPAIR_TYPE_MASK; -+ fault_addr >>= FAULT_ADDR_REPAIR_TYPE_LEN; -+ info_struct->reserved = fault_addr & FAULT_ADDR_RESERVED_MASK; -+ fault_addr >>= FAULT_ADDR_RESERVED_LEN; -+ info_struct->crc8 = (uint32_t)fault_addr; -+} -+ -+static bool variable_existed(char *name, char *guid) -+{ -+ char filename[PATH_MAX]; -+ int fd; -+ -+ snprintf(filename, PATH_MAX - 1, "%s/%s-%s", EFIVARFS_PATH, name, guid); -+ -+ // open var file -+ fd = open(filename, O_RDONLY); -+ if (fd < 0) { -+ log(LOG_WARNING, "open file %s failed\n", filename); -+ return false; -+ } -+ close(fd); -+ return true; -+} -+ -+static uint32_t read_variable_attribute(char *name, char *guid) { -+ char filename[PATH_MAX]; -+ int fd; -+ size_t readsize; -+ uint32_t attribute = (uint32_t)-1; -+ -+ snprintf(filename, PATH_MAX - 1, "%s/%s-%s", EFIVARFS_PATH, name, guid); -+ -+ // open var file -+ fd = open(filename, O_RDONLY); -+ if (fd < 0) { -+ log(LOG_ERROR, "open %s failed\n", filename); -+ return attribute; -+ } -+ -+ // read attributes from first 4 bytes -+ readsize = read(fd, &attribute, sizeof(uint32_t)); -+ if (readsize != sizeof(uint32_t)) { -+ log(LOG_ERROR, "read attribute of %s failed\n", filename); -+ } -+ -+ close(fd); -+ return attribute; -+} -+ -+static int efivarfs_set_mutable(char *name, char *guid, bool mutable) -+{ -+ unsigned long orig_attrs, new_attrs; -+ char filename[PATH_MAX]; -+ int fd; -+ -+ snprintf(filename, PATH_MAX - 1, "%s/%s-%s", EFIVARFS_PATH, name, guid); -+ -+ fd = open(filename, O_RDONLY); -+ if (fd < 0) { -+ log(LOG_ERROR, "open %s failed\n", filename); -+ goto err; -+ } -+ -+ if (ioctl(fd, FS_IOC_GETFLAGS, &orig_attrs) == -1) { -+ log(LOG_ERROR, "ioctl FS_IOC_GETFLAGS failed\n"); -+ goto err; -+ } -+ -+ if (mutable) -+ new_attrs = orig_attrs & ~(unsigned long)FS_IMMUTABLE_FL; -+ else -+ new_attrs = orig_attrs | FS_IMMUTABLE_FL; -+ -+ if (new_attrs == orig_attrs) { -+ close(fd); -+ return 0; -+ } -+ -+ if (ioctl(fd, FS_IOC_SETFLAGS, &new_attrs) == -1) { -+ log(LOG_ERROR, "ioctl FS_IOC_SETFLAGS failed\n"); -+ goto err; -+ } -+ close(fd); -+ return 0; -+err: -+ if (fd >= 0) -+ close(fd); -+ return -1; -+} -+ -+static int write_variable(char *name, char *guid, void *value, unsigned long size, uint32_t attribute) { -+ int fd, mode; -+ size_t writesize; -+ void *buffer; -+ unsigned long total; -+ char filename[PATH_MAX]; -+ -+ snprintf(filename, PATH_MAX - 1, "%s/%s-%s", EFIVARFS_PATH, name, guid); -+ -+ // prepare attributes(size 4 bytes) and data -+ total = size + sizeof(uint32_t); -+ buffer = malloc(total); -+ if (buffer == NULL) { -+ log(LOG_ERROR, "malloc data for %s failed\n", filename); -+ goto err; -+ } -+ memcpy(buffer, &attribute, sizeof(uint32_t)); -+ memcpy(buffer + sizeof(uint32_t), value, size); -+ -+ // change attr -+ if (efivarfs_set_mutable(name, guid, 1) != 0) { -+ log(LOG_ERROR, "set mutable for %s failed\n", filename); -+ goto err; -+ } -+ -+ mode = O_WRONLY; -+ if (attribute & EFI_VARIABLE_APPEND_WRITE) -+ mode |= O_APPEND; -+ else -+ mode |= O_CREAT; -+ -+ // open var file -+ fd = open(filename, mode, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); -+ if (fd < 0) { -+ log(LOG_ERROR, "open %s failed\n", filename); -+ goto err; -+ } -+ -+ // write to var file -+ writesize = write(fd, buffer, total); -+ if (writesize != total) { -+ log(LOG_ERROR, "write %s failed\n", filename); -+ goto err; -+ } -+ -+ close(fd); -+ free(buffer); -+ if (efivarfs_set_mutable(name, guid, 0) != 0) { -+ log(LOG_ERROR, "set immutable for %s failed\n", filename); -+ } -+ return 0; -+err: -+ if (fd >= 0) -+ close(fd); -+ if (buffer) -+ free(buffer); -+ if (efivarfs_set_mutable(name, guid, 0) != 0) { -+ log(LOG_ERROR, "set immutable for %s failed\n", filename); -+ } -+ return -1; -+} -+ -+static int append_variable(char *name, char *guid, void *data, unsigned long size) { -+ // prepare append attribute -+ uint32_t attribute = read_variable_attribute(name, guid); -+ if (attribute == (uint32_t)-1) { -+ log(LOG_ERROR, "read %s-%s attribute failed\n", name, guid); -+ return -1; -+ } -+ attribute |= EFI_VARIABLE_APPEND_WRITE; -+ -+ return write_variable(name, guid, data, size, attribute); -+} -+ -+static size_t get_var_size(char *name, char *guid) { -+ char filename[PATH_MAX]; -+ int fd; -+ struct stat stat; -+ -+ snprintf(filename, PATH_MAX - 1, "%s/%s-%s", EFIVARFS_PATH, name, guid); -+ -+ // open var file -+ fd = open(filename, O_RDONLY); -+ if (fd < 0) { -+ log(LOG_WARNING, "open %s failed\n", filename); -+ goto err; -+ } -+ // read stat -+ if (fstat(fd, &stat) != 0) { -+ log(LOG_WARNING, "fstat %s failed\n", filename); -+ goto err; -+ } -+ close(fd); -+ return stat.st_size; -+err: -+ if (fd >= 0) -+ close(fd); -+ return (size_t)-1; -+} -+ -+int init_all_flash() { -+ for (int i = 0; i < FLASH_ENTRY_NUM; i++) { -+ // check existed entry -+ if (variable_existed(flash_names[i], flash_guids[i])) { -+ total_size += get_var_size(flash_names[i], flash_guids[i]); -+ continue; -+ } -+ // create new entry -+ uint32_t attribute = EFI_VARIABLE_NON_VOLATILE | -+ EFI_VARIABLE_BOOTSERVICE_ACCESS | -+ EFI_VARIABLE_RUNTIME_ACCESS; -+ char *data = ""; -+ unsigned long size = 1; -+ int ret = write_variable(flash_names[i], flash_guids[i], data, size, attribute); -+ if (ret) { -+ log(LOG_ERROR, "init %s-%s failed, fault info storage funtion not enabled\n", flash_names[i], flash_guids[i]); -+ return -1; -+ } -+ total_size += sizeof(uint32_t) + 1; -+ } -+ // check total entry size -+ log(LOG_DEBUG, "current fault info total size: %luKB, flash max threshold: %uKB\n", -+ total_size / KB_SIZE, MAX_VAR_SIZE / KB_SIZE); -+ if (total_size > MAX_VAR_SIZE) { -+ log(LOG_ERROR, "fault info storage reach threshold, cannot save new record\n"); -+ } -+ return 0; -+} -+ -+static int write_fault_info_to_flash(const struct hisi_common_error_section *err) { -+ int ret, guid_index; -+ uint32_t reg_size; -+ uint64_t fault_addr; -+ -+ // check flash usage threshold -+ if (total_size + sizeof(uint64_t) > MAX_VAR_SIZE) { -+ log(LOG_WARNING, "fault info storage reach threshold, cannot save new record into flash\n"); -+ return -1; -+ } -+ -+ // parse physical addr -+ reg_size = err->reg_array_size / sizeof(uint32_t); -+ fault_addr = err->reg_array[reg_size - 1]; -+ fault_addr <<= TYPE_UINT32_WIDTH; -+ fault_addr += err->reg_array[reg_size - 2]; -+ -+ // get guid -+ struct fault_addr_info info_struct; -+ parse_fault_addr_info(&info_struct, fault_addr); -+ guid_index = get_guid_index(info_struct.processer_id, info_struct.error_type); -+ if (guid_index < 0) { -+ log(LOG_ERROR, "invalid fault info\n"); -+ return -1; -+ } -+ // record physical addr in flash -+ ret = append_variable(flash_names[guid_index], flash_guids[guid_index], &fault_addr, sizeof(uint64_t)); -+ if (ret < 0) { -+ log(LOG_ERROR, "append to %s-%s failed\n", flash_names[guid_index], flash_guids[guid_index]); -+ return -1; -+ } -+ total_size += sizeof(uint64_t); -+ log(LOG_INFO, "write hbm fault info to flash success\n"); -+ return 0; -+} -+ -+static int write_file(char *path, const char *name, unsigned long long value) -+{ -+ char fname[MAX_PATH]; -+ char buf[20]; -+ int ret; -+ int fd; -+ -+ snprintf(fname, MAX_PATH, "%s/%s", path, name); -+ -+ fd = open(fname, O_WRONLY); -+ if (fd < 0) { -+ log(LOG_WARNING, "HBM ACLS: Cannot to open '%s': %s\n", -+ fname, strerror(errno)); -+ return -errno; -+ } -+ -+ snprintf(buf, sizeof(buf), "0x%llx\n", value); -+ ret = write(fd, buf, strlen(buf)); -+ if (ret <= 0) -+ log(LOG_WARNING, "HBM ACLS: Failed to set %s (0x%llx): %s\n", -+ fname, value, strerror(errno)); -+ -+ close(fd); -+ return ret > 0 ? 0 : -errno; -+} -+ -+static int get_hardware_corrupted_size() -+{ -+ FILE *fp; -+ char line[256]; -+ int hardware_corrupted_size = -1; -+ char *key = "HardwareCorrupted:"; -+ -+ fp = fopen("/proc/meminfo", "r"); -+ if (fp == NULL) { -+ log(LOG_ERROR, "Failed to open /proc/meminfo\n"); -+ return -1; -+ } -+ -+ while (fgets(line, sizeof(line), fp) != NULL) { -+ char *pos; -+ if ((pos = strstr(line, key)) != NULL) { -+ sscanf(pos, "HardwareCorrupted: %5d kB\n", &hardware_corrupted_size); -+ break; -+ } -+ } -+ -+ fclose(fp); -+ return hardware_corrupted_size; -+} -+ -+static uint8_t get_repair_result_code(int ret) -+{ -+ if (ret == -ENOSPC) { -+ return REPAIR_FAILED_NO_RESOURCE; -+ } else if (ret == -EIO) { -+ return REPAIR_FAILED_OTHER_REASON; -+ } else if (ret == -ENXIO || ret == -EINVAL) { -+ return REPAIR_FAILED_INVALID_PARAM; -+ } -+ return REPAIR_FAILED_OTHER_REASON; -+} -+ -+static int notice_BMC(const struct hisi_common_error_section *err, uint8_t repair_result_code) -+{ -+ int sockfd; -+ struct sockaddr_un addr; -+ char bmc_msg[sizeof(BMC_REPORT_FORMAT)] = {0}; -+ uint8_t repair_type_code, isolation_type_code; -+ uint32_t repair_type; -+ unsigned long long fault_addr; -+ -+ sockfd = socket(AF_UNIX, SOCK_STREAM, 0); -+ if (sockfd < 0) { -+ log(LOG_ERROR, "Failed to create BMC notice socket\n"); -+ return -1; -+ } -+ -+ memset(&addr, 0, sizeof(struct sockaddr_un)); -+ addr.sun_family = AF_UNIX; -+ strncpy(addr.sun_path, BMC_SOCKET_PATH, sizeof(addr.sun_path) - 1); -+ if (connect(sockfd, (struct sockaddr *)&addr, sizeof(struct sockaddr_un)) < 0) { -+ log(LOG_ERROR, "Failed to connect BMC notice socket\n"); -+ close(sockfd); -+ return -1; -+ } -+ -+ /* assemble bmc specific msg */ -+ repair_type_code = 0; -+ isolation_type_code = 0; -+ repair_type = err->reg_array[HBM_REPAIR_REQ_TYPE]; -+ if (repair_type & HBM_CE_ACLS) { -+ repair_type_code = 0; -+ isolation_type_code = SINGLE_ADDR_FAULT; -+ } else if (repair_type & HBM_PSUE_ACLS) { -+ repair_type_code = 1; -+ isolation_type_code = SINGLE_ADDR_FAULT; -+ } else if (repair_type & HBM_CE_SPPR) { -+ repair_type_code = 2; -+ isolation_type_code = ROW_FAULT; -+ } else if (repair_type & HBM_PSUE_SPPR) { -+ repair_type_code = 3; -+ isolation_type_code = ROW_FAULT; -+ } -+ -+ const uint32_t reg_size = err->reg_array_size / sizeof(uint32_t); -+ -+ fault_addr = err->reg_array[reg_size - 1]; -+ fault_addr <<= TYPE_UINT32_WIDTH; -+ fault_addr += err->reg_array[reg_size - 2]; -+ -+ log(LOG_DEBUG, "Get the fault addr is %llu\n", fault_addr); -+ -+ struct fault_addr_info info_struct; -+ parse_fault_addr_info(&info_struct, fault_addr); -+ -+ log(LOG_DEBUG, "info_struct.processer_id is %u\n", info_struct.processer_id); -+ log(LOG_DEBUG, "info_struct.die_id is %u\n", info_struct.die_id); -+ log(LOG_DEBUG, "info_struct.stack_id is %u\n", info_struct.stack_id); -+ log(LOG_DEBUG, "info_struct.sid is %u\n", info_struct.sid); -+ log(LOG_DEBUG, "info_struct.channel_id is %u\n", info_struct.channel_id); -+ log(LOG_DEBUG, "info_struct.bankgroup_id is %u\n", info_struct.bankgroup_id); -+ log(LOG_DEBUG, "info_struct.bank_id is %u\n", info_struct.bank_id); -+ log(LOG_DEBUG, "info_struct.row_id is %u\n", info_struct.row_id); -+ log(LOG_DEBUG, "info_struct.column_id is %u\n", info_struct.column_id); -+ log(LOG_DEBUG, "info_struct.error_type is %u\n", info_struct.error_type); -+ log(LOG_DEBUG, "info_struct.repair_type is %u\n", info_struct.repair_type); -+ log(LOG_DEBUG, "info_struct.reserved is %u\n", info_struct.reserved); -+ log(LOG_DEBUG, "info_struct.crc8 is %u\n", info_struct.crc8); -+ -+ snprintf(bmc_msg, sizeof(BMC_REPORT_FORMAT), BMC_REPORT_FORMAT, -+ repair_type_code, -+ repair_result_code, -+ isolation_type_code, -+ info_struct.processer_id, -+ info_struct.die_id, -+ info_struct.stack_id, -+ info_struct.sid, -+ info_struct.channel_id, -+ info_struct.bankgroup_id, -+ info_struct.bank_id, -+ info_struct.row_id, -+ info_struct.column_id -+ ); -+ -+ log(LOG_DEBUG, "Send msg to sysSentry, bmc msg is %s\n", bmc_msg); -+ -+ if (write(sockfd, bmc_msg, strlen(bmc_msg)) <= 0) { -+ log(LOG_ERROR, "Failed to send data to BMC notice socket\n"); -+ close(sockfd); -+ return -1; -+ } -+ -+ close(sockfd); -+ return 0; -+} -+ -+static int hbmc_hbm_page_isolate(const struct hisi_common_error_section *err) -+{ -+ unsigned long long paddr; -+ int ret; -+ bool is_acls = err->reg_array[HBM_REPAIR_REQ_TYPE] & (HBM_CE_ACLS | HBM_PSUE_ACLS); -+ int required_isolate_size = (is_acls ? HBM_ACLS_ADDR_NUM : HBM_SPPR_ADDR_NUM) * DEFAULT_PAGE_SIZE_KB; -+ int hardware_corrupted_size = get_hardware_corrupted_size(); -+ if (hardware_corrupted_size < 0) { -+ log(LOG_ERROR, "Page isolate failed: Get hardware_corrupted_size failed"); -+ notice_BMC(err, ISOLATE_FAILED_OTHER_REASON); -+ return -1; -+ } -+ if ((required_isolate_size + hardware_corrupted_size) > page_isolation_threshold) { -+ log(LOG_INFO, "Page isolate failed: the isolation resource is not enough\n"); -+ notice_BMC(err, ISOLATE_FAILED_OVER_THRESHOLD); -+ return -1; -+ } -+ if (is_acls) { -+ /* ACLS */ -+ paddr = err->reg_array[HBM_ADDH]; -+ paddr <<= TYPE_UINT32_WIDTH; -+ paddr += err->reg_array[HBM_ADDL]; -+ -+ ret = write_file("/sys/kernel/page_eject", "offline_page", paddr); -+ if (ret < 0) { -+ notice_BMC(err, ISOLATE_FAILED_OTHER_REASON); -+ log(LOG_WARNING, "HBM: ACLS offline failed, address is 0x%llx \n", paddr); -+ return ret; -+ } -+ } else { -+ /* SPPR */ -+ bool all_success = true; -+ uint32_t i; -+ for (i = 0; i < HBM_SPPR_ADDR_NUM; i++) { -+ paddr = err->reg_array[2 * i + HBM_ADDH]; -+ paddr <<= TYPE_UINT32_WIDTH; -+ paddr += err->reg_array[2 * i + HBM_ADDL]; -+ ret = write_file("/sys/kernel/page_eject", "offline_page", paddr); -+ if (ret < 0) { -+ all_success = false; -+ log(LOG_WARNING, "HBM: SPPR offline failed, address is 0x%llx \n", paddr); -+ continue; -+ } -+ } -+ if (!all_success) { -+ notice_BMC(err, ISOLATE_FAILED_OTHER_REASON); -+ ret = -1; -+ } -+ } -+ return ret < 0 ? ret : 0; -+} -+ -+static int hbmc_hbm_after_repair(bool is_acls, const int repair_ret, const unsigned long long paddr) -+{ -+ int ret; -+ if (repair_ret < 0) { -+ log(LOG_WARNING, "HBM %s: Keep page (0x%llx) offline\n", is_acls ? "ACLS" : "SPPR", paddr); -+ /* not much we can do about errors here */ -+ (void)write_file("/sys/kernel/page_eject", "remove_page", paddr); -+ return get_repair_result_code(repair_ret); -+ } -+ -+ ret = write_file("/sys/kernel/page_eject", "online_page", paddr); -+ if (ret < 0) { -+ log(LOG_WARNING, "HBM %s: Page (0x%llx) online failed\n",is_acls ? "ACLS" : "SPPR", paddr); -+ return ONLINE_PAGE_FAILED; -+ } else { -+ log(LOG_INFO, "HBM %s: Page (0x%llx) repair and online success\n",is_acls ? "ACLS" : "SPPR", paddr); -+ return ISOLATE_REPAIR_ONLINE_SUCCESS; -+ } -+} -+ -+static uint8_t hbmc_hbm_repair(const struct hisi_common_error_section *err, char *path) -+{ -+ unsigned long long paddr; -+ int ret; -+ uint8_t repair_result_code; -+ bool is_acls; -+ -+ /* Both ACLS and SPPR only repair the first address */ -+ paddr = err->reg_array[HBM_ADDH]; -+ paddr <<= TYPE_UINT32_WIDTH; -+ paddr += err->reg_array[HBM_ADDL]; -+ -+ is_acls = err->reg_array[HBM_REPAIR_REQ_TYPE] & HBM_CE_ACLS || -+ err->reg_array[HBM_REPAIR_REQ_TYPE] & HBM_PSUE_ACLS; -+ -+ ret = write_file(path, is_acls ? "acls_query" : "sppr_query", paddr); -+ if (ret < 0) { -+ notice_BMC(err, get_repair_result_code(ret)); -+ log(LOG_WARNING, "HBM: Address 0x%llx is not supported to %s repair\n", paddr, is_acls ? "ACLS" : "SPPR"); -+ return ret; -+ } -+ -+ ret = write_file(path, is_acls ? "acls_repair" : "sppr_repair", paddr); -+ -+ if (is_acls) { -+ /* ACLS */ -+ repair_result_code = hbmc_hbm_after_repair(is_acls, ret, paddr); -+ notice_BMC(err, repair_result_code); -+ return ret; -+ } else { -+ /* SPPR */ -+ bool all_online_success = true; -+ uint32_t i; -+ for (i = 0; i < HBM_SPPR_ADDR_NUM; i++) { -+ paddr = err->reg_array[2 * i + HBM_ADDH]; -+ paddr <<= TYPE_UINT32_WIDTH; -+ paddr += err->reg_array[2 * i + HBM_ADDL]; -+ -+ repair_result_code = hbmc_hbm_after_repair(is_acls, ret, paddr); -+ if (repair_result_code != ISOLATE_REPAIR_ONLINE_SUCCESS) { -+ all_online_success = false; -+ } -+ } -+ if (ret < 0) { -+ notice_BMC(err, get_repair_result_code(ret)); -+ return ret; -+ } else if (all_online_success) { -+ notice_BMC(err, ISOLATE_REPAIR_ONLINE_SUCCESS); -+ return 0; -+ } else { -+ notice_BMC(err, ONLINE_PAGE_FAILED); -+ return ret; -+ } -+ } -+ /* The final return code is not necessary */ -+ return ret < 0 ? ret : 0; -+} -+ -+static int hbmc_get_memory_type(char *path) -+{ -+ int type = HBM_UNKNOWN; -+ char fname[MAX_PATH]; -+ char buf[128]; -+ FILE *file; -+ -+ snprintf(fname, MAX_PATH, "%s/%s", path, "memory_type"); -+ file = fopen(fname, "r"); -+ if (!file) { -+ log(LOG_WARNING, "HBM: Cannot to open '%s': %s\n", -+ fname, strerror(errno)); -+ return -errno; -+ } -+ -+ if (!fgets(buf, sizeof(buf), file)) { -+ log(LOG_WARNING, "HBM: Failed to read %s\n", fname); -+ goto err; -+ } -+ -+ /* Remove the last '\n' */ -+ buf[strlen(buf) - 1] = 0; -+ -+ if (strcmp(buf, "HBM") == 0) -+ type = HBM_HBM_MEMORY; -+ else if (strcmp(buf, "DDR") == 0) -+ type = HBM_DDR_MEMORY; -+ -+err: -+ fclose(file); -+ return type; -+} -+ -+static void hbm_repair_handler(const struct hisi_common_error_section *err) -+{ -+ log(LOG_DEBUG, "Received ACLS/SPPR flat mode repair request, try to repair\n"); -+ char *sys_dev_path = "/sys/devices/platform"; -+ char path[MAX_PATH]; -+ struct dirent *dent; -+ DIR *dir; -+ int ret; -+ bool find_device = false, find_hbm_mem = false; -+ -+ ret = hbmc_hbm_page_isolate(err); -+ if (ret < 0) { -+ return; -+ } -+ -+ dir = opendir(sys_dev_path); -+ if (!dir) { -+ log(LOG_WARNING, "Can't read '%s': %s\n", -+ sys_dev_path, strerror(errno)); -+ notice_BMC(err, REPAIR_FAILED_OTHER_REASON); -+ return; -+ } -+ -+ while ((dent = readdir(dir))) { -+ if (!strstr(dent->d_name, HBM_MEM_RAS_NAME)) -+ continue; -+ find_device = true; -+ -+ snprintf(path, MAX_PATH, "%s/%s", sys_dev_path, dent->d_name); -+ -+ if (hbmc_get_memory_type(path) == HBM_HBM_MEMORY) { -+ find_hbm_mem = true; -+ ret = hbmc_hbm_repair(err, path); -+ if (ret != -ENXIO) -+ break; -+ } -+ } -+ if (!find_device) { -+ log(LOG_ERROR, "Repair driver is not loaded, skip error, error_type is %u\n", -+ err->reg_array[HBM_REPAIR_REQ_TYPE] & HBM_ERROR_MASK); -+ notice_BMC(err, REPAIR_FAILED_OTHER_REASON); -+ } else if (!find_hbm_mem) { -+ log(LOG_ERROR, "No HBM device memory type found, skip error, error_type is %u\n", -+ err->reg_array[HBM_REPAIR_REQ_TYPE] & HBM_ERROR_MASK); -+ notice_BMC(err, REPAIR_FAILED_OTHER_REASON); -+ } -+ -+ closedir(dir); -+} -+ -+static bool hbm_repair_validate(const struct hisi_common_error_section *err) -+{ -+ if (!((err->val_bits & BIT(COMMON_VALID_MODULE_ID)) && -+ (err->val_bits & BIT(COMMON_VALID_SUBMODULE_ID)) && -+ (err->val_bits & BIT(COMMON_VALID_REG_ARRAY_SIZE)) -+ )) { -+ log(LOG_DEBUG, "Err val_bits validate failed, val_bits is %u\n", err->val_bits); -+ return false; -+ } -+ log(LOG_DEBUG, "err->module_id: %u\n", err->module_id); -+ log(LOG_DEBUG, "err->submodule_id: %u\n", err->submodule_id); -+ log(LOG_DEBUG, "err->val_bits: 0x%x\n", err->val_bits); -+ log(LOG_DEBUG, "err->reg_array_size: %u\n", err->reg_array_size); -+ -+ if (err->module_id != HBMC_MODULE_ID || -+ err->submodule_id != HBMC_SUBMOD_HBM_REPAIR) { -+ log(LOG_DEBUG, "err module_id or sub_module id doesn't not match\n"); -+ return false; -+ } -+ -+ uint32_t hbm_repair_reg_type = err->reg_array[HBM_REPAIR_REQ_TYPE] & HBM_ERROR_MASK; -+ bool is_acls_valid = (hbm_repair_reg_type & (HBM_CE_ACLS | HBM_PSUE_ACLS)) && -+ (err->reg_array_size == HBM_ACLS_ARRAY_SIZE); -+ bool is_sppr_valid = (hbm_repair_reg_type & (HBM_CE_SPPR | HBM_PSUE_SPPR)) && -+ (err->reg_array_size == HBM_SPPR_ARRAY_SIZE); -+ bool is_cache_mode = (hbm_repair_reg_type & HBM_CACHE_MODE) && -+ (err->reg_array_size == HBM_CACHE_ARRAY_SIZE); -+ -+ if (!(is_acls_valid || is_sppr_valid || is_cache_mode)) { -+ log(LOG_DEBUG, "err type (%u) is unknown or address array length (%u) is invalid\n", -+ hbm_repair_reg_type, err->reg_array_size); -+ return false; -+ } -+ -+ log(LOG_INFO, "Received ACLS/SPPR repair request\n"); -+ return true; -+} -+ -+static bool hbm_flat_mode_validate(const struct hisi_common_error_section *err) -+{ -+ uint32_t hbm_repair_reg_type = err->reg_array[HBM_REPAIR_REQ_TYPE] & HBM_ERROR_MASK; -+ return !(hbm_repair_reg_type & HBM_CACHE_MODE); -+} -+ -+int decode_hisi_common_section(struct ras_non_standard_event *event) -+{ -+ const struct hisi_common_error_section *err = (struct hisi_common_error_section *)event->error; -+ -+ if (hbm_repair_validate(err)) { -+ write_fault_info_to_flash(err); -+ if (hbm_flat_mode_validate(err)) { -+ hbm_repair_handler(err); -+ } -+ } -+ -+ return 0; -+} -diff --git a/src/c/hbm_online_repair/non-standard-hbm-repair.h b/src/c/hbm_online_repair/non-standard-hbm-repair.h -new file mode 100644 -index 0000000..7e8e448 ---- /dev/null -+++ b/src/c/hbm_online_repair/non-standard-hbm-repair.h -@@ -0,0 +1,89 @@ -+#ifndef __NON_STANDARD_HBM_REPAIR -+#define __NON_STANDARD_HBM_REPAIR -+ -+#include "ras-non-standard-handler.h" -+ -+#define DEFAULT_PAGE_SIZE_KB 4 -+#define HBM_MEM_RAS_NAME "HISI0521" -+#define HBM_UNKNOWN 0 -+#define HBM_HBM_MEMORY 1 -+#define HBM_DDR_MEMORY 2 -+ -+#define TYPE_UINT32_WIDTH 32 -+#define HBM_REPAIR_REQ_TYPE 0 -+#define HBM_CE_ACLS BIT(0) -+#define HBM_PSUE_ACLS BIT(1) -+#define HBM_CE_SPPR BIT(2) -+#define HBM_PSUE_SPPR BIT(3) -+#define HBM_CACHE_MODE (BIT(4) | BIT(5) | BIT(6) | BIT(7)) -+#define HBM_ERROR_MASK 0b11111111 -+#define HBM_ADDL 1 -+#define HBM_ADDH 2 -+#define HBM_ERROR_TYPE_SIZE 4 -+#define HBM_ADDR_SIZE 8 -+#define HBM_ACLS_ADDR_NUM 1 -+#define HBM_SPPR_ADDR_NUM 16 -+#define HBM_ACLS_ARRAY_SIZE (HBM_ERROR_TYPE_SIZE + HBM_ADDR_SIZE * HBM_ACLS_ADDR_NUM + HBM_ADDR_SIZE) -+#define HBM_SPPR_ARRAY_SIZE (HBM_ERROR_TYPE_SIZE + HBM_ADDR_SIZE * HBM_SPPR_ADDR_NUM + HBM_ADDR_SIZE) -+#define HBM_CACHE_ARRAY_SIZE (HBM_ERROR_TYPE_SIZE + HBM_ADDR_SIZE) -+#define HBMC_MODULE_ID 0x28 -+#define HBMC_SUBMOD_HBM_REPAIR 6 -+#define COMMON_VALID_MODULE_ID 5 -+#define COMMON_VALID_SUBMODULE_ID 6 -+#define COMMON_VALID_REG_ARRAY_SIZE 12 -+ -+#define BMC_SOCKET_PATH "/var/run/sysSentry/bmc.sock" -+#define BMC_REPORT_FORMAT "REP00%02x%02x%02x0000000000000000%02x%02x%02x00%02x00%02x%02x%02x%08x%08x0000000000" -+ -+#define ISOLATE_FAILED_OVER_THRESHOLD 0b10000001 -+#define ISOLATE_FAILED_OTHER_REASON 0b10000010 -+#define REPAIR_FAILED_NO_RESOURCE 0b10010100 -+#define REPAIR_FAILED_INVALID_PARAM 0b10011000 -+#define REPAIR_FAILED_OTHER_REASON 0b10011100 -+#define ONLINE_PAGE_FAILED 0b10100000 -+#define ISOLATE_REPAIR_ONLINE_SUCCESS 0b00000000 -+ -+#define ROW_FAULT 1 -+#define SINGLE_ADDR_FAULT 6 -+ -+#define FAULT_ADDR_PROCESSOR_ID_LEN 2 -+#define FAULT_ADDR_DIE_ID_LEN 1 -+#define FAULT_ADDR_STACK_ID_LEN 3 -+#define FAULT_ADDR_SID_LEN 3 -+#define FAULT_ADDR_CHANNEL_ID_LEN 8 -+#define FAULT_ADDR_BANKGROUP_ID_LEN 3 -+#define FAULT_ADDR_BANK_ID_LEN 3 -+#define FAULT_ADDR_ROW_ID_LEN 17 -+#define FAULT_ADDR_COLUMN_ID_LEN 10 -+#define FAULT_ADDR_ERROR_TYPE_LEN 2 -+#define FAULT_ADDR_REPAIR_TYPE_LEN 2 -+#define FAULT_ADDR_RESERVED_LEN 2 -+#define FAULT_ADDR_CRC8_LEN 8 -+ -+#define FAULT_ADDR_PROCESSOR_ID_MASK ((1 << FAULT_ADDR_PROCESSOR_ID_LEN ) - 1) -+#define FAULT_ADDR_DIE_ID_MASK ((1 << FAULT_ADDR_DIE_ID_LEN ) - 1) -+#define FAULT_ADDR_STACK_ID_MASK ((1 << FAULT_ADDR_STACK_ID_LEN ) - 1) -+#define FAULT_ADDR_SID_MASK ((1 << FAULT_ADDR_SID_LEN ) - 1) -+#define FAULT_ADDR_CHANNEL_ID_MASK ((1 << FAULT_ADDR_CHANNEL_ID_LEN ) - 1) -+#define FAULT_ADDR_BANKGROUP_ID_MASK ((1 << FAULT_ADDR_BANKGROUP_ID_LEN ) - 1) -+#define FAULT_ADDR_BANK_ID_MASK ((1 << FAULT_ADDR_BANK_ID_LEN ) - 1) -+#define FAULT_ADDR_ROW_ID_MASK ((1 << FAULT_ADDR_ROW_ID_LEN ) - 1) -+#define FAULT_ADDR_COLUMN_ID_MASK ((1 << FAULT_ADDR_COLUMN_ID_LEN ) - 1) -+#define FAULT_ADDR_ERROR_TYPE_MASK ((1 << FAULT_ADDR_ERROR_TYPE_LEN ) - 1) -+#define FAULT_ADDR_REPAIR_TYPE_MASK ((1 << FAULT_ADDR_REPAIR_TYPE_LEN ) - 1) -+#define FAULT_ADDR_RESERVED_MASK ((1 << FAULT_ADDR_RESERVED_LEN ) - 1) -+#define FAULT_ADDR_CRC8_MASK ((1 << FAULT_ADDR_CRC8_LEN ) - 1) -+ -+#define EFI_VARIABLE_NON_VOLATILE 0x1 -+#define EFI_VARIABLE_BOOTSERVICE_ACCESS 0x2 -+#define EFI_VARIABLE_RUNTIME_ACCESS 0x4 -+#define EFI_VARIABLE_APPEND_WRITE 0x40 -+ -+#define EFIVARFS_PATH "/sys/firmware/efi/efivars" -+#define MAX_VAR_SIZE (128 * 1024) -+#define FLASH_ENTRY_NUM 8 -+#define KB_SIZE 1024 -+ -+extern int init_all_flash(); -+ -+#endif -diff --git a/src/c/hbm_online_repair/ras-events.c b/src/c/hbm_online_repair/ras-events.c -new file mode 100644 -index 0000000..0b12329 ---- /dev/null -+++ b/src/c/hbm_online_repair/ras-events.c -@@ -0,0 +1,534 @@ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+#include -+#include "ras-non-standard-handler.h" -+#include "logger.h" -+ -+/* -+ * Polling time, if read() doesn't block. Currently, trace_pipe_raw never -+ * blocks on read(). So, we need to sleep for a while, to avoid spending -+ * too much CPU cycles. A fix for it is expected for 3.10. -+ */ -+#define POLLING_TIME 3 -+ -+/* Test for a little-endian machine */ -+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -+ #define ENDIAN KBUFFER_ENDIAN_LITTLE -+#else -+ #define ENDIAN KBUFFER_ENDIAN_BIG -+#endif -+ -+static int get_debugfs_dir(char *debugfs_dir, size_t len) -+{ -+ FILE *fp; -+ char line[MAX_PATH + 1 + 256]; -+ -+ fp = fopen("/proc/mounts","r"); -+ if (!fp) { -+ log(LOG_INFO, "Can't open /proc/mounts"); -+ return errno; -+ } -+ -+ do { -+ char *p, *type, *dir; -+ if (!fgets(line, sizeof(line), fp)) -+ break; -+ -+ p = strtok(line, " \t"); -+ if (!p) -+ break; -+ -+ dir = strtok(NULL, " \t"); -+ if (!dir) -+ break; -+ -+ type = strtok(NULL, " \t"); -+ if (!type) -+ break; -+ -+ if (!strcmp(type, "debugfs")) { -+ fclose(fp); -+ strncpy(debugfs_dir, dir, len - 1); -+ debugfs_dir[len - 1] = '\0'; -+ return 0; -+ } -+ } while(1); -+ -+ fclose(fp); -+ log(LOG_INFO, "Can't find debugfs\n"); -+ return ENOENT; -+} -+ -+ -+static int open_trace(char *trace_dir, char *name, int flags) -+{ -+ int ret; -+ char fname[MAX_PATH + 1]; -+ -+ strcpy(fname, trace_dir); -+ strcat(fname, "/"); -+ strcat(fname, name); -+ -+ ret = open(fname, flags); -+ if (ret < 0) -+ log(LOG_WARNING, "open_trace() failed, fname=%s ret=%d errno=%d\n", fname, ret, errno); -+ -+ return ret; -+} -+ -+static int create_trace_instance(char *trace_instance_dir) -+{ -+ char fname[MAX_PATH + 1]; -+ int rc; -+ -+ get_debugfs_dir(fname, sizeof(fname)); -+ strcat(fname, "/tracing/instances/"TOOL_NAME); -+ rc = mkdir(fname, S_IRWXU); -+ if (rc < 0 && errno != EEXIST) { -+ log(LOG_INFO, "Unable to create " TOOL_NAME " instance at %s\n", fname); -+ return -1; -+ } -+ strcpy(trace_instance_dir, fname); -+ return 0; -+} -+ -+struct ras_events *init_trace_instance(void) -+{ -+ struct ras_events *ras = calloc(1, sizeof(*ras)); -+ if (!ras) { -+ log(LOG_ERROR, "Can't allocate memory for ras struct\n"); -+ return NULL; -+ } -+ int rc = create_trace_instance(ras->tracing); -+ if (rc < 0) { -+ free(ras); -+ return NULL; -+ } -+ return ras; -+} -+ -+/* -+ * Tracing enable/disable code -+ */ -+int toggle_ras_event(char *trace_dir, char *group, char *event, int enable) -+{ -+ int fd, rc; -+ char fname[MAX_PATH + 1]; -+ -+ snprintf(fname, sizeof(fname), "%s%s:%s\n", -+ enable ? "" : "!", -+ group, event); -+ -+ /* Enable RAS events */ -+ fd = open_trace(trace_dir, "set_event", O_RDWR | O_APPEND); -+ if (fd < 0) { -+ log(LOG_WARNING, "Can't open set_event\n"); -+ rc = -errno; -+ goto err; -+ } -+ -+ rc = write(fd, fname, strlen(fname)); -+ close(fd); -+ if (rc <= 0) { -+ log(LOG_WARNING, "Can't write to set_event\n"); -+ rc = -EIO; -+ goto err; -+ } -+ -+ log(LOG_INFO, "%s:%s event %s\n", -+ group, event, -+ enable ? "enabled" : "disabled"); -+ return 0; -+err: -+ log(LOG_ERROR, "Can't %s %s:%s tracing\n", -+ enable ? "enable" : "disable", group, event); -+ return rc; -+} -+ -+static int parse_header_page(struct ras_events *ras, struct tep_handle *pevent) -+{ -+ int fd, len, page_size = DEFAULT_PAGE_SIZE; -+ char buf[page_size]; -+ -+ fd = open_trace(ras->tracing, "events/header_page", O_RDONLY); -+ if (fd < 0) { -+ log(LOG_WARNING, "Open event header page failed\n"); -+ return -1; -+ } -+ -+ len = read(fd, buf, page_size); -+ close(fd); -+ if (len <= 0) { -+ log(LOG_WARNING, "Read event header page failed\n"); -+ return -1; -+ } -+ -+ if (tep_parse_header_page(pevent, buf, len, sizeof(long))) { -+ log(LOG_WARNING, "Parse event header page failed\n"); -+ return -1; -+ } -+ -+ return 0; -+} -+ -+static void parse_ras_data(struct pcpu_data *pdata, struct kbuffer *kbuf, -+ void *data, unsigned long long time_stamp) -+{ -+ struct tep_record record; -+ struct trace_seq s; -+ -+ record.ts = time_stamp; -+ record.size = kbuffer_event_size(kbuf); -+ record.data = data; -+ record.offset = kbuffer_curr_offset(kbuf); -+ record.cpu = pdata->cpu; -+ -+ /* note offset is just offset in subbuffer */ -+ record.missed_events = kbuffer_missed_events(kbuf); -+ record.record_size = kbuffer_curr_size(kbuf); -+ -+ trace_seq_init(&s); -+ tep_print_event(pdata->ras->pevent, &s, &record, "%s-%s-%d-%s", -+ TEP_PRINT_NAME, TEP_PRINT_COMM, TEP_PRINT_TIME, TEP_PRINT_INFO); -+ trace_seq_do_printf(&s); -+ fflush(stdout); -+ trace_seq_destroy(&s); -+} -+ -+static int get_num_cpus() -+{ -+ return sysconf(_SC_NPROCESSORS_ONLN); -+} -+ -+static int set_buffer_percent(struct ras_events *ras, int percent) -+{ -+ int res = 0; -+ int fd; -+ -+ fd = open_trace(ras->tracing, "buffer_percent", O_WRONLY); -+ if (fd >= 0) { -+ char buf[16]; -+ ssize_t size; -+ snprintf(buf, sizeof(buf), "%d", percent); -+ size = write(fd, buf, strlen(buf)); -+ if (size <= 0) { -+ log(LOG_WARNING, "can't write to buffer_percent\n"); -+ res = -1; -+ } -+ close(fd); -+ } else { -+ log(LOG_WARNING, "Can't open buffer_percent\n"); -+ res = -1; -+ } -+ -+ return res; -+} -+ -+static int read_ras_event_all_cpus(struct pcpu_data *pdata, -+ unsigned n_cpus) -+{ -+ ssize_t size; -+ unsigned long long time_stamp; -+ void *data; -+ int ready, i, count_nready; -+ struct kbuffer *kbuf; -+ void *page; -+ struct pollfd fds[n_cpus + 1]; -+ struct signalfd_siginfo fdsiginfo; -+ sigset_t mask; -+ int warnonce[n_cpus]; -+ char pipe_raw[PATH_MAX]; -+ -+ memset(&warnonce, 0, sizeof(warnonce)); -+ -+ page = malloc(pdata[0].ras->page_size); -+ if (!page) { -+ log(LOG_ERROR, "Can't allocate page\n"); -+ return -ENOMEM; -+ } -+ -+ kbuf = kbuffer_alloc(KBUFFER_LSIZE_8, ENDIAN); -+ if (!kbuf) { -+ log(LOG_ERROR, "Can't allocate kbuf\n"); -+ free(page); -+ return -ENOMEM; -+ } -+ -+ /* Fix for poll() on the per_cpu trace_pipe and trace_pipe_raw blocks -+ * indefinitely with the default buffer_percent in the kernel trace system, -+ * which is introduced by the following change in the kernel. -+ * https://lore.kernel.org/all/20221020231427.41be3f26@gandalf.local.home/T/#u. -+ * Set buffer_percent to 0 so that poll() will return immediately -+ * when the trace data is available in the ras per_cpu trace pipe_raw -+ */ -+ if (set_buffer_percent(pdata[0].ras, 0)) -+ log(LOG_WARNING, "Set buffer_percent failed\n"); -+ -+ for (i = 0; i < (n_cpus + 1); i++) -+ fds[i].fd = -1; -+ -+ for (i = 0; i < n_cpus; i++) { -+ fds[i].events = POLLIN; -+ -+ snprintf(pipe_raw, sizeof(pipe_raw), -+ "per_cpu/cpu%d/trace_pipe_raw", i); -+ -+ fds[i].fd = open_trace(pdata[0].ras->tracing, pipe_raw, O_RDONLY); -+ if (fds[i].fd < 0) { -+ log(LOG_ERROR, "Can't open trace_pipe_raw\n"); -+ goto error; -+ } -+ } -+ -+ sigemptyset(&mask); -+ sigaddset(&mask, SIGINT); -+ sigaddset(&mask, SIGTERM); -+ sigaddset(&mask, SIGHUP); -+ sigaddset(&mask, SIGQUIT); -+ if (sigprocmask(SIG_BLOCK, &mask, NULL) == -1) -+ log(LOG_WARNING, "sigprocmask\n"); -+ fds[n_cpus].events = POLLIN; -+ fds[n_cpus].fd = signalfd(-1, &mask, 0); -+ if (fds[n_cpus].fd < 0) { -+ log(LOG_WARNING, "signalfd\n"); -+ goto error; -+ } -+ -+ log(LOG_INFO, "Listening to events for cpus 0 to %u\n", n_cpus - 1); -+ -+ do { -+ ready = poll(fds, (n_cpus + 1), -1); -+ if (ready < 0) { -+ log(LOG_WARNING, "poll\n"); -+ } -+ -+ /* check for the signal */ -+ if (fds[n_cpus].revents & POLLIN) { -+ size = read(fds[n_cpus].fd, &fdsiginfo, -+ sizeof(struct signalfd_siginfo)); -+ if (size != sizeof(struct signalfd_siginfo)) { -+ log(LOG_WARNING, "signalfd read\n"); -+ continue; -+ } -+ -+ if (fdsiginfo.ssi_signo == SIGINT || -+ fdsiginfo.ssi_signo == SIGTERM || -+ fdsiginfo.ssi_signo == SIGHUP || -+ fdsiginfo.ssi_signo == SIGQUIT) { -+ log(LOG_INFO, "Recevied signal=%d\n", -+ fdsiginfo.ssi_signo); -+ goto error; -+ } else { -+ log(LOG_INFO, -+ "Received unexpected signal=%d\n", -+ fdsiginfo.ssi_signo); -+ continue; -+ } -+ } -+ -+ count_nready = 0; -+ for (i = 0; i < n_cpus; i++) { -+ if (fds[i].revents & POLLERR) { -+ if (!warnonce[i]) { -+ log(LOG_INFO, -+ "Error on CPU %i\n", i); -+ warnonce[i]++; -+ } -+ continue; -+ } -+ if (!(fds[i].revents & POLLIN)) { -+ count_nready++; -+ continue; -+ } -+ size = read(fds[i].fd, page, pdata[i].ras->page_size); -+ if (size < 0) { -+ log(LOG_WARNING, "read\n"); -+ goto error; -+ } else if (size > 0) { -+ log(LOG_DEBUG, "cpu %d receive %ld bytes data\n", i, size); -+ kbuffer_load_subbuffer(kbuf, page); -+ -+ while ((data = kbuffer_read_event(kbuf, &time_stamp))) { -+ if (kbuffer_curr_size(kbuf) < 0) { -+ log(LOG_ERROR, "invalid kbuf data, discard\n"); -+ break; -+ } -+ -+ log(LOG_DEBUG, "parse_ras_data\n"); -+ parse_ras_data(&pdata[i], -+ kbuf, data, time_stamp); -+ -+ /* increment to read next event */ -+ log(LOG_DEBUG, "kbuffer_next_event\n"); -+ kbuffer_next_event(kbuf, NULL); -+ } -+ } else { -+ count_nready++; -+ } -+ } -+ -+ /* -+ * If count_nready == n_cpus, there is no cpu fd in POLLIN state, -+ * so we need to break the cycle -+ */ -+ if (count_nready == n_cpus) { -+ log(LOG_ERROR, "no cpu fd in POLLIN state, stop running\n"); -+ break; -+ } -+ } while (1); -+ -+error: -+ kbuffer_free(kbuf); -+ free(page); -+ sigprocmask(SIG_UNBLOCK, &mask, NULL); -+ -+ for (i = 0; i < (n_cpus + 1); i++) { -+ if (fds[i].fd > 0) -+ close(fds[i].fd); -+ } -+ -+ return -1; -+} -+ -+static int init_header_page(struct ras_events *ras, struct tep_handle *pevent) -+{ -+ int rc; -+ -+ rc = parse_header_page(ras, pevent); -+ if (rc) { -+ log(LOG_ERROR, "cannot read trace header_page: %d\n", rc); -+ return rc; -+ } -+ return 0; -+} -+ -+static int init_event_format(struct ras_events *ras, struct tep_handle *pevent, -+ char *group, char *event) -+{ -+ char *page, fname[MAX_PATH + 1]; -+ int fd, size, rc, page_size = DEFAULT_PAGE_SIZE; -+ -+ // read one page from format -+ snprintf(fname, sizeof(fname), "events/%s/%s/format", group, event); -+ fd = open_trace(ras->tracing, fname, O_RDONLY); -+ if (fd < 0) { -+ log(LOG_ERROR, -+ "Can't get %s:%s traces. Perhaps this feature is not supported on your system.\n", -+ group, event); -+ return errno; -+ } -+ -+ log(LOG_INFO, "page_size: %d\n", page_size); -+ ras->page_size = page_size; -+ page = malloc(page_size); -+ if (!page) { -+ log(LOG_ERROR, "Can't allocate page to read %s:%s format\n", -+ group, event); -+ rc = errno; -+ close(fd); -+ return rc; -+ } -+ -+ size = read(fd, page, page_size); -+ close(fd); -+ if (size < 0) { -+ log(LOG_ERROR, "Can't read format\n"); -+ free(page); -+ return size; -+ } -+ -+ // parse event format -+ rc = tep_parse_event(pevent, page, size, group); -+ if (rc) { -+ log(LOG_ERROR, "Can't parse event %s:%s\n", group, event); -+ free(page); -+ return EINVAL; -+ } -+ return 0; -+} -+ -+static int add_event_handler(struct ras_events *ras, struct tep_handle *pevent, -+ char *group, char *event, -+ tep_event_handler_func func) -+{ -+ int rc; -+ -+ rc = init_event_format(ras, pevent, group, event); -+ if (rc) { -+ log(LOG_ERROR, "init_event_format for %s:%s failed\n", group, event); -+ return rc; -+ } -+ -+ /* Registers the special event handlers */ -+ rc = tep_register_event_handler(pevent, -1, group, event, func, ras); -+ if (rc < 0) { -+ log(LOG_ERROR, "Can't register event handler for %s:%s\n", -+ group, event); -+ return EINVAL; -+ } -+ -+ return 0; -+} -+ -+int handle_ras_events(struct ras_events *ras) -+{ -+ int rc, i; -+ unsigned cpus; -+ struct tep_handle *pevent = NULL; -+ struct pcpu_data *data = NULL; -+ -+ pevent = tep_alloc(); -+ if (!pevent) { -+ log(LOG_ERROR, "Can't allocate pevent\n"); -+ rc = errno; -+ goto err; -+ } -+ ras->pevent = pevent; -+ -+ rc = init_header_page(ras, pevent); -+ if (rc) { -+ log(LOG_ERROR, "init_header_page failed\n"); -+ goto err; -+ } -+ -+ rc = add_event_handler(ras, pevent, "ras", "non_standard_event", -+ ras_non_standard_event_handler); -+ if (rc) { -+ log(LOG_ERROR, "Can't get traces from %s:%s\n", -+ "ras", "non_standard_event"); -+ goto err; -+ } -+ log(LOG_INFO, "add_event_handler done\n"); -+ -+ cpus = get_num_cpus(); -+ data = calloc(sizeof(*data), cpus); -+ if (!data) -+ goto err; -+ -+ for (i = 0; i < cpus; i++) { -+ data[i].ras = ras; -+ data[i].cpu = i; -+ } -+ rc = read_ras_event_all_cpus(data, cpus); -+ -+err: -+ if (data) -+ free(data); -+ if (pevent) -+ tep_free(pevent); -+ return rc; -+} -diff --git a/src/c/hbm_online_repair/ras-events.h b/src/c/hbm_online_repair/ras-events.h -new file mode 100644 -index 0000000..4218d93 ---- /dev/null -+++ b/src/c/hbm_online_repair/ras-events.h -@@ -0,0 +1,28 @@ -+#ifndef __RAS_EVENTS_H -+#define __RAS_EVENTS_H -+ -+#include -+#include -+ -+#define MAX_PATH 1024 -+ -+#define DEFAULT_PAGE_SIZE 4096 -+ -+struct ras_events { -+ char tracing[MAX_PATH + 1]; -+ struct tep_handle *pevent; -+ int page_size; -+}; -+ -+struct pcpu_data { -+ struct tep_handle *pevent; -+ struct ras_events *ras; -+ int cpu; -+}; -+ -+/* Function prototypes */ -+int toggle_ras_event(char *trace_dir, char *group, char *event, int enable); -+int handle_ras_events(struct ras_events *ras); -+struct ras_events *init_trace_instance(void); -+ -+#endif -diff --git a/src/c/hbm_online_repair/ras-non-standard-handler.c b/src/c/hbm_online_repair/ras-non-standard-handler.c -new file mode 100644 -index 0000000..1d1fd04 ---- /dev/null -+++ b/src/c/hbm_online_repair/ras-non-standard-handler.c -@@ -0,0 +1,81 @@ -+#include -+#include -+#include -+#include -+#include -+#include -+#include "ras-non-standard-handler.h" -+#include "logger.h" -+ -+static char *uuid_le(const char *uu) -+{ -+ static char uuid[sizeof("xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx")]; -+ if (!uu) { -+ log(LOG_ERROR, "uuid_le failed: uu is empty"); -+ return uuid; -+ } -+ size_t uu_len = strlen(uu); -+ if (uu_len < SECTION_TYPE_UUID_LEN) { -+ log(LOG_ERROR, "uuid_le failed: uu is too short"); -+ return uuid; -+ } -+ -+ char *p = uuid; -+ int i; -+ static const unsigned char le[16] = {3,2,1,0,5,4,7,6,8,9,10,11,12,13,14,15}; -+ -+ for (i = 0; i < 16; i++) { -+ p += sprintf(p, "%.2x", (unsigned char) uu[le[i]]); -+ switch (i) { -+ case 3: -+ case 5: -+ case 7: -+ case 9: -+ *p++ = '-'; -+ break; -+ } -+ } -+ -+ *p = 0; -+ -+ return uuid; -+} -+ -+int ras_non_standard_event_handler(struct trace_seq *s, -+ struct tep_record *record, -+ struct tep_event *event, void *context) -+{ -+ int len; -+ unsigned long long val; -+ struct ras_non_standard_event ev; -+ -+ ev.sec_type = tep_get_field_raw(s, event, "sec_type", -+ record, &len, 1); -+ if(!ev.sec_type) { -+ log(LOG_WARNING, "get event section type failed"); -+ return -1; -+ } -+ -+ trace_seq_printf(s, "\n"); -+ trace_seq_printf(s, "sec_type: %s\n", uuid_le(ev.sec_type)); -+ -+ if (tep_get_field_val(s, event, "len", record, &val, 1) < 0) { -+ log(LOG_WARNING, "tep get field val failed"); -+ return -1; -+ } -+ -+ ev.length = val; -+ trace_seq_printf(s, "length: %d\n", ev.length); -+ -+ ev.error = tep_get_field_raw(s, event, "buf", record, &len, 1); -+ if(!ev.error || ev.length != len) { -+ log(LOG_WARNING, "get event error failed"); -+ return -1; -+ } -+ -+ if (strcmp(uuid_le(ev.sec_type), HISI_COMMON_SECTION_TYPE_UUID) == 0) { -+ decode_hisi_common_section(&ev); -+ } -+ -+ return 0; -+} -diff --git a/src/c/hbm_online_repair/ras-non-standard-handler.h b/src/c/hbm_online_repair/ras-non-standard-handler.h -new file mode 100644 -index 0000000..0272dc1 ---- /dev/null -+++ b/src/c/hbm_online_repair/ras-non-standard-handler.h -@@ -0,0 +1,25 @@ -+#ifndef __RAS_NON_STANDARD_HANDLER_H -+#define __RAS_NON_STANDARD_HANDLER_H -+ -+#include -+#include "ras-events.h" -+ -+#define BIT(nr) (1UL << (nr)) -+ -+#define SECTION_TYPE_UUID_LEN 16 -+#define HISI_COMMON_SECTION_TYPE_UUID "c8b328a8-9917-4af6-9a13-2e08ab2e7586" -+ -+struct ras_non_standard_event { -+ char timestamp[64]; -+ const char *sec_type; -+ const uint8_t *error; -+ uint32_t length; -+}; -+ -+int ras_non_standard_event_handler(struct trace_seq *s, -+ struct tep_record *record, -+ struct tep_event *event, void *context); -+ -+int decode_hisi_common_section(struct ras_non_standard_event *event); -+ -+#endif -diff --git a/src/python/.gitignore b/src/python/.gitignore -new file mode 100644 -index 0000000..58200d4 ---- /dev/null -+++ b/src/python/.gitignore -@@ -0,0 +1 @@ -+__pycache__/ -diff --git a/src/python/syssentry/bmc_alarm.py b/src/python/syssentry/bmc_alarm.py -new file mode 100644 -index 0000000..5956538 ---- /dev/null -+++ b/src/python/syssentry/bmc_alarm.py -@@ -0,0 +1,159 @@ -+import logging -+import socket -+from enum import Enum -+ -+from .utils import execute_command -+ -+HEX_CHAR_LEN = 2 -+SOCKET_RECEIVE_LEN = 128 -+BMC_DATA_HEAD = "REP" -+BMC_REPORT_TYPE_BIT = 0 -+HBMC_REPAIR_TYPE_BIT = 1 -+HBMC_REPAIR_RESULT_BIT = 2 -+HBMC_ISOLATION_TYPE_BIT = 3 -+HBMC_SEND_HEAD_LEN = 4 # "ipmtool", "raw", "0x30", "0x92" -+HBMC_SEND_ROW_BIT = 26 + HBMC_SEND_HEAD_LEN -+HBMC_SEND_COL_BIT = 30 + HBMC_SEND_HEAD_LEN -+HBMC_REPAIR_TYPE_OFFSET = 7 -+ -+HBMC_SEND_SUCCESS_CODE = "db 07 00" -+ -+ -+class ReportType(Enum): -+ HBMC_REPAIR_BMC = 0x00 -+ -+ -+class HBMCRepairType(Enum): -+ CE_ACLS = 7 -+ PS_UCE_ACLS = 8 -+ CE_SPPR = 9 -+ PS_UCE_SPPR = 10 -+ -+ -+class HBMCRepairResultType(Enum): -+ ISOLATE_FAILED_OVER_THRESHOLD = 0b10000001 -+ ISOLATE_FAILED_OTHER_REASON = 0b10000010 -+ REPAIR_FAILED_NO_RESOURCE = 0b10010100 -+ REPAIR_FAILED_INVALID_PARAM = 0b10011000 -+ REPAIR_FAILED_OTHER_REASON = 0b10011100 -+ ONLINE_PAGE_FAILED = 0b10100000 -+ ISOLATE_REPAIR_ONLINE_SUCCESS = 0b00000000 -+ -+ -+class HBMCIsolationType(Enum): -+ ROW_FAULT = 1 -+ SINGLE_ADDR_FAULT = 6 -+ -+ -+def find_value_is_in_enum(value: int, enum: Enum): -+ for item in enum: -+ if value == item.value: -+ return True -+ return False -+ -+ -+def convert_hex_char_to_int(data, bit): -+ if len(data) < (bit+1)*HEX_CHAR_LEN: -+ logging.error(f"Data {data} len is too short, current convert bit is {bit}") -+ char = data[bit*HEX_CHAR_LEN:(bit+1)*HEX_CHAR_LEN] -+ try: -+ value = int(char, 16) -+ except ValueError: -+ logging.error(f"Cannot convert char [{char}] to int") -+ raise ValueError -+ return value -+ -+ -+def reverse_byte(data): -+ return data[3], data[2], data[1], data[0] -+ -+ -+def parse_hbmc_report(data: str): -+ logging.debug(f"bmc receive raw data is {data}") -+ repair_type = convert_hex_char_to_int(data, HBMC_REPAIR_TYPE_BIT) -+ repair_type += HBMC_REPAIR_TYPE_OFFSET -+ if not find_value_is_in_enum(repair_type, HBMCRepairType): -+ logging.warning(f"HBMC msg repair type ({repair_type}) is unknown") -+ raise ValueError -+ -+ repair_result = convert_hex_char_to_int(data, HBMC_REPAIR_RESULT_BIT) -+ if not find_value_is_in_enum(repair_result, HBMCRepairResultType): -+ logging.warning(f"HBMC msg repair result ({repair_result}) is unknown") -+ raise ValueError -+ -+ isolation_type = convert_hex_char_to_int(data, HBMC_ISOLATION_TYPE_BIT) -+ if not find_value_is_in_enum(isolation_type, HBMCIsolationType): -+ logging.warning(f"HBMC msg isolation type ({isolation_type}) is unknown") -+ raise ValueError -+ -+ cmd_list = [ -+ "ipmitool", -+ "raw", -+ "0x30", # Netfn -+ "0x92", # cmd -+ "0xdb", -+ "0x07", -+ "0x00", -+ "0x65", # sub command -+ "0x01", # SystemId -+ "0x00", # LocalSystemId -+ "{:#04X}".format(repair_type), -+ "{:#04X}".format(repair_result), -+ "{:#04X}".format(isolation_type), -+ ] -+ # send the remain data directly -+ data = data[(HBMC_ISOLATION_TYPE_BIT + 1) * HEX_CHAR_LEN:] -+ other_info_str = [] -+ for i in range(len(data) // 2): -+ other_info_str.append("{:#04X}".format(convert_hex_char_to_int(data, i))) -+ cmd_list.extend(other_info_str) -+ -+ cmd_list[HBMC_SEND_ROW_BIT:HBMC_SEND_ROW_BIT + 4] = reverse_byte(cmd_list[HBMC_SEND_ROW_BIT:HBMC_SEND_ROW_BIT + 4]) -+ cmd_list[HBMC_SEND_COL_BIT:HBMC_SEND_COL_BIT + 4] = reverse_byte(cmd_list[HBMC_SEND_COL_BIT:HBMC_SEND_COL_BIT + 4]) -+ -+ logging.info(f"Send bmc alarm command is {cmd_list}") -+ -+ ret = execute_command(cmd_list) -+ if HBMC_SEND_SUCCESS_CODE not in ret: -+ logging.warning(f"Send bmc alarm failed, error code is {ret}") -+ raise ValueError -+ logging.debug("Send bmc alarm success") -+ -+ -+PARSE_REPORT_MSG_FUNC_DICT = { -+ ReportType.HBMC_REPAIR_BMC.value: parse_hbmc_report, -+} -+ -+ -+def bmc_recv(server_socket: socket.socket): -+ logging.debug("Get hbm socket connection request") -+ try: -+ client_socket, _ = server_socket.accept() -+ logging.debug("cpu alarm fd listen ok") -+ -+ data = client_socket.recv(SOCKET_RECEIVE_LEN) -+ data = data.decode() -+ -+ data_head = data[0:len(BMC_DATA_HEAD)] -+ if data_head != BMC_DATA_HEAD: -+ logging.warning(f"The head of the msg is incorrect, head is {data_head}") -+ raise ValueError -+ -+ # remove the data head -+ data = data[len(BMC_DATA_HEAD):] -+ logging.info(f"Remove head data is {data}") -+ -+ report_type = convert_hex_char_to_int(data, BMC_REPORT_TYPE_BIT) -+ if report_type not in PARSE_REPORT_MSG_FUNC_DICT.keys(): -+ logging.warning(f"The type of the msg ({report_type}) is unknown") -+ raise ValueError -+ -+ PARSE_REPORT_MSG_FUNC_DICT[report_type](data) -+ -+ except socket.error: -+ logging.error("socket error") -+ return -+ except (ValueError, OSError, UnicodeError, TypeError, NotImplementedError): -+ logging.error("server recv bmc msg failed!") -+ client_socket.close() -+ return -diff --git a/src/python/syssentry/syssentry.py b/src/python/syssentry/syssentry.py -index ea09095..3829849 100644 ---- a/src/python/syssentry/syssentry.py -+++ b/src/python/syssentry/syssentry.py -@@ -48,6 +48,12 @@ try: - except ImportError: - CPU_EXIST = False - -+BMC_EXIST = True -+try: -+ from .bmc_alarm import bmc_recv -+except ImportError: -+ BMC_EXIST = False -+ - - INSPECTOR = None - -@@ -89,6 +95,9 @@ RESULT_SOCKET_PATH = "/var/run/sysSentry/result.sock" - - CPU_ALARM_SOCKET_PATH = "/var/run/sysSentry/report.sock" - -+BMC_SOCKET_PATH = "/var/run/sysSentry/bmc.sock" -+ -+fd_list = [] - - def msg_data_process(msg_data): - """message data process""" -@@ -334,6 +343,41 @@ def cpu_alarm_fd_create(): - - return cpu_alarm_fd - -+def bmc_fd_create(): -+ """create bmc fd""" -+ if not os.path.exists(SENTRY_RUN_DIR): -+ logging.debug("%s not exist", SENTRY_RUN_DIR) -+ return None -+ -+ try: -+ bmc_fd = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) -+ except socket.error: -+ logging.error("bmc fd create failed") -+ return None -+ -+ bmc_fd.setblocking(False) -+ if os.path.exists(BMC_SOCKET_PATH): -+ os.remove(BMC_SOCKET_PATH) -+ -+ try: -+ bmc_fd.bind(BMC_SOCKET_PATH) -+ except OSError: -+ logging.error("bmc fd bind failed") -+ bmc_fd.close() -+ return None -+ -+ os.chmod(BMC_SOCKET_PATH, 0o600) -+ try: -+ bmc_fd.listen(5) -+ except OSError: -+ logging.error("bmc fd listen failed") -+ bmc_fd.close() -+ return None -+ -+ logging.debug("%s bind and listen", BMC_SOCKET_PATH) -+ -+ return bmc_fd -+ - - def server_result_recv(server_socket: socket.socket): - """server result receive""" -@@ -407,35 +451,47 @@ def server_result_fd_create(): - return server_result_fd - - -+def close_all_fd(): -+ for fd in fd_list: -+ fd.close() -+ -+ - def main_loop(): - """main loop""" -+ - server_fd = server_fd_create() - if not server_fd: -+ close_all_fd() - return -+ fd_list.append(server_fd) - - server_result_fd = server_result_fd_create() - if not server_result_fd: -- server_fd.close() -+ close_all_fd() - return -+ fd_list.append(server_result_fd) - - heartbeat_fd = heartbeat_fd_create() - if not heartbeat_fd: -- server_fd.close() -- server_result_fd.close() -+ close_all_fd() - return -+ fd_list.append(heartbeat_fd) - - cpu_alarm_fd = cpu_alarm_fd_create() - if not cpu_alarm_fd: -- server_fd.close() -- heartbeat_fd.close() -- server_result_fd.close() -+ close_all_fd() -+ return -+ fd_list.append(cpu_alarm_fd) -+ -+ bmc_fd = bmc_fd_create() -+ if not bmc_fd: -+ close_all_fd() - return -+ fd_list.append(bmc_fd) - - epoll_fd = select.epoll() -- epoll_fd.register(server_fd.fileno(), select.EPOLLIN) -- epoll_fd.register(server_result_fd.fileno(), select.EPOLLIN) -- epoll_fd.register(heartbeat_fd.fileno(), select.EPOLLIN) -- epoll_fd.register(cpu_alarm_fd.fileno(), select.EPOLLIN) -+ for fd in fd_list: -+ epoll_fd.register(fd.fileno(), select.EPOLLIN) - - logging.debug("start main loop") - # onstart_tasks_handle() -@@ -458,6 +514,8 @@ def main_loop(): - heartbeat_recv(heartbeat_fd) - elif CPU_EXIST and event_fd == cpu_alarm_fd.fileno(): - cpu_alarm_recv(cpu_alarm_fd) -+ elif BMC_EXIST and event_fd == bmc_fd.fileno(): -+ bmc_recv(bmc_fd) - else: - continue - --- -2.27.0 - diff --git a/add-log-for-improving-maintainability.patch b/add-log-for-improving-maintainability.patch deleted file mode 100644 index a9a4527..0000000 --- a/add-log-for-improving-maintainability.patch +++ /dev/null @@ -1,251 +0,0 @@ -From a8418093bb37482da7ccaac0c950f2ed8d0ba2fa Mon Sep 17 00:00:00 2001 -From: gaoruoshu -Date: Thu, 10 Oct 2024 15:07:29 +0800 -Subject: [PATCH] add log for improving maintainability - ---- - .../avg_block_io/avg_block_io.py | 4 +- - .../sentryPlugins/avg_block_io/module_conn.py | 57 ++++++++++------- - .../avg_block_io/stage_window.py | 8 +++ - .../sentryPlugins/avg_block_io/utils.py | 63 +++++++++++++++++-- - 4 files changed, 103 insertions(+), 29 deletions(-) - -diff --git a/src/python/sentryPlugins/avg_block_io/avg_block_io.py b/src/python/sentryPlugins/avg_block_io/avg_block_io.py -index 26a60c5..cf2ded3 100644 ---- a/src/python/sentryPlugins/avg_block_io/avg_block_io.py -+++ b/src/python/sentryPlugins/avg_block_io/avg_block_io.py -@@ -194,11 +194,11 @@ def init_io_win(io_dic, config, common_param): - - if avg_lim_value and avg_time_value and tot_lim_value: - io_data[disk_name][stage_name][rw]["latency"] = IoWindow(window_size=io_dic["win_size"], window_threshold=io_dic["win_threshold"], abnormal_multiple=avg_time_value, abnormal_multiple_lim=avg_lim_value, abnormal_time=tot_lim_value) -- logging.debug("Successfully create {}-{}-{} latency window".format(disk_name, stage_name, rw)) -+ logging.debug("Successfully create {}-{}-{}-latency window".format(disk_name, stage_name, rw)) - - if iodump_lim_value is not None: - io_data[disk_name][stage_name][rw]["iodump"] = IoDumpWindow(window_size=io_dic["win_size"], window_threshold=io_dic["win_threshold"], abnormal_time=iodump_lim_value) -- logging.debug("Successfully create {}-{}-{} iodump window".format(disk_name, stage_name, rw)) -+ logging.debug("Successfully create {}-{}-{}-iodump window".format(disk_name, stage_name, rw)) - return io_data, io_avg_value - - -diff --git a/src/python/sentryPlugins/avg_block_io/module_conn.py b/src/python/sentryPlugins/avg_block_io/module_conn.py -index 2fc5a83..40b3fcc 100644 ---- a/src/python/sentryPlugins/avg_block_io/module_conn.py -+++ b/src/python/sentryPlugins/avg_block_io/module_conn.py -@@ -13,7 +13,7 @@ import logging - import sys - import time - --from .utils import is_abnormal -+from .utils import is_abnormal, get_win_data, log_slow_win - from sentryCollector.collect_plugin import is_iocollect_valid, get_io_data, Result_Messages - from syssentry.result import ResultLevel, report_result - from xalarm.sentry_notify import xalarm_report, MINOR_ALM, ALARM_TYPE_OCCUR -@@ -66,36 +66,51 @@ def report_alarm_fail(alarm_info): - - def process_report_data(disk_name, rw, io_data): - """check abnormal window and report to xalarm""" -- if not is_abnormal((disk_name, 'bio', rw), io_data): -+ abnormal, abnormal_list = is_abnormal((disk_name, 'bio', rw), io_data) -+ if not abnormal: - return - -- msg = {"alarm_source": TASK_NAME, "driver_name": disk_name, "io_type": rw} -+ msg = { -+ "alarm_source": TASK_NAME, "driver_name": disk_name, "io_type": rw, -+ "reason": "unknown", "block_stack": "bio", "alarm_type": abnormal_list, -+ "details": get_win_data(disk_name, rw, io_data) -+ } - -+ # io press - ctrl_stage = ['throtl', 'wbt', 'iocost', 'bfq'] - for stage_name in ctrl_stage: -- if is_abnormal((disk_name, stage_name, rw), io_data): -- msg["reason"] = "IO press slow" -- msg["block_stack"] = f"bio,{stage_name}" -- logging.warning("{} - {} report IO press slow".format(disk_name, rw)) -- xalarm_report(1002, MINOR_ALM, ALARM_TYPE_OCCUR, json.dumps(msg)) -- return -- -- if is_abnormal((disk_name, 'rq_driver', rw), io_data): -+ abnormal, abnormal_list = is_abnormal((disk_name, 'bio', rw), io_data) -+ if not abnormal: -+ continue -+ msg["reason"] = "IO press" -+ msg["block_stack"] = f"bio,{stage_name}" -+ msg["alarm_type"] = abnormal_list -+ log_slow_win(msg, "IO press") -+ xalarm_report(1002, MINOR_ALM, ALARM_TYPE_OCCUR, json.dumps(msg)) -+ return -+ -+ # driver slow -+ abnormal, abnormal_list = is_abnormal((disk_name, 'rq_driver', rw), io_data) -+ if abnormal: - msg["reason"] = "driver slow" - msg["block_stack"] = "bio,rq_driver" -- logging.warning("{} - {} report driver slow".format(disk_name, rw)) -+ msg["alarm_type"] = abnormal_list -+ log_slow_win(msg, "driver slow") - xalarm_report(1002, MINOR_ALM, ALARM_TYPE_OCCUR, json.dumps(msg)) - return - -+ # kernel slow - kernel_stage = ['gettag', 'plug', 'deadline', 'hctx', 'requeue'] - for stage_name in kernel_stage: -- if is_abnormal((disk_name, stage_name, rw), io_data): -- msg["reason"] = "kernel slow" -- msg["block_stack"] = f"bio,{stage_name}" -- logging.warning("{} - {} report kernel slow".format(disk_name, rw)) -- xalarm_report(1002, MINOR_ALM, ALARM_TYPE_OCCUR, json.dumps(msg)) -- return -- msg["reason"] = "unknown" -- msg["block_stack"] = "bio" -- logging.warning("{} - {} report UNKNOWN slow".format(disk_name, rw)) -+ abnormal, abnormal_list = is_abnormal((disk_name, stage_name, rw), io_data) -+ if not abnormal: -+ continue -+ msg["reason"] = "kernel slow" -+ msg["block_stack"] = f"bio,{stage_name}" -+ msg["alarm_type"] = abnormal_list -+ log_slow_win(msg, "kernel slow") -+ xalarm_report(1002, MINOR_ALM, ALARM_TYPE_OCCUR, json.dumps(msg)) -+ return -+ -+ log_slow_win(msg, "unknown") - xalarm_report(1002, MINOR_ALM, ALARM_TYPE_OCCUR, json.dumps(msg)) -diff --git a/src/python/sentryPlugins/avg_block_io/stage_window.py b/src/python/sentryPlugins/avg_block_io/stage_window.py -index 9b0ce79..5113782 100644 ---- a/src/python/sentryPlugins/avg_block_io/stage_window.py -+++ b/src/python/sentryPlugins/avg_block_io/stage_window.py -@@ -14,6 +14,11 @@ class AbnormalWindowBase: - self.window_size = window_size - self.window_threshold = window_threshold - self.abnormal_window = [False] * window_size -+ self.window_data = [-1] * window_size -+ -+ def append_new_data(self, ab_res): -+ self.window_data.pop(0) -+ self.window_data.append(ab_res) - - def append_new_period(self, ab_res, avg_val=0): - self.abnormal_window.pop(0) -@@ -25,6 +30,9 @@ class AbnormalWindowBase: - def is_abnormal_window(self): - return sum(self.abnormal_window) > self.window_threshold - -+ def window_data_to_string(self): -+ return ",".join(str(x) for x in self.window_data) -+ - - class IoWindow(AbnormalWindowBase): - def __init__(self, window_size=10, window_threshold=7, abnormal_multiple=5, abnormal_multiple_lim=30, abnormal_time=40): -diff --git a/src/python/sentryPlugins/avg_block_io/utils.py b/src/python/sentryPlugins/avg_block_io/utils.py -index 2de9a46..3b7f027 100644 ---- a/src/python/sentryPlugins/avg_block_io/utils.py -+++ b/src/python/sentryPlugins/avg_block_io/utils.py -@@ -65,15 +65,32 @@ def set_nested_value(data, keys, value): - return True - - -+def get_win_data(disk_name, rw, io_data): -+ """get latency and iodump win data""" -+ latency = '' -+ iodump = '' -+ for stage_name in io_data[disk_name]: -+ if 'latency' in io_data[disk_name][stage_name][rw]: -+ latency_list = io_data[disk_name][stage_name][rw]['latency'].window_data_to_string() -+ latency += f'{stage_name}: [{latency_list}], ' -+ if 'iodump' in io_data[disk_name][stage_name][rw]: -+ iodump_list = io_data[disk_name][stage_name][rw]['iodump'].window_data_to_string() -+ iodump += f'{stage_name}: [{iodump_list}], ' -+ return {"latency": latency[:-2], "iodump": iodump[:-2]} -+ -+ - def is_abnormal(io_key, io_data): - """check if latency and iodump win abnormal""" -+ abnormal_list = '' - for key in ['latency', 'iodump']: - all_keys = get_nested_value(io_data, io_key) - if all_keys and key in all_keys: - win = get_nested_value(io_data, io_key + (key,)) - if win and win.is_abnormal_window(): -- return True -- return False -+ abnormal_list += key + ', ' -+ if not abnormal_list: -+ return False, abnormal_list -+ return True, abnormal_list[:-2] - - - def update_io_avg(old_avg, period_value, win_size): -@@ -87,8 +104,8 @@ def update_io_avg(old_avg, period_value, win_size): - return [new_avg_value, new_avg_count] - - --def update_io_data(old_avg, period_value, win_size, io_data, io_key): -- """update data of latency and iodump window""" -+def update_io_period(old_avg, period_value, io_data, io_key): -+ """update period of latency and iodump window""" - all_wins = get_nested_value(io_data, io_key) - if all_wins and "latency" in all_wins: - io_data[io_key[0]][io_key[1]][io_key[2]]["latency"].append_new_period(period_value[0], old_avg[AVG_VALUE]) -@@ -96,20 +113,54 @@ def update_io_data(old_avg, period_value, win_size, io_data, io_key): - io_data[io_key[0]][io_key[1]][io_key[2]]["iodump"].append_new_period(period_value[1]) - - -+def update_io_data(period_value, io_data, io_key): -+ """update data of latency and iodump window""" -+ all_wins = get_nested_value(io_data, io_key) -+ if all_wins and "latency" in all_wins: -+ io_data[io_key[0]][io_key[1]][io_key[2]]["latency"].append_new_data(period_value[0]) -+ if all_wins and "iodump" in all_wins: -+ io_data[io_key[0]][io_key[1]][io_key[2]]["iodump"].append_new_data(period_value[1]) -+ -+ -+def log_abnormal_period(old_avg, period_value, io_data, io_key): -+ """record log of abnormal period""" -+ all_wins = get_nested_value(io_data, io_key) -+ if all_wins and "latency" in all_wins: -+ if all_wins["latency"].is_abnormal_period(period_value[0], old_avg[AVG_VALUE]): -+ logging.info(f"[abnormal_period] disk: {io_key[0]}, stage: {io_key[1]}, iotype: {io_key[2]}, " -+ f"type: latency, avg: {round(old_avg[AVG_VALUE], 3)}, curr_val: {period_value[0]}") -+ if all_wins and "iodump" in all_wins: -+ if all_wins["iodump"].is_abnormal_period(period_value[1]): -+ logging.info(f"[abnormal_period] disk: {io_key[0]}, stage: {io_key[1]}, iotype: {io_key[2]}, " -+ f"type: iodump, curr_val: {period_value[1]}") -+ -+ -+def log_slow_win(msg, reason): -+ """record log of slow win""" -+ logging.warning(f"[SLOW IO] disk: {msg['driver_name']}, stage: {msg['block_stack']}, " -+ f"iotype: {msg['io_type']}, type: {msg['alarm_type']}, reason: {reason}") -+ logging.info(f"latency: {msg['details']['latency']}") -+ logging.info(f"iodump: {msg['details']['iodump']}") -+ -+ - def update_avg_and_check_abnormal(data, io_key, win_size, io_avg_value, io_data): - """update avg and check abonrmal, return true if win_size full""" - period_value = get_nested_value(data, io_key) - old_avg = get_nested_value(io_avg_value, io_key) - - # 更新avg数据 -+ update_io_data(period_value, io_data, io_key) - if old_avg[AVG_COUNT] < win_size: - set_nested_value(io_avg_value, io_key, update_io_avg(old_avg, period_value, win_size)) - return False - -+ # 打印异常周期数据 -+ log_abnormal_period(old_avg, period_value, io_data, io_key) -+ - # 更新win数据 -- 判断异常周期 -- update_io_data(old_avg, period_value, win_size, io_data, io_key) -+ update_io_period(old_avg, period_value, io_data, io_key) - all_wins = get_nested_value(io_data, io_key) -- if all_wins and 'latency' not in all_wins: -+ if not all_wins or 'latency' not in all_wins: - return True - period = get_nested_value(io_data, io_key + ("latency",)) - if period and period.is_abnormal_period(period_value[0], old_avg[AVG_VALUE]): --- -2.27.0 - diff --git a/add-log-for-xalarm-when-sending-msg-and-clean-invali.patch b/add-log-for-xalarm-when-sending-msg-and-clean-invali.patch deleted file mode 100644 index b8a762f..0000000 --- a/add-log-for-xalarm-when-sending-msg-and-clean-invali.patch +++ /dev/null @@ -1,24 +0,0 @@ -From ef3aad0ca57d35b0a4fe29a0205596021bae0227 Mon Sep 17 00:00:00 2001 -From: caixiaomeng -Date: Fri, 11 Oct 2024 17:59:54 +0800 -Subject: [PATCH] add log for xalarm when sending msg and clean invalid client - socket - ---- - src/python/xalarm/xalarm_transfer.py | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/src/python/xalarm/xalarm_transfer.py b/src/python/xalarm/xalarm_transfer.py -index 42137d8..9e867cc 100644 ---- a/src/python/xalarm/xalarm_transfer.py -+++ b/src/python/xalarm/xalarm_transfer.py -@@ -117,4 +117,5 @@ def transmit_alarm(server_sock, epoll, fd_to_socket, bin_data): - epoll.unregister(fileno) - fd_to_socket[fileno].close() - del fd_to_socket[fileno] -+ logging.info(f"cleaned up connection {fileno} for client lost connection.") - --- -2.27.0 - - diff --git a/add-log-level-and-change-log-format.patch b/add-log-level-and-change-log-format.patch deleted file mode 100644 index 219c86c..0000000 --- a/add-log-level-and-change-log-format.patch +++ /dev/null @@ -1,522 +0,0 @@ -From c1ab550a3f817826ac6f279de97e6d3820901275 Mon Sep 17 00:00:00 2001 -From: gaoruoshu -Date: Fri, 27 Sep 2024 14:10:18 +0800 -Subject: [PATCH] add log level and change log format - ---- - config/collector.conf | 5 ++- - config/inspect.conf | 5 ++- - config/plugins/avg_block_io.ini | 5 ++- - config/xalarm.conf | 3 ++ - src/python/sentryCollector/collect_config.py | 29 ++++++++++++++++ - src/python/sentryCollector/collect_io.py | 15 ++------- - src/python/sentryCollector/collect_plugin.py | 32 +++++++++--------- - src/python/sentryCollector/collectd.py | 6 ++-- - .../avg_block_io/avg_block_io.py | 7 ++-- - .../sentryPlugins/avg_block_io/utils.py | 32 ++++++++++++++++++ - src/python/syssentry/sentry_config.py | 28 ++++++++++++++++ - src/python/syssentry/syssentry.py | 7 ++-- - src/python/xalarm/xalarm_config.py | 33 +++++++++++++++++-- - src/python/xalarm/xalarm_daemon.py | 7 ++-- - 14 files changed, 172 insertions(+), 42 deletions(-) - -diff --git a/config/collector.conf b/config/collector.conf -index 9baa086..56b0ed1 100644 ---- a/config/collector.conf -+++ b/config/collector.conf -@@ -4,4 +4,7 @@ modules=io - [io] - period_time=1 - max_save=10 --disk=default -\ No newline at end of file -+disk=default -+ -+[log] -+level=info -\ No newline at end of file -diff --git a/config/inspect.conf b/config/inspect.conf -index 071cca1..f451d9e 100644 ---- a/config/inspect.conf -+++ b/config/inspect.conf -@@ -1,2 +1,5 @@ - [inspect] --Interval=3 -\ No newline at end of file -+Interval=3 -+ -+[log] -+level=info -diff --git a/config/plugins/avg_block_io.ini b/config/plugins/avg_block_io.ini -index bc33dde..858db18 100644 ---- a/config/plugins/avg_block_io.ini -+++ b/config/plugins/avg_block_io.ini -@@ -1,8 +1,11 @@ -+[log] -+level=info -+ - [common] - disk=default - stage=default - iotype=read,write --period_time=1 -+period_time=1 - - [algorithm] - win_size=30 -diff --git a/config/xalarm.conf b/config/xalarm.conf -index 14c6d39..323d2dd 100644 ---- a/config/xalarm.conf -+++ b/config/xalarm.conf -@@ -1,2 +1,5 @@ - [filter] - id_mask = 1001-1128 -+ -+[log] -+level=info -diff --git a/src/python/sentryCollector/collect_config.py b/src/python/sentryCollector/collect_config.py -index 0fdd9f0..5aa38ec 100644 ---- a/src/python/sentryCollector/collect_config.py -+++ b/src/python/sentryCollector/collect_config.py -@@ -32,6 +32,35 @@ CONF_IO_PERIOD_TIME_DEFAULT = 1 - CONF_IO_MAX_SAVE_DEFAULT = 10 - CONF_IO_DISK_DEFAULT = "default" - -+# log -+CONF_LOG = 'log' -+CONF_LOG_LEVEL = 'level' -+LogLevel = { -+ "debug": logging.DEBUG, -+ "info": logging.INFO, -+ "warning": logging.WARNING, -+ "error": logging.ERROR, -+ "critical": logging.CRITICAL -+} -+ -+ -+def get_log_level(filename=COLLECT_CONF_PATH): -+ if not os.path.exists(filename): -+ return logging.INFO -+ -+ try: -+ config = configparser.ConfigParser() -+ config.read(filename) -+ if not config.has_option(CONF_LOG, CONF_LOG_LEVEL): -+ return logging.INFO -+ log_level = config.get(CONF_LOG, CONF_LOG_LEVEL) -+ if log_level.lower() in LogLevel: -+ return LogLevel.get(log_level.lower()) -+ return logging.INFO -+ except configparser.Error: -+ return logging.INFO -+ -+ - class CollectConfig: - def __init__(self, filename=COLLECT_CONF_PATH): - -diff --git a/src/python/sentryCollector/collect_io.py b/src/python/sentryCollector/collect_io.py -index 9c8dae7..019d174 100644 ---- a/src/python/sentryCollector/collect_io.py -+++ b/src/python/sentryCollector/collect_io.py -@@ -163,18 +163,6 @@ class CollectIo(): - logging.error("An error occurred2: %s", e) - return column_names - -- def task_loop(self): -- if self.stop_event.is_set(): -- logging.info("collect io thread exit") -- return -- -- for disk_name, stage_list in self.disk_map_stage.items(): -- if self.get_blk_io_hierarchy(disk_name, stage_list) < 0: -- continue -- self.append_period_lat(disk_name, stage_list) -- -- threading.Timer(self.period_time, self.task_loop).start() -- - def is_kernel_avaliable(self): - base_path = '/sys/kernel/debug/block' - all_disk = [] -@@ -191,6 +179,9 @@ class CollectIo(): - if file_name == 'stats': - all_disk.append(disk_name) - -+ if self.loop_all: -+ self.disk_list = all_disk -+ - for disk_name in self.disk_list: - if not self.loop_all and disk_name not in all_disk: - logging.warning("the %s disk not exist!", disk_name) -diff --git a/src/python/sentryCollector/collect_plugin.py b/src/python/sentryCollector/collect_plugin.py -index 1faa5e3..3e2cf4c 100644 ---- a/src/python/sentryCollector/collect_plugin.py -+++ b/src/python/sentryCollector/collect_plugin.py -@@ -75,14 +75,14 @@ def client_send_and_recv(request_data, data_str_len, protocol): - try: - client_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - except socket.error: -- print("collect_plugin: client create socket error") -+ logging.error("collect_plugin: client create socket error") - return None - - try: - client_socket.connect(COLLECT_SOCKET_PATH) - except OSError: - client_socket.close() -- print("collect_plugin: client connect error") -+ logging.error("collect_plugin: client connect error") - return None - - req_data_len = len(request_data) -@@ -94,23 +94,23 @@ def client_send_and_recv(request_data, data_str_len, protocol): - res_data = res_data.decode() - except (OSError, UnicodeError): - client_socket.close() -- print("collect_plugin: client communicate error") -+ logging.error("collect_plugin: client communicate error") - return None - - res_magic = res_data[:CLT_MSG_MAGIC_LEN] - if res_magic != "RES": -- print("res msg format error") -+ logging.error("res msg format error") - return None - - protocol_str = res_data[CLT_MSG_MAGIC_LEN:CLT_MSG_MAGIC_LEN+CLT_MSG_PRO_LEN] - try: - protocol_id = int(protocol_str) - except ValueError: -- print("recv msg protocol id is invalid %s", protocol_str) -+ logging.error("recv msg protocol id is invalid %s", protocol_str) - return None - - if protocol_id >= ClientProtocol.PRO_END: -- print("protocol id is invalid") -+ logging.error("protocol id is invalid") - return None - - try: -@@ -119,7 +119,7 @@ def client_send_and_recv(request_data, data_str_len, protocol): - res_msg_data = res_msg_data.decode() - return res_msg_data - except (OSError, ValueError, UnicodeError): -- print("collect_plugin: client recv res msg error") -+ logging.error("collect_plugin: client recv res msg error") - finally: - client_socket.close() - -@@ -128,30 +128,30 @@ def client_send_and_recv(request_data, data_str_len, protocol): - def validate_parameters(param, len_limit, char_limit): - ret = ResultMessage.RESULT_SUCCEED - if not param: -- print("param is invalid") -+ logging.error("param is invalid, param = %s", param) - ret = ResultMessage.RESULT_NOT_PARAM - return [False, ret] - - if not isinstance(param, list): -- print(f"{param} is not list type.") -+ logging.error("%s is not list type.", param) - ret = ResultMessage.RESULT_NOT_PARAM - return [False, ret] - - if len(param) <= 0: -- print(f"{param} length is 0.") -+ logging.error("%s length is 0.", param) - ret = ResultMessage.RESULT_INVALID_LENGTH - return [False, ret] - - pattern = r'^[a-zA-Z0-9_-]+$' - for info in param: - if not re.match(pattern, info): -- print(f"{info} is invalid char") -+ logging.error("%s is invalid char", info) - ret = ResultMessage.RESULT_INVALID_CHAR - return [False, ret] - - # length of len_limit is exceeded, keep len_limit - if len(param) > len_limit: -- print(f"{param} length more than {len_limit}, keep the first {len_limit}") -+ logging.error("%s length more than %d, keep the first %d", param, len_limit, len_limit) - param[:] = param[0:len_limit] - - # only keep elements under the char_limit length -@@ -202,13 +202,13 @@ def inter_is_iocollect_valid(period, disk_list=None, stage=None): - request_message = json.dumps(req_msg_struct) - result_message = client_send_and_recv(request_message, CLT_MSG_LEN_LEN, ClientProtocol.IS_IOCOLLECT_VALID) - if not result_message: -- print("collect_plugin: client_send_and_recv failed") -+ logging.error("collect_plugin: client_send_and_recv failed") - return result - - try: - json.loads(result_message) - except json.JSONDecodeError: -- print("is_iocollect_valid: json decode error") -+ logging.error("is_iocollect_valid: json decode error") - result['ret'] = ResultMessage.RESULT_PARSE_FAILED - return result - -@@ -260,12 +260,12 @@ def inter_get_io_data(period, disk_list, stage, iotype): - request_message = json.dumps(req_msg_struct) - result_message = client_send_and_recv(request_message, CLT_MSG_LEN_LEN, ClientProtocol.GET_IO_DATA) - if not result_message: -- print("collect_plugin: client_send_and_recv failed") -+ logging.error("collect_plugin: client_send_and_recv failed") - return result - try: - json.loads(result_message) - except json.JSONDecodeError: -- print("get_io_data: json decode error") -+ logging.error("get_io_data: json decode error") - result['ret'] = ResultMessage.RESULT_PARSE_FAILED - return result - -diff --git a/src/python/sentryCollector/collectd.py b/src/python/sentryCollector/collectd.py -index d9d8862..33f4b04 100644 ---- a/src/python/sentryCollector/collectd.py -+++ b/src/python/sentryCollector/collectd.py -@@ -26,7 +26,7 @@ import threading - - from .collect_io import CollectIo - from .collect_server import CollectServer --from .collect_config import CollectConfig -+from .collect_config import CollectConfig, get_log_level - - SENTRY_RUN_DIR = "/var/run/sysSentry" - COLLECT_SOCKET_PATH = "/var/run/sysSentry/collector.sock" -@@ -57,7 +57,9 @@ def main(): - os.mkdir(SENTRY_RUN_DIR) - os.chmod(SENTRY_RUN_DIR, mode=SENTRY_RUN_DIR_PERM) - -- logging.basicConfig(filename=COLLECT_LOG_FILE, level=logging.INFO) -+ log_level = get_log_level() -+ log_format = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s" -+ logging.basicConfig(filename=COLLECT_LOG_FILE, level=log_level, format=log_format) - os.chmod(COLLECT_LOG_FILE, 0o600) - - try: -diff --git a/src/python/sentryPlugins/avg_block_io/avg_block_io.py b/src/python/sentryPlugins/avg_block_io/avg_block_io.py -index ac35be2..b6b3b28 100644 ---- a/src/python/sentryPlugins/avg_block_io/avg_block_io.py -+++ b/src/python/sentryPlugins/avg_block_io/avg_block_io.py -@@ -15,7 +15,7 @@ import time - - from .stage_window import IoWindow, IoDumpWindow - from .module_conn import avg_is_iocollect_valid, avg_get_io_data, report_alarm_fail, process_report_data, sig_handler --from .utils import update_avg_and_check_abnormal -+from .utils import update_avg_and_check_abnormal, get_log_level - - CONFIG_FILE = "/etc/sysSentry/plugins/avg_block_io.ini" - -@@ -283,7 +283,10 @@ def main(): - signal.signal(signal.SIGINT, sig_handler) - signal.signal(signal.SIGTERM, sig_handler) - -- logging.basicConfig(level=logging.INFO) -+ log_level = get_log_level(CONFIG_FILE) -+ log_format = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s" -+ -+ logging.basicConfig(level=log_level, format=log_format) - - # 初始化配置读取 - config = configparser.ConfigParser(comment_prefixes=('#', ';')) -diff --git a/src/python/sentryPlugins/avg_block_io/utils.py b/src/python/sentryPlugins/avg_block_io/utils.py -index 54ed080..2de9a46 100644 ---- a/src/python/sentryPlugins/avg_block_io/utils.py -+++ b/src/python/sentryPlugins/avg_block_io/utils.py -@@ -8,9 +8,41 @@ - # IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR - # PURPOSE. - # See the Mulan PSL v2 for more details. -+import configparser -+import logging -+import os -+ - AVG_VALUE = 0 - AVG_COUNT = 1 - -+CONF_LOG = 'log' -+CONF_LOG_LEVEL = 'level' -+LogLevel = { -+ "debug": logging.DEBUG, -+ "info": logging.INFO, -+ "warning": logging.WARNING, -+ "error": logging.ERROR, -+ "critical": logging.CRITICAL -+} -+ -+ -+def get_log_level(filename): -+ if not os.path.exists(filename): -+ return logging.INFO -+ -+ try: -+ config = configparser.ConfigParser() -+ config.read(filename) -+ if not config.has_option(CONF_LOG, CONF_LOG_LEVEL): -+ return logging.INFO -+ log_level = config.get(CONF_LOG, CONF_LOG_LEVEL) -+ -+ if log_level.lower() in LogLevel: -+ return LogLevel.get(log_level.lower()) -+ return logging.INFO -+ except configparser.Error: -+ return logging.INFO -+ - - def get_nested_value(data, keys): - """get data from nested dict""" -diff --git a/src/python/syssentry/sentry_config.py b/src/python/syssentry/sentry_config.py -index a0e7b79..1169887 100644 ---- a/src/python/syssentry/sentry_config.py -+++ b/src/python/syssentry/sentry_config.py -@@ -21,6 +21,34 @@ import sys - DEFAULT_INSPECT_DELAY = 3 - INSPECT_CONF_PATH = "/etc/sysSentry/inspect.conf" - -+CONF_LOG = 'log' -+CONF_LOG_LEVEL = 'level' -+LogLevel = { -+ "debug": logging.DEBUG, -+ "info": logging.INFO, -+ "warning": logging.WARNING, -+ "error": logging.ERROR, -+ "critical": logging.CRITICAL -+} -+ -+ -+def get_log_level(filename=INSPECT_CONF_PATH): -+ if not os.path.exists(filename): -+ return logging.INFO -+ -+ try: -+ config = configparser.ConfigParser() -+ config.read(filename) -+ if not config.has_option(CONF_LOG, CONF_LOG_LEVEL): -+ return logging.INFO -+ log_level = config.get(CONF_LOG, CONF_LOG_LEVEL) -+ -+ if log_level.lower() in LogLevel: -+ return LogLevel.get(log_level.lower()) -+ return logging.INFO -+ except configparser.Error: -+ return logging.INFO -+ - - class SentryConfig: - """ -diff --git a/src/python/syssentry/syssentry.py b/src/python/syssentry/syssentry.py -index 776971f..9ef0203 100644 ---- a/src/python/syssentry/syssentry.py -+++ b/src/python/syssentry/syssentry.py -@@ -23,7 +23,7 @@ import fcntl - - import select - --from .sentry_config import SentryConfig -+from .sentry_config import SentryConfig, get_log_level - - from .task_map import TasksMap - from .global_values import SENTRY_RUN_DIR, CTL_SOCKET_PATH, SENTRY_RUN_DIR_PERM -@@ -563,7 +563,10 @@ def main(): - os.mkdir(SENTRY_RUN_DIR) - os.chmod(SENTRY_RUN_DIR, mode=SENTRY_RUN_DIR_PERM) - -- logging.basicConfig(filename=SYSSENTRY_LOG_FILE, level=logging.INFO) -+ log_level = get_log_level() -+ log_format = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s" -+ -+ logging.basicConfig(filename=SYSSENTRY_LOG_FILE, level=log_level, format=log_format) - os.chmod(SYSSENTRY_LOG_FILE, 0o600) - - if not chk_and_set_pidfile(): -diff --git a/src/python/xalarm/xalarm_config.py b/src/python/xalarm/xalarm_config.py -index 8e56d10..754a816 100644 ---- a/src/python/xalarm/xalarm_config.py -+++ b/src/python/xalarm/xalarm_config.py -@@ -15,9 +15,10 @@ Create: 2023-11-02 - """ - - import re -+import os - import dataclasses - import logging --from configparser import ConfigParser -+import configparser - - - MAIN_CONFIG_PATH = '/etc/sysSentry/xalarm.conf' -@@ -27,6 +28,34 @@ MIN_ID_NUMBER = 1001 - MAX_ID_NUMBER = 1128 - MAX_ID_MASK_CAPACITY = 128 - -+# log -+CONF_LOG = 'log' -+CONF_LOG_LEVEL = 'level' -+LogLevel = { -+ "debug": logging.DEBUG, -+ "info": logging.INFO, -+ "warning": logging.WARNING, -+ "error": logging.ERROR, -+ "critical": logging.CRITICAL -+} -+ -+ -+def get_log_level(filename=MAIN_CONFIG_PATH): -+ if not os.path.exists(filename): -+ return logging.INFO -+ -+ try: -+ config = configparser.ConfigParser() -+ config.read(filename) -+ if not config.has_option(CONF_LOG, CONF_LOG_LEVEL): -+ return logging.INFO -+ log_level = config.get(CONF_LOG, CONF_LOG_LEVEL) -+ if log_level.lower() in LogLevel: -+ return LogLevel.get(log_level.lower()) -+ return logging.INFO -+ except configparser.Error: -+ return logging.INFO -+ - - @dataclasses.dataclass - class AlarmConfig: -@@ -106,7 +135,7 @@ def config_init(): - """ - alarm_config = AlarmConfig() - -- cfg = ConfigParser() -+ cfg = configparser.ConfigParser() - cfg.read(MAIN_CONFIG_PATH) - - id_mask = parse_id_mask(cfg) -diff --git a/src/python/xalarm/xalarm_daemon.py b/src/python/xalarm/xalarm_daemon.py -index 00e8886..3ab211c 100644 ---- a/src/python/xalarm/xalarm_daemon.py -+++ b/src/python/xalarm/xalarm_daemon.py -@@ -21,7 +21,7 @@ import signal - import fcntl - import socket - --from .xalarm_config import config_init -+from .xalarm_config import config_init, get_log_level - from .xalarm_server import server_loop, SOCK_FILE - - ALARM_DIR = "/var/run/xalarm" -@@ -120,9 +120,10 @@ def alarm_process_create(): - os.mkdir(ALARM_DIR) - os.chmod(ALARM_DIR, ALARM_DIR_PERMISSION) - -+ log_level = get_log_level() -+ log_format = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s" - -- logging.basicConfig(filename=ALARM_LOGFILE, level=logging.INFO, -- format='%(asctime)s|%(levelname)s| %(message)s') -+ logging.basicConfig(filename=ALARM_LOGFILE, level=log_level, format=log_format) - - signal.signal(signal.SIGTERM, signal_handler) - --- -2.23.0 - diff --git a/add-parameter-time_range-alarm_id-and-alarm_clear_ti.patch b/add-parameter-time_range-alarm_id-and-alarm_clear_ti.patch deleted file mode 100644 index ee9e234..0000000 --- a/add-parameter-time_range-alarm_id-and-alarm_clear_ti.patch +++ /dev/null @@ -1,104 +0,0 @@ -From 0a4bd4097690bee7250676a0c262a830c7a8fbcf Mon Sep 17 00:00:00 2001 -From: jinsaihang -Date: Fri, 11 Oct 2024 15:35:43 +0800 -Subject: [PATCH] add parameter time_range ,alarm_id and alarm_clear_time - validation - -Signed-off-by: jinsaihang ---- - sysSentry-1.0.2/src/python/syssentry/alarm.py | 19 +++++++++++++++++++ - .../src/python/syssentry/load_mods.py | 6 ++---- - .../src/python/syssentry/sentryctl | 4 +++- - 3 files changed, 24 insertions(+), 5 deletions(-) - -diff --git a/src/python/syssentry/alarm.py b/src/python/syssentry/alarm.py -index d5337d3..43c1065 100644 ---- a/src/python/syssentry/alarm.py -+++ b/src/python/syssentry/alarm.py -@@ -18,6 +18,7 @@ from datetime import datetime - import time - import logging - import json -+import sys - - from xalarm.register_xalarm import xalarm_register,xalarm_getid,xalarm_getlevel,xalarm_gettype,xalarm_gettime,xalarm_getdesc - from xalarm.xalarm_api import Xalarm -@@ -41,9 +42,15 @@ id_base = 1001 - clientId = -1 - - MILLISECONDS_UNIT_SECONDS = 1000 -+MAX_NUM_OF_ALARM_ID = 128 -+MIN_ALARM_ID = 1001 -+MAX_ALARM_ID = (MIN_ALARM_ID + MAX_NUM_OF_ALARM_ID - 1) - - def update_alarm_list(alarm_info: Xalarm): - alarm_id = xalarm_getid(alarm_info) -+ if alarm_id < MIN_ALARM_ID or alarm_id > MAX_ALARM_ID: -+ logging.warnning(f"Invalid alarm_id {alarm_id}") -+ return - timestamp = xalarm_gettime(alarm_info) - if not timestamp: - logging.error("Retrieve timestamp failed") -@@ -77,7 +84,19 @@ def alarm_register(): - logging.info(f"alarm_register: {task_name} is registered") - task = TasksMap.tasks_dict[task_type][task_name] - alarm_id = task.alarm_id -+ if alarm_id < MIN_ALARM_ID or alarm_id > MAX_ALARM_ID: -+ logging.warnning(f"Invalid alarm_id {alarm_id}: ignore {task_name} alarm") -+ continue - alarm_clear_time = task.alarm_clear_time -+ try: -+ alarm_clear_time = int(alarm_clear_time) -+ if alarm_clear_time <= 0: -+ raise ValueError("Not a positive integer") -+ if alarm_clear_time > sys.maxsize: -+ raise ValueError("Exceeds maximum value for int") -+ except (ValueError, OverflowError, TypeError) as e: -+ logging.warnning(f"Invalid alarm_clear_time {alarm_clear_time}: ignore {task_name} alarm") -+ continue - alarm_list_dict[alarm_id] = [] - task_alarm_id_dict[task_name] = alarm_id - if alarm_id not in alarm_id_clear_time_dict: -diff --git a/src/python/syssentry/load_mods.py b/src/python/syssentry/load_mods.py -index ae05e57..7daf17d 100644 ---- a/src/python/syssentry/load_mods.py -+++ b/src/python/syssentry/load_mods.py -@@ -203,11 +203,9 @@ def parse_mod_conf(mod_name, mod_conf): - if not (MIN_ALARM_ID <= task.alarm_id <= MAX_ALARM_ID): - raise ValueError("Invalid alarm_id") - except ValueError: -- task.alarm_id = -1 -- logging.warning("Invalid alarm_id, set to -1") -+ logging.warning("Invalid alarm_id") - except configparser.NoOptionError: -- task.alarm_id = -1 -- logging.warning("Unset alarm_id and alarm_clear_time, use -1 and 15s as default") -+ logging.warning("Unset alarm_clear_time, use 15s as default") - - if CONF_ONSTART in mod_conf.options(CONF_TASK): - is_onstart = (mod_conf.get(CONF_TASK, CONF_ONSTART) == 'yes') -diff --git a/src/python/syssentry/sentryctl b/src/python/syssentry/sentryctl -index 3de93d0..c2e3cef 100644 ---- a/src/python/syssentry/sentryctl -+++ b/src/python/syssentry/sentryctl -@@ -136,7 +136,7 @@ if __name__ == '__main__': - parser_get_result.add_argument('task_name') - parser_get_alarm = subparsers.add_parser('get_alarm', help='get task alarm') - parser_get_alarm.add_argument('task_name') -- parser_get_alarm.add_argument('-s', '--time_range', type=str, default=DEFAULT_ALARM_TIME_RANGE, help='Specified time range') -+ parser_get_alarm.add_argument('-s', '--time_range', type=int, default=DEFAULT_ALARM_TIME_RANGE, help='Specified time range') - parser_get_alarm.add_argument('-d', '--detailed', action='store_true', help='Print Detailed Information') - parser_list = subparsers.add_parser('list', help='show all loaded task mod') - -@@ -153,6 +153,8 @@ if __name__ == '__main__': - elif client_args.cmd_type == 'get_result': - req_msg_struct = {"type": "get_result", "data": client_args.task_name} - elif client_args.cmd_type == 'get_alarm': -+ if not isinstance(client_args.time_range, int) or client_args.time_range <= 0: -+ print(f"time_range is not a positive integer: {client_args.time_range}") - req_msg_struct = { - "type": "get_alarm", - "data": { --- -2.27.0 - diff --git a/add-pyxalarm-and-pySentryNotify-add-multi-users-supp.patch b/add-pyxalarm-and-pySentryNotify-add-multi-users-supp.patch deleted file mode 100644 index 5b27e69..0000000 --- a/add-pyxalarm-and-pySentryNotify-add-multi-users-supp.patch +++ /dev/null @@ -1,678 +0,0 @@ -From a18ea2e94fef78334a56dce1ea3f67ee649732f3 Mon Sep 17 00:00:00 2001 -From: PshySimon -Date: Thu, 26 Sep 2024 16:12:25 +0800 -Subject: [PATCH] add pyxalarm and pySentryNotify, add multi users support for - xalarmd and adapt libxalarm - ---- - src/libso/xalarm/register_xalarm.c | 41 ++---- - src/libso/xalarm/register_xalarm.h | 10 +- - src/python/xalarm/register_xalarm.py | 192 +++++++++++++++++++++++++++ - src/python/xalarm/sentry_notify.py | 71 ++++++++++ - src/python/xalarm/xalarm_api.py | 18 ++- - src/python/xalarm/xalarm_server.py | 40 +++++- - src/python/xalarm/xalarm_transfer.py | 96 ++++++++++++-- - 7 files changed, 408 insertions(+), 60 deletions(-) - create mode 100644 src/python/xalarm/register_xalarm.py - create mode 100644 src/python/xalarm/sentry_notify.py - -diff --git a/src/libso/xalarm/register_xalarm.c b/src/libso/xalarm/register_xalarm.c -index 152c078..21a419f 100644 ---- a/src/libso/xalarm/register_xalarm.c -+++ b/src/libso/xalarm/register_xalarm.c -@@ -35,7 +35,7 @@ - #define ALARM_SOCKET_PERMISSION 0700 - #define TIME_UNIT_MILLISECONDS 1000 - --#define MAX_PARAS_LEN 511 -+#define MAX_PARAS_LEN 1023 - #define MIN_ALARM_ID 1001 - #define MAX_ALARM_ID (MIN_ALARM_ID + MAX_NUM_OF_ALARM_ID - 1) - -@@ -91,7 +91,7 @@ static int create_unix_socket(const char *path) - return -1; - } - -- fd = socket(AF_UNIX, SOCK_DGRAM, 0); -+ fd = socket(AF_UNIX, SOCK_STREAM, 0); - if (fd < 0) { - printf("socket failed:%s\n", strerror(errno)); - return -1; -@@ -103,14 +103,6 @@ static int create_unix_socket(const char *path) - goto release_socket; - } - -- if (access(PATH_REG_ALARM, F_OK) == 0) { -- ret = unlink(PATH_REG_ALARM); -- if (ret != 0) { -- printf("unlink register socket file failed\n"); -- goto release_socket; -- } -- } -- - if (access(DIR_XALARM, F_OK) == -1) { - if (mkdir(DIR_XALARM, ALARM_DIR_PERMISSION) == -1) { - printf("mkdir %s failed\n", DIR_XALARM); -@@ -120,32 +112,22 @@ static int create_unix_socket(const char *path) - - if (memset(&alarm_addr, 0, sizeof(alarm_addr)) == NULL) { - printf("create_unix_socket: memset alarm_addr failed, ret: %d\n", ret); -- goto remove_dir; -+ goto release_socket; - } - alarm_addr.sun_family = AF_UNIX; - strncpy(alarm_addr.sun_path, path, sizeof(alarm_addr.sun_path) - 1); - -- if (bind(fd, (struct sockaddr *)&alarm_addr, sizeof(alarm_addr.sun_family) + strlen(alarm_addr.sun_path)) < 0) { -- printf("bind socket failed:%s\n", strerror(errno)); -- goto remove_dir; -+ if (connect(fd, (struct sockaddr*)&alarm_addr, sizeof(alarm_addr)) == -1) { -+ printf("create_unix_socket: connect alarm_addr failed, ret: %d\n", ret); -+ goto release_socket; - } - if (chmod(path, ALARM_SOCKET_PERMISSION) < 0) { - printf("chmod %s failed: %s\n", path, strerror(errno)); -- goto unlink_sockfile; -+ goto release_socket; - } - - return fd; - --unlink_sockfile: -- ret = unlink(PATH_REG_ALARM); -- if (ret != 0) { -- printf("unlink register socket file failed\n"); -- } --remove_dir: -- ret = rmdir(DIR_XALARM); -- if (ret != 0) { -- printf("rmdir %s failed: %s\n", path, strerror(errno)); -- } - release_socket: - (void)close(fd); - -@@ -271,8 +253,6 @@ int xalarm_Register(alarm_callback_func callback, struct alarm_subscription_info - - void xalarm_UnRegister(int client_id) - { -- int ret; -- - if (!g_register_info.is_registered) { - printf("%s: alarm has not registered\n", __func__); - return; -@@ -292,10 +272,6 @@ void xalarm_UnRegister(int client_id) - if (g_register_info.register_fd != -1) { - (void)close(g_register_info.register_fd); - g_register_info.register_fd = -1; -- ret = unlink(PATH_REG_ALARM); -- if (ret != 0) { -- printf("%s: unlink register socket file failed\n", __func__); -- } - } - - memset(g_register_info.alarm_enable_bitmap, 0, MAX_NUM_OF_ALARM_ID * sizeof(char)); -@@ -357,7 +333,7 @@ int xalarm_Report(unsigned short usAlarmId, unsigned char ucAlarmLevel, - struct sockaddr_un alarm_addr; - - if ((usAlarmId < MIN_ALARM_ID || usAlarmId > MAX_ALARM_ID) || -- (ucAlarmLevel < ALARM_LEVEL_FATAL || ucAlarmLevel > ALARM_LEVEL_DEBUG) || -+ (ucAlarmLevel < MINOR_ALM || ucAlarmLevel > CRITICAL_ALM) || - (ucAlarmType < ALARM_TYPE_OCCUR || ucAlarmType > ALARM_TYPE_RECOVER)) { - fprintf(stderr, "%s: alarm info invalid\n", __func__); - return -1; -@@ -666,3 +642,4 @@ int report_result(const char *task_name, enum RESULT_LEVEL result_level, const c - return RETURE_CODE_SUCCESS; - } - -+ -diff --git a/src/libso/xalarm/register_xalarm.h b/src/libso/xalarm/register_xalarm.h -index 1f26c6a..fef9482 100644 ---- a/src/libso/xalarm/register_xalarm.h -+++ b/src/libso/xalarm/register_xalarm.h -@@ -11,7 +11,7 @@ - #include - #include - --#define ALARM_INFO_MAX_PARAS_LEN 512 -+#define ALARM_INFO_MAX_PARAS_LEN 1024 - #define MAX_STRERROR_SIZE 1024 - #define MAX_ALARM_TYEPS 1024 - #define MIN_ALARM_ID 1001 -@@ -19,11 +19,9 @@ - - #define MEMORY_ALARM_ID 1001 - --#define ALARM_LEVEL_FATAL 1 --#define ALARM_LEVEL_ERROR 2 --#define ALARM_LEVEL_WARNING 3 --#define ALARM_LEVEL_INFO 4 --#define ALARM_LEVEL_DEBUG 5 -+#define MINOR_ALM 1 -+#define MAJOR_ALM 2 -+#define CRITICAL_ALM 3 - - #define ALARM_TYPE_OCCUR 1 - #define ALARM_TYPE_RECOVER 2 -diff --git a/src/python/xalarm/register_xalarm.py b/src/python/xalarm/register_xalarm.py -new file mode 100644 -index 0000000..e58343d ---- /dev/null -+++ b/src/python/xalarm/register_xalarm.py -@@ -0,0 +1,192 @@ -+import os -+import sys -+import socket -+import logging -+import threading -+import time -+import fcntl -+import inspect -+from struct import error as StructParseError -+ -+from .xalarm_api import Xalarm, alarm_bin2stu -+ -+ -+ALARM_REPORT_LEN = 1048 -+MAX_NUM_OF_ALARM_ID=128 -+MIN_ALARM_ID = 1001 -+MAX_ALARM_ID = (MIN_ALARM_ID + MAX_NUM_OF_ALARM_ID - 1) -+DIR_XALARM = "/var/run/xalarm" -+PATH_REG_ALARM = "/var/run/xalarm/alarm" -+PATH_REPORT_ALARM = "/var/run/xalarm/report" -+ALARM_DIR_PERMISSION = 0o0750 -+ALARM_REG_SOCK_PERMISSION = 0o0700 -+ALARM_SOCKET_PERMISSION = 0o0700 -+TIME_UNIT_MILLISECONDS = 1000 -+ALARM_REGISTER_INFO = None -+ -+ -+class AlarmRegister: -+ def __init__(self, id_filter: list[bool], callback: callable): -+ self.id_filter = id_filter -+ self.callback = callback -+ self.socket = self.create_unix_socket() -+ self.is_registered = True -+ self.thread = threading.Thread(target=self.alarm_recv) -+ self.thread_should_stop = False -+ -+ def check_params(self) -> bool: -+ if (len(self.id_filter) != MAX_NUM_OF_ALARM_ID): -+ sys.stderr.write("check_params: invalid param id_filter\n") -+ return False -+ -+ sig = inspect.signature(self.callback) -+ if len(sig.parameters) != 1: -+ sys.stderr.write("check_params: invalid param callback\n") -+ return False -+ -+ if self.socket is None: -+ sys.stderr.write("check_params: scoket create failed\n") -+ return False -+ return True -+ -+ def set_id_filter(self, id_filter: list[bool]) -> bool: -+ if (len(id_filter) > MAX_NUM_OF_ALARM_ID): -+ sys.stderr.write("set_id_filter: invalid param id_filter\n") -+ return False -+ self.id_filter = id_filter -+ -+ def id_is_registered(self, alarm_id) -> bool: -+ if alarm_id < MIN_ALARM_ID or alarm_id > MAX_ALARM_ID: -+ return False -+ return self.id_filter[alarm_id - MIN_ALARM_ID] -+ -+ def put_alarm_info(self, alarm_info: Xalarm) -> None: -+ if not self.callback or not alarm_info: -+ return -+ if not self.id_is_registered(alarm_info.alarm_id): -+ return -+ self.callback(alarm_info) -+ -+ def create_unix_socket(self) -> socket.socket: -+ try: -+ sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) -+ sock.setblocking(False) -+ -+ if not os.access(DIR_XALARM, os.F_OK): -+ os.makedirs(DIR_XALARM) -+ os.chmod(DIR_XALARM, ALARM_DIR_PERMISSION) -+ -+ sock.connect(PATH_REG_ALARM) -+ return sock -+ except (IOError, OSError, FileNotFoundError) as e: -+ sock.close() -+ sys.stderr.write(f"create_unix_socket: create socket error:{e}\n") -+ return None -+ -+ def alarm_recv(self): -+ while not self.thread_should_stop: -+ try: -+ data = self.socket.recv(ALARM_REPORT_LEN) -+ if not data: -+ sys.stderr.write("connection closed by xalarmd, maybe connections reach max num or service stopped.\n") -+ self.thread_should_stop = True -+ break -+ if len(data) != ALARM_REPORT_LEN: -+ sys.stderr.write(f"server receive report msg length wrong {len(data)}\n") -+ continue -+ -+ alarm_info = alarm_bin2stu(data) -+ self.put_alarm_info(alarm_info) -+ except (BlockingIOError) as e: -+ time.sleep(0.1) -+ except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError): -+ sys.stderr.write("Connection closed by the server.\n") -+ self.thread_should_stop = True -+ except (ValueError, StructParseError, InterruptedError) as e: -+ sys.stderr.write(f"{e}\n") -+ except Exception as e: -+ sys.stderr.write(f"{e}\n") -+ self.thread_should_stop = True -+ -+ def start_thread(self) -> None: -+ self.thread.daemon = True -+ self.thread.start() -+ -+ def stop_thread(self) -> None: -+ self.thread_should_stop = True -+ self.thread.join() -+ self.socket.close() -+ -+ -+def xalarm_register(callback: callable, id_filter: list[bool]) -> int: -+ global ALARM_REGISTER_INFO -+ -+ if ALARM_REGISTER_INFO is not None: -+ sys.stderr.write("xalarm_register: alarm has registered\n") -+ return -1 -+ -+ ALARM_REGISTER_INFO = AlarmRegister(id_filter, callback) -+ if not ALARM_REGISTER_INFO.check_params(): -+ return -1 -+ -+ ALARM_REGISTER_INFO.start_thread() -+ -+ return 0 -+ -+ -+def xalarm_unregister(clientId: int) -> None: -+ global ALARM_REGISTER_INFO -+ if clientId < 0: -+ sys.stderr.write("xalarm_unregister: invalid client\n") -+ return -+ -+ if ALARM_REGISTER_INFO is None: -+ sys.stderr.write("xalarm_unregister: alarm has not registered\n") -+ return -+ -+ ALARM_REGISTER_INFO.stop_thread() -+ ALARM_REGISTER_INFO = None -+ -+ -+def xalarm_upgrade(clientId: int, id_filter: list[bool]) -> None: -+ global ALARM_REGISTER_INFO -+ if clientId < 0: -+ sys.stderr.write("xalarm_unregister: invalid client\n") -+ return -+ if ALARM_REGISTER_INFO is None: -+ sys.stderr.write("xalarm_unregister: alarm has not registered\n") -+ return -+ ALARM_REGISTER_INFO.id_filter = id_filter -+ -+ -+def xalarm_getid(alarm_info: Xalarm) -> int: -+ if not alarm_info: -+ return 0 -+ return alarm_info.alarm_id -+ -+ -+def xalarm_getlevel(alarm_info: Xalarm) -> int: -+ if not alarm_info: -+ return 0 -+ return alarm_info.alarm_level -+ -+ -+def xalarm_gettype(alarm_info: Xalarm) -> int: -+ if not alarm_info: -+ return 0 -+ return alarm_info.alarm_type -+ -+ -+def xalarm_gettime(alarm_info: Xalarm) -> int: -+ if not alarm_info: -+ return 0 -+ return alarm_info.timetamp.tv_sec * TIME_UNIT_MILLISECONDS + alarm_info.timetamp.tv_usec / TIME_UNIT_MILLISECONDS -+ -+def xalarm_getdesc(alarm_info: Xalarm) -> str: -+ if not alarm_info: -+ return None -+ try: -+ desc_str = alarm_info.msg1.rstrip(b'\x00').decode('utf-8') -+ except UnicodeError: -+ desc_str = None -+ return desc_str -diff --git a/src/python/xalarm/sentry_notify.py b/src/python/xalarm/sentry_notify.py -new file mode 100644 -index 0000000..a19e5b3 ---- /dev/null -+++ b/src/python/xalarm/sentry_notify.py -@@ -0,0 +1,71 @@ -+import os -+import sys -+import time -+import socket -+from struct import error as StructParseError -+ -+from .xalarm_api import alarm_stu2bin, Xalarm -+ -+MAX_NUM_OF_ALARM_ID = 128 -+MIN_ALARM_ID = 1001 -+MAX_ALARM_ID = (MIN_ALARM_ID + MAX_NUM_OF_ALARM_ID - 1) -+ -+MINOR_ALM = 1 -+MAJOR_ALM = 2 -+CRITICAL_ALM = 3 -+ -+ALARM_TYPE_OCCUR = 1 -+ALARM_TYPE_RECOVER = 2 -+ -+MAX_PUC_PARAS_LEN = 1024 -+ -+DIR_XALARM = "/var/run/xalarm" -+PATH_REPORT_ALARM = "/var/run/xalarm/report" -+ALARM_DIR_PERMISSION = 0o750 -+ALARM_SOCKET_PERMISSION = 0o700 -+ -+ -+def check_params(alarm_id, alarm_level, alarm_type, puc_paras) -> bool: -+ if not os.path.exists(DIR_XALARM): -+ sys.stderr.write(f"check_params: {DIR_XALARM} not exist, failed") -+ return False -+ -+ if not os.path.exists(PATH_REPORT_ALARM): -+ sys.stderr.write(f"check_params: {PATH_REPORT_ALARM} not exist, failed") -+ return False -+ -+ if (alarm_id < MIN_ALARM_ID or alarm_id > MAX_ALARM_ID or -+ alarm_level < MINOR_ALM or alarm_level > CRITICAL_ALM or -+ alarm_type < ALARM_TYPE_OCCUR or alarm_type > ALARM_TYPE_RECOVER): -+ sys.stderr.write("check_params: alarm info invalid\n") -+ return False -+ -+ if len(puc_paras) >= MAX_PUC_PARAS_LEN: -+ sys.stderr.write(f"check_params: alarm msg should be less than {MAX_PUC_PARAS_LEN}\n") -+ return False -+ -+ return True -+ -+def xalarm_report(alarm_id, alarm_level, alarm_type, puc_paras) -> bool: -+ if not check_params(alarm_id, alarm_level, alarm_type, puc_paras): -+ return False -+ -+ try: -+ sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) -+ -+ current_time = time.time() -+ current_time_seconds = int(current_time) -+ current_microseconds = int((current_time - current_time_seconds) * 1_000_000) -+ alarm_info = Xalarm(alarm_id, alarm_type, alarm_level, -+ current_time_seconds, current_microseconds, puc_paras) -+ -+ sock.sendto(alarm_stu2bin(alarm_info), PATH_REPORT_ALARM) -+ except (FileNotFoundError, StructParseError, socket.error, OSError, UnicodeError) as e: -+ sys.stderr.write(f"check_params: error occurs when sending msg.{e}\n") -+ return False -+ finally: -+ sock.close() -+ -+ return True -+ -+ -diff --git a/src/python/xalarm/xalarm_api.py b/src/python/xalarm/xalarm_api.py -index 94d7638..99eabf5 100644 ---- a/src/python/xalarm/xalarm_api.py -+++ b/src/python/xalarm/xalarm_api.py -@@ -23,6 +23,7 @@ ALARM_LEVELS = (1, 2, 3, 4, 5) - ALARM_SOCK_PATH = "/var/run/xalarm/report" - MIN_ALARM_ID = 1001 - MAX_ALARM_ID = 1128 -+MAX_MSG_LEN = 1024 - - - @dataclasses.dataclass -@@ -97,15 +98,15 @@ class Xalarm: - def msg1(self, msg): - """msg1 setter - """ -- if len(msg) > 512: -- raise ValueError("msg1 length must below 255") -+ if len(msg) > MAX_MSG_LEN: -+ raise ValueError(f"msg1 length must below {MAX_MSG_LEN}") - self._msg1 = msg - - - def alarm_bin2stu(bin_data): - """alarm binary to struct - """ -- struct_data = struct.unpack("@HBBll512s", bin_data) -+ struct_data = struct.unpack(f"@HBBll{MAX_MSG_LEN}s", bin_data) - - alarm_info = Xalarm(1001, 2, 1, 0, 0, "") - alarm_info.alarm_id = struct_data[0] -@@ -116,3 +117,14 @@ def alarm_bin2stu(bin_data): - alarm_info.msg1 = struct_data[5] - - return alarm_info -+ -+ -+def alarm_stu2bin(alarm_info: Xalarm): -+ return struct.pack( -+ f'@HBBll{MAX_MSG_LEN}s', -+ alarm_info.alarm_id, -+ alarm_info.alarm_level, -+ alarm_info.alarm_type, -+ alarm_info.timetamp.tv_sec, -+ alarm_info.timetamp.tv_usec, -+ alarm_info.msg1.encode('utf-8')) -diff --git a/src/python/xalarm/xalarm_server.py b/src/python/xalarm/xalarm_server.py -index 84db273..fcaf393 100644 ---- a/src/python/xalarm/xalarm_server.py -+++ b/src/python/xalarm/xalarm_server.py -@@ -17,16 +17,20 @@ Create: 2023-11-02 - import socket - import os - import logging -+import select -+import threading - from struct import error as StructParseError - - from .xalarm_api import alarm_bin2stu --from .xalarm_transfer import check_filter, transmit_alarm -+from .xalarm_transfer import check_filter, transmit_alarm, wait_for_connection - - - ALARM_DIR = "/var/run/xalarm" -+USER_RECV_SOCK = "/var/run/xalarm/alarm" - SOCK_FILE = "/var/run/xalarm/report" --ALARM_REPORT_LEN = 536 -+ALARM_REPORT_LEN = 1048 - ALARM_DIR_PERMISSION = 0o750 -+ALARM_LISTEN_QUEUE_LEN = 5 - - - def clear_sock_path(): -@@ -37,6 +41,8 @@ def clear_sock_path(): - os.chmod(ALARM_DIR, ALARM_DIR_PERMISSION) - if os.path.exists(SOCK_FILE): - os.unlink(SOCK_FILE) -+ if os.path.exists(USER_RECV_SOCK): -+ os.unlink(USER_RECV_SOCK) - - - def server_loop(alarm_config): -@@ -49,6 +55,21 @@ def server_loop(alarm_config): - sock.bind(SOCK_FILE) - os.chmod(SOCK_FILE, 0o600) - -+ alarm_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) -+ alarm_sock.bind(USER_RECV_SOCK) -+ os.chmod(USER_RECV_SOCK, 0o600) -+ alarm_sock.listen(ALARM_LISTEN_QUEUE_LEN) -+ alarm_sock.setblocking(False) -+ -+ epoll = select.epoll() -+ epoll.register(alarm_sock.fileno(), select.EPOLLIN) -+ fd_to_socket = {alarm_sock.fileno(): alarm_sock,} -+ thread_should_stop = False -+ -+ thread = threading.Thread(target=wait_for_connection, args=(alarm_sock, epoll, fd_to_socket, thread_should_stop)) -+ thread.daemon = True -+ thread.start() -+ - while True: - try: - data, _ = sock.recvfrom(ALARM_REPORT_LEN) -@@ -58,14 +79,21 @@ def server_loop(alarm_config): - logging.debug("server receive report msg length wrong %d", - len(data)) - continue -- - alarm_info = alarm_bin2stu(data) - logging.debug("server bin2stu msg") - if not check_filter(alarm_info, alarm_config): - continue -+ transmit_alarm(alarm_sock, epoll, fd_to_socket, data) -+ except Exception as e: -+ logging.error(f"Error server:{e}") -+ -+ thread_should_stop = True -+ thread.join() - -- transmit_alarm(data) -- except (ValueError, StructParseError): -- pass -+ epoll.unregister(alarm_sock.fileno()) -+ epoll.close() -+ alarm_sock.close() -+ os.unlink(USER_RECV_SOCK) - - sock.close() -+ -diff --git a/src/python/xalarm/xalarm_transfer.py b/src/python/xalarm/xalarm_transfer.py -index b590b43..42137d8 100644 ---- a/src/python/xalarm/xalarm_transfer.py -+++ b/src/python/xalarm/xalarm_transfer.py -@@ -16,10 +16,12 @@ Create: 2023-11-02 - - import socket - import logging -+import select - --USER_RECV_SOCK = "/var/run/xalarm/alarm" - MIN_ID_NUMBER = 1001 - MAX_ID_NUMBER = 1128 -+MAX_CONNECTION_NUM = 100 -+TEST_CONNECT_BUFFER_SIZE = 32 - - - def check_filter(alarm_info, alarm_filter): -@@ -35,16 +37,84 @@ def check_filter(alarm_info, alarm_filter): - return True - - --def transmit_alarm(bin_data): -- """forward alarm message -+def cleanup_closed_connections(server_sock, epoll, fd_to_socket): - """ -- sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) -- try: -- sock.sendto(bin_data, USER_RECV_SOCK) -- logging.debug("transfer alarm success") -- except ConnectionRefusedError: -- logging.debug("transfer sendto failed") -- except FileNotFoundError: -- logging.debug("transfer sendto failed") -- finally: -- sock.close() -+ clean invalid client socket connections saved in 'fd_to_socket' -+ :param server_sock: server socket instance of alarm -+ :param epoll: epoll instance, used to unregister invalid client connections -+ :param fd_to_socket: dict instance, used to hold client connections and server connections -+ """ -+ to_remove = [] -+ for fileno, connection in fd_to_socket.items(): -+ if connection is server_sock: -+ continue -+ try: -+ # test whether connection still alive, use MSG_DONTWAIT to avoid blocking thread -+ # use MSG_PEEK to avoid consuming buffer data -+ data = connection.recv(TEST_CONNECT_BUFFER_SIZE, socket.MSG_DONTWAIT | socket.MSG_PEEK) -+ if not data: -+ to_remove.append(fileno) -+ except BlockingIOError: -+ pass -+ except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError): -+ to_remove.append(fileno) -+ -+ for fileno in to_remove: -+ epoll.unregister(fileno) -+ fd_to_socket[fileno].close() -+ del fd_to_socket[fileno] -+ logging.info(f"cleaned up connection {fileno} for client lost connection.") -+ -+ -+def wait_for_connection(server_sock, epoll, fd_to_socket, thread_should_stop): -+ """ -+ thread function for catch and save client connection -+ :param server_sock: server socket instance of alarm -+ :param epoll: epoll instance, used to unregister invalid client connections -+ :param fd_to_socket: dict instance, used to hold client connections and server connections -+ :param thread_should_stop: bool instance -+ """ -+ while not thread_should_stop: -+ try: -+ events = epoll.poll(1) -+ -+ for fileno, event in events: -+ if fileno == server_sock.fileno(): -+ connection, client_address = server_sock.accept() -+ # if reach max connection, cleanup closed connections -+ if len(fd_to_socket) - 1 >= MAX_CONNECTION_NUM: -+ cleanup_closed_connections(server_sock, epoll, fd_to_socket) -+ # if connections still reach max num, close this connection automatically -+ if len(fd_to_socket) - 1 >= MAX_CONNECTION_NUM: -+ logging.info(f"connection reach max num of {MAX_CONNECTION_NUM}, closed current connection!") -+ connection.close() -+ continue -+ epoll.register(connection.fileno(), select.EPOLLOUT) -+ fd_to_socket[connection.fileno()] = connection -+ except socket.error as e: -+ logging.debug(f"socket error, reason is {e}") -+ break -+ except (KeyError, OSError, ValueError) as e: -+ logging.debug(f"wait for connection failed {e}") -+ -+ -+def transmit_alarm(server_sock, epoll, fd_to_socket, bin_data): -+ """ -+ this function is to broadcast alarm data to client, if fail to send data, remove connections held by fd_to_socket -+ :param server_sock: server socket instance of alarm -+ :param epoll: epoll instance, used to unregister invalid client connections -+ :param fd_to_socket: dict instance, used to hold client connections and server connections -+ :param bin_data: binary instance, alarm info data in C-style struct format defined in xalarm_api.py -+ """ -+ to_remove = [] -+ for fileno, connection in fd_to_socket.items(): -+ if connection is not server_sock: -+ try: -+ connection.sendall(bin_data) -+ except (BrokenPipeError, ConnectionResetError): -+ to_remove.append(fileno) -+ for fileno in to_remove: -+ epoll.unregister(fileno) -+ fd_to_socket[fileno].close() -+ del fd_to_socket[fileno] -+ --- -2.27.0 - diff --git a/add-root-cause-analysis.patch b/add-root-cause-analysis.patch deleted file mode 100644 index 94de7ff..0000000 --- a/add-root-cause-analysis.patch +++ /dev/null @@ -1,1253 +0,0 @@ -From 24f8eddad364e83cfc5b6b1607462ffe524b59f1 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?=E8=B4=BA=E6=9C=89=E5=BF=97?= <1037617413@qq.com> -Date: Sat, 12 Oct 2024 21:59:18 +0800 -Subject: [PATCH] add root cause analysis - ---- - config/plugins/ai_block_io.ini | 15 +- - .../sentryPlugins/ai_block_io/ai_block_io.py | 133 +++-- - .../ai_block_io/config_parser.py | 465 +++++++++++------- - .../sentryPlugins/ai_block_io/data_access.py | 1 + - .../sentryPlugins/ai_block_io/detector.py | 54 +- - .../sentryPlugins/ai_block_io/io_data.py | 32 +- - .../ai_block_io/sliding_window.py | 57 ++- - src/python/sentryPlugins/ai_block_io/utils.py | 44 +- - 8 files changed, 491 insertions(+), 310 deletions(-) - -diff --git a/config/plugins/ai_block_io.ini b/config/plugins/ai_block_io.ini -index a814d52..422cfa3 100644 ---- a/config/plugins/ai_block_io.ini -+++ b/config/plugins/ai_block_io.ini -@@ -2,7 +2,6 @@ - level=info - - [common] --absolute_threshold=40 - slow_io_detect_frequency=1 - disk=default - stage=bio -@@ -18,4 +17,16 @@ n_sigma_parameter=3 - [sliding_window] - sliding_window_type=not_continuous - window_size=30 --window_minimum_threshold=6 -\ No newline at end of file -+window_minimum_threshold=6 -+ -+[latency_sata_ssd] -+read_tot_lim=50000 -+write_tot_lim=50000 -+ -+[latency_nvme_ssd] -+read_tot_lim=500 -+write_tot_lim=500 -+ -+[latency_sata_hdd] -+read_tot_lim=50000 -+write_tot_lim=50000 -\ No newline at end of file -diff --git a/src/python/sentryPlugins/ai_block_io/ai_block_io.py b/src/python/sentryPlugins/ai_block_io/ai_block_io.py -index e1052ec..dd661a1 100644 ---- a/src/python/sentryPlugins/ai_block_io/ai_block_io.py -+++ b/src/python/sentryPlugins/ai_block_io/ai_block_io.py -@@ -12,13 +12,18 @@ - import time - import signal - import logging -+from collections import defaultdict - - from .detector import Detector, DiskDetector --from .threshold import ThresholdFactory, AbsoluteThreshold -+from .threshold import ThresholdFactory - from .sliding_window import SlidingWindowFactory - from .utils import get_data_queue_size_and_update_size - from .config_parser import ConfigParser --from .data_access import get_io_data_from_collect_plug, check_collect_valid -+from .data_access import ( -+ get_io_data_from_collect_plug, -+ check_collect_valid, -+ get_disk_type, -+) - from .io_data import MetricName - from .alarm_report import Xalarm, Report - -@@ -34,7 +39,7 @@ def sig_handler(signum, frame): - class SlowIODetection: - _config_parser = None - _disk_list = None -- _detector_name_list = {} -+ _detector_name_list = defaultdict(list) - _disk_detectors = {} - - def __init__(self, config_parser: ConfigParser): -@@ -43,9 +48,13 @@ class SlowIODetection: - self.__init_detector() - - def __init_detector_name_list(self): -- self._disk_list = check_collect_valid(self._config_parser.slow_io_detect_frequency) -+ self._disk_list = check_collect_valid( -+ self._config_parser.slow_io_detect_frequency -+ ) - if self._disk_list is None: -- Report.report_pass("get available disk error, please check if the collector plug is enable. exiting...") -+ Report.report_pass( -+ "get available disk error, please check if the collector plug is enable. exiting..." -+ ) - exit(1) - - logging.info(f"ai_block_io plug has found disks: {self._disk_list}") -@@ -56,27 +65,45 @@ class SlowIODetection: - # 情况2：is not None and len = 0，则不启动任何磁盘检测 - # 情况3：len ！= 0，则取交集 - if disks is None: -- logging.warning("you not specify any disk or use default, so ai_block_io will enable all available disk.") -- for disk in self._disk_list: -- for stage in stages: -- for iotype in iotypes: -- if disk not in self._detector_name_list: -- self._detector_name_list[disk] = [] -- self._detector_name_list[disk].append(MetricName(disk, stage, iotype, "latency")) -- else: -- for disk in disks: -- if disk in self._disk_list: -- for stage in stages: -- for iotype in iotypes: -- if disk not in self._detector_name_list: -- self._detector_name_list[disk] = [] -- self._detector_name_list[disk].append(MetricName(disk, stage, iotype, "latency")) -- else: -- logging.warning("disk: [%s] not in available disk list, so it will be ignored.", disk) -- if len(self._detector_name_list) == 0: -- logging.critical("the disks to detection is empty, ai_block_io will exit.") -- Report.report_pass("the disks to detection is empty, ai_block_io will exit.") -- exit(1) -+ logging.warning( -+ "you not specify any disk or use default, so ai_block_io will enable all available disk." -+ ) -+ for disk in self._disk_list: -+ if disks is not None: -+ if disk not in disks: -+ continue -+ disks.remove(disk) -+ -+ disk_type_result = get_disk_type(disk) -+ if disk_type_result["ret"] == 0 and disk_type_result["message"] in ( -+ '0', -+ '1', -+ '2', -+ ): -+ disk_type = int(disk_type_result["message"]) -+ else: -+ logging.warning( -+ "%s get disk type error, return %s, so it will be ignored.", -+ disk, -+ disk_type_result, -+ ) -+ continue -+ for stage in stages: -+ for iotype in iotypes: -+ self._detector_name_list[disk].append( -+ MetricName(disk, disk_type, stage, iotype, "latency") -+ ) -+ if disks: -+ logging.warning( -+ "disks: %s not in available disk list, so they will be ignored.", -+ disks, -+ ) -+ if not self._detector_name_list: -+ logging.critical("the disks to detection is empty, ai_block_io will exit.") -+ Report.report_pass( -+ "the disks to detection is empty, ai_block_io will exit." -+ ) -+ exit(1) - - def __init_detector(self): - train_data_duration, train_update_duration = ( -@@ -88,26 +115,39 @@ class SlowIODetection: - train_data_duration, train_update_duration, slow_io_detection_frequency - ) - sliding_window_type = self._config_parser.sliding_window_type -- window_size, window_threshold = (self._config_parser.get_window_size_and_window_minimum_threshold()) -+ window_size, window_threshold = ( -+ self._config_parser.get_window_size_and_window_minimum_threshold() -+ ) - - for disk, metric_name_list in self._detector_name_list.items(): -- threshold = ThresholdFactory().get_threshold( -- threshold_type, -- boxplot_parameter=self._config_parser.boxplot_parameter, -- n_sigma_paramter=self._config_parser.n_sigma_parameter, -- data_queue_size=data_queue_size, -- data_queue_update_size=update_size, -- ) -- sliding_window = SlidingWindowFactory().get_sliding_window( -- sliding_window_type, -- queue_length=window_size, -- threshold=window_threshold, -- ) - disk_detector = DiskDetector(disk) - for metric_name in metric_name_list: -+ threshold = ThresholdFactory().get_threshold( -+ threshold_type, -+ boxplot_parameter=self._config_parser.boxplot_parameter, -+ n_sigma_paramter=self._config_parser.n_sigma_parameter, -+ data_queue_size=data_queue_size, -+ data_queue_update_size=update_size, -+ ) -+ abs_threshold = self._config_parser.get_tot_lim( -+ metric_name.disk_type, metric_name.io_access_type_name -+ ) -+ if abs_threshold is None: -+ logging.warning( -+ "disk %s, disk type %s, io type %s, get tot lim error, so it will be ignored.", -+ disk, -+ metric_name.disk_type, -+ metric_name.io_access_type_name, -+ ) -+ sliding_window = SlidingWindowFactory().get_sliding_window( -+ sliding_window_type, -+ queue_length=window_size, -+ threshold=window_threshold, -+ abs_threshold=abs_threshold, -+ ) - detector = Detector(metric_name, threshold, sliding_window) - disk_detector.add_detector(detector) -- logging.info(f'disk: [{disk}] add detector:\n [{disk_detector}]') -+ logging.info(f"disk: [{disk}] add detector:\n [{disk_detector}]") - self._disk_detectors[disk] = disk_detector - - def launch(self): -@@ -138,14 +178,17 @@ class SlowIODetection: - logging.debug("step3. Report slow io event to sysSentry.") - for slow_io_event in slow_io_event_list: - metric_name: MetricName = slow_io_event[1] -+ window_info = slow_io_event[2] -+ root_cause = slow_io_event[3] - alarm_content = { -- "driver_name": f"{metric_name.get_disk_name()}", -- "reason": "disk_slow", -- "block_stack": f"{metric_name.get_stage_name()}", -- "io_type": f"{metric_name.get_io_access_type_name()}", -+ "driver_name": f"{metric_name.disk_name}", -+ "reason": root_cause, -+ "block_stack": f"{metric_name.stage_name}", -+ "io_type": f"{metric_name.io_access_type_name}", - "alarm_source": "ai_block_io", - "alarm_type": "latency", -- "details": f"current window is: {slow_io_event[2]}, threshold is: {slow_io_event[3]}.", -+ "details": f"disk type: {metric_name.disk_type}, current window: {window_info[1]}, " -+ f"ai threshold: {window_info[2]}, abs threshold: {window_info[3]}.", - } - Xalarm.major(alarm_content) - logging.warning(alarm_content) -diff --git a/src/python/sentryPlugins/ai_block_io/config_parser.py b/src/python/sentryPlugins/ai_block_io/config_parser.py -index a357766..3388cd4 100644 ---- a/src/python/sentryPlugins/ai_block_io/config_parser.py -+++ b/src/python/sentryPlugins/ai_block_io/config_parser.py -@@ -20,59 +20,62 @@ from .utils import get_threshold_type_enum, get_sliding_window_type_enum, get_lo - - LOG_FORMAT = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s" - --ALL_STAGE_LIST = ['throtl', 'wbt', 'gettag', 'plug', 'deadline', 'hctx', 'requeue', 'rq_driver', 'bio'] --ALL_IOTPYE_LIST = ['read', 'write'] -+ALL_STAGE_LIST = [ -+ "throtl", -+ "wbt", -+ "gettag", -+ "plug", -+ "deadline", -+ "hctx", -+ "requeue", -+ "rq_driver", -+ "bio", -+] -+ALL_IOTPYE_LIST = ["read", "write"] -+DISK_TYPE_MAP = { -+ 0: "nvme_ssd", -+ 1: "sata_ssd", -+ 2: "sata_hdd", -+} - - - def init_log_format(log_level: str): - logging.basicConfig(level=get_log_level(log_level.lower()), format=LOG_FORMAT) - if log_level.lower() not in ("info", "warning", "error", "debug"): - logging.warning( -- f"the log_level: {log_level} you set is invalid, use default value: info." -+ "the log_level: %s you set is invalid, use default value: info.", log_level - ) - - - class ConfigParser: -- DEFAULT_ABSOLUTE_THRESHOLD = 40 -- DEFAULT_SLOW_IO_DETECTION_FREQUENCY = 1 -- DEFAULT_LOG_LEVEL = "info" -- -- DEFAULT_STAGE = 'throtl,wbt,gettag,plug,deadline,hctx,requeue,rq_driver,bio' -- DEFAULT_IOTYPE = 'read,write' -- -- DEFAULT_ALGORITHM_TYPE = "boxplot" -- DEFAULT_TRAIN_DATA_DURATION = 24 -- DEFAULT_TRAIN_UPDATE_DURATION = 2 -- DEFAULT_BOXPLOT_PARAMETER = 1.5 -- DEFAULT_N_SIGMA_PARAMETER = 3 -- -- DEFAULT_SLIDING_WINDOW_TYPE = "not_continuous" -- DEFAULT_WINDOW_SIZE = 30 -- DEFAULT_WINDOW_MINIMUM_THRESHOLD = 6 -+ DEFAULT_CONF = { -+ "log": {"level": "info"}, -+ "common": { -+ "slow_io_detect_frequency": 1, -+ "disk": None, -+ "stage": "throtl,wbt,gettag,plug,deadline,hctx,requeue,rq_driver,bio", -+ "iotype": "read,write", -+ }, -+ "algorithm": { -+ "train_data_duration": 24.0, -+ "train_update_duration": 2.0, -+ "algorithm_type": get_threshold_type_enum("boxplot"), -+ "boxplot_parameter": 1.5, -+ "n_sigma_parameter": 3.0, -+ }, -+ "sliding_window": { -+ "sliding_window_type": get_sliding_window_type_enum("not_continuous"), -+ "window_size": 30, -+ "window_minimum_threshold": 6, -+ }, -+ "latency_sata_ssd": {"read_tot_lim": 50000, "write_tot_lim": 50000}, -+ "latency_nvme_ssd": {"read_tot_lim": 500, "write_tot_lim": 500}, -+ "latency_sata_hdd": {"read_tot_lim": 50000, "write_tot_lim": 50000}, -+ } - - def __init__(self, config_file_name): -- self.__absolute_threshold = ConfigParser.DEFAULT_ABSOLUTE_THRESHOLD -- self.__slow_io_detect_frequency = ( -- ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY -- ) -- self.__log_level = ConfigParser.DEFAULT_LOG_LEVEL -- self.__disks_to_detection = None -- self.__stage = ConfigParser.DEFAULT_STAGE -- self.__iotype = ConfigParser.DEFAULT_IOTYPE -- -- self.__algorithm_type = get_threshold_type_enum( -- ConfigParser.DEFAULT_ALGORITHM_TYPE -- ) -- self.__train_data_duration = ConfigParser.DEFAULT_TRAIN_UPDATE_DURATION -- self.__train_update_duration = ConfigParser.DEFAULT_TRAIN_UPDATE_DURATION -- self.__boxplot_parameter = ConfigParser.DEFAULT_BOXPLOT_PARAMETER -- self.__n_sigma_parameter = ConfigParser.DEFAULT_N_SIGMA_PARAMETER -- -- self.__sliding_window_type = ConfigParser.DEFAULT_SLIDING_WINDOW_TYPE -- self.__window_size = ConfigParser.DEFAULT_WINDOW_SIZE -- self.__window_minimum_threshold = ConfigParser.DEFAULT_WINDOW_MINIMUM_THRESHOLD -- -- self.__config_file_name = config_file_name -+ self._conf = ConfigParser.DEFAULT_CONF -+ self._config_file_name = config_file_name - - def _get_config_value( - self, -@@ -156,30 +159,21 @@ class ConfigParser: - - return value - -- def __read_absolute_threshold(self, items_common: dict): -- self.__absolute_threshold = self._get_config_value( -- items_common, -- "absolute_threshold", -- float, -- ConfigParser.DEFAULT_ABSOLUTE_THRESHOLD, -- gt=0, -- ) -- -- def __read__slow_io_detect_frequency(self, items_common: dict): -- self.__slow_io_detect_frequency = self._get_config_value( -+ def _read_slow_io_detect_frequency(self, items_common: dict): -+ self._conf["common"]["slow_io_detect_frequency"] = self._get_config_value( - items_common, - "slow_io_detect_frequency", - int, -- ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY, -+ self.DEFAULT_CONF["common"]["slow_io_detect_frequency"], - gt=0, - le=300, - ) - -- def __read__disks_to_detect(self, items_common: dict): -+ def _read_disks_to_detect(self, items_common: dict): - disks_to_detection = items_common.get("disk") - if disks_to_detection is None: - logging.warning("config of disk not found, the default value will be used.") -- self.__disks_to_detection = None -+ self._conf["common"]["disk"] = None - return - disks_to_detection = disks_to_detection.strip() - if not disks_to_detection: -@@ -189,40 +183,46 @@ class ConfigParser: - ) - exit(1) - disk_list = disks_to_detection.split(",") -+ disk_list = [disk.strip() for disk in disk_list] - if len(disk_list) == 1 and disk_list[0] == "default": -- self.__disks_to_detection = None -+ self._conf["common"]["disk"] = None - return -- self.__disks_to_detection = disk_list -+ self._conf["common"]["disk"] = disk_list - -- def __read__train_data_duration(self, items_algorithm: dict): -- self.__train_data_duration = self._get_config_value( -+ def _read_train_data_duration(self, items_algorithm: dict): -+ self._conf["common"]["train_data_duration"] = self._get_config_value( - items_algorithm, - "train_data_duration", - float, -- ConfigParser.DEFAULT_TRAIN_DATA_DURATION, -+ self.DEFAULT_CONF["algorithm"]["train_data_duration"], - gt=0, - le=720, - ) - -- def __read__train_update_duration(self, items_algorithm: dict): -- default_train_update_duration = ConfigParser.DEFAULT_TRAIN_UPDATE_DURATION -- if default_train_update_duration > self.__train_data_duration: -- default_train_update_duration = self.__train_data_duration / 2 -- self.__train_update_duration = self._get_config_value( -+ def _read_train_update_duration(self, items_algorithm: dict): -+ default_train_update_duration = self.DEFAULT_CONF["algorithm"][ -+ "train_update_duration" -+ ] -+ if default_train_update_duration > self._conf["common"]["train_data_duration"]: -+ default_train_update_duration = ( -+ self._conf["common"]["train_data_duration"] / 2 -+ ) -+ self._conf["common"]["train_update_duration"] = self._get_config_value( - items_algorithm, - "train_update_duration", - float, - default_train_update_duration, - gt=0, -- le=self.__train_data_duration, -+ le=self._conf["common"]["train_data_duration"], - ) - -- def __read__algorithm_type_and_parameter(self, items_algorithm: dict): -- algorithm_type = items_algorithm.get( -- "algorithm_type", ConfigParser.DEFAULT_ALGORITHM_TYPE -- ) -- self.__algorithm_type = get_threshold_type_enum(algorithm_type) -- if self.__algorithm_type is None: -+ def _read_algorithm_type_and_parameter(self, items_algorithm: dict): -+ algorithm_type = items_algorithm.get("algorithm_type") -+ if algorithm_type is not None: -+ self._conf["algorithm"]["algorithm_type"] = get_threshold_type_enum( -+ algorithm_type -+ ) -+ if self._conf["algorithm"]["algorithm_type"] is None: - logging.critical( - "the algorithm_type: %s you set is invalid. ai_block_io plug will exit.", - algorithm_type, -@@ -231,129 +231,175 @@ class ConfigParser: - f"the algorithm_type: {algorithm_type} you set is invalid. ai_block_io plug will exit." - ) - exit(1) -- -- if self.__algorithm_type == ThresholdType.NSigmaThreshold: -- self.__n_sigma_parameter = self._get_config_value( -+ elif self._conf["algorithm"]["algorithm_type"] == ThresholdType.NSigmaThreshold: -+ self._conf["algorithm"]["n_sigma_parameter"] = self._get_config_value( - items_algorithm, - "n_sigma_parameter", - float, -- ConfigParser.DEFAULT_N_SIGMA_PARAMETER, -+ self.DEFAULT_CONF["algorithm"]["n_sigma_parameter"], - gt=0, - le=10, - ) -- elif self.__algorithm_type == ThresholdType.BoxplotThreshold: -- self.__boxplot_parameter = self._get_config_value( -+ elif ( -+ self._conf["algorithm"]["algorithm_type"] == ThresholdType.BoxplotThreshold -+ ): -+ self._conf["algorithm"]["boxplot_parameter"] = self._get_config_value( - items_algorithm, - "boxplot_parameter", - float, -- ConfigParser.DEFAULT_BOXPLOT_PARAMETER, -+ self.DEFAULT_CONF["algorithm"]["boxplot_parameter"], - gt=0, - le=10, - ) - -- def __read__stage(self, items_algorithm: dict): -- stage_str = items_algorithm.get('stage', ConfigParser.DEFAULT_STAGE) -- stage_list = stage_str.split(',') -- if len(stage_list) == 1 and stage_list[0] == '': -- logging.critical('stage value not allow is empty, exiting...') -+ def _read_stage(self, items_algorithm: dict): -+ stage_str = items_algorithm.get( -+ "stage", self.DEFAULT_CONF["common"]["stage"] -+ ).strip() -+ stage_list = stage_str.split(",") -+ stage_list = [stage.strip() for stage in stage_list] -+ if len(stage_list) == 1 and stage_list[0] == "": -+ logging.critical("stage value not allow is empty, exiting...") - exit(1) -- if len(stage_list) == 1 and stage_list[0] == 'default': -- logging.warning(f'stage will enable default value: {ConfigParser.DEFAULT_STAGE}') -- self.__stage = ALL_STAGE_LIST -+ if len(stage_list) == 1 and stage_list[0] == "default": -+ logging.warning( -+ "stage will enable default value: %s", -+ self.DEFAULT_CONF["common"]["stage"], -+ ) -+ self._conf["common"]["stage"] = ALL_STAGE_LIST - return - for stage in stage_list: - if stage not in ALL_STAGE_LIST: -- logging.critical(f'stage: {stage} is not valid stage, ai_block_io will exit...') -+ logging.critical( -+ "stage: %s is not valid stage, ai_block_io will exit...", stage -+ ) - exit(1) - dup_stage_list = set(stage_list) -- if 'bio' not in dup_stage_list: -- logging.critical('stage must contains bio stage, exiting...') -+ if "bio" not in dup_stage_list: -+ logging.critical("stage must contains bio stage, exiting...") - exit(1) -- self.__stage = dup_stage_list -- -- def __read__iotype(self, items_algorithm: dict): -- iotype_str = items_algorithm.get('iotype', ConfigParser.DEFAULT_IOTYPE) -- iotype_list = iotype_str.split(',') -- if len(iotype_list) == 1 and iotype_list[0] == '': -- logging.critical('iotype value not allow is empty, exiting...') -+ self._conf["common"]["stage"] = dup_stage_list -+ -+ def _read_iotype(self, items_algorithm: dict): -+ iotype_str = items_algorithm.get( -+ "iotype", self.DEFAULT_CONF["common"]["iotype"] -+ ).strip() -+ iotype_list = iotype_str.split(",") -+ iotype_list = [iotype.strip() for iotype in iotype_list] -+ if len(iotype_list) == 1 and iotype_list[0] == "": -+ logging.critical("iotype value not allow is empty, exiting...") - exit(1) -- if len(iotype_list) == 1 and iotype_list[0] == 'default': -- logging.warning(f'iotype will enable default value: {ConfigParser.DEFAULT_IOTYPE}') -- self.__iotype = ALL_IOTPYE_LIST -+ if len(iotype_list) == 1 and iotype_list[0] == "default": -+ logging.warning( -+ "iotype will enable default value: %s", -+ self.DEFAULT_CONF["common"]["iotype"], -+ ) -+ self._conf["common"]["iotype"] = ALL_IOTPYE_LIST - return - for iotype in iotype_list: - if iotype not in ALL_IOTPYE_LIST: -- logging.critical(f'iotype: {iotype} is not valid iotype, ai_block_io will exit...') -+ logging.critical( -+ "iotype: %s is not valid iotype, ai_block_io will exit...", iotype -+ ) - exit(1) - dup_iotype_list = set(iotype_list) -- self.__iotype = dup_iotype_list -+ self._conf["common"]["iotype"] = dup_iotype_list -+ -+ def _read_sliding_window_type(self, items_sliding_window: dict): -+ sliding_window_type = items_sliding_window.get("sliding_window_type") -+ if sliding_window_type is not None: -+ self._conf["sliding_window"]["sliding_window_type"] = ( -+ get_sliding_window_type_enum(sliding_window_type) -+ ) -+ if self._conf["sliding_window"]["sliding_window_type"] is None: -+ logging.critical( -+ "the sliding_window_type: %s you set is invalid. ai_block_io plug will exit.", -+ sliding_window_type, -+ ) -+ Report.report_pass( -+ f"the sliding_window_type: {sliding_window_type} you set is invalid. ai_block_io plug will exit." -+ ) -+ exit(1) - -- def __read__window_size(self, items_sliding_window: dict): -- self.__window_size = self._get_config_value( -+ def _read_window_size(self, items_sliding_window: dict): -+ self._conf["sliding_window"]["window_size"] = self._get_config_value( - items_sliding_window, - "window_size", - int, -- ConfigParser.DEFAULT_WINDOW_SIZE, -+ self.DEFAULT_CONF["sliding_window"]["window_size"], - gt=0, - le=3600, - ) - -- def __read__window_minimum_threshold(self, items_sliding_window: dict): -- default_window_minimum_threshold = ConfigParser.DEFAULT_WINDOW_MINIMUM_THRESHOLD -- if default_window_minimum_threshold > self.__window_size: -- default_window_minimum_threshold = self.__window_size / 2 -- self.__window_minimum_threshold = self._get_config_value( -- items_sliding_window, -- "window_minimum_threshold", -- int, -- default_window_minimum_threshold, -- gt=0, -- le=self.__window_size, -+ def _read_window_minimum_threshold(self, items_sliding_window: dict): -+ default_window_minimum_threshold = self.DEFAULT_CONF["sliding_window"][ -+ "window_minimum_threshold" -+ ] -+ if ( -+ default_window_minimum_threshold -+ > self._conf["sliding_window"]["window_size"] -+ ): -+ default_window_minimum_threshold = ( -+ self._conf["sliding_window"]["window_size"] / 2 -+ ) -+ self._conf["sliding_window"]["window_minimum_threshold"] = ( -+ self._get_config_value( -+ items_sliding_window, -+ "window_minimum_threshold", -+ int, -+ default_window_minimum_threshold, -+ gt=0, -+ le=self._conf["sliding_window"]["window_size"], -+ ) - ) - - def read_config_from_file(self): -- if not os.path.exists(self.__config_file_name): -- init_log_format(self.__log_level) -+ if not os.path.exists(self._config_file_name): -+ init_log_format(self._conf["log"]["level"]) - logging.critical( - "config file %s not found, ai_block_io plug will exit.", -- self.__config_file_name, -+ self._config_file_name, - ) - Report.report_pass( -- f"config file {self.__config_file_name} not found, ai_block_io plug will exit." -+ f"config file {self._config_file_name} not found, ai_block_io plug will exit." - ) - exit(1) - - con = configparser.ConfigParser() - try: -- con.read(self.__config_file_name, encoding="utf-8") -+ con.read(self._config_file_name, encoding="utf-8") - except configparser.Error as e: -- init_log_format(self.__log_level) -+ init_log_format(self._conf["log"]["level"]) - logging.critical( -- f"config file read error: %s, ai_block_io plug will exit.", e -+ "config file read error: %s, ai_block_io plug will exit.", e - ) - Report.report_pass( - f"config file read error: {e}, ai_block_io plug will exit." - ) - exit(1) - -- if con.has_section('log'): -- items_log = dict(con.items('log')) -+ if con.has_section("log"): -+ items_log = dict(con.items("log")) - # 情况一：没有log，则使用默认值 - # 情况二：有log，值为空或异常，使用默认值 - # 情况三：有log，值正常，则使用该值 -- self.__log_level = items_log.get('level', ConfigParser.DEFAULT_LOG_LEVEL) -- init_log_format(self.__log_level) -+ self._conf["log"]["level"] = items_log.get( -+ "level", self.DEFAULT_CONF["log"]["level"] -+ ) -+ init_log_format(self._conf["log"]["level"]) - else: -- init_log_format(self.__log_level) -- logging.warning(f"log section parameter not found, it will be set to default value.") -+ init_log_format(self._conf["log"]["level"]) -+ logging.warning( -+ "log section parameter not found, it will be set to default value." -+ ) - - if con.has_section("common"): - items_common = dict(con.items("common")) -- self.__read_absolute_threshold(items_common) -- self.__read__slow_io_detect_frequency(items_common) -- self.__read__disks_to_detect(items_common) -- self.__read__stage(items_common) -- self.__read__iotype(items_common) -+ -+ self._read_slow_io_detect_frequency(items_common) -+ self._read_disks_to_detect(items_common) -+ self._read_stage(items_common) -+ self._read_iotype(items_common) - else: - logging.warning( - "common section parameter not found, it will be set to default value." -@@ -361,9 +407,9 @@ class ConfigParser: - - if con.has_section("algorithm"): - items_algorithm = dict(con.items("algorithm")) -- self.__read__train_data_duration(items_algorithm) -- self.__read__train_update_duration(items_algorithm) -- self.__read__algorithm_type_and_parameter(items_algorithm) -+ self._read_train_data_duration(items_algorithm) -+ self._read_train_update_duration(items_algorithm) -+ self._read_algorithm_type_and_parameter(items_algorithm) - else: - logging.warning( - "algorithm section parameter not found, it will be set to default value." -@@ -371,101 +417,162 @@ class ConfigParser: - - if con.has_section("sliding_window"): - items_sliding_window = dict(con.items("sliding_window")) -- sliding_window_type = items_sliding_window.get( -- "sliding_window_type", ConfigParser.DEFAULT_SLIDING_WINDOW_TYPE -+ -+ self._read_window_size(items_sliding_window) -+ self._read_window_minimum_threshold(items_sliding_window) -+ else: -+ logging.warning( -+ "sliding_window section parameter not found, it will be set to default value." -+ ) -+ -+ if con.has_section("latency_sata_ssd"): -+ items_latency_sata_ssd = dict(con.items("latency_sata_ssd")) -+ self._conf["latency_sata_ssd"]["read_tot_lim"] = self._get_config_value( -+ items_latency_sata_ssd, -+ "read_tot_lim", -+ int, -+ self.DEFAULT_CONF["latency_sata_ssd"]["read_tot_lim"], -+ gt=0, - ) -- self.__sliding_window_type = get_sliding_window_type_enum( -- sliding_window_type -+ self._conf["latency_sata_ssd"]["write_tot_lim"] = self._get_config_value( -+ items_latency_sata_ssd, -+ "write_tot_lim", -+ int, -+ self.DEFAULT_CONF["latency_sata_ssd"]["write_tot_lim"], -+ gt=0, - ) -- self.__read__window_size(items_sliding_window) -- self.__read__window_minimum_threshold(items_sliding_window) - else: - logging.warning( -- "sliding_window section parameter not found, it will be set to default value." -+ "latency_sata_ssd section parameter not found, it will be set to default value." -+ ) -+ if con.has_section("latency_nvme_ssd"): -+ items_latency_nvme_ssd = dict(con.items("latency_nvme_ssd")) -+ self._conf["latency_nvme_ssd"]["read_tot_lim"] = self._get_config_value( -+ items_latency_nvme_ssd, -+ "read_tot_lim", -+ int, -+ self.DEFAULT_CONF["latency_nvme_ssd"]["read_tot_lim"], -+ gt=0, -+ ) -+ self._conf["latency_nvme_ssd"]["write_tot_lim"] = self._get_config_value( -+ items_latency_nvme_ssd, -+ "write_tot_lim", -+ int, -+ self.DEFAULT_CONF["latency_nvme_ssd"]["write_tot_lim"], -+ gt=0, -+ ) -+ else: -+ logging.warning( -+ "latency_nvme_ssd section parameter not found, it will be set to default value." -+ ) -+ if con.has_section("latency_sata_hdd"): -+ items_latency_sata_hdd = dict(con.items("latency_sata_hdd")) -+ self._conf["latency_sata_hdd"]["read_tot_lim"] = self._get_config_value( -+ items_latency_sata_hdd, -+ "read_tot_lim", -+ int, -+ self.DEFAULT_CONF["latency_sata_hdd"]["read_tot_lim"], -+ gt=0, -+ ) -+ self._conf["latency_sata_hdd"]["write_tot_lim"] = self._get_config_value( -+ items_latency_sata_hdd, -+ "write_tot_lim", -+ int, -+ self.DEFAULT_CONF["latency_sata_hdd"]["write_tot_lim"], -+ gt=0, -+ ) -+ else: -+ logging.warning( -+ "latency_sata_hdd section parameter not found, it will be set to default value." - ) - - self.__print_all_config_value() - -- def __repr__(self): -- config_str = { -- 'log.level': self.__log_level, -- 'common.absolute_threshold': self.__absolute_threshold, -- 'common.slow_io_detect_frequency': self.__slow_io_detect_frequency, -- 'common.disk': self.__disks_to_detection, -- 'common.stage': self.__stage, -- 'common.iotype': self.__iotype, -- 'algorithm.train_data_duration': self.__train_data_duration, -- 'algorithm.train_update_duration': self.__train_update_duration, -- 'algorithm.algorithm_type': self.__algorithm_type, -- 'algorithm.boxplot_parameter': self.__boxplot_parameter, -- 'algorithm.n_sigma_parameter': self.__n_sigma_parameter, -- 'sliding_window.sliding_window_type': self.__sliding_window_type, -- 'sliding_window.window_size': self.__window_size, -- 'sliding_window.window_minimum_threshold': self.__window_minimum_threshold -- } -- return str(config_str) -+ def __repr__(self) -> str: -+ return str(self._conf) -+ -+ def __str__(self) -> str: -+ return str(self._conf) - - def __print_all_config_value(self): -- logging.info(f"all config is follow:\n {self}") -+ logging.info("all config is follow:\n %s", self) -+ -+ def get_tot_lim(self, disk_type, io_type): -+ if io_type == "read": -+ return self._conf.get( -+ f"latency_{DISK_TYPE_MAP.get(disk_type, '')}", {} -+ ).get("read_tot_lim", None) -+ elif io_type == "write": -+ return self._conf.get( -+ f"latency_{DISK_TYPE_MAP.get(disk_type, '')}", {} -+ ).get("write_tot_lim", None) -+ else: -+ return None - - def get_train_data_duration_and_train_update_duration(self): -- return self.__train_data_duration, self.__train_update_duration -+ return ( -+ self._conf["common"]["train_data_duration"], -+ self._conf["common"]["train_update_duration"], -+ ) - - def get_window_size_and_window_minimum_threshold(self): -- return self.__window_size, self.__window_minimum_threshold -+ return ( -+ self._conf["sliding_window"]["window_size"], -+ self._conf["sliding_window"]["window_minimum_threshold"], -+ ) - - @property - def slow_io_detect_frequency(self): -- return self.__slow_io_detect_frequency -+ return self._conf["common"]["slow_io_detect_frequency"] - - @property - def algorithm_type(self): -- return self.__algorithm_type -+ return self._conf["algorithm"]["algorithm_type"] - - @property - def sliding_window_type(self): -- return self.__sliding_window_type -+ return self._conf["sliding_window"]["sliding_window_type"] - - @property - def train_data_duration(self): -- return self.__train_data_duration -+ return self._conf["common"]["train_data_duration"] - - @property - def train_update_duration(self): -- return self.__train_update_duration -+ return self._conf["common"]["train_update_duration"] - - @property - def window_size(self): -- return self.__window_size -+ return self._conf["sliding_window"]["window_size"] - - @property - def window_minimum_threshold(self): -- return self.__window_minimum_threshold -+ return self._conf["sliding_window"]["window_minimum_threshold"] - - @property - def absolute_threshold(self): -- return self.__absolute_threshold -+ return self._conf["common"]["absolute_threshold"] - - @property - def log_level(self): -- return self.__log_level -+ return self._conf["log"]["level"] - - @property - def disks_to_detection(self): -- return self.__disks_to_detection -+ return self._conf["common"]["disk"] - - @property - def stage(self): -- return self.__stage -+ return self._conf["common"]["stage"] - - @property - def iotype(self): -- return self.__iotype -+ return self._conf["common"]["iotype"] - - @property - def boxplot_parameter(self): -- return self.__boxplot_parameter -+ return self._conf["algorithm"]["boxplot_parameter"] - - @property - def n_sigma_parameter(self): -- return self.__n_sigma_parameter -+ return self._conf["algorithm"]["n_sigma_parameter"] -diff --git a/src/python/sentryPlugins/ai_block_io/data_access.py b/src/python/sentryPlugins/ai_block_io/data_access.py -index ed997e6..1bc5ed8 100644 ---- a/src/python/sentryPlugins/ai_block_io/data_access.py -+++ b/src/python/sentryPlugins/ai_block_io/data_access.py -@@ -16,6 +16,7 @@ from sentryCollector.collect_plugin import ( - Result_Messages, - get_io_data, - is_iocollect_valid, -+ get_disk_type - ) - - -diff --git a/src/python/sentryPlugins/ai_block_io/detector.py b/src/python/sentryPlugins/ai_block_io/detector.py -index e710ddd..87bd1dd 100644 ---- a/src/python/sentryPlugins/ai_block_io/detector.py -+++ b/src/python/sentryPlugins/ai_block_io/detector.py -@@ -17,9 +17,6 @@ from .utils import get_metric_value_from_io_data_dict_by_metric_name - - - class Detector: -- _metric_name: MetricName = None -- _threshold: Threshold = None -- _slidingWindow: SlidingWindow = None - - def __init__(self, metric_name: MetricName, threshold: Threshold, sliding_window: SlidingWindow): - self._metric_name = metric_name -@@ -40,18 +37,24 @@ class Detector: - metric_value = get_metric_value_from_io_data_dict_by_metric_name(io_data_dict_with_disk_name, self._metric_name) - if metric_value is None: - logging.debug('not found metric value, so return None.') -- return False, None, None -+ return (False, False), None, None, None - logging.debug(f'input metric value: {str(metric_value)}') - self._threshold.push_latest_data_to_queue(metric_value) - detection_result = self._slidingWindow.is_slow_io_event(metric_value) -- logging.debug(f'Detection result: {str(detection_result)}') -+ # 检测到慢周期，由Detector负责打印info级别日志 -+ if detection_result[0][1]: -+ logging.info(f'[abnormal period happen]: disk info: {self._metric_name}, window: {detection_result[1]}, ' -+ f'current value: {metric_value}, ai threshold: {detection_result[2]}, ' -+ f'absolute threshold: {detection_result[3]}') -+ else: -+ logging.debug(f'Detection result: {str(detection_result)}') - logging.debug(f'exit Detector: {self}') - return detection_result - - def __repr__(self): -- return (f'disk_name: {self._metric_name.get_disk_name()}, stage_name: {self._metric_name.get_stage_name()},' -- f' io_type_name: {self._metric_name.get_io_access_type_name()},' -- f' metric_name: {self._metric_name.get_metric_name()}, threshold_type: {self._threshold},' -+ return (f'disk_name: {self._metric_name.disk_name}, stage_name: {self._metric_name.stage_name},' -+ f' io_type_name: {self._metric_name.io_access_type_name},' -+ f' metric_name: {self._metric_name.metric_name}, threshold_type: {self._threshold},' - f' sliding_window_type: {self._slidingWindow}') - - -@@ -65,13 +68,38 @@ class DiskDetector: - self._detector_list.append(detector) - - def is_slow_io_event(self, io_data_dict_with_disk_name: dict): -- # 只有bio阶段发生异常，就认为发生了慢IO事件 -- # todo：根因诊断 -+ """ -+ 根因诊断逻辑：只有bio阶段发生异常，才认为发生了慢IO事件，即bio阶段异常是慢IO事件的必要条件 -+ 情况一：bio异常，rq_driver也异常，则慢盘 -+ 情况二：bio异常，rq_driver无异常，且有内核IO栈任意阶段异常，则IO栈异常 -+ 情况三：bio异常，rq_driver无异常，且无内核IO栈任意阶段异常，则IO压力大 -+ 情况四：bio异常，则UNKNOWN -+ """ -+ diagnosis_info = {"bio": [], "rq_driver": [], "io_stage": []} - for detector in self._detector_list: -+ # result返回内容：(是否检测到慢IO，是否检测到慢周期)、窗口、ai阈值、绝对阈值 -+ # 示例： (False, False), self._io_data_queue, self._ai_threshold, self._abs_threshold - result = detector.is_slow_io_event(io_data_dict_with_disk_name) -- if result[0] and detector.get_metric_name().get_stage_name() == 'bio': -- return result[0], detector.get_metric_name(), result[1], result[2] -- return False, None, None, None -+ if result[0][0]: -+ if detector.get_metric_name().stage_name == "bio": -+ diagnosis_info["bio"].append((detector.get_metric_name(), result)) -+ elif detector.get_metric_name().stage_name == "rq_driver": -+ diagnosis_info["rq_driver"].append((detector.get_metric_name(), result)) -+ else: -+ diagnosis_info["io_stage"].append((detector.get_metric_name(), result)) -+ -+ # 返回内容：（1）是否检测到慢IO事件、（2）MetricName、（3）滑动窗口及阈值、（4）慢IO事件根因 -+ root_cause = None -+ if len(diagnosis_info["bio"]) == 0: -+ return False, None, None, None -+ elif len(diagnosis_info["rq_driver"]) != 0: -+ root_cause = "[Root Cause：disk slow]" -+ elif len(diagnosis_info["io_stage"]) != 0: -+ stage = diagnosis_info["io_stage"][0][1].get_stage_name() -+ root_cause = f"[Root Cause：io stage slow, stage: {stage}]" -+ if root_cause is None: -+ root_cause = "[Root Cause：high io pressure]" -+ return True, diagnosis_info["bio"][0][0], diagnosis_info["bio"][0][1], root_cause - - def __repr__(self): - msg = f'disk: {self._disk_name}, ' -diff --git a/src/python/sentryPlugins/ai_block_io/io_data.py b/src/python/sentryPlugins/ai_block_io/io_data.py -index 0e17051..d341b55 100644 ---- a/src/python/sentryPlugins/ai_block_io/io_data.py -+++ b/src/python/sentryPlugins/ai_block_io/io_data.py -@@ -45,30 +45,10 @@ class IOData: - time_stamp: float = field(default_factory=lambda: datetime.now().timestamp()) - - -+@dataclass(frozen=True) - class MetricName: -- _disk_name: str = None -- _stage_name: str = None -- _io_access_type_name: str = None -- _metric_name: str = None -- -- def __init__(self, disk_name: str, stage_name: str, io_access_type_name: str, metric_name: str): -- self._disk_name = disk_name -- self._stage_name = stage_name -- self._io_access_type_name = io_access_type_name -- self._metric_name = metric_name -- -- def get_disk_name(self): -- return self._disk_name -- -- def get_stage_name(self): -- return self._stage_name -- -- def get_io_access_type_name(self): -- return self._io_access_type_name -- -- def get_metric_name(self): -- return self._metric_name -- -- def __repr__(self): -- return (f'disk: {self._disk_name}, stage: {self._stage_name}, io_access_type: {self._io_access_type_name},' -- f'metric: {self._metric_name}') -+ disk_name: str -+ disk_type: str -+ stage_name: str -+ io_access_type_name: str -+ metric_name: str -diff --git a/src/python/sentryPlugins/ai_block_io/sliding_window.py b/src/python/sentryPlugins/ai_block_io/sliding_window.py -index 89191e5..d7c402a 100644 ---- a/src/python/sentryPlugins/ai_block_io/sliding_window.py -+++ b/src/python/sentryPlugins/ai_block_io/sliding_window.py -@@ -21,15 +21,11 @@ class SlidingWindowType(Enum): - - - class SlidingWindow: -- _ai_threshold = None -- _queue_length = None -- _queue_threshold = None -- _io_data_queue: list = None -- _io_data_queue_abnormal_tag: list = None -- -- def __init__(self, queue_length: int, threshold: int): -+ def __init__(self, queue_length: int, threshold: int, abs_threshold: int = None): - self._queue_length = queue_length - self._queue_threshold = threshold -+ self._ai_threshold = None -+ self._abs_threshold = abs_threshold - self._io_data_queue = [] - self._io_data_queue_abnormal_tag = [] - -@@ -38,7 +34,12 @@ class SlidingWindow: - self._io_data_queue.pop(0) - self._io_data_queue_abnormal_tag.pop(0) - self._io_data_queue.append(data) -- self._io_data_queue_abnormal_tag.append(data >= self._ai_threshold if self._ai_threshold is not None else False) -+ tag = False -+ if ((self._ai_threshold is not None and data >= self._ai_threshold) or -+ (self._abs_threshold is not None and data >= self._abs_threshold)): -+ tag = True -+ self._io_data_queue_abnormal_tag.append(tag) -+ return tag - - def update(self, threshold): - if self._ai_threshold == threshold: -@@ -49,7 +50,7 @@ class SlidingWindow: - self._io_data_queue_abnormal_tag.append(data >= self._ai_threshold) - - def is_slow_io_event(self, data): -- return False, None, None -+ return False, None, None, None - - def __repr__(self): - return "[SlidingWindow]" -@@ -57,12 +58,13 @@ class SlidingWindow: - - class NotContinuousSlidingWindow(SlidingWindow): - def is_slow_io_event(self, data): -- super().push(data) -- if len(self._io_data_queue) < self._queue_length or self._ai_threshold is None: -- return False, self._io_data_queue, self._ai_threshold -+ is_abnormal_period = super().push(data) -+ is_slow_io_event = False -+ if len(self._io_data_queue) < self._queue_length or (self._ai_threshold is None and self._abs_threshold is None): -+ is_slow_io_event = False - if self._io_data_queue_abnormal_tag.count(True) >= self._queue_threshold: -- return True, self._io_data_queue, self._ai_threshold -- return False, self._io_data_queue, self._ai_threshold -+ is_slow_io_event = True -+ return (is_slow_io_event, is_abnormal_period), self._io_data_queue, self._ai_threshold, self._abs_threshold - - def __repr__(self): - return f"[NotContinuousSlidingWindow, window size: {self._queue_length}, threshold: {self._queue_threshold}]" -@@ -70,18 +72,20 @@ class NotContinuousSlidingWindow(SlidingWindow): - - class ContinuousSlidingWindow(SlidingWindow): - def is_slow_io_event(self, data): -- super().push(data) -- if len(self._io_data_queue) < self._queue_length or self._ai_threshold is None: -- return False, self._io_data_queue, self._ai_threshold -+ is_abnormal_period = super().push(data) -+ is_slow_io_event = False -+ if len(self._io_data_queue) < self._queue_length or (self._ai_threshold is None and self._abs_threshold is None): -+ is_slow_io_event = False - consecutive_count = 0 - for tag in self._io_data_queue_abnormal_tag: - if tag: - consecutive_count += 1 - if consecutive_count >= self._queue_threshold: -- return True, self._io_data_queue, self._ai_threshold -+ is_slow_io_event = True -+ break - else: - consecutive_count = 0 -- return False, self._io_data_queue, self._ai_threshold -+ return (is_slow_io_event, is_abnormal_period), self._io_data_queue, self._ai_threshold, self._abs_threshold - - def __repr__(self): - return f"[ContinuousSlidingWindow, window size: {self._queue_length}, threshold: {self._queue_threshold}]" -@@ -89,20 +93,23 @@ class ContinuousSlidingWindow(SlidingWindow): - - class MedianSlidingWindow(SlidingWindow): - def is_slow_io_event(self, data): -- super().push(data) -- if len(self._io_data_queue) < self._queue_length or self._ai_threshold is None: -- return False, self._io_data_queue, self._ai_threshold -+ is_abnormal_period = super().push(data) -+ is_slow_io_event = False -+ if len(self._io_data_queue) < self._queue_length or (self._ai_threshold is None and self._abs_threshold is None): -+ is_slow_io_event = False - median = np.median(self._io_data_queue) - if median >= self._ai_threshold: -- return True, self._io_data_queue, self._ai_threshold -- return False, self._io_data_queue, self._ai_threshold -+ is_slow_io_event = True -+ return (is_slow_io_event, is_abnormal_period), self._io_data_queue, self._ai_threshold, self._abs_threshold - - def __repr__(self): - return f"[MedianSlidingWindow, window size: {self._queue_length}]" - - - class SlidingWindowFactory: -- def get_sliding_window(self, sliding_window_type: SlidingWindowType, *args, **kwargs): -+ def get_sliding_window( -+ self, sliding_window_type: SlidingWindowType, *args, **kwargs -+ ): - if sliding_window_type == SlidingWindowType.NotContinuousSlidingWindow: - return NotContinuousSlidingWindow(*args, **kwargs) - elif sliding_window_type == SlidingWindowType.ContinuousSlidingWindow: -diff --git a/src/python/sentryPlugins/ai_block_io/utils.py b/src/python/sentryPlugins/ai_block_io/utils.py -index 0ed37b9..d6f4067 100644 ---- a/src/python/sentryPlugins/ai_block_io/utils.py -+++ b/src/python/sentryPlugins/ai_block_io/utils.py -@@ -19,53 +19,57 @@ from .io_data import MetricName, IOData - - - def get_threshold_type_enum(algorithm_type: str): -- if algorithm_type.lower() == 'absolute': -+ if algorithm_type.lower() == "absolute": - return ThresholdType.AbsoluteThreshold -- if algorithm_type.lower() == 'boxplot': -+ if algorithm_type.lower() == "boxplot": - return ThresholdType.BoxplotThreshold -- if algorithm_type.lower() == 'n_sigma': -+ if algorithm_type.lower() == "n_sigma": - return ThresholdType.NSigmaThreshold - return None - - - def get_sliding_window_type_enum(sliding_window_type: str): -- if sliding_window_type.lower() == 'not_continuous': -+ if sliding_window_type.lower() == "not_continuous": - return SlidingWindowType.NotContinuousSlidingWindow -- if sliding_window_type.lower() == 'continuous': -+ if sliding_window_type.lower() == "continuous": - return SlidingWindowType.ContinuousSlidingWindow -- if sliding_window_type.lower() == 'median': -+ if sliding_window_type.lower() == "median": - return SlidingWindowType.MedianSlidingWindow -- logging.warning(f"the sliding window type: {sliding_window_type} you set is invalid, use default value: not_continuous") -- return SlidingWindowType.NotContinuousSlidingWindow -+ return None - - --def get_metric_value_from_io_data_dict_by_metric_name(io_data_dict: dict, metric_name: MetricName): -+def get_metric_value_from_io_data_dict_by_metric_name( -+ io_data_dict: dict, metric_name: MetricName -+): - try: -- io_data: IOData = io_data_dict[metric_name.get_disk_name()] -- io_stage_data = asdict(io_data)[metric_name.get_stage_name()] -- base_data = io_stage_data[metric_name.get_io_access_type_name()] -- metric_value = base_data[metric_name.get_metric_name()] -+ io_data: IOData = io_data_dict[metric_name.disk_name] -+ io_stage_data = asdict(io_data)[metric_name.stage_name] -+ base_data = io_stage_data[metric_name.io_access_type_name] -+ metric_value = base_data[metric_name.metric_name] - return metric_value - except KeyError: - return None - - --def get_data_queue_size_and_update_size(training_data_duration: float, train_update_duration: float, -- slow_io_detect_frequency: int): -+def get_data_queue_size_and_update_size( -+ training_data_duration: float, -+ train_update_duration: float, -+ slow_io_detect_frequency: int, -+): - data_queue_size = int(training_data_duration * 60 * 60 / slow_io_detect_frequency) - update_size = int(train_update_duration * 60 * 60 / slow_io_detect_frequency) - return data_queue_size, update_size - - - def get_log_level(log_level: str): -- if log_level.lower() == 'debug': -+ if log_level.lower() == "debug": - return logging.DEBUG -- elif log_level.lower() == 'info': -+ elif log_level.lower() == "info": - return logging.INFO -- elif log_level.lower() == 'warning': -+ elif log_level.lower() == "warning": - return logging.WARNING -- elif log_level.lower() == 'error': -+ elif log_level.lower() == "error": - return logging.ERROR -- elif log_level.lower() == 'critical': -+ elif log_level.lower() == "critical": - return logging.CRITICAL - return logging.INFO --- -2.23.0 - diff --git a/add-sentryctl-get_alarm-module_name-s-time_range-d.patch b/add-sentryctl-get_alarm-module_name-s-time_range-d.patch deleted file mode 100644 index 0003219..0000000 --- a/add-sentryctl-get_alarm-module_name-s-time_range-d.patch +++ /dev/null @@ -1,438 +0,0 @@ -From 8fa9389a85763831ea85d94f179a305d7f95d585 Mon Sep 17 00:00:00 2001 -From: jinsaihang -Date: Sun, 29 Sep 2024 02:04:52 +0000 -Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=91=8A=E8=AD=A6=E4=BA=8B?= - =?UTF-8?q?=E4=BB=B6=E6=9F=A5=E8=AF=A2=E5=8A=9F=E8=83=BD=EF=BC=9Asentryctl?= - =?UTF-8?q?=20get=5Falarm=20=20-s=20=20-d?= -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -Signed-off-by: jinsaihang ---- - src/python/syssentry/alarm.py | 142 ++++++++++++++++++ - .../src/python/syssentry/callbacks.py | 17 +++ - .../src/python/syssentry/global_values.py | 4 + - .../src/python/syssentry/load_mods.py | 16 ++ - .../src/python/syssentry/sentryctl | 20 ++- - .../src/python/syssentry/syssentry.py | 13 +- - .../src/python/syssentry/task_map.py | 5 +- - 7 files changed, 212 insertions(+), 5 deletions(-) - create mode 100644 src/python/syssentry/alarm.py - -diff --git a/src/python/syssentry/alarm.py b/src/python/syssentry/alarm.py -new file mode 100644 -index 0000000..74a2716 ---- /dev/null -+++ b/src/python/syssentry/alarm.py -@@ -0,0 +1,142 @@ -+# coding: utf-8 -+# Copyright (c) 2024 Huawei Technologies Co., Ltd. -+# sysSentry is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+ -+""" -+use for report alarm -+""" -+import threading -+from typing import Dict, List -+from datetime import datetime -+import time -+import logging -+import json -+ -+from xalarm.register_xalarm import xalarm_register,xalarm_getid,xalarm_getlevel,xalarm_gettype,xalarm_gettime,xalarm_getdesc -+from xalarm.xalarm_api import Xalarm -+ -+from .global_values import InspectTask -+from .task_map import TasksMap -+ -+# 告警ID映射字典，key为插件名，value为告警ID（类型为数字） -+task_alarm_id_dict: Dict[str, int] = {} -+ -+# 告警老化时间字典，key为告警ID，value为老化时间（类型为数字，单位为秒） -+alarm_id_clear_time_dict: Dict[int, int] = {} -+ -+# 告警事件列表，key为告警ID，value为告警ID对应的告警事件列表（类型为list） -+alarm_list_dict: Dict[int, List[Xalarm]] = {} -+# 告警事件列表锁 -+alarm_list_lock = threading.Lock() -+ -+id_filter = [] -+id_base = 1001 -+clientId = -1 -+ -+MILLISECONDS_UNIT_SECONDS = 1000 -+ -+def update_alarm_list(alarm_info: Xalarm): -+ alarm_id = xalarm_getid(alarm_info) -+ timestamp = xalarm_gettime(alarm_info) -+ if not timestamp: -+ logging.error("Retrieve timestamp failed") -+ return -+ alarm_list_lock.acquire() -+ try: -+ # new alarm is inserted into list head -+ if alarm_id not in alarm_list_dict: -+ logging.warning(f"update_alarm_list: alarm_id {alarm_id} not found in alarm_list_dict") -+ return -+ alarm_list = alarm_list_dict[alarm_id] -+ -+ alarm_list.insert(0, alarm_info) -+ # clear alarm_info older than clear time threshold -+ clear_index = -1 -+ clear_time = alarm_id_clear_time_dict[alarm_id] -+ for i in range(len(alarm_list)): -+ if (timestamp - xalarm_gettime(alarm_list[i])) / MILLISECONDS_UNIT_SECONDS > clear_time: -+ clear_index = i -+ break -+ if clear_index >= 0: -+ alarm_list_dict[alarm_id] = alarm_list[:clear_index] -+ finally: -+ alarm_list_lock.release() -+ -+def alarm_register(): -+ logging.debug(f"alarm_register: enter") -+ # 初始化告警ID映射字典、告警老化时间字典 -+ for task_type in TasksMap.tasks_dict: -+ for task_name in TasksMap.tasks_dict[task_type]: -+ logging.info(f"alarm_register: {task_name} is registered") -+ task = TasksMap.tasks_dict[task_type][task_name] -+ alarm_id = task.alarm_id -+ alarm_clear_time = task.alarm_clear_time -+ alarm_list_dict[alarm_id] = [] -+ task_alarm_id_dict[task_name] = alarm_id -+ if alarm_id not in alarm_id_clear_time_dict: -+ alarm_id_clear_time_dict[alarm_id] = alarm_clear_time -+ else: -+ alarm_id_clear_time_dict[alarm_id] = max(alarm_clear_time, alarm_id_clear_time_dict[alarm_id]) -+ # 注册告警回调 -+ id_filter = [True] * 128 -+ clientId = xalarm_register(update_alarm_list, id_filter) -+ if clientId < 0: -+ logging.info(f'register xalarm: failed') -+ return clientId -+ logging.info('register xalarm: success') -+ return clientId -+ -+def get_alarm_result(task_name: str, time_range: int, detailed: bool) -> List[Dict]: -+ alarm_list_lock.acquire() -+ try: -+ if task_name not in task_alarm_id_dict: -+ logging.debug("task_name does not exist") -+ return [] -+ alarm_id = task_alarm_id_dict[task_name] -+ if alarm_id not in alarm_list_dict: -+ logging.debug("alarm_id does not exist") -+ return [] -+ alarm_list = alarm_list_dict[alarm_id] -+ logging.debug(f"get_alarm_result: alarm_list of {alarm_id} has {len(alarm_list)} elements") -+ # clear alarm_info older than clear time threshold -+ stop_index = -1 -+ timestamp = int(datetime.now().timestamp()) -+ for i in range(len(alarm_list)): -+ logging.debug(f"timestamp, alarm_list[{i}].timestamp: {timestamp}, {xalarm_gettime(alarm_list[i])}") -+ if timestamp - (xalarm_gettime(alarm_list[i])) / MILLISECONDS_UNIT_SECONDS > int(time_range): -+ stop_index = i -+ break -+ if stop_index >= 0: -+ alarm_list = alarm_list[:stop_index] -+ logging.debug(f"get_alarm_result: final alarm_list of {alarm_id} has {len(alarm_list)} elements") -+ -+ def xalarm_to_dict(alarm_info: Xalarm) -> dict: -+ return { -+ 'alarm_id': xalarm_getid(alarm_info), -+ 'alarm_type': xalarm_gettype(alarm_info), -+ 'alarm_level': xalarm_getlevel(alarm_info), -+ 'timetamp': xalarm_gettime(alarm_info), -+ 'msg1': xalarm_getdesc(alarm_info) -+ } -+ -+ alarm_list = [xalarm_to_dict(alarm) for alarm in alarm_list] -+ -+ # keep detail -+ for alarm in alarm_list: -+ alarm_info = alarm['msg1'] -+ alarm_info = json.loads(alarm_info) -+ if not detailed: -+ if 'details' in alarm_info: -+ alarm_info.pop('details', None) -+ alarm.pop('msg1', None) -+ alarm['alarm_info'] = alarm_info -+ return alarm_list -+ finally: -+ alarm_list_lock.release() -diff --git a/src/python/syssentry/callbacks.py b/src/python/syssentry/callbacks.py -index b38b381..6ec2c29 100644 ---- a/src/python/syssentry/callbacks.py -+++ b/src/python/syssentry/callbacks.py -@@ -18,6 +18,7 @@ import logging - - from .task_map import TasksMap, ONESHOT_TYPE, PERIOD_TYPE - from .mod_status import EXITED_STATUS, RUNNING_STATUS, WAITING_STATUS, set_runtime_status -+from .alarm import get_alarm_result - - - def task_get_status(mod_name): -@@ -41,6 +42,22 @@ def task_get_result(mod_name): - - return "success", task.get_result() - -+def task_get_alarm(data): -+ """get alarm by mod name""" -+ task_name = data['task_name'] -+ time_range = data['time_range'] -+ try: -+ detailed = data['detailed'] -+ except KeyError: -+ logging.debug("Key 'detailed' does not exist in the dictionary") -+ detailed = None -+ task = TasksMap.get_task_by_name(task_name) -+ if not task: -+ return "failed", f"cannot find task by name {task_name}" -+ if not task.load_enabled: -+ return "failed", f"mod {task_name} is not enabled" -+ -+ return "success", get_alarm_result(task_name, time_range, detailed) - - def task_stop(mod_name): - """stop by mod name""" -diff --git a/src/python/syssentry/global_values.py b/src/python/syssentry/global_values.py -index 483d544..b123b2d 100644 ---- a/src/python/syssentry/global_values.py -+++ b/src/python/syssentry/global_values.py -@@ -27,6 +27,7 @@ CTL_SOCKET_PATH = "/var/run/sysSentry/control.sock" - SYSSENTRY_CONF_PATH = "/etc/sysSentry" - INSPECT_CONF_PATH = "/etc/sysSentry/inspect.conf" - TASK_LOG_DIR = "/var/log/sysSentry" -+DEFAULT_ALARM_CLEAR_TIME = 15 - - SENTRY_RUN_DIR_PERM = 0o750 - -@@ -76,6 +77,9 @@ class InspectTask: - self.env_file = "" - # start mode - self.conflict = "up" -+ # alarm id -+ self.alarm_id = -1 -+ self.alarm_clear_time = DEFAULT_ALARM_CLEAR_TIME - - def start(self): - """ -diff --git a/src/python/syssentry/load_mods.py b/src/python/syssentry/load_mods.py -index 48d7e66..ae05e57 100644 ---- a/src/python/syssentry/load_mods.py -+++ b/src/python/syssentry/load_mods.py -@@ -24,6 +24,7 @@ from .task_map import TasksMap, ONESHOT_TYPE, PERIOD_TYPE - from .cron_process import PeriodTask - from .mod_status import set_task_status - -+from xalarm.register_xalarm import MIN_ALARM_ID, MAX_ALARM_ID - ONESHOT_CONF = 'oneshot' - PERIOD_CONF = 'period' - -@@ -41,6 +42,8 @@ CONF_TASK_RESTART = 'task_restart' - CONF_ONSTART = 'onstart' - CONF_ENV_FILE = 'env_file' - CONF_CONFLICT = 'conflict' -+CONF_ALARM_ID = 'alarm_id' -+CONF_ALARM_CLEAR_TIME = 'alarm_clear_time' - - MOD_FILE_SUFFIX = '.mod' - MOD_SUFFIX_LEN = 4 -@@ -194,6 +197,18 @@ def parse_mod_conf(mod_name, mod_conf): - task.heartbeat_interval = heartbeat_interval - task.load_enabled = is_enabled - -+ try: -+ task.alarm_id = int(mod_conf.get(CONF_TASK, CONF_ALARM_ID)) -+ task.alarm_clear_time = int(mod_conf.get(CONF_TASK, CONF_ALARM_CLEAR_TIME)) -+ if not (MIN_ALARM_ID <= task.alarm_id <= MAX_ALARM_ID): -+ raise ValueError("Invalid alarm_id") -+ except ValueError: -+ task.alarm_id = -1 -+ logging.warning("Invalid alarm_id, set to -1") -+ except configparser.NoOptionError: -+ task.alarm_id = -1 -+ logging.warning("Unset alarm_id and alarm_clear_time, use -1 and 15s as default") -+ - if CONF_ONSTART in mod_conf.options(CONF_TASK): - is_onstart = (mod_conf.get(CONF_TASK, CONF_ONSTART) == 'yes') - if task_type == PERIOD_CONF: -@@ -327,3 +342,4 @@ def reload_single_mod(mod_name): - res, ret = reload_mod_by_name(mod_name) - - return res, ret -+ -diff --git a/src/python/syssentry/sentryctl b/src/python/syssentry/sentryctl -index e94491f..675c17a 100644 ---- a/src/python/syssentry/sentryctl -+++ b/src/python/syssentry/sentryctl -@@ -25,6 +25,7 @@ MAX_PARAM_LENGTH = 256 - - RESULT_MSG_DATA_LEN = 4 - CTL_MSG_LEN_LEN = 3 -+DEFAULT_ALARM_TIME_RANGE = 10 - - def status_output_format(res_data): - """format output""" -@@ -57,6 +58,8 @@ def res_output_handle(res_struct, req_type): - status_output_format(res_struct['data']) - elif req_type == 'get_result': - result_output_format(res_struct['data']) -+ elif req_type == 'get_alarm': -+ result_output_format(res_struct['data']) - elif res_struct['ret'] == "failed": - print(res_struct['data']) - -@@ -75,6 +78,7 @@ def client_send_and_recv(request_data, data_str_len): - print("sentryctl: client creat socket error") - return None - -+ # connect to syssentry - try: - client_socket.connect(CTL_SOCKET_PATH) - except OSError: -@@ -82,6 +86,7 @@ def client_send_and_recv(request_data, data_str_len): - print("sentryctl: client connect error") - return None - -+ # msg: CTL{len}{data} - req_data_len = len(request_data) - request_msg = "CTL" + str(req_data_len).zfill(3) + request_data - -@@ -94,8 +99,8 @@ def client_send_and_recv(request_data, data_str_len): - print("sentryctl: client communicate error") - return None - -+ # res: RES{len}{data} - res_magic = res_data[:3] -- - if res_magic != "RES": - print("res msg format error") - return None -@@ -128,6 +133,10 @@ if __name__ == '__main__': - parser_status.add_argument('task_name') - parser_get_result = subparsers.add_parser('get_result', help='get task result') - parser_get_result.add_argument('task_name') -+ parser_get_alarm = subparsers.add_parser('get_alarm', help='get task alarm') -+ parser_get_alarm.add_argument('task_name') -+ parser_get_alarm.add_argument('-s', '--time_range', type=str, default=DEFAULT_ALARM_TIME_RANGE, help='Specified time range') -+ parser_get_alarm.add_argument('-d', '--detailed', action='store_true', help='Print Detailed Information') - parser_list = subparsers.add_parser('list', help='show all loaded task mod') - - client_args = parser.parse_args() -@@ -142,6 +151,15 @@ if __name__ == '__main__': - req_msg_struct = {"type": "get_status", "data": client_args.task_name} - elif client_args.cmd_type == 'get_result': - req_msg_struct = {"type": "get_result", "data": client_args.task_name} -+ elif client_args.cmd_type == 'get_alarm': -+ req_msg_struct = { -+ "type": "get_alarm", -+ "data": { -+ 'task_name': client_args.task_name, -+ 'time_range': client_args.time_range, -+ 'detailed': client_args.detailed, -+ } -+ } - elif client_args.cmd_type == 'reload': - req_msg_struct = {"type": "reload", "data": client_args.task_name} - else: -diff --git a/src/python/syssentry/syssentry.py b/src/python/syssentry/syssentry.py -index 9ef0203..c2dee85 100644 ---- a/src/python/syssentry/syssentry.py -+++ b/src/python/syssentry/syssentry.py -@@ -28,7 +28,7 @@ from .sentry_config import SentryConfig, get_log_level - from .task_map import TasksMap - from .global_values import SENTRY_RUN_DIR, CTL_SOCKET_PATH, SENTRY_RUN_DIR_PERM - from .cron_process import period_tasks_handle --from .callbacks import mod_list_show, task_start, task_get_status, task_stop, task_get_result -+from .callbacks import mod_list_show, task_start, task_get_status, task_stop, task_get_result, task_get_alarm - from .mod_status import get_task_by_pid, set_runtime_status - from .load_mods import load_tasks, reload_single_mod - from .heartbeat import (heartbeat_timeout_chk, heartbeat_fd_create, -@@ -36,7 +36,11 @@ from .heartbeat import (heartbeat_timeout_chk, heartbeat_fd_create, - from .result import RESULT_MSG_HEAD_LEN, RESULT_MSG_MAGIC_LEN, RESULT_MAGIC - from .result import RESULT_LEVEL_ERR_MSG_DICT, ResultLevel - from .utils import get_current_time_string -+from .alarm import alarm_register - -+from xalarm.register_xalarm import xalarm_unregister -+ -+clientId = -1 - - CPU_EXIST = True - try: -@@ -62,6 +66,7 @@ type_func = { - 'stop': task_stop, - 'get_status': task_get_status, - 'get_result': task_get_result, -+ 'get_alarm': task_get_alarm, - 'reload': reload_single_mod - } - -@@ -107,11 +112,12 @@ def msg_data_process(msg_data): - return "Invaild cmd type" - - cmd_param = data_struct['data'] -- logging.debug("msg_data_process cmd_type:%s cmd_param:%s", cmd_type, cmd_param) -+ logging.debug("msg_data_process cmd_type:%s cmd_param:%s", cmd_type, str(cmd_param)) - if cmd_type in type_func: - ret, res_data = type_func[cmd_type](cmd_param) - else: - ret, res_data = type_func_void[cmd_type]() -+ logging.debug("msg_data_process res_data:%s",str(res_data)) - res_msg_struct = {"ret": ret, "data": res_data} - res_msg = json.dumps(res_msg_struct) - -@@ -584,10 +590,13 @@ def main(): - _ = SentryConfig.init_param() - TasksMap.init_task_map() - load_tasks() -+ clientId = alarm_register() - main_loop() - - except Exception: - logging.error('%s', traceback.format_exc()) - finally: -+ if clientId != -1: -+ xalarm_unregister(clientId) - release_pidfile() - -diff --git a/src/python/syssentry/task_map.py b/src/python/syssentry/task_map.py -index 70aa19d..27e97ff 100644 ---- a/src/python/syssentry/task_map.py -+++ b/src/python/syssentry/task_map.py -@@ -13,16 +13,16 @@ - tasks map class and initialize function. - """ - import logging -+from typing import Dict - - ONESHOT_TYPE = "ONESHOT" - PERIOD_TYPE = "PERIOD" - - TASKS_MAP = None - -- - class TasksMap: - """task map class""" -- tasks_dict = {} -+ tasks_dict: Dict[str, Dict] = {} - - @classmethod - def init_task_map(cls): -@@ -65,3 +65,4 @@ class TasksMap: - logging.debug("getting task by name: %s", res) - break - return res -+ --- -2.27.0 - diff --git a/add-xalarm-cleanup-invalid-server-socket-peroidly.patch b/add-xalarm-cleanup-invalid-server-socket-peroidly.patch deleted file mode 100644 index 5ee845a..0000000 --- a/add-xalarm-cleanup-invalid-server-socket-peroidly.patch +++ /dev/null @@ -1,90 +0,0 @@ -From 4fa9b250f56dc3f4f431fc091e25d8f2558a9bb2 Mon Sep 17 00:00:00 2001 -From: caixiaomeng -Date: Fri, 11 Oct 2024 18:12:21 +0800 -Subject: [PATCH] add xalarm cleanup invalid server socket peroidly - ---- - src/python/xalarm/xalarm_server.py | 20 +++++++++++++++----- - src/python/xalarm/xalarm_transfer.py | 8 ++++++++ - 2 files changed, 23 insertions(+), 5 deletions(-) - -diff --git a/src/python/xalarm/xalarm_server.py b/src/python/xalarm/xalarm_server.py -index 2882609..f90a0e2 100644 ---- a/src/python/xalarm/xalarm_server.py -+++ b/src/python/xalarm/xalarm_server.py -@@ -22,7 +22,12 @@ import threading - from struct import error as StructParseError - - from .xalarm_api import alarm_bin2stu --from .xalarm_transfer import check_filter, transmit_alarm, wait_for_connection -+from .xalarm_transfer import ( -+ check_filter, -+ transmit_alarm, -+ wait_for_connection, -+ peroid_task_to_cleanup_connections -+) - - - ALARM_DIR = "/var/run/xalarm" -@@ -66,9 +71,13 @@ def server_loop(alarm_config): - fd_to_socket = {alarm_sock.fileno(): alarm_sock,} - thread_should_stop = False - -- thread = threading.Thread(target=wait_for_connection, args=(alarm_sock, epoll, fd_to_socket, thread_should_stop)) -- thread.daemon = True -- thread.start() -+ conn_thread = threading.Thread(target=wait_for_connection, args=(alarm_sock, epoll, fd_to_socket, thread_should_stop)) -+ conn_thread.daemon = True -+ conn_thread.start() -+ -+ cleanup_thread = threading.Thread(target=peroid_task_to_cleanup_connections, args=(alarm_sock, epoll, fd_to_socket, thread_should_stop)) -+ cleanup_thread.daemon = True -+ cleanup_thread.start() - - while True: - try: -@@ -88,7 +97,8 @@ def server_loop(alarm_config): - logging.error(f"Error server:{e}") - - thread_should_stop = True -- thread.join() -+ conn_thread.join() -+ cleanup_thread.join() - - epoll.unregister(alarm_sock.fileno()) - epoll.close() -diff --git a/src/python/xalarm/xalarm_transfer.py b/src/python/xalarm/xalarm_transfer.py -index 90dccbc..75807e0 100644 ---- a/src/python/xalarm/xalarm_transfer.py -+++ b/src/python/xalarm/xalarm_transfer.py -@@ -17,11 +17,13 @@ Create: 2023-11-02 - import socket - import logging - import select -+from time import sleep - - MIN_ID_NUMBER = 1001 - MAX_ID_NUMBER = 1128 - MAX_CONNECTION_NUM = 100 - TEST_CONNECT_BUFFER_SIZE = 32 -+PEROID_SCANN_TIME = 60 - - - def check_filter(alarm_info, alarm_filter): -@@ -66,6 +68,12 @@ def cleanup_closed_connections(server_sock, epoll, fd_to_socket): - logging.info(f"cleaned up connection {fileno} for client lost connection.") - - -+def peroid_task_to_cleanup_connections(server_sock, epoll, fd_to_socket, thread_should_stop): -+ while not thread_should_stop: -+ sleep(PEROID_SCANN_TIME) -+ cleanup_closed_connections(server_sock, epoll, fd_to_socket) -+ -+ - def wait_for_connection(server_sock, epoll, fd_to_socket, thread_should_stop): - """ - thread function for catch and save client connection --- -2.27.0 - - diff --git a/ai_block_io-adapt-alarm-module.patch b/ai_block_io-adapt-alarm-module.patch deleted file mode 100644 index f24974b..0000000 --- a/ai_block_io-adapt-alarm-module.patch +++ /dev/null @@ -1,221 +0,0 @@ -From 367f8ab8a5ad26d80caf1bc4529c79d279ef0fb1 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?=E8=B4=BA=E6=9C=89=E5=BF=97?= <1037617413@qq.com> -Date: Thu, 10 Oct 2024 17:21:48 +0800 -Subject: [PATCH] ai_block_io adapt alarm module - ---- - config/tasks/ai_block_io.mod | 4 +- - .../sentryPlugins/ai_block_io/ai_block_io.py | 28 +++++--- - .../sentryPlugins/ai_block_io/alarm_report.py | 65 ++++++++++++++----- - .../sentryPlugins/ai_block_io/data_access.py | 5 +- - .../sentryPlugins/ai_block_io/detector.py | 2 +- - 5 files changed, 73 insertions(+), 31 deletions(-) - -diff --git a/config/tasks/ai_block_io.mod b/config/tasks/ai_block_io.mod -index 1971d7d..82f4f0b 100644 ---- a/config/tasks/ai_block_io.mod -+++ b/config/tasks/ai_block_io.mod -@@ -2,4 +2,6 @@ - enabled=yes - task_start=/usr/bin/python3 /usr/bin/ai_block_io - task_stop=pkill -f /usr/bin/ai_block_io --type=oneshot -\ No newline at end of file -+type=oneshot -+alarm_id=1002 -+alarm_clear_time=5 -\ No newline at end of file -diff --git a/src/python/sentryPlugins/ai_block_io/ai_block_io.py b/src/python/sentryPlugins/ai_block_io/ai_block_io.py -index 3b00ef3..77104a9 100644 ---- a/src/python/sentryPlugins/ai_block_io/ai_block_io.py -+++ b/src/python/sentryPlugins/ai_block_io/ai_block_io.py -@@ -20,14 +20,14 @@ from .utils import get_data_queue_size_and_update_size - from .config_parser import ConfigParser - from .data_access import get_io_data_from_collect_plug, check_collect_valid - from .io_data import MetricName --from .alarm_report import AlarmReport -+from .alarm_report import Xalarm, Report - - CONFIG_FILE = "/etc/sysSentry/plugins/ai_block_io.ini" - - - def sig_handler(signum, frame): - logging.info("receive signal: %d", signum) -- AlarmReport().report_fail(f"receive signal: {signum}") -+ Report.report_pass(f"receive signal: {signum}, exiting...") - exit(signum) - - -@@ -44,6 +44,10 @@ class SlowIODetection: - - def __init_detector_name_list(self): - self._disk_list = check_collect_valid(self._config_parser.get_slow_io_detect_frequency()) -+ if self._disk_list is None: -+ Report.report_pass("get available disk error, please check if the collector plug is enable. exiting...") -+ exit(1) -+ - logging.info(f"ai_block_io plug has found disks: {self._disk_list}") - disks_to_detection: list = self._config_parser.get_disks_to_detection() - # 情况1：None，则启用所有磁盘检测 -@@ -101,7 +105,8 @@ class SlowIODetection: - ) - logging.debug(f'step1. Get io data: {str(io_data_dict_with_disk_name)}') - if io_data_dict_with_disk_name is None: -- continue -+ Report.report_pass("get io data error, please check if the collector plug is enable. exitting...") -+ exit(1) - - # Step2：慢IO检测 - logging.debug('step2. Start to detection slow io event.') -@@ -117,13 +122,16 @@ class SlowIODetection: - for slow_io_event in slow_io_event_list: - metric_name: MetricName = slow_io_event[0] - result = slow_io_event[1] -- alarm_content = (f"disk {metric_name.get_disk_name()} has slow io event. " -- f"stage is: {metric_name.get_stage_name()}, " -- f"io access type is: {metric_name.get_io_access_type_name()}, " -- f"metric is: {metric_name.get_metric_name()}, " -- f"current window is: {result[1]}, " -- f"threshold is: {result[2]}") -- AlarmReport.report_major_alm(alarm_content) -+ alarm_content = { -+ "driver_name": f"{metric_name.get_disk_name()}", -+ "reason": "disk_slow", -+ "block_stack": f"{metric_name.get_stage_name()}", -+ "io_type": f"{metric_name.get_io_access_type_name()}", -+ "alarm_source": "ai_block_io", -+ "alarm_type": "latency", -+ "details": f"current window is: {result[1]}, threshold is: {result[2]}.", -+ } -+ Xalarm.major(alarm_content) - logging.warning(alarm_content) - - # Step4：等待检测时间 -diff --git a/src/python/sentryPlugins/ai_block_io/alarm_report.py b/src/python/sentryPlugins/ai_block_io/alarm_report.py -index 230c8cd..92bd6e3 100644 ---- a/src/python/sentryPlugins/ai_block_io/alarm_report.py -+++ b/src/python/sentryPlugins/ai_block_io/alarm_report.py -@@ -9,41 +9,72 @@ - # PURPOSE. - # See the Mulan PSL v2 for more details. - --from syssentry.result import ResultLevel, report_result - import logging - import json - -+from xalarm.sentry_notify import ( -+ xalarm_report, -+ MINOR_ALM, -+ MAJOR_ALM, -+ CRITICAL_ALM, -+ ALARM_TYPE_OCCUR, -+ ALARM_TYPE_RECOVER, -+) -+ -+from syssentry.result import ResultLevel, report_result -+ - --class AlarmReport: -+class Report: - TASK_NAME = "ai_block_io" - - @staticmethod - def report_pass(info: str): -- report_result(AlarmReport.TASK_NAME, ResultLevel.PASS, json.dumps({"msg": info})) -- logging.info(f'Report {AlarmReport.TASK_NAME} PASS: {info}') -+ report_result(Report.TASK_NAME, ResultLevel.PASS, json.dumps({"msg": info})) -+ logging.info(f'Report {Report.TASK_NAME} PASS: {info}') - - @staticmethod - def report_fail(info: str): -- report_result(AlarmReport.TASK_NAME, ResultLevel.FAIL, json.dumps({"msg": info})) -- logging.info(f'Report {AlarmReport.TASK_NAME} FAIL: {info}') -+ report_result(Report.TASK_NAME, ResultLevel.FAIL, json.dumps({"msg": info})) -+ logging.info(f'Report {Report.TASK_NAME} FAIL: {info}') - - @staticmethod - def report_skip(info: str): -- report_result(AlarmReport.TASK_NAME, ResultLevel.SKIP, json.dumps({"msg": info})) -- logging.info(f'Report {AlarmReport.TASK_NAME} SKIP: {info}') -+ report_result(Report.TASK_NAME, ResultLevel.SKIP, json.dumps({"msg": info})) -+ logging.info(f'Report {Report.TASK_NAME} SKIP: {info}') -+ -+ -+class Xalarm: -+ ALARM_ID = 1002 - - @staticmethod -- def report_minor_alm(info: str): -- report_result(AlarmReport.TASK_NAME, ResultLevel.MINOR_ALM, json.dumps({"msg": info})) -- logging.info(f'Report {AlarmReport.TASK_NAME} MINOR_ALM: {info}') -+ def minor(info: dict): -+ info_str = json.dumps(info) -+ xalarm_report(Xalarm.ALARM_ID, MINOR_ALM, ALARM_TYPE_OCCUR, info_str) -+ logging.info(f"Report {Xalarm.ALARM_ID} MINOR_ALM: {info_str}") - - @staticmethod -- def report_major_alm(info: str): -- report_result(AlarmReport.TASK_NAME, ResultLevel.MAJOR_ALM, json.dumps({"msg": info})) -- logging.info(f'Report {AlarmReport.TASK_NAME} MAJOR_ALM: {info}') -+ def major(info: dict): -+ info_str = json.dumps(info) -+ xalarm_report(Xalarm.ALARM_ID, MAJOR_ALM, ALARM_TYPE_OCCUR, info_str) -+ logging.info(f"Report {Xalarm.ALARM_ID} MAJOR_ALM: {info_str}") - - @staticmethod -- def report_critical_alm(info: str): -- report_result(AlarmReport.TASK_NAME, ResultLevel.CRITICAL_ALM, json.dumps({"msg": info})) -- logging.info(f'Report {AlarmReport.TASK_NAME} CRITICAL_ALM: {info}') -+ def critical(info: dict): -+ info_str = json.dumps(info) -+ xalarm_report(Xalarm.ALARM_ID, CRITICAL_ALM, ALARM_TYPE_OCCUR, info_str) -+ logging.info(f"Report {Xalarm.ALARM_ID} CRITICAL_ALM: {info_str}") -+ -+ def minor_recover(info: dict): -+ info_str = json.dumps(info) -+ xalarm_report(Xalarm.ALARM_ID, MINOR_ALM, ALARM_TYPE_RECOVER, info_str) -+ logging.info(f"Report {Xalarm.ALARM_ID} MINOR_ALM Recover: {info_str}") -+ -+ def major_recover(info: dict): -+ info_str = json.dumps(info) -+ xalarm_report(Xalarm.ALARM_ID, MAJOR_ALM, ALARM_TYPE_RECOVER, info_str) -+ logging.info(f"Report {Xalarm.ALARM_ID} MAJOR_ALM Recover: {info_str}") - -+ def critical_recover(info: dict): -+ info_str = json.dumps(info) -+ xalarm_report(Xalarm.ALARM_ID, CRITICAL_ALM, ALARM_TYPE_RECOVER, info_str) -+ logging.info(f"Report {Xalarm.ALARM_ID} CRITICAL_ALM Recover: {info_str}") -diff --git a/src/python/sentryPlugins/ai_block_io/data_access.py b/src/python/sentryPlugins/ai_block_io/data_access.py -index 01c5315..c7679cd 100644 ---- a/src/python/sentryPlugins/ai_block_io/data_access.py -+++ b/src/python/sentryPlugins/ai_block_io/data_access.py -@@ -42,10 +42,11 @@ def check_collect_valid(period): - data = json.loads(data_raw["message"]) - except Exception as e: - logging.warning(f"get io data failed, {e}") -- return [] -+ return None - return [k for k in data.keys()] - else: -- return [] -+ logging.warning(f"get io data failed, return {data_raw}") -+ return None - - - def _get_raw_data(period, disk_list): -diff --git a/src/python/sentryPlugins/ai_block_io/detector.py b/src/python/sentryPlugins/ai_block_io/detector.py -index a48144f..0ed282b 100644 ---- a/src/python/sentryPlugins/ai_block_io/detector.py -+++ b/src/python/sentryPlugins/ai_block_io/detector.py -@@ -35,7 +35,7 @@ class Detector: - self._count += 1 - if self._count % 15 == 0: - self._count = 0 -- logging.info(f"({self._metric_name}) 's latest threshold is: {self._threshold.get_threshold()}.") -+ logging.debug(f"({self._metric_name}) 's latest threshold is: {self._threshold.get_threshold()}.") - logging.debug(f'enter Detector: {self}') - metric_value = get_metric_value_from_io_data_dict_by_metric_name(io_data_dict_with_disk_name, self._metric_name) - if metric_value is None: --- -2.23.0 - diff --git a/ai_block_io-fix-some-bugs.patch b/ai_block_io-fix-some-bugs.patch deleted file mode 100644 index b82b44d..0000000 --- a/ai_block_io-fix-some-bugs.patch +++ /dev/null @@ -1,235 +0,0 @@ -From 1e13bc31ae3aa94f36aa124eefdfc8773221eacd Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?=E8=B4=BA=E6=9C=89=E5=BF=97?= <1037617413@qq.com> -Date: Mon, 14 Oct 2024 23:16:46 +0800 -Subject: [PATCH] ai_block_io fix some bugs - ---- - .../sentryPlugins/ai_block_io/ai_block_io.py | 1 + - .../ai_block_io/config_parser.py | 20 ++++++++++--------- - .../sentryPlugins/ai_block_io/detector.py | 18 ++++++++++++----- - .../sentryPlugins/ai_block_io/io_data.py | 2 +- - .../sentryPlugins/ai_block_io/threshold.py | 17 +++++++++------- - 5 files changed, 36 insertions(+), 22 deletions(-) - -diff --git a/src/python/sentryPlugins/ai_block_io/ai_block_io.py b/src/python/sentryPlugins/ai_block_io/ai_block_io.py -index dd661a1..4eecd43 100644 ---- a/src/python/sentryPlugins/ai_block_io/ai_block_io.py -+++ b/src/python/sentryPlugins/ai_block_io/ai_block_io.py -@@ -55,6 +55,7 @@ class SlowIODetection: - Report.report_pass( - "get available disk error, please check if the collector plug is enable. exiting..." - ) -+ logging.critical("get available disk error, please check if the collector plug is enable. exiting...") - exit(1) - - logging.info(f"ai_block_io plug has found disks: {self._disk_list}") -diff --git a/src/python/sentryPlugins/ai_block_io/config_parser.py b/src/python/sentryPlugins/ai_block_io/config_parser.py -index 3388cd4..7b0cd29 100644 ---- a/src/python/sentryPlugins/ai_block_io/config_parser.py -+++ b/src/python/sentryPlugins/ai_block_io/config_parser.py -@@ -190,7 +190,7 @@ class ConfigParser: - self._conf["common"]["disk"] = disk_list - - def _read_train_data_duration(self, items_algorithm: dict): -- self._conf["common"]["train_data_duration"] = self._get_config_value( -+ self._conf["algorithm"]["train_data_duration"] = self._get_config_value( - items_algorithm, - "train_data_duration", - float, -@@ -203,17 +203,17 @@ class ConfigParser: - default_train_update_duration = self.DEFAULT_CONF["algorithm"][ - "train_update_duration" - ] -- if default_train_update_duration > self._conf["common"]["train_data_duration"]: -+ if default_train_update_duration > self._conf["algorithm"]["train_data_duration"]: - default_train_update_duration = ( -- self._conf["common"]["train_data_duration"] / 2 -+ self._conf["algorithm"]["train_data_duration"] / 2 - ) -- self._conf["common"]["train_update_duration"] = self._get_config_value( -+ self._conf["algorithm"]["train_update_duration"] = self._get_config_value( - items_algorithm, - "train_update_duration", - float, - default_train_update_duration, - gt=0, -- le=self._conf["common"]["train_data_duration"], -+ le=self._conf["algorithm"]["train_data_duration"], - ) - - def _read_algorithm_type_and_parameter(self, items_algorithm: dict): -@@ -401,6 +401,8 @@ class ConfigParser: - self._read_stage(items_common) - self._read_iotype(items_common) - else: -+ self._conf["common"]["stage"] = ALL_STAGE_LIST -+ self._conf["common"]["iotype"] = ALL_IOTPYE_LIST - logging.warning( - "common section parameter not found, it will be set to default value." - ) -@@ -511,8 +513,8 @@ class ConfigParser: - - def get_train_data_duration_and_train_update_duration(self): - return ( -- self._conf["common"]["train_data_duration"], -- self._conf["common"]["train_update_duration"], -+ self._conf["algorithm"]["train_data_duration"], -+ self._conf["algorithm"]["train_update_duration"], - ) - - def get_window_size_and_window_minimum_threshold(self): -@@ -535,11 +537,11 @@ class ConfigParser: - - @property - def train_data_duration(self): -- return self._conf["common"]["train_data_duration"] -+ return self._conf["algorithm"]["train_data_duration"] - - @property - def train_update_duration(self): -- return self._conf["common"]["train_update_duration"] -+ return self._conf["algorithm"]["train_update_duration"] - - @property - def window_size(self): -diff --git a/src/python/sentryPlugins/ai_block_io/detector.py b/src/python/sentryPlugins/ai_block_io/detector.py -index 87bd1dd..5b21714 100644 ---- a/src/python/sentryPlugins/ai_block_io/detector.py -+++ b/src/python/sentryPlugins/ai_block_io/detector.py -@@ -9,6 +9,7 @@ - # PURPOSE. - # See the Mulan PSL v2 for more details. - import logging -+from datetime import datetime - - from .io_data import MetricName - from .threshold import Threshold -@@ -21,18 +22,25 @@ class Detector: - def __init__(self, metric_name: MetricName, threshold: Threshold, sliding_window: SlidingWindow): - self._metric_name = metric_name - self._threshold = threshold -+ # for when threshold update, it can print latest threshold with metric name -+ self._threshold.set_metric_name(self._metric_name) - self._slidingWindow = sliding_window - self._threshold.attach_observer(self._slidingWindow) -- self._count = 0 -+ self._count = None - - def get_metric_name(self): - return self._metric_name - - def is_slow_io_event(self, io_data_dict_with_disk_name: dict): -- self._count += 1 -- if self._count % 15 == 0: -- self._count = 0 -- logging.debug(f"({self._metric_name}) 's latest threshold is: {self._threshold.get_threshold()}.") -+ if self._count is None: -+ self._count = datetime.now() -+ else: -+ now_time = datetime.now() -+ time_diff = (now_time - self._count).total_seconds() -+ if time_diff >= 60: -+ logging.info(f"({self._metric_name}) 's latest threshold is: {self._threshold.get_threshold()}.") -+ self._count = None -+ - logging.debug(f'enter Detector: {self}') - metric_value = get_metric_value_from_io_data_dict_by_metric_name(io_data_dict_with_disk_name, self._metric_name) - if metric_value is None: -diff --git a/src/python/sentryPlugins/ai_block_io/io_data.py b/src/python/sentryPlugins/ai_block_io/io_data.py -index d341b55..6042911 100644 ---- a/src/python/sentryPlugins/ai_block_io/io_data.py -+++ b/src/python/sentryPlugins/ai_block_io/io_data.py -@@ -48,7 +48,7 @@ class IOData: - @dataclass(frozen=True) - class MetricName: - disk_name: str -- disk_type: str -+ disk_type: int - stage_name: str - io_access_type_name: str - metric_name: str -diff --git a/src/python/sentryPlugins/ai_block_io/threshold.py b/src/python/sentryPlugins/ai_block_io/threshold.py -index 3b7a5a8..600d041 100644 ---- a/src/python/sentryPlugins/ai_block_io/threshold.py -+++ b/src/python/sentryPlugins/ai_block_io/threshold.py -@@ -23,11 +23,6 @@ class ThresholdState(Enum): - - - class Threshold: -- threshold = None -- data_queue: queue.Queue = None -- data_queue_update_size: int = None -- new_data_size: int = None -- threshold_state: ThresholdState = None - - def __init__(self, data_queue_size: int = 10000, data_queue_update_size: int = 1000): - self._observer = None -@@ -36,12 +31,16 @@ class Threshold: - self.new_data_size = 0 - self.threshold_state = ThresholdState.INIT - self.threshold = math.inf -+ self.metric_name = None - - def set_threshold(self, threshold): - self.threshold = threshold - self.threshold_state = ThresholdState.START - self.notify_observer() - -+ def set_metric_name(self, metric_name): -+ self.metric_name = metric_name -+ - def get_threshold(self): - if self.threshold_state == ThresholdState.INIT: - return None -@@ -84,6 +83,7 @@ class BoxplotThreshold(Threshold): - self.parameter = boxplot_parameter - - def _update_threshold(self): -+ old_threshold = self.threshold - data = list(self.data_queue.queue) - q1 = np.percentile(data, 25) - q3 = np.percentile(data, 75) -@@ -91,6 +91,7 @@ class BoxplotThreshold(Threshold): - self.threshold = q3 + self.parameter * iqr - if self.threshold_state == ThresholdState.INIT: - self.threshold_state = ThresholdState.START -+ logging.info(f"MetricName: [{self.metric_name}]'s threshold update, old is: {old_threshold} -> new is: {self.threshold}") - self.notify_observer() - - def push_latest_data_to_queue(self, data): -@@ -109,7 +110,7 @@ class BoxplotThreshold(Threshold): - self.new_data_size = 0 - - def __repr__(self): -- return f"[BoxplotThreshold, param is: {self.parameter}]" -+ return f"[BoxplotThreshold, param is: {self.parameter}, train_size: {self.data_queue.maxsize}, update_size: {self.data_queue_update_size}]" - - - class NSigmaThreshold(Threshold): -@@ -118,12 +119,14 @@ class NSigmaThreshold(Threshold): - self.parameter = n_sigma_parameter - - def _update_threshold(self): -+ old_threshold = self.threshold - data = list(self.data_queue.queue) - mean = np.mean(data) - std = np.std(data) - self.threshold = mean + self.parameter * std - if self.threshold_state == ThresholdState.INIT: - self.threshold_state = ThresholdState.START -+ logging.info(f"MetricName: [{self.metric_name}]'s threshold update, old is: {old_threshold} -> new is: {self.threshold}") - self.notify_observer() - - def push_latest_data_to_queue(self, data): -@@ -142,7 +145,7 @@ class NSigmaThreshold(Threshold): - self.new_data_size = 0 - - def __repr__(self): -- return f"[NSigmaThreshold, param is: {self.parameter}]" -+ return f"[NSigmaThreshold, param is: {self.parameter}, train_size: {self.data_queue.maxsize}, update_size: {self.data_queue_update_size}]" - - - class ThresholdType(Enum): --- -2.23.0 - diff --git a/ai_block_io-fix-some-config-parameters-parse-bug.patch b/ai_block_io-fix-some-config-parameters-parse-bug.patch deleted file mode 100644 index bb84cad..0000000 --- a/ai_block_io-fix-some-config-parameters-parse-bug.patch +++ /dev/null @@ -1,626 +0,0 @@ -From f3a0738061e852c8125513f6222b4a5d6ea73270 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?=E8=B4=BA=E6=9C=89=E5=BF=97?= <1037617413@qq.com> -Date: Fri, 25 Oct 2024 15:34:25 +0800 -Subject: [PATCH] ai_block_io fix some config parameters parse bug - ---- - .../sentryPlugins/ai_block_io/ai_block_io.py | 70 +++++---- - .../ai_block_io/config_parser.py | 135 ++++++++++++++---- - .../sentryPlugins/ai_block_io/data_access.py | 14 ++ - .../sentryPlugins/ai_block_io/detector.py | 16 ++- - .../ai_block_io/sliding_window.py | 2 +- - .../sentryPlugins/ai_block_io/threshold.py | 14 +- - src/python/sentryPlugins/ai_block_io/utils.py | 2 - - 7 files changed, 180 insertions(+), 73 deletions(-) - -diff --git a/src/python/sentryPlugins/ai_block_io/ai_block_io.py b/src/python/sentryPlugins/ai_block_io/ai_block_io.py -index 74f246a..14f740d 100644 ---- a/src/python/sentryPlugins/ai_block_io/ai_block_io.py -+++ b/src/python/sentryPlugins/ai_block_io/ai_block_io.py -@@ -23,6 +23,7 @@ from .data_access import ( - get_io_data_from_collect_plug, - check_collect_valid, - get_disk_type, -+ check_disk_is_available - ) - from .io_data import MetricName - from .alarm_report import Xalarm, Report -@@ -31,14 +32,14 @@ CONFIG_FILE = "/etc/sysSentry/plugins/ai_block_io.ini" - - - def sig_handler(signum, frame): -- logging.info("receive signal: %d", signum) - Report.report_pass(f"receive signal: {signum}, exiting...") -+ logging.info("Finished ai_block_io plugin running.") - exit(signum) - - - class SlowIODetection: - _config_parser = None -- _disk_list = None -+ _disk_list = [] - _detector_name_list = defaultdict(list) - _disk_detectors = {} - -@@ -48,32 +49,30 @@ class SlowIODetection: - self.__init_detector() - - def __init_detector_name_list(self): -- self._disk_list = check_collect_valid( -- self._config_parser.period_time -- ) -- if self._disk_list is None: -- Report.report_pass( -- "get available disk error, please check if the collector plug is enable. exiting..." -- ) -- logging.critical("get available disk error, please check if the collector plug is enable. exiting...") -- exit(1) -- -- logging.info(f"ai_block_io plug has found disks: {self._disk_list}") - disks: list = self._config_parser.disks_to_detection - stages: list = self._config_parser.stage - iotypes: list = self._config_parser.iotype -- # 情况1：None，则启用所有磁盘检测 -- # 情况2：is not None and len = 0，则不启动任何磁盘检测 -- # 情况3：len ！= 0，则取交集 -+ - if disks is None: -- logging.warning( -- "you not specify any disk or use default, so ai_block_io will enable all available disk." -- ) -- for disk in self._disk_list: -- if disks is not None: -- if disk not in disks: -- continue -- disks.remove(disk) -+ logging.warning("you not specify any disk or use default, so ai_block_io will enable all available disk.") -+ all_available_disk_list = check_collect_valid(self._config_parser.period_time) -+ if all_available_disk_list is None: -+ Report.report_pass("get available disk error, please check if the collector plug is enable. exiting...") -+ logging.critical("get available disk error, please check if the collector plug is enable. exiting...") -+ exit(1) -+ if len(all_available_disk_list) == 0: -+ Report.report_pass("not found available disk. exiting...") -+ logging.critical("not found available disk. exiting...") -+ exit(1) -+ disks = all_available_disk_list -+ logging.info(f"available disk list is follow: {disks}.") -+ -+ for disk in disks: -+ tmp_disk = [disk] -+ ret = check_disk_is_available(self._config_parser.period_time, tmp_disk) -+ if not ret: -+ logging.warning(f"disk: {disk} is not available, it will be ignored.") -+ continue - - disk_type_result = get_disk_type(disk) - if disk_type_result["ret"] == 0 and disk_type_result["message"] in ( -@@ -89,20 +88,15 @@ class SlowIODetection: - disk_type_result, - ) - continue -+ self._disk_list.append(disk) - for stage in stages: - for iotype in iotypes: - self._detector_name_list[disk].append(MetricName(disk, disk_type, stage, iotype, "latency")) - self._detector_name_list[disk].append(MetricName(disk, disk_type, stage, iotype, "io_dump")) -- if disks: -- logging.warning( -- "disks: %s not in available disk list, so they will be ignored.", -- disks, -- ) -+ - if not self._detector_name_list: -+ Report.report_pass("the disks to detection is empty, ai_block_io will exit.") - logging.critical("the disks to detection is empty, ai_block_io will exit.") -- Report.report_pass( -- "the disks to detection is empty, ai_block_io will exit." -- ) - exit(1) - - def __init_detector(self): -@@ -202,16 +196,20 @@ class SlowIODetection: - logging.debug("step3. Report slow io event to sysSentry.") - for slow_io_event in slow_io_event_list: - alarm_content = { -+ "alarm_source": "ai_block_io", - "driver_name": slow_io_event[1], -+ "io_type": slow_io_event[4], - "reason": slow_io_event[2], - "block_stack": slow_io_event[3], -- "io_type": slow_io_event[4], -- "alarm_source": "ai_block_io", - "alarm_type": slow_io_event[5], -- "details": slow_io_event[6], -+ "details": slow_io_event[6] - } - Xalarm.major(alarm_content) -- logging.warning("[SLOW IO] " + str(alarm_content)) -+ tmp_alarm_content = alarm_content.copy() -+ del tmp_alarm_content["details"] -+ logging.warning("[SLOW IO] " + str(tmp_alarm_content)) -+ logging.warning(f"latency: " + str(alarm_content.get("details").get("latency"))) -+ logging.warning(f"iodump: " + str(alarm_content.get("details").get("iodump"))) - - # Step4：等待检测时间 - logging.debug("step4. Wait to start next slow io event detection loop.") -diff --git a/src/python/sentryPlugins/ai_block_io/config_parser.py b/src/python/sentryPlugins/ai_block_io/config_parser.py -index 91ec5c6..3049db2 100644 ---- a/src/python/sentryPlugins/ai_block_io/config_parser.py -+++ b/src/python/sentryPlugins/ai_block_io/config_parser.py -@@ -105,21 +105,26 @@ class ConfigParser: - ge=None, - lt=None, - le=None, -+ section=None - ): -+ if section is not None: -+ print_key = section + "." + key -+ else: -+ print_key = key - value = config_items.get(key) - if value is None: - logging.warning( - "config of %s not found, the default value %s will be used.", -- key, -+ print_key, - default_value, - ) - value = default_value - if not value: - logging.critical( -- "the value of %s is empty, ai_block_io plug will exit.", key -+ "the value of %s is empty, ai_block_io plug will exit.", print_key - ) - Report.report_pass( -- f"the value of {key} is empty, ai_block_io plug will exit." -+ f"the value of {print_key} is empty, ai_block_io plug will exit." - ) - exit(1) - try: -@@ -127,51 +132,51 @@ class ConfigParser: - except ValueError: - logging.critical( - "the value of %s is not a valid %s, ai_block_io plug will exit.", -- key, -+ print_key, - value_type, - ) - Report.report_pass( -- f"the value of {key} is not a valid {value_type}, ai_block_io plug will exit." -+ f"the value of {print_key} is not a valid {value_type}, ai_block_io plug will exit." - ) - exit(1) - if gt is not None and value <= gt: - logging.critical( - "the value of %s is not greater than %s, ai_block_io plug will exit.", -- key, -+ print_key, - gt, - ) - Report.report_pass( -- f"the value of {key} is not greater than {gt}, ai_block_io plug will exit." -+ f"the value of {print_key} is not greater than {gt}, ai_block_io plug will exit." - ) - exit(1) - if ge is not None and value < ge: - logging.critical( - "the value of %s is not greater than or equal to %s, ai_block_io plug will exit.", -- key, -+ print_key, - ge, - ) - Report.report_pass( -- f"the value of {key} is not greater than or equal to {ge}, ai_block_io plug will exit." -+ f"the value of {print_key} is not greater than or equal to {ge}, ai_block_io plug will exit." - ) - exit(1) - if lt is not None and value >= lt: - logging.critical( - "the value of %s is not less than %s, ai_block_io plug will exit.", -- key, -+ print_key, - lt, - ) - Report.report_pass( -- f"the value of {key} is not less than {lt}, ai_block_io plug will exit." -+ f"the value of {print_key} is not less than {lt}, ai_block_io plug will exit." - ) - exit(1) - if le is not None and value > le: - logging.critical( - "the value of %s is not less than or equal to %s, ai_block_io plug will exit.", -- key, -+ print_key, - le, - ) - Report.report_pass( -- f"the value of {key} is not less than or equal to {le}, ai_block_io plug will exit." -+ f"the value of {print_key} is not less than or equal to {le}, ai_block_io plug will exit." - ) - exit(1) - -@@ -188,7 +193,7 @@ class ConfigParser: - frequency = self._conf["common"]["period_time"] - ret = check_detect_frequency_is_valid(frequency) - if ret is None: -- log = f"period_time: {frequency} is valid, "\ -+ log = f"period_time: {frequency} is invalid, "\ - f"Check whether the value range is too large or is not an "\ - f"integer multiple of period_time.. exiting..." - Report.report_pass(log) -@@ -202,6 +207,7 @@ class ConfigParser: - self._conf["common"]["disk"] = None - return - disks_to_detection = disks_to_detection.strip() -+ disks_to_detection = disks_to_detection.lower() - if not disks_to_detection: - logging.critical("the value of disk is empty, ai_block_io plug will exit.") - Report.report_pass( -@@ -213,7 +219,18 @@ class ConfigParser: - if len(disk_list) == 1 and disk_list[0] == "default": - self._conf["common"]["disk"] = None - return -- self._conf["common"]["disk"] = disk_list -+ if len(disk_list) > 10: -+ ten_disk_list = disk_list[0:10] -+ other_disk_list = disk_list[10:] -+ logging.warning(f"disk only support maximum is 10, disks: {ten_disk_list} will be retained, other: {other_disk_list} will be ignored.") -+ else: -+ ten_disk_list = disk_list -+ set_ten_disk_list = set(ten_disk_list) -+ if len(ten_disk_list) > len(set_ten_disk_list): -+ tmp = ten_disk_list -+ ten_disk_list = list(set_ten_disk_list) -+ logging.warning(f"disk exist duplicate, it will be deduplicate, before: {tmp}, after: {ten_disk_list}") -+ self._conf["common"]["disk"] = ten_disk_list - - def _read_train_data_duration(self, items_algorithm: dict): - self._conf["algorithm"]["train_data_duration"] = self._get_config_value( -@@ -244,10 +261,12 @@ class ConfigParser: - - def _read_algorithm_type_and_parameter(self, items_algorithm: dict): - algorithm_type = items_algorithm.get("algorithm_type") -- if algorithm_type is not None: -- self._conf["algorithm"]["algorithm_type"] = get_threshold_type_enum( -- algorithm_type -- ) -+ if algorithm_type is None: -+ default_algorithm_type = self._conf["algorithm"]["algorithm_type"] -+ logging.warning(f"algorithm_type not found, it will be set default: {default_algorithm_type}") -+ else: -+ self._conf["algorithm"]["algorithm_type"] = get_threshold_type_enum(algorithm_type) -+ - if self._conf["algorithm"]["algorithm_type"] is None: - logging.critical( - "the algorithm_type: %s you set is invalid. ai_block_io plug will exit.", -@@ -257,6 +276,7 @@ class ConfigParser: - f"the algorithm_type: {algorithm_type} you set is invalid. ai_block_io plug will exit." - ) - exit(1) -+ - elif self._conf["algorithm"]["algorithm_type"] == ThresholdType.NSigmaThreshold: - self._conf["algorithm"]["n_sigma_parameter"] = self._get_config_value( - items_algorithm, -@@ -279,9 +299,14 @@ class ConfigParser: - ) - - def _read_stage(self, items_algorithm: dict): -- stage_str = items_algorithm.get( -- "stage", self.DEFAULT_CONF["common"]["stage"] -- ).strip() -+ stage_str = items_algorithm.get("stage") -+ if stage_str is None: -+ stage_str = self.DEFAULT_CONF["common"]["stage"] -+ logging.warning(f"stage not found, it will be set default: {stage_str}") -+ else: -+ stage_str = stage_str.strip() -+ -+ stage_str = stage_str.lower() - stage_list = stage_str.split(",") - stage_list = [stage.strip() for stage in stage_list] - if len(stage_list) == 1 and stage_list[0] == "": -@@ -307,9 +332,14 @@ class ConfigParser: - self._conf["common"]["stage"] = dup_stage_list - - def _read_iotype(self, items_algorithm: dict): -- iotype_str = items_algorithm.get( -- "iotype", self.DEFAULT_CONF["common"]["iotype"] -- ).strip() -+ iotype_str = items_algorithm.get("iotype") -+ if iotype_str is None: -+ iotype_str = self.DEFAULT_CONF["common"]["iotype"] -+ logging.warning(f"iotype not found, it will be set default: {iotype_str}") -+ else: -+ iotype_str = iotype_str.strip() -+ -+ iotype_str = iotype_str.lower() - iotype_list = iotype_str.split(",") - iotype_list = [iotype.strip() for iotype in iotype_list] - if len(iotype_list) == 1 and iotype_list[0] == "": -@@ -333,6 +363,13 @@ class ConfigParser: - - def _read_sliding_window_type(self, items_sliding_window: dict): - sliding_window_type = items_sliding_window.get("win_type") -+ -+ if sliding_window_type is None: -+ default_sliding_window_type = self._conf["algorithm"]["win_type"] -+ logging.warning(f"win_type not found, it will be set default: {default_sliding_window_type}") -+ return -+ -+ sliding_window_type = sliding_window_type.strip() - if sliding_window_type is not None: - self._conf["algorithm"]["win_type"] = ( - get_sliding_window_type_enum(sliding_window_type) -@@ -439,6 +476,7 @@ class ConfigParser: - int, - self.DEFAULT_CONF["latency_sata_ssd"]["read_tot_lim"], - gt=0, -+ section="latency_sata_ssd" - ) - self._conf["latency_sata_ssd"]["write_tot_lim"] = self._get_config_value( - items_latency_sata_ssd, -@@ -446,21 +484,32 @@ class ConfigParser: - int, - self.DEFAULT_CONF["latency_sata_ssd"]["write_tot_lim"], - gt=0, -+ section="latency_sata_ssd" - ) - self._conf["latency_sata_ssd"]["read_avg_lim"] = self._get_config_value( - items_latency_sata_ssd, - "read_avg_lim", - int, - self.DEFAULT_CONF["latency_sata_ssd"]["read_avg_lim"], -- gt=0 -+ gt=0, -+ section="latency_sata_ssd" - ) - self._conf["latency_sata_ssd"]["write_avg_lim"] = self._get_config_value( - items_latency_sata_ssd, - "write_avg_lim", - int, - self.DEFAULT_CONF["latency_sata_ssd"]["write_avg_lim"], -- gt=0 -+ gt=0, -+ section="latency_sata_ssd" - ) -+ if self._conf["latency_sata_ssd"]["read_avg_lim"] >= self._conf["latency_sata_ssd"]["read_tot_lim"]: -+ Report.report_pass("latency_sata_ssd.read_avg_lim must < latency_sata_ssd.read_tot_lim . exiting...") -+ logging.critical("latency_sata_ssd.read_avg_lim must < latency_sata_ssd.read_tot_lim . exiting...") -+ exit(1) -+ if self._conf["latency_sata_ssd"]["write_avg_lim"] >= self._conf["latency_sata_ssd"]["write_tot_lim"]: -+ Report.report_pass("latency_sata_ssd.write_avg_lim must < latency_sata_ssd.write_tot_lim . exiting...") -+ logging.critical("latency_sata_ssd.read_avg_lim must < latency_sata_ssd.read_tot_lim . exiting...") -+ exit(1) - else: - Report.report_pass("not found latency_sata_ssd section. exiting...") - logging.critical("not found latency_sata_ssd section. exiting...") -@@ -474,6 +523,7 @@ class ConfigParser: - int, - self.DEFAULT_CONF["latency_nvme_ssd"]["read_tot_lim"], - gt=0, -+ section="latency_nvme_ssd" - ) - self._conf["latency_nvme_ssd"]["write_tot_lim"] = self._get_config_value( - items_latency_nvme_ssd, -@@ -481,21 +531,32 @@ class ConfigParser: - int, - self.DEFAULT_CONF["latency_nvme_ssd"]["write_tot_lim"], - gt=0, -+ section="latency_nvme_ssd" - ) - self._conf["latency_nvme_ssd"]["read_avg_lim"] = self._get_config_value( - items_latency_nvme_ssd, - "read_avg_lim", - int, - self.DEFAULT_CONF["latency_nvme_ssd"]["read_avg_lim"], -- gt=0 -+ gt=0, -+ section="latency_nvme_ssd" - ) - self._conf["latency_nvme_ssd"]["write_avg_lim"] = self._get_config_value( - items_latency_nvme_ssd, - "write_avg_lim", - int, - self.DEFAULT_CONF["latency_nvme_ssd"]["write_avg_lim"], -- gt=0 -+ gt=0, -+ section="latency_nvme_ssd" - ) -+ if self._conf["latency_nvme_ssd"]["read_avg_lim"] >= self._conf["latency_nvme_ssd"]["read_tot_lim"]: -+ Report.report_pass("latency_nvme_ssd.read_avg_lim must < latency_nvme_ssd.read_tot_lim . exiting...") -+ logging.critical("latency_nvme_ssd.read_avg_lim must < latency_nvme_ssd.read_tot_lim . exiting...") -+ exit(1) -+ if self._conf["latency_nvme_ssd"]["write_avg_lim"] >= self._conf["latency_nvme_ssd"]["write_tot_lim"]: -+ Report.report_pass("latency_nvme_ssd.write_avg_lim must < latency_nvme_ssd.write_tot_lim . exiting...") -+ logging.critical("latency_nvme_ssd.write_avg_lim must < latency_nvme_ssd.write_tot_lim . exiting...") -+ exit(1) - else: - Report.report_pass("not found latency_nvme_ssd section. exiting...") - logging.critical("not found latency_nvme_ssd section. exiting...") -@@ -509,6 +570,7 @@ class ConfigParser: - int, - self.DEFAULT_CONF["latency_sata_hdd"]["read_tot_lim"], - gt=0, -+ section="latency_sata_hdd" - ) - self._conf["latency_sata_hdd"]["write_tot_lim"] = self._get_config_value( - items_latency_sata_hdd, -@@ -516,21 +578,32 @@ class ConfigParser: - int, - self.DEFAULT_CONF["latency_sata_hdd"]["write_tot_lim"], - gt=0, -+ section="latency_sata_hdd" - ) - self._conf["latency_sata_hdd"]["read_avg_lim"] = self._get_config_value( - items_latency_sata_hdd, - "read_avg_lim", - int, - self.DEFAULT_CONF["latency_sata_hdd"]["read_avg_lim"], -- gt=0 -+ gt=0, -+ section="latency_sata_hdd" - ) - self._conf["latency_sata_hdd"]["write_avg_lim"] = self._get_config_value( - items_latency_sata_hdd, - "write_avg_lim", - int, - self.DEFAULT_CONF["latency_sata_hdd"]["write_avg_lim"], -- gt=0 -+ gt=0, -+ section="latency_sata_hdd" - ) -+ if self._conf["latency_sata_hdd"]["read_avg_lim"] >= self._conf["latency_sata_hdd"]["read_tot_lim"]: -+ Report.report_pass("latency_sata_hdd.read_avg_lim must < latency_sata_hdd.read_tot_lim . exiting...") -+ logging.critical("latency_sata_hdd.read_avg_lim must < latency_sata_hdd.read_tot_lim . exiting...") -+ exit(1) -+ if self._conf["latency_sata_hdd"]["write_avg_lim"] >= self._conf["latency_sata_hdd"]["write_tot_lim"]: -+ Report.report_pass("latency_sata_hdd.write_avg_lim must < latency_sata_hdd.write_tot_lim . exiting...") -+ logging.critical("latency_sata_hdd.write_avg_lim must < latency_sata_hdd.write_tot_lim . exiting...") -+ exit(1) - else: - Report.report_pass("not found latency_sata_hdd section. exiting...") - logging.critical("not found latency_sata_hdd section. exiting...") -diff --git a/src/python/sentryPlugins/ai_block_io/data_access.py b/src/python/sentryPlugins/ai_block_io/data_access.py -index e4869d5..2f2d607 100644 ---- a/src/python/sentryPlugins/ai_block_io/data_access.py -+++ b/src/python/sentryPlugins/ai_block_io/data_access.py -@@ -67,6 +67,20 @@ def check_detect_frequency_is_valid(period): - return None - - -+def check_disk_is_available(period_time, disk): -+ data_raw = is_iocollect_valid(period_time, disk) -+ if data_raw["ret"] == 0: -+ try: -+ data = json.loads(data_raw["message"]) -+ except Exception as e: -+ return False -+ if not data: -+ return False -+ return True -+ else: -+ return False -+ -+ - def _get_raw_data(period, disk_list): - return get_io_data( - period, -diff --git a/src/python/sentryPlugins/ai_block_io/detector.py b/src/python/sentryPlugins/ai_block_io/detector.py -index e3a0952..496e032 100644 ---- a/src/python/sentryPlugins/ai_block_io/detector.py -+++ b/src/python/sentryPlugins/ai_block_io/detector.py -@@ -75,6 +75,18 @@ class Detector: - f' sliding_window_type: {self._slidingWindow}') - - -+def set_to_str(parameter: set): -+ ret = "" -+ parameter = list(parameter) -+ length = len(parameter) -+ for i in range(length): -+ if i == 0: -+ ret += parameter[i] -+ else: -+ ret += "," + parameter[i] -+ return ret -+ -+ - class DiskDetector: - - def __init__(self, disk_name: str): -@@ -124,7 +136,7 @@ class DiskDetector: - alarm_type.add(metric_name.metric_name) - - latency_wins, iodump_wins = self.get_detector_list_window() -- details = f"latency: {latency_wins}, iodump: {iodump_wins}" -+ details = {"latency": latency_wins, "iodump": iodump_wins} - - io_press = {"throtl", "wbt", "iocost", "bfq"} - driver_slow = {"rq_driver"} -@@ -137,7 +149,7 @@ class DiskDetector: - elif not kernel_slow.isdisjoint(block_stack): - reason = "kernel_slow" - -- return True, driver_name, reason, str(block_stack), str(io_type), str(alarm_type), details -+ return True, driver_name, reason, set_to_str(block_stack), set_to_str(io_type), set_to_str(alarm_type), details - - def __repr__(self): - msg = f'disk: {self._disk_name}, ' -diff --git a/src/python/sentryPlugins/ai_block_io/sliding_window.py b/src/python/sentryPlugins/ai_block_io/sliding_window.py -index 4083c43..ff3fa3b 100644 ---- a/src/python/sentryPlugins/ai_block_io/sliding_window.py -+++ b/src/python/sentryPlugins/ai_block_io/sliding_window.py -@@ -107,7 +107,7 @@ class MedianSlidingWindow(SlidingWindow): - if len(self._io_data_queue) < self._queue_length or (self._ai_threshold is None and self._abs_threshold is None): - is_slow_io_event = False - median = np.median(self._io_data_queue) -- if median >= self._ai_threshold: -+ if (self._ai_threshold is not None and median > self._ai_threshold) or (self._abs_threshold is not None and median > self._abs_threshold): - is_slow_io_event = True - return (is_slow_io_event, is_abnormal_period), self._io_data_queue, self._ai_threshold, self._abs_threshold, self._avg_lim - -diff --git a/src/python/sentryPlugins/ai_block_io/threshold.py b/src/python/sentryPlugins/ai_block_io/threshold.py -index 600d041..e202bb8 100644 ---- a/src/python/sentryPlugins/ai_block_io/threshold.py -+++ b/src/python/sentryPlugins/ai_block_io/threshold.py -@@ -65,9 +65,12 @@ class Threshold: - def __repr__(self): - return "Threshold" - -+ def __str__(self): -+ return "Threshold" -+ - - class AbsoluteThreshold(Threshold): -- def __init__(self, data_queue_size: int = 10000, data_queue_update_size: int = 1000): -+ def __init__(self, data_queue_size: int = 10000, data_queue_update_size: int = 1000, **kwargs): - super().__init__(data_queue_size, data_queue_update_size) - - def push_latest_data_to_queue(self, data): -@@ -76,6 +79,9 @@ class AbsoluteThreshold(Threshold): - def __repr__(self): - return "[AbsoluteThreshold]" - -+ def __str__(self): -+ return "absolute" -+ - - class BoxplotThreshold(Threshold): - def __init__(self, boxplot_parameter: float = 1.5, data_queue_size: int = 10000, data_queue_update_size: int = 1000, **kwargs): -@@ -112,6 +118,9 @@ class BoxplotThreshold(Threshold): - def __repr__(self): - return f"[BoxplotThreshold, param is: {self.parameter}, train_size: {self.data_queue.maxsize}, update_size: {self.data_queue_update_size}]" - -+ def __str__(self): -+ return "boxplot" -+ - - class NSigmaThreshold(Threshold): - def __init__(self, n_sigma_parameter: float = 3.0, data_queue_size: int = 10000, data_queue_update_size: int = 1000, **kwargs): -@@ -147,6 +156,9 @@ class NSigmaThreshold(Threshold): - def __repr__(self): - return f"[NSigmaThreshold, param is: {self.parameter}, train_size: {self.data_queue.maxsize}, update_size: {self.data_queue_update_size}]" - -+ def __str__(self): -+ return "n_sigma" -+ - - class ThresholdType(Enum): - AbsoluteThreshold = 0 -diff --git a/src/python/sentryPlugins/ai_block_io/utils.py b/src/python/sentryPlugins/ai_block_io/utils.py -index d6f4067..7d2390b 100644 ---- a/src/python/sentryPlugins/ai_block_io/utils.py -+++ b/src/python/sentryPlugins/ai_block_io/utils.py -@@ -19,8 +19,6 @@ from .io_data import MetricName, IOData - - - def get_threshold_type_enum(algorithm_type: str): -- if algorithm_type.lower() == "absolute": -- return ThresholdType.AbsoluteThreshold - if algorithm_type.lower() == "boxplot": - return ThresholdType.BoxplotThreshold - if algorithm_type.lower() == "n_sigma": --- -2.23.0 - diff --git a/ai_block_io-lack-section-exit.patch b/ai_block_io-lack-section-exit.patch deleted file mode 100644 index c226ee1..0000000 --- a/ai_block_io-lack-section-exit.patch +++ /dev/null @@ -1,98 +0,0 @@ -From 8e4f39897dc8dc51cfa0bbf24667be1688876c15 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?=E8=B4=BA=E6=9C=89=E5=BF=97?= <1037617413@qq.com> -Date: Mon, 21 Oct 2024 14:18:20 +0800 -Subject: [PATCH] ai_block_io lack section exit - ---- - .../ai_block_io/config_parser.py | 40 +++++++++---------- - 1 file changed, 20 insertions(+), 20 deletions(-) - -diff --git a/src/python/sentryPlugins/ai_block_io/config_parser.py b/src/python/sentryPlugins/ai_block_io/config_parser.py -index 7b0cd29..447eccd 100644 ---- a/src/python/sentryPlugins/ai_block_io/config_parser.py -+++ b/src/python/sentryPlugins/ai_block_io/config_parser.py -@@ -401,11 +401,9 @@ class ConfigParser: - self._read_stage(items_common) - self._read_iotype(items_common) - else: -- self._conf["common"]["stage"] = ALL_STAGE_LIST -- self._conf["common"]["iotype"] = ALL_IOTPYE_LIST -- logging.warning( -- "common section parameter not found, it will be set to default value." -- ) -+ Report.report_pass("not found common section. exiting...") -+ logging.critical("not found common section. exiting...") -+ exit(1) - - if con.has_section("algorithm"): - items_algorithm = dict(con.items("algorithm")) -@@ -413,9 +411,9 @@ class ConfigParser: - self._read_train_update_duration(items_algorithm) - self._read_algorithm_type_and_parameter(items_algorithm) - else: -- logging.warning( -- "algorithm section parameter not found, it will be set to default value." -- ) -+ Report.report_pass("not found algorithm section. exiting...") -+ logging.critical("not found algorithm section. exiting...") -+ exit(1) - - if con.has_section("sliding_window"): - items_sliding_window = dict(con.items("sliding_window")) -@@ -423,9 +421,9 @@ class ConfigParser: - self._read_window_size(items_sliding_window) - self._read_window_minimum_threshold(items_sliding_window) - else: -- logging.warning( -- "sliding_window section parameter not found, it will be set to default value." -- ) -+ Report.report_pass("not found sliding_window section. exiting...") -+ logging.critical("not found sliding_window section. exiting...") -+ exit(1) - - if con.has_section("latency_sata_ssd"): - items_latency_sata_ssd = dict(con.items("latency_sata_ssd")) -@@ -444,9 +442,10 @@ class ConfigParser: - gt=0, - ) - else: -- logging.warning( -- "latency_sata_ssd section parameter not found, it will be set to default value." -- ) -+ Report.report_pass("not found latency_sata_ssd section. exiting...") -+ logging.critical("not found latency_sata_ssd section. exiting...") -+ exit(1) -+ - if con.has_section("latency_nvme_ssd"): - items_latency_nvme_ssd = dict(con.items("latency_nvme_ssd")) - self._conf["latency_nvme_ssd"]["read_tot_lim"] = self._get_config_value( -@@ -464,9 +463,10 @@ class ConfigParser: - gt=0, - ) - else: -- logging.warning( -- "latency_nvme_ssd section parameter not found, it will be set to default value." -- ) -+ Report.report_pass("not found latency_nvme_ssd section. exiting...") -+ logging.critical("not found latency_nvme_ssd section. exiting...") -+ exit(1) -+ - if con.has_section("latency_sata_hdd"): - items_latency_sata_hdd = dict(con.items("latency_sata_hdd")) - self._conf["latency_sata_hdd"]["read_tot_lim"] = self._get_config_value( -@@ -484,9 +484,9 @@ class ConfigParser: - gt=0, - ) - else: -- logging.warning( -- "latency_sata_hdd section parameter not found, it will be set to default value." -- ) -+ Report.report_pass("not found latency_sata_hdd section. exiting...") -+ logging.critical("not found latency_sata_hdd section. exiting...") -+ exit(1) - - self.__print_all_config_value() - --- -2.23.0 - diff --git a/ai_block_io-support-absolute-threshold-lower-limit.patch b/ai_block_io-support-absolute-threshold-lower-limit.patch deleted file mode 100644 index ccd8f17..0000000 --- a/ai_block_io-support-absolute-threshold-lower-limit.patch +++ /dev/null @@ -1,728 +0,0 @@ -From cedd862d4e4a97a6c4fa13cbff2af452910ea5b4 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?=E8=B4=BA=E6=9C=89=E5=BF=97?= <1037617413@qq.com> -Date: Thu, 24 Oct 2024 09:39:16 +0800 -Subject: [PATCH] ai_block_io support absolute threshold lower limit - ---- - config/plugins/ai_block_io.ini | 19 +- - .../sentryPlugins/ai_block_io/ai_block_io.py | 36 ++-- - .../sentryPlugins/ai_block_io/alarm_report.py | 18 +- - .../ai_block_io/config_parser.py | 168 ++++++++++++------ - .../sentryPlugins/ai_block_io/detector.py | 92 ++++++---- - .../ai_block_io/sliding_window.py | 21 ++- - 6 files changed, 222 insertions(+), 132 deletions(-) - -diff --git a/config/plugins/ai_block_io.ini b/config/plugins/ai_block_io.ini -index 040237d..d0b1e74 100644 ---- a/config/plugins/ai_block_io.ini -+++ b/config/plugins/ai_block_io.ini -@@ -2,9 +2,9 @@ - level=info - - [common] --slow_io_detect_frequency=1 -+period_time=1 - disk=default --stage=bio -+stage=default - iotype=read,write - - [algorithm] -@@ -12,22 +12,25 @@ train_data_duration=24 - train_update_duration=2 - algorithm_type=boxplot - boxplot_parameter=1.5 --n_sigma_parameter=3 -- --[sliding_window] --sliding_window_type=not_continuous --window_size=30 --window_minimum_threshold=6 -+win_type=not_continuous -+win_size=30 -+win_threshold=6 - - [latency_sata_ssd] -+read_avg_lim=10000 -+write_avg_lim=10000 - read_tot_lim=50000 - write_tot_lim=50000 - - [latency_nvme_ssd] -+read_avg_lim=300 -+write_avg_lim=300 - read_tot_lim=500 - write_tot_lim=500 - - [latency_sata_hdd] -+read_avg_lim=15000 -+write_avg_lim=15000 - read_tot_lim=50000 - write_tot_lim=50000 - -diff --git a/src/python/sentryPlugins/ai_block_io/ai_block_io.py b/src/python/sentryPlugins/ai_block_io/ai_block_io.py -index f25e6d5..74f246a 100644 ---- a/src/python/sentryPlugins/ai_block_io/ai_block_io.py -+++ b/src/python/sentryPlugins/ai_block_io/ai_block_io.py -@@ -49,7 +49,7 @@ class SlowIODetection: - - def __init_detector_name_list(self): - self._disk_list = check_collect_valid( -- self._config_parser.slow_io_detect_frequency -+ self._config_parser.period_time - ) - if self._disk_list is None: - Report.report_pass( -@@ -109,7 +109,7 @@ class SlowIODetection: - train_data_duration, train_update_duration = ( - self._config_parser.get_train_data_duration_and_train_update_duration() - ) -- slow_io_detection_frequency = self._config_parser.slow_io_detect_frequency -+ slow_io_detection_frequency = self._config_parser.period_time - threshold_type = self._config_parser.algorithm_type - data_queue_size, update_size = get_data_queue_size_and_update_size( - train_data_duration, train_update_duration, slow_io_detection_frequency -@@ -131,10 +131,13 @@ class SlowIODetection: - data_queue_size=data_queue_size, - data_queue_update_size=update_size, - ) -- abs_threshold = self._config_parser.get_tot_lim( -+ tot_lim = self._config_parser.get_tot_lim( - metric_name.disk_type, metric_name.io_access_type_name - ) -- if abs_threshold is None: -+ avg_lim = self._config_parser.get_avg_lim( -+ metric_name.disk_type, metric_name.io_access_type_name -+ ) -+ if tot_lim is None: - logging.warning( - "disk %s, disk type %s, io type %s, get tot lim error, so it will be ignored.", - disk, -@@ -145,7 +148,8 @@ class SlowIODetection: - sliding_window_type, - queue_length=window_size, - threshold=window_threshold, -- abs_threshold=abs_threshold, -+ abs_threshold=tot_lim, -+ avg_lim=avg_lim - ) - detector = Detector(metric_name, threshold, sliding_window) - disk_detector.add_detector(detector) -@@ -176,7 +180,7 @@ class SlowIODetection: - - # Step1：获取IO数据 - io_data_dict_with_disk_name = get_io_data_from_collect_plug( -- self._config_parser.slow_io_detect_frequency, self._disk_list -+ self._config_parser.period_time, self._disk_list - ) - logging.debug(f"step1. Get io data: {str(io_data_dict_with_disk_name)}") - if io_data_dict_with_disk_name is None: -@@ -197,25 +201,21 @@ class SlowIODetection: - # Step3：慢IO事件上报 - logging.debug("step3. Report slow io event to sysSentry.") - for slow_io_event in slow_io_event_list: -- metric_name: MetricName = slow_io_event[1] -- window_info = slow_io_event[2] -- root_cause = slow_io_event[3] - alarm_content = { -- "driver_name": f"{metric_name.disk_name}", -- "reason": root_cause, -- "block_stack": f"{metric_name.stage_name}", -- "io_type": f"{metric_name.io_access_type_name}", -+ "driver_name": slow_io_event[1], -+ "reason": slow_io_event[2], -+ "block_stack": slow_io_event[3], -+ "io_type": slow_io_event[4], - "alarm_source": "ai_block_io", -- "alarm_type": "latency", -- "details": f"disk type: {metric_name.disk_type}, current window: {window_info[1]}, " -- f"ai threshold: {window_info[2]}, abs threshold: {window_info[3]}.", -+ "alarm_type": slow_io_event[5], -+ "details": slow_io_event[6], - } - Xalarm.major(alarm_content) -- logging.warning(alarm_content) -+ logging.warning("[SLOW IO] " + str(alarm_content)) - - # Step4：等待检测时间 - logging.debug("step4. Wait to start next slow io event detection loop.") -- time.sleep(self._config_parser.slow_io_detect_frequency) -+ time.sleep(self._config_parser.period_time) - - - def main(): -diff --git a/src/python/sentryPlugins/ai_block_io/alarm_report.py b/src/python/sentryPlugins/ai_block_io/alarm_report.py -index 92bd6e3..61bb145 100644 ---- a/src/python/sentryPlugins/ai_block_io/alarm_report.py -+++ b/src/python/sentryPlugins/ai_block_io/alarm_report.py -@@ -30,17 +30,17 @@ class Report: - @staticmethod - def report_pass(info: str): - report_result(Report.TASK_NAME, ResultLevel.PASS, json.dumps({"msg": info})) -- logging.info(f'Report {Report.TASK_NAME} PASS: {info}') -+ logging.debug(f'Report {Report.TASK_NAME} PASS: {info}') - - @staticmethod - def report_fail(info: str): - report_result(Report.TASK_NAME, ResultLevel.FAIL, json.dumps({"msg": info})) -- logging.info(f'Report {Report.TASK_NAME} FAIL: {info}') -+ logging.debug(f'Report {Report.TASK_NAME} FAIL: {info}') - - @staticmethod - def report_skip(info: str): - report_result(Report.TASK_NAME, ResultLevel.SKIP, json.dumps({"msg": info})) -- logging.info(f'Report {Report.TASK_NAME} SKIP: {info}') -+ logging.debug(f'Report {Report.TASK_NAME} SKIP: {info}') - - - class Xalarm: -@@ -50,31 +50,31 @@ class Xalarm: - def minor(info: dict): - info_str = json.dumps(info) - xalarm_report(Xalarm.ALARM_ID, MINOR_ALM, ALARM_TYPE_OCCUR, info_str) -- logging.info(f"Report {Xalarm.ALARM_ID} MINOR_ALM: {info_str}") -+ logging.debug(f"Report {Xalarm.ALARM_ID} MINOR_ALM: {info_str}") - - @staticmethod - def major(info: dict): - info_str = json.dumps(info) - xalarm_report(Xalarm.ALARM_ID, MAJOR_ALM, ALARM_TYPE_OCCUR, info_str) -- logging.info(f"Report {Xalarm.ALARM_ID} MAJOR_ALM: {info_str}") -+ logging.debug(f"Report {Xalarm.ALARM_ID} MAJOR_ALM: {info_str}") - - @staticmethod - def critical(info: dict): - info_str = json.dumps(info) - xalarm_report(Xalarm.ALARM_ID, CRITICAL_ALM, ALARM_TYPE_OCCUR, info_str) -- logging.info(f"Report {Xalarm.ALARM_ID} CRITICAL_ALM: {info_str}") -+ logging.debug(f"Report {Xalarm.ALARM_ID} CRITICAL_ALM: {info_str}") - - def minor_recover(info: dict): - info_str = json.dumps(info) - xalarm_report(Xalarm.ALARM_ID, MINOR_ALM, ALARM_TYPE_RECOVER, info_str) -- logging.info(f"Report {Xalarm.ALARM_ID} MINOR_ALM Recover: {info_str}") -+ logging.debug(f"Report {Xalarm.ALARM_ID} MINOR_ALM Recover: {info_str}") - - def major_recover(info: dict): - info_str = json.dumps(info) - xalarm_report(Xalarm.ALARM_ID, MAJOR_ALM, ALARM_TYPE_RECOVER, info_str) -- logging.info(f"Report {Xalarm.ALARM_ID} MAJOR_ALM Recover: {info_str}") -+ logging.debug(f"Report {Xalarm.ALARM_ID} MAJOR_ALM Recover: {info_str}") - - def critical_recover(info: dict): - info_str = json.dumps(info) - xalarm_report(Xalarm.ALARM_ID, CRITICAL_ALM, ALARM_TYPE_RECOVER, info_str) -- logging.info(f"Report {Xalarm.ALARM_ID} CRITICAL_ALM Recover: {info_str}") -+ logging.debug(f"Report {Xalarm.ALARM_ID} CRITICAL_ALM Recover: {info_str}") -diff --git a/src/python/sentryPlugins/ai_block_io/config_parser.py b/src/python/sentryPlugins/ai_block_io/config_parser.py -index 1117939..91ec5c6 100644 ---- a/src/python/sentryPlugins/ai_block_io/config_parser.py -+++ b/src/python/sentryPlugins/ai_block_io/config_parser.py -@@ -52,7 +52,7 @@ class ConfigParser: - DEFAULT_CONF = { - "log": {"level": "info"}, - "common": { -- "slow_io_detect_frequency": 1, -+ "period_time": 1, - "disk": None, - "stage": "throtl,wbt,gettag,plug,deadline,hctx,requeue,rq_driver,bio", - "iotype": "read,write", -@@ -63,16 +63,32 @@ class ConfigParser: - "algorithm_type": get_threshold_type_enum("boxplot"), - "boxplot_parameter": 1.5, - "n_sigma_parameter": 3.0, -+ "win_type": get_sliding_window_type_enum("not_continuous"), -+ "win_size": 30, -+ "win_threshold": 6, - }, -- "sliding_window": { -- "sliding_window_type": get_sliding_window_type_enum("not_continuous"), -- "window_size": 30, -- "window_minimum_threshold": 6, -+ "latency_sata_ssd": { -+ "read_avg_lim": 10000, -+ "write_avg_lim": 10000, -+ "read_tot_lim": 50000, -+ "write_tot_lim": 50000 - }, -- "latency_sata_ssd": {"read_tot_lim": 50000, "write_tot_lim": 50000}, -- "latency_nvme_ssd": {"read_tot_lim": 500, "write_tot_lim": 500}, -- "latency_sata_hdd": {"read_tot_lim": 50000, "write_tot_lim": 50000}, -- "iodump": {"read_iodump_lim": 0, "write_iodump_lim": 0} -+ "latency_nvme_ssd": { -+ "read_avg_lim": 300, -+ "write_avg_lim": 300, -+ "read_tot_lim": 500, -+ "write_tot_lim": 500 -+ }, -+ "latency_sata_hdd": { -+ "read_avg_lim": 15000, -+ "write_avg_lim": 15000, -+ "read_tot_lim": 50000, -+ "write_tot_lim": 50000 -+ }, -+ "iodump": { -+ "read_iodump_lim": 0, -+ "write_iodump_lim": 0 -+ } - } - - def __init__(self, config_file_name): -@@ -161,18 +177,18 @@ class ConfigParser: - - return value - -- def _read_slow_io_detect_frequency(self, items_common: dict): -- self._conf["common"]["slow_io_detect_frequency"] = self._get_config_value( -+ def _read_period_time(self, items_common: dict): -+ self._conf["common"]["period_time"] = self._get_config_value( - items_common, -- "slow_io_detect_frequency", -+ "period_time", - int, -- self.DEFAULT_CONF["common"]["slow_io_detect_frequency"], -+ self.DEFAULT_CONF["common"]["period_time"], - gt=0 - ) -- frequency = self._conf["common"]["slow_io_detect_frequency"] -+ frequency = self._conf["common"]["period_time"] - ret = check_detect_frequency_is_valid(frequency) - if ret is None: -- log = f"slow io detect frequency: {frequency} is valid, "\ -+ log = f"period_time: {frequency} is valid, "\ - f"Check whether the value range is too large or is not an "\ - f"integer multiple of period_time.. exiting..." - Report.report_pass(log) -@@ -316,50 +332,41 @@ class ConfigParser: - self._conf["common"]["iotype"] = dup_iotype_list - - def _read_sliding_window_type(self, items_sliding_window: dict): -- sliding_window_type = items_sliding_window.get("sliding_window_type") -+ sliding_window_type = items_sliding_window.get("win_type") - if sliding_window_type is not None: -- self._conf["sliding_window"]["sliding_window_type"] = ( -+ self._conf["algorithm"]["win_type"] = ( - get_sliding_window_type_enum(sliding_window_type) - ) -- if self._conf["sliding_window"]["sliding_window_type"] is None: -+ if self._conf["algorithm"]["win_type"] is None: - logging.critical( -- "the sliding_window_type: %s you set is invalid. ai_block_io plug will exit.", -+ "the win_type: %s you set is invalid. ai_block_io plug will exit.", - sliding_window_type, - ) - Report.report_pass( -- f"the sliding_window_type: {sliding_window_type} you set is invalid. ai_block_io plug will exit." -+ f"the win_type: {sliding_window_type} you set is invalid. ai_block_io plug will exit." - ) - exit(1) - - def _read_window_size(self, items_sliding_window: dict): -- self._conf["sliding_window"]["window_size"] = self._get_config_value( -+ self._conf["algorithm"]["win_size"] = self._get_config_value( - items_sliding_window, -- "window_size", -+ "win_size", - int, -- self.DEFAULT_CONF["sliding_window"]["window_size"], -+ self.DEFAULT_CONF["algorithm"]["win_size"], - gt=0, -- le=3600, -+ le=300, - ) - - def _read_window_minimum_threshold(self, items_sliding_window: dict): -- default_window_minimum_threshold = self.DEFAULT_CONF["sliding_window"][ -- "window_minimum_threshold" -- ] -- if ( -- default_window_minimum_threshold -- > self._conf["sliding_window"]["window_size"] -- ): -- default_window_minimum_threshold = ( -- self._conf["sliding_window"]["window_size"] / 2 -- ) -- self._conf["sliding_window"]["window_minimum_threshold"] = ( -+ default_window_minimum_threshold = self.DEFAULT_CONF["algorithm"]["win_threshold"] -+ self._conf["algorithm"]["win_threshold"] = ( - self._get_config_value( - items_sliding_window, -- "window_minimum_threshold", -+ "win_threshold", - int, - default_window_minimum_threshold, - gt=0, -- le=self._conf["sliding_window"]["window_size"], -+ le=self._conf["algorithm"]["win_size"], - ) - ) - -@@ -406,7 +413,7 @@ class ConfigParser: - if con.has_section("common"): - items_common = dict(con.items("common")) - -- self._read_slow_io_detect_frequency(items_common) -+ self._read_period_time(items_common) - self._read_disks_to_detect(items_common) - self._read_stage(items_common) - self._read_iotype(items_common) -@@ -420,20 +427,9 @@ class ConfigParser: - self._read_train_data_duration(items_algorithm) - self._read_train_update_duration(items_algorithm) - self._read_algorithm_type_and_parameter(items_algorithm) -- else: -- Report.report_pass("not found algorithm section. exiting...") -- logging.critical("not found algorithm section. exiting...") -- exit(1) -- -- if con.has_section("sliding_window"): -- items_sliding_window = dict(con.items("sliding_window")) -- -- self._read_window_size(items_sliding_window) -- self._read_window_minimum_threshold(items_sliding_window) -- else: -- Report.report_pass("not found sliding_window section. exiting...") -- logging.critical("not found sliding_window section. exiting...") -- exit(1) -+ self._read_sliding_window_type(items_algorithm) -+ self._read_window_size(items_algorithm) -+ self._read_window_minimum_threshold(items_algorithm) - - if con.has_section("latency_sata_ssd"): - items_latency_sata_ssd = dict(con.items("latency_sata_ssd")) -@@ -451,6 +447,20 @@ class ConfigParser: - self.DEFAULT_CONF["latency_sata_ssd"]["write_tot_lim"], - gt=0, - ) -+ self._conf["latency_sata_ssd"]["read_avg_lim"] = self._get_config_value( -+ items_latency_sata_ssd, -+ "read_avg_lim", -+ int, -+ self.DEFAULT_CONF["latency_sata_ssd"]["read_avg_lim"], -+ gt=0 -+ ) -+ self._conf["latency_sata_ssd"]["write_avg_lim"] = self._get_config_value( -+ items_latency_sata_ssd, -+ "write_avg_lim", -+ int, -+ self.DEFAULT_CONF["latency_sata_ssd"]["write_avg_lim"], -+ gt=0 -+ ) - else: - Report.report_pass("not found latency_sata_ssd section. exiting...") - logging.critical("not found latency_sata_ssd section. exiting...") -@@ -472,6 +482,20 @@ class ConfigParser: - self.DEFAULT_CONF["latency_nvme_ssd"]["write_tot_lim"], - gt=0, - ) -+ self._conf["latency_nvme_ssd"]["read_avg_lim"] = self._get_config_value( -+ items_latency_nvme_ssd, -+ "read_avg_lim", -+ int, -+ self.DEFAULT_CONF["latency_nvme_ssd"]["read_avg_lim"], -+ gt=0 -+ ) -+ self._conf["latency_nvme_ssd"]["write_avg_lim"] = self._get_config_value( -+ items_latency_nvme_ssd, -+ "write_avg_lim", -+ int, -+ self.DEFAULT_CONF["latency_nvme_ssd"]["write_avg_lim"], -+ gt=0 -+ ) - else: - Report.report_pass("not found latency_nvme_ssd section. exiting...") - logging.critical("not found latency_nvme_ssd section. exiting...") -@@ -493,6 +517,20 @@ class ConfigParser: - self.DEFAULT_CONF["latency_sata_hdd"]["write_tot_lim"], - gt=0, - ) -+ self._conf["latency_sata_hdd"]["read_avg_lim"] = self._get_config_value( -+ items_latency_sata_hdd, -+ "read_avg_lim", -+ int, -+ self.DEFAULT_CONF["latency_sata_hdd"]["read_avg_lim"], -+ gt=0 -+ ) -+ self._conf["latency_sata_hdd"]["write_avg_lim"] = self._get_config_value( -+ items_latency_sata_hdd, -+ "write_avg_lim", -+ int, -+ self.DEFAULT_CONF["latency_sata_hdd"]["write_avg_lim"], -+ gt=0 -+ ) - else: - Report.report_pass("not found latency_sata_hdd section. exiting...") - logging.critical("not found latency_sata_hdd section. exiting...") -@@ -542,6 +580,18 @@ class ConfigParser: - else: - return None - -+ def get_avg_lim(self, disk_type, io_type): -+ if io_type == "read": -+ return self._conf.get( -+ f"latency_{DISK_TYPE_MAP.get(disk_type, '')}", {} -+ ).get("read_avg_lim", None) -+ elif io_type == "write": -+ return self._conf.get( -+ f"latency_{DISK_TYPE_MAP.get(disk_type, '')}", {} -+ ).get("write_avg_lim", None) -+ else: -+ return None -+ - def get_train_data_duration_and_train_update_duration(self): - return ( - self._conf["algorithm"]["train_data_duration"], -@@ -550,13 +600,13 @@ class ConfigParser: - - def get_window_size_and_window_minimum_threshold(self): - return ( -- self._conf["sliding_window"]["window_size"], -- self._conf["sliding_window"]["window_minimum_threshold"], -+ self._conf["algorithm"]["win_size"], -+ self._conf["algorithm"]["win_threshold"], - ) - - @property -- def slow_io_detect_frequency(self): -- return self._conf["common"]["slow_io_detect_frequency"] -+ def period_time(self): -+ return self._conf["common"]["period_time"] - - @property - def algorithm_type(self): -@@ -564,7 +614,7 @@ class ConfigParser: - - @property - def sliding_window_type(self): -- return self._conf["sliding_window"]["sliding_window_type"] -+ return self._conf["algorithm"]["win_type"] - - @property - def train_data_duration(self): -@@ -576,11 +626,11 @@ class ConfigParser: - - @property - def window_size(self): -- return self._conf["sliding_window"]["window_size"] -+ return self._conf["algorithm"]["win_size"] - - @property - def window_minimum_threshold(self): -- return self._conf["sliding_window"]["window_minimum_threshold"] -+ return self._conf["algorithm"]["win_threshold"] - - @property - def absolute_threshold(self): -diff --git a/src/python/sentryPlugins/ai_block_io/detector.py b/src/python/sentryPlugins/ai_block_io/detector.py -index 8536f7a..e3a0952 100644 ---- a/src/python/sentryPlugins/ai_block_io/detector.py -+++ b/src/python/sentryPlugins/ai_block_io/detector.py -@@ -28,9 +28,13 @@ class Detector: - self._threshold.attach_observer(self._slidingWindow) - self._count = None - -- def get_metric_name(self): -+ @property -+ def metric_name(self): - return self._metric_name - -+ def get_sliding_window_data(self): -+ return self._slidingWindow.get_data() -+ - def is_slow_io_event(self, io_data_dict_with_disk_name: dict): - if self._count is None: - self._count = datetime.now() -@@ -38,22 +42,27 @@ class Detector: - now_time = datetime.now() - time_diff = (now_time - self._count).total_seconds() - if time_diff >= 60: -- logging.info(f"({self._metric_name}) 's latest threshold is: {self._threshold.get_threshold()}.") -+ logging.info(f"({self._metric_name}) 's latest ai threshold is: {self._threshold.get_threshold()}.") - self._count = None - - logging.debug(f'enter Detector: {self}') - metric_value = get_metric_value_from_io_data_dict_by_metric_name(io_data_dict_with_disk_name, self._metric_name) - if metric_value is None: - logging.debug('not found metric value, so return None.') -- return (False, False), None, None, None -+ return (False, False), None, None, None, None - logging.debug(f'input metric value: {str(metric_value)}') - self._threshold.push_latest_data_to_queue(metric_value) - detection_result = self._slidingWindow.is_slow_io_event(metric_value) - # 检测到慢周期，由Detector负责打印info级别日志 - if detection_result[0][1]: -- logging.info(f'[abnormal period happen]: disk info: {self._metric_name}, window: {detection_result[1]}, ' -- f'current value: {metric_value}, ai threshold: {detection_result[2]}, ' -- f'absolute threshold: {detection_result[3]}') -+ logging.info(f'[abnormal_period]: disk: {self._metric_name.disk_name}, ' -+ f'stage: {self._metric_name.stage_name}, ' -+ f'iotype: {self._metric_name.io_access_type_name}, ' -+ f'metric: {self._metric_name.metric_name}, ' -+ f'current value: {metric_value}, ' -+ f'ai threshold: {detection_result[2]}, ' -+ f'absolute threshold upper limit: {detection_result[3]}, ' -+ f'lower limit: {detection_result[4]}') - else: - logging.debug(f'Detection result: {str(detection_result)}') - logging.debug(f'exit Detector: {self}') -@@ -75,41 +84,60 @@ class DiskDetector: - def add_detector(self, detector: Detector): - self._detector_list.append(detector) - -+ def get_detector_list_window(self): -+ latency_wins = {"read": {}, "write": {}} -+ iodump_wins = {"read": {}, "write": {}} -+ for detector in self._detector_list: -+ if detector.metric_name.metric_name == 'latency': -+ latency_wins[detector.metric_name.io_access_type_name][detector.metric_name.stage_name] = detector.get_sliding_window_data() -+ elif detector.metric_name.metric_name == 'io_dump': -+ iodump_wins[detector.metric_name.io_access_type_name][detector.metric_name.stage_name] = detector.get_sliding_window_data() -+ return latency_wins, iodump_wins -+ - def is_slow_io_event(self, io_data_dict_with_disk_name: dict): -- """ -- 根因诊断逻辑：只有bio阶段发生异常，才认为发生了慢IO事件，即bio阶段异常是慢IO事件的必要条件 -- 情况一：bio异常，rq_driver也异常，则慢盘 -- 情况二：bio异常，rq_driver无异常，且有内核IO栈任意阶段异常，则IO栈异常 -- 情况三：bio异常，rq_driver无异常，且无内核IO栈任意阶段异常，则IO压力大 -- 情况四：bio异常，则UNKNOWN -- """ -- diagnosis_info = {"bio": [], "rq_driver": [], "io_stage": []} -+ diagnosis_info = {"bio": [], "rq_driver": [], "kernel_stack": []} - for detector in self._detector_list: - # result返回内容：(是否检测到慢IO，是否检测到慢周期)、窗口、ai阈值、绝对阈值 - # 示例： (False, False), self._io_data_queue, self._ai_threshold, self._abs_threshold - result = detector.is_slow_io_event(io_data_dict_with_disk_name) - if result[0][0]: -- if detector.get_metric_name().stage_name == "bio": -- diagnosis_info["bio"].append((detector.get_metric_name(), result)) -- elif detector.get_metric_name().stage_name == "rq_driver": -- diagnosis_info["rq_driver"].append((detector.get_metric_name(), result)) -+ if detector.metric_name.stage_name == "bio": -+ diagnosis_info["bio"].append(detector.metric_name) -+ elif detector.metric_name.stage_name == "rq_driver": -+ diagnosis_info["rq_driver"].append(detector.metric_name) - else: -- diagnosis_info["io_stage"].append((detector.get_metric_name(), result)) -+ diagnosis_info["kernel_stack"].append(detector.metric_name) - -- # 返回内容：（1）是否检测到慢IO事件、（2）MetricName、（3）滑动窗口及阈值、（4）慢IO事件根因 -- root_cause = None - if len(diagnosis_info["bio"]) == 0: -- return False, None, None, None -- elif len(diagnosis_info["rq_driver"]) != 0: -- root_cause = "[Root Cause: disk slow]" -- elif len(diagnosis_info["io_stage"]) != 0: -- stage_list = [] -- for io_stage in diagnosis_info["io_stage"]: -- stage_list.append(io_stage[0].stage_name) -- root_cause = f"[Root Cause: io stage slow, stage: {stage_list}]" -- if root_cause is None: -- root_cause = "[Root Cause: high io pressure]" -- return True, diagnosis_info["bio"][0][0], diagnosis_info["bio"][0][1], root_cause -+ return False, None, None, None, None, None, None -+ -+ driver_name = self._disk_name -+ reason = "unknown" -+ block_stack = set() -+ io_type = set() -+ alarm_type = set() -+ -+ for key, value in diagnosis_info.items(): -+ for metric_name in value: -+ block_stack.add(metric_name.stage_name) -+ io_type.add(metric_name.io_access_type_name) -+ alarm_type.add(metric_name.metric_name) -+ -+ latency_wins, iodump_wins = self.get_detector_list_window() -+ details = f"latency: {latency_wins}, iodump: {iodump_wins}" -+ -+ io_press = {"throtl", "wbt", "iocost", "bfq"} -+ driver_slow = {"rq_driver"} -+ kernel_slow = {"gettag", "plug", "deadline", "hctx", "requeue"} -+ -+ if not io_press.isdisjoint(block_stack): -+ reason = "io_press" -+ elif not driver_slow.isdisjoint(block_stack): -+ reason = "driver_slow" -+ elif not kernel_slow.isdisjoint(block_stack): -+ reason = "kernel_slow" -+ -+ return True, driver_name, reason, str(block_stack), str(io_type), str(alarm_type), details - - def __repr__(self): - msg = f'disk: {self._disk_name}, ' -diff --git a/src/python/sentryPlugins/ai_block_io/sliding_window.py b/src/python/sentryPlugins/ai_block_io/sliding_window.py -index cebe41f..4083c43 100644 ---- a/src/python/sentryPlugins/ai_block_io/sliding_window.py -+++ b/src/python/sentryPlugins/ai_block_io/sliding_window.py -@@ -21,11 +21,12 @@ class SlidingWindowType(Enum): - - - class SlidingWindow: -- def __init__(self, queue_length: int, threshold: int, abs_threshold: int = None): -+ def __init__(self, queue_length: int, threshold: int, abs_threshold: int = None, avg_lim: int = None): - self._queue_length = queue_length - self._queue_threshold = threshold - self._ai_threshold = None - self._abs_threshold = abs_threshold -+ self._avg_lim = avg_lim - self._io_data_queue = [] - self._io_data_queue_abnormal_tag = [] - -@@ -35,8 +36,13 @@ class SlidingWindow: - self._io_data_queue_abnormal_tag.pop(0) - self._io_data_queue.append(data) - tag = False -- if ((self._ai_threshold is not None and data > self._ai_threshold) or -- (self._abs_threshold is not None and data > self._abs_threshold)): -+ if self._avg_lim is not None and data < self._avg_lim: -+ tag = False -+ self._io_data_queue_abnormal_tag.append(tag) -+ return tag -+ if self._ai_threshold is not None and data > self._ai_threshold: -+ tag = True -+ if self._abs_threshold is not None and data > self._abs_threshold: - tag = True - self._io_data_queue_abnormal_tag.append(tag) - return tag -@@ -52,6 +58,9 @@ class SlidingWindow: - def is_slow_io_event(self, data): - return False, None, None, None - -+ def get_data(self): -+ return self._io_data_queue -+ - def __repr__(self): - return "[SlidingWindow]" - -@@ -64,7 +73,7 @@ class NotContinuousSlidingWindow(SlidingWindow): - is_slow_io_event = False - if self._io_data_queue_abnormal_tag.count(True) >= self._queue_threshold: - is_slow_io_event = True -- return (is_slow_io_event, is_abnormal_period), self._io_data_queue, self._ai_threshold, self._abs_threshold -+ return (is_slow_io_event, is_abnormal_period), self._io_data_queue, self._ai_threshold, self._abs_threshold, self._avg_lim - - def __repr__(self): - return f"[NotContinuousSlidingWindow, window size: {self._queue_length}, threshold: {self._queue_threshold}]" -@@ -85,7 +94,7 @@ class ContinuousSlidingWindow(SlidingWindow): - break - else: - consecutive_count = 0 -- return (is_slow_io_event, is_abnormal_period), self._io_data_queue, self._ai_threshold, self._abs_threshold -+ return (is_slow_io_event, is_abnormal_period), self._io_data_queue, self._ai_threshold, self._abs_threshold, self._avg_lim - - def __repr__(self): - return f"[ContinuousSlidingWindow, window size: {self._queue_length}, threshold: {self._queue_threshold}]" -@@ -100,7 +109,7 @@ class MedianSlidingWindow(SlidingWindow): - median = np.median(self._io_data_queue) - if median >= self._ai_threshold: - is_slow_io_event = True -- return (is_slow_io_event, is_abnormal_period), self._io_data_queue, self._ai_threshold, self._abs_threshold -+ return (is_slow_io_event, is_abnormal_period), self._io_data_queue, self._ai_threshold, self._abs_threshold, self._avg_lim - - def __repr__(self): - return f"[MedianSlidingWindow, window size: {self._queue_length}]" --- -2.23.0 - diff --git a/ai_block_io-support-iodump.patch b/ai_block_io-support-iodump.patch deleted file mode 100644 index 990995e..0000000 --- a/ai_block_io-support-iodump.patch +++ /dev/null @@ -1,200 +0,0 @@ -From db97139c411e86d6dc07fe0e91ae38c1bef17a8d Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?=E8=B4=BA=E6=9C=89=E5=BF=97?= <1037617413@qq.com> -Date: Tue, 22 Oct 2024 16:37:52 +0800 -Subject: [PATCH] ai_block_io support iodump - ---- - config/plugins/ai_block_io.ini | 6 +- - .../sentryPlugins/ai_block_io/ai_block_io.py | 75 ++++++++++++------- - .../ai_block_io/config_parser.py | 30 ++++++++ - .../ai_block_io/sliding_window.py | 4 +- - 4 files changed, 84 insertions(+), 31 deletions(-) - -diff --git a/config/plugins/ai_block_io.ini b/config/plugins/ai_block_io.ini -index 422cfa3..040237d 100644 ---- a/config/plugins/ai_block_io.ini -+++ b/config/plugins/ai_block_io.ini -@@ -29,4 +29,8 @@ write_tot_lim=500 - - [latency_sata_hdd] - read_tot_lim=50000 --write_tot_lim=50000 -\ No newline at end of file -+write_tot_lim=50000 -+ -+[iodump] -+read_iodump_lim=0 -+write_iodump_lim=0 -\ No newline at end of file -diff --git a/src/python/sentryPlugins/ai_block_io/ai_block_io.py b/src/python/sentryPlugins/ai_block_io/ai_block_io.py -index 4eecd43..f25e6d5 100644 ---- a/src/python/sentryPlugins/ai_block_io/ai_block_io.py -+++ b/src/python/sentryPlugins/ai_block_io/ai_block_io.py -@@ -15,7 +15,7 @@ import logging - from collections import defaultdict - - from .detector import Detector, DiskDetector --from .threshold import ThresholdFactory -+from .threshold import ThresholdFactory, ThresholdType - from .sliding_window import SlidingWindowFactory - from .utils import get_data_queue_size_and_update_size - from .config_parser import ConfigParser -@@ -91,9 +91,8 @@ class SlowIODetection: - continue - for stage in stages: - for iotype in iotypes: -- self._detector_name_list[disk].append( -- MetricName(disk, disk_type, stage, iotype, "latency") -- ) -+ self._detector_name_list[disk].append(MetricName(disk, disk_type, stage, iotype, "latency")) -+ self._detector_name_list[disk].append(MetricName(disk, disk_type, stage, iotype, "io_dump")) - if disks: - logging.warning( - "disks: %s not in available disk list, so they will be ignored.", -@@ -123,31 +122,51 @@ class SlowIODetection: - for disk, metric_name_list in self._detector_name_list.items(): - disk_detector = DiskDetector(disk) - for metric_name in metric_name_list: -- threshold = ThresholdFactory().get_threshold( -- threshold_type, -- boxplot_parameter=self._config_parser.boxplot_parameter, -- n_sigma_paramter=self._config_parser.n_sigma_parameter, -- data_queue_size=data_queue_size, -- data_queue_update_size=update_size, -- ) -- abs_threshold = self._config_parser.get_tot_lim( -- metric_name.disk_type, metric_name.io_access_type_name -- ) -- if abs_threshold is None: -- logging.warning( -- "disk %s, disk type %s, io type %s, get tot lim error, so it will be ignored.", -- disk, -- metric_name.disk_type, -- metric_name.io_access_type_name, -+ -+ if metric_name.metric_name == 'latency': -+ threshold = ThresholdFactory().get_threshold( -+ threshold_type, -+ boxplot_parameter=self._config_parser.boxplot_parameter, -+ n_sigma_paramter=self._config_parser.n_sigma_parameter, -+ data_queue_size=data_queue_size, -+ data_queue_update_size=update_size, - ) -- sliding_window = SlidingWindowFactory().get_sliding_window( -- sliding_window_type, -- queue_length=window_size, -- threshold=window_threshold, -- abs_threshold=abs_threshold, -- ) -- detector = Detector(metric_name, threshold, sliding_window) -- disk_detector.add_detector(detector) -+ abs_threshold = self._config_parser.get_tot_lim( -+ metric_name.disk_type, metric_name.io_access_type_name -+ ) -+ if abs_threshold is None: -+ logging.warning( -+ "disk %s, disk type %s, io type %s, get tot lim error, so it will be ignored.", -+ disk, -+ metric_name.disk_type, -+ metric_name.io_access_type_name, -+ ) -+ sliding_window = SlidingWindowFactory().get_sliding_window( -+ sliding_window_type, -+ queue_length=window_size, -+ threshold=window_threshold, -+ abs_threshold=abs_threshold, -+ ) -+ detector = Detector(metric_name, threshold, sliding_window) -+ disk_detector.add_detector(detector) -+ continue -+ -+ elif metric_name.metric_name == 'io_dump': -+ threshold = ThresholdFactory().get_threshold(ThresholdType.AbsoluteThreshold) -+ abs_threshold = None -+ if metric_name.io_access_type_name == 'read': -+ abs_threshold = self._config_parser.read_iodump_lim -+ elif metric_name.io_access_type_name == 'write': -+ abs_threshold = self._config_parser.write_iodump_lim -+ sliding_window = SlidingWindowFactory().get_sliding_window( -+ sliding_window_type, -+ queue_length=window_size, -+ threshold=window_threshold -+ ) -+ detector = Detector(metric_name, threshold, sliding_window) -+ threshold.set_threshold(abs_threshold) -+ disk_detector.add_detector(detector) -+ - logging.info(f"disk: [{disk}] add detector:\n [{disk_detector}]") - self._disk_detectors[disk] = disk_detector - -diff --git a/src/python/sentryPlugins/ai_block_io/config_parser.py b/src/python/sentryPlugins/ai_block_io/config_parser.py -index 274a31e..1117939 100644 ---- a/src/python/sentryPlugins/ai_block_io/config_parser.py -+++ b/src/python/sentryPlugins/ai_block_io/config_parser.py -@@ -72,6 +72,7 @@ class ConfigParser: - "latency_sata_ssd": {"read_tot_lim": 50000, "write_tot_lim": 50000}, - "latency_nvme_ssd": {"read_tot_lim": 500, "write_tot_lim": 500}, - "latency_sata_hdd": {"read_tot_lim": 50000, "write_tot_lim": 50000}, -+ "iodump": {"read_iodump_lim": 0, "write_iodump_lim": 0} - } - - def __init__(self, config_file_name): -@@ -497,6 +498,27 @@ class ConfigParser: - logging.critical("not found latency_sata_hdd section. exiting...") - exit(1) - -+ if con.has_section("iodump"): -+ items_iodump = dict(con.items("iodump")) -+ self._conf["iodump"]["read_iodump_lim"] = self._get_config_value( -+ items_iodump, -+ "read_iodump_lim", -+ int, -+ self.DEFAULT_CONF["iodump"]["read_iodump_lim"], -+ ge=0 -+ ) -+ self._conf["iodump"]["write_iodump_lim"] = self._get_config_value( -+ items_iodump, -+ "write_iodump_lim", -+ int, -+ self.DEFAULT_CONF["iodump"]["write_iodump_lim"], -+ ge=0 -+ ) -+ else: -+ Report.report_pass("not found iodump section. exiting...") -+ logging.critical("not found iodump section. exiting...") -+ exit(1) -+ - self.__print_all_config_value() - - def __repr__(self) -> str: -@@ -587,3 +609,11 @@ class ConfigParser: - @property - def n_sigma_parameter(self): - return self._conf["algorithm"]["n_sigma_parameter"] -+ -+ @property -+ def read_iodump_lim(self): -+ return self._conf["iodump"]["read_iodump_lim"] -+ -+ @property -+ def write_iodump_lim(self): -+ return self._conf["iodump"]["write_iodump_lim"] -\ No newline at end of file -diff --git a/src/python/sentryPlugins/ai_block_io/sliding_window.py b/src/python/sentryPlugins/ai_block_io/sliding_window.py -index d7c402a..cebe41f 100644 ---- a/src/python/sentryPlugins/ai_block_io/sliding_window.py -+++ b/src/python/sentryPlugins/ai_block_io/sliding_window.py -@@ -35,8 +35,8 @@ class SlidingWindow: - self._io_data_queue_abnormal_tag.pop(0) - self._io_data_queue.append(data) - tag = False -- if ((self._ai_threshold is not None and data >= self._ai_threshold) or -- (self._abs_threshold is not None and data >= self._abs_threshold)): -+ if ((self._ai_threshold is not None and data > self._ai_threshold) or -+ (self._abs_threshold is not None and data > self._abs_threshold)): - tag = True - self._io_data_queue_abnormal_tag.append(tag) - return tag --- -2.23.0 - diff --git a/ai_block_io-support-stage-and-iotype.patch b/ai_block_io-support-stage-and-iotype.patch deleted file mode 100644 index 1fd7505..0000000 --- a/ai_block_io-support-stage-and-iotype.patch +++ /dev/null @@ -1,906 +0,0 @@ -From 13dc3712b4530a312aa43610f7696a4a62f30e96 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?=E8=B4=BA=E6=9C=89=E5=BF=97?= <1037617413@qq.com> -Date: Fri, 11 Oct 2024 21:50:32 +0800 -Subject: [PATCH] ai_block_io support stage and iotype - ---- - config/plugins/ai_block_io.ini | 7 +- - .../sentryPlugins/ai_block_io/ai_block_io.py | 126 +++-- - .../ai_block_io/config_parser.py | 471 +++++++++++++----- - .../sentryPlugins/ai_block_io/data_access.py | 11 +- - .../sentryPlugins/ai_block_io/detector.py | 25 + - src/python/sentryPlugins/ai_block_io/utils.py | 3 +- - 6 files changed, 453 insertions(+), 190 deletions(-) - -diff --git a/config/plugins/ai_block_io.ini b/config/plugins/ai_block_io.ini -index 01ce266..a814d52 100644 ---- a/config/plugins/ai_block_io.ini -+++ b/config/plugins/ai_block_io.ini -@@ -1,7 +1,12 @@ -+[log] -+level=info -+ - [common] - absolute_threshold=40 - slow_io_detect_frequency=1 --log_level=info -+disk=default -+stage=bio -+iotype=read,write - - [algorithm] - train_data_duration=24 -diff --git a/src/python/sentryPlugins/ai_block_io/ai_block_io.py b/src/python/sentryPlugins/ai_block_io/ai_block_io.py -index 77104a9..e1052ec 100644 ---- a/src/python/sentryPlugins/ai_block_io/ai_block_io.py -+++ b/src/python/sentryPlugins/ai_block_io/ai_block_io.py -@@ -13,7 +13,7 @@ import time - import signal - import logging - --from .detector import Detector -+from .detector import Detector, DiskDetector - from .threshold import ThresholdFactory, AbsoluteThreshold - from .sliding_window import SlidingWindowFactory - from .utils import get_data_queue_size_and_update_size -@@ -34,8 +34,8 @@ def sig_handler(signum, frame): - class SlowIODetection: - _config_parser = None - _disk_list = None -- _detector_name_list = [] -- _detectors = {} -+ _detector_name_list = {} -+ _disk_detectors = {} - - def __init__(self, config_parser: ConfigParser): - self._config_parser = config_parser -@@ -43,85 +43,101 @@ class SlowIODetection: - self.__init_detector() - - def __init_detector_name_list(self): -- self._disk_list = check_collect_valid(self._config_parser.get_slow_io_detect_frequency()) -+ self._disk_list = check_collect_valid(self._config_parser.slow_io_detect_frequency) - if self._disk_list is None: - Report.report_pass("get available disk error, please check if the collector plug is enable. exiting...") - exit(1) - - logging.info(f"ai_block_io plug has found disks: {self._disk_list}") -- disks_to_detection: list = self._config_parser.get_disks_to_detection() -+ disks: list = self._config_parser.disks_to_detection -+ stages: list = self._config_parser.stage -+ iotypes: list = self._config_parser.iotype - # 情况1：None，则启用所有磁盘检测 - # 情况2：is not None and len = 0，则不启动任何磁盘检测 - # 情况3：len ！= 0，则取交集 -- if disks_to_detection is None: -+ if disks is None: - logging.warning("you not specify any disk or use default, so ai_block_io will enable all available disk.") - for disk in self._disk_list: -- self._detector_name_list.append(MetricName(disk, "bio", "read", "latency")) -- self._detector_name_list.append(MetricName(disk, "bio", "write", "latency")) -- elif len(disks_to_detection) == 0: -- logging.warning('please attention: conf file not specify any disk to detection, so it will not start ai block io.') -+ for stage in stages: -+ for iotype in iotypes: -+ if disk not in self._detector_name_list: -+ self._detector_name_list[disk] = [] -+ self._detector_name_list[disk].append(MetricName(disk, stage, iotype, "latency")) - else: -- for disk_to_detection in disks_to_detection: -- if disk_to_detection in self._disk_list: -- self._detector_name_list.append(MetricName(disk_to_detection, "bio", "read", "latency")) -- self._detector_name_list.append(MetricName(disk_to_detection, "bio", "write", "latency")) -+ for disk in disks: -+ if disk in self._disk_list: -+ for stage in stages: -+ for iotype in iotypes: -+ if disk not in self._detector_name_list: -+ self._detector_name_list[disk] = [] -+ self._detector_name_list[disk].append(MetricName(disk, stage, iotype, "latency")) - else: -- logging.warning(f"disk：[{disk_to_detection}] not in available disk list, so it will be ignored.") -- logging.info(f'start to detection follow disk and it\'s metric: {self._detector_name_list}') -+ logging.warning("disk: [%s] not in available disk list, so it will be ignored.", disk) -+ if len(self._detector_name_list) == 0: -+ logging.critical("the disks to detection is empty, ai_block_io will exit.") -+ Report.report_pass("the disks to detection is empty, ai_block_io will exit.") -+ exit(1) - - def __init_detector(self): -- train_data_duration, train_update_duration = (self._config_parser. -- get_train_data_duration_and_train_update_duration()) -- slow_io_detection_frequency = self._config_parser.get_slow_io_detect_frequency() -- threshold_type = self._config_parser.get_algorithm_type() -- data_queue_size, update_size = get_data_queue_size_and_update_size(train_data_duration, -- train_update_duration, -- slow_io_detection_frequency) -- sliding_window_type = self._config_parser.get_sliding_window_type() -- window_size, window_threshold = self._config_parser.get_window_size_and_window_minimum_threshold() -- -- for detector_name in self._detector_name_list: -- threshold = ThresholdFactory().get_threshold(threshold_type, -- boxplot_parameter=self._config_parser.get_boxplot_parameter(), -- n_sigma_paramter=self._config_parser.get_n_sigma_parameter(), -- data_queue_size=data_queue_size, -- data_queue_update_size=update_size) -- sliding_window = SlidingWindowFactory().get_sliding_window(sliding_window_type, queue_length=window_size, -- threshold=window_threshold) -- detector = Detector(detector_name, threshold, sliding_window) -- # 绝对阈值的阈值初始化 -- if isinstance(threshold, AbsoluteThreshold): -- threshold.set_threshold(self._config_parser.get_absolute_threshold()) -- self._detectors[detector_name] = detector -- logging.info(f"add detector: {detector}") -+ train_data_duration, train_update_duration = ( -+ self._config_parser.get_train_data_duration_and_train_update_duration() -+ ) -+ slow_io_detection_frequency = self._config_parser.slow_io_detect_frequency -+ threshold_type = self._config_parser.algorithm_type -+ data_queue_size, update_size = get_data_queue_size_and_update_size( -+ train_data_duration, train_update_duration, slow_io_detection_frequency -+ ) -+ sliding_window_type = self._config_parser.sliding_window_type -+ window_size, window_threshold = (self._config_parser.get_window_size_and_window_minimum_threshold()) -+ -+ for disk, metric_name_list in self._detector_name_list.items(): -+ threshold = ThresholdFactory().get_threshold( -+ threshold_type, -+ boxplot_parameter=self._config_parser.boxplot_parameter, -+ n_sigma_paramter=self._config_parser.n_sigma_parameter, -+ data_queue_size=data_queue_size, -+ data_queue_update_size=update_size, -+ ) -+ sliding_window = SlidingWindowFactory().get_sliding_window( -+ sliding_window_type, -+ queue_length=window_size, -+ threshold=window_threshold, -+ ) -+ disk_detector = DiskDetector(disk) -+ for metric_name in metric_name_list: -+ detector = Detector(metric_name, threshold, sliding_window) -+ disk_detector.add_detector(detector) -+ logging.info(f'disk: [{disk}] add detector:\n [{disk_detector}]') -+ self._disk_detectors[disk] = disk_detector - - def launch(self): - while True: -- logging.debug('step0. AI threshold slow io event detection is looping.') -+ logging.debug("step0. AI threshold slow io event detection is looping.") - - # Step1：获取IO数据 - io_data_dict_with_disk_name = get_io_data_from_collect_plug( -- self._config_parser.get_slow_io_detect_frequency(), self._disk_list -+ self._config_parser.slow_io_detect_frequency, self._disk_list - ) -- logging.debug(f'step1. Get io data: {str(io_data_dict_with_disk_name)}') -+ logging.debug(f"step1. Get io data: {str(io_data_dict_with_disk_name)}") - if io_data_dict_with_disk_name is None: -- Report.report_pass("get io data error, please check if the collector plug is enable. exitting...") -+ Report.report_pass( -+ "get io data error, please check if the collector plug is enable. exitting..." -+ ) - exit(1) - - # Step2：慢IO检测 -- logging.debug('step2. Start to detection slow io event.') -+ logging.debug("step2. Start to detection slow io event.") - slow_io_event_list = [] -- for metric_name, detector in self._detectors.items(): -- result = detector.is_slow_io_event(io_data_dict_with_disk_name) -+ for disk, disk_detector in self._disk_detectors.items(): -+ result = disk_detector.is_slow_io_event(io_data_dict_with_disk_name) - if result[0]: -- slow_io_event_list.append((detector.get_metric_name(), result)) -- logging.debug('step2. End to detection slow io event.') -+ slow_io_event_list.append(result) -+ logging.debug("step2. End to detection slow io event.") - - # Step3：慢IO事件上报 -- logging.debug('step3. Report slow io event to sysSentry.') -+ logging.debug("step3. Report slow io event to sysSentry.") - for slow_io_event in slow_io_event_list: -- metric_name: MetricName = slow_io_event[0] -- result = slow_io_event[1] -+ metric_name: MetricName = slow_io_event[1] - alarm_content = { - "driver_name": f"{metric_name.get_disk_name()}", - "reason": "disk_slow", -@@ -129,14 +145,14 @@ class SlowIODetection: - "io_type": f"{metric_name.get_io_access_type_name()}", - "alarm_source": "ai_block_io", - "alarm_type": "latency", -- "details": f"current window is: {result[1]}, threshold is: {result[2]}.", -+ "details": f"current window is: {slow_io_event[2]}, threshold is: {slow_io_event[3]}.", - } - Xalarm.major(alarm_content) - logging.warning(alarm_content) - - # Step4：等待检测时间 -- logging.debug('step4. Wait to start next slow io event detection loop.') -- time.sleep(self._config_parser.get_slow_io_detect_frequency()) -+ logging.debug("step4. Wait to start next slow io event detection loop.") -+ time.sleep(self._config_parser.slow_io_detect_frequency) - - - def main(): -diff --git a/src/python/sentryPlugins/ai_block_io/config_parser.py b/src/python/sentryPlugins/ai_block_io/config_parser.py -index 354c122..a357766 100644 ---- a/src/python/sentryPlugins/ai_block_io/config_parser.py -+++ b/src/python/sentryPlugins/ai_block_io/config_parser.py -@@ -9,44 +9,60 @@ - # PURPOSE. - # See the Mulan PSL v2 for more details. - -+import os - import configparser - import logging - -+from .alarm_report import Report - from .threshold import ThresholdType - from .utils import get_threshold_type_enum, get_sliding_window_type_enum, get_log_level - - - LOG_FORMAT = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s" - -+ALL_STAGE_LIST = ['throtl', 'wbt', 'gettag', 'plug', 'deadline', 'hctx', 'requeue', 'rq_driver', 'bio'] -+ALL_IOTPYE_LIST = ['read', 'write'] -+ - - def init_log_format(log_level: str): - logging.basicConfig(level=get_log_level(log_level.lower()), format=LOG_FORMAT) -- if log_level.lower() not in ('info', 'warning', 'error', 'debug'): -- logging.warning(f'the log_level: {log_level} you set is invalid, use default value: info.') -+ if log_level.lower() not in ("info", "warning", "error", "debug"): -+ logging.warning( -+ f"the log_level: {log_level} you set is invalid, use default value: info." -+ ) - - - class ConfigParser: - DEFAULT_ABSOLUTE_THRESHOLD = 40 - DEFAULT_SLOW_IO_DETECTION_FREQUENCY = 1 -- DEFAULT_LOG_LEVEL = 'info' -+ DEFAULT_LOG_LEVEL = "info" -+ -+ DEFAULT_STAGE = 'throtl,wbt,gettag,plug,deadline,hctx,requeue,rq_driver,bio' -+ DEFAULT_IOTYPE = 'read,write' - -- DEFAULT_ALGORITHM_TYPE = 'boxplot' -+ DEFAULT_ALGORITHM_TYPE = "boxplot" - DEFAULT_TRAIN_DATA_DURATION = 24 - DEFAULT_TRAIN_UPDATE_DURATION = 2 - DEFAULT_BOXPLOT_PARAMETER = 1.5 - DEFAULT_N_SIGMA_PARAMETER = 3 - -- DEFAULT_SLIDING_WINDOW_TYPE = 'not_continuous' -+ DEFAULT_SLIDING_WINDOW_TYPE = "not_continuous" - DEFAULT_WINDOW_SIZE = 30 - DEFAULT_WINDOW_MINIMUM_THRESHOLD = 6 - - def __init__(self, config_file_name): - self.__absolute_threshold = ConfigParser.DEFAULT_ABSOLUTE_THRESHOLD -- self.__slow_io_detect_frequency = ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY -+ self.__slow_io_detect_frequency = ( -+ ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY -+ ) - self.__log_level = ConfigParser.DEFAULT_LOG_LEVEL - self.__disks_to_detection = None -+ self.__stage = ConfigParser.DEFAULT_STAGE -+ self.__iotype = ConfigParser.DEFAULT_IOTYPE - -- self.__algorithm_type = ConfigParser.DEFAULT_ALGORITHM_TYPE -+ self.__algorithm_type = get_threshold_type_enum( -+ ConfigParser.DEFAULT_ALGORITHM_TYPE -+ ) - self.__train_data_duration = ConfigParser.DEFAULT_TRAIN_UPDATE_DURATION - self.__train_update_duration = ConfigParser.DEFAULT_TRAIN_UPDATE_DURATION - self.__boxplot_parameter = ConfigParser.DEFAULT_BOXPLOT_PARAMETER -@@ -58,199 +74,398 @@ class ConfigParser: - - self.__config_file_name = config_file_name - -- def __read_absolute_threshold(self, items_common: dict): -+ def _get_config_value( -+ self, -+ config_items: dict, -+ key: str, -+ value_type, -+ default_value=None, -+ gt=None, -+ ge=None, -+ lt=None, -+ le=None, -+ ): -+ value = config_items.get(key) -+ if value is None: -+ logging.warning( -+ "config of %s not found, the default value %s will be used.", -+ key, -+ default_value, -+ ) -+ value = default_value -+ if not value: -+ logging.critical( -+ "the value of %s is empty, ai_block_io plug will exit.", key -+ ) -+ Report.report_pass( -+ f"the value of {key} is empty, ai_block_io plug will exit." -+ ) -+ exit(1) - try: -- self.__absolute_threshold = float(items_common.get('absolute_threshold', -- ConfigParser.DEFAULT_ABSOLUTE_THRESHOLD)) -- if self.__absolute_threshold <= 0: -- logging.warning( -- f'the_absolute_threshold: {self.__absolute_threshold} you set is invalid, use default value: {ConfigParser.DEFAULT_ABSOLUTE_THRESHOLD}.') -- self.__absolute_threshold = ConfigParser.DEFAULT_ABSOLUTE_THRESHOLD -+ value = value_type(value) - except ValueError: -- self.__absolute_threshold = ConfigParser.DEFAULT_ABSOLUTE_THRESHOLD -- logging.warning( -- f'the_absolute_threshold type conversion has error, use default value: {self.__absolute_threshold}.') -+ logging.critical( -+ "the value of %s is not a valid %s, ai_block_io plug will exit.", -+ key, -+ value_type, -+ ) -+ Report.report_pass( -+ f"the value of {key} is not a valid {value_type}, ai_block_io plug will exit." -+ ) -+ exit(1) -+ if gt is not None and value <= gt: -+ logging.critical( -+ "the value of %s is not greater than %s, ai_block_io plug will exit.", -+ key, -+ gt, -+ ) -+ Report.report_pass( -+ f"the value of {key} is not greater than {gt}, ai_block_io plug will exit." -+ ) -+ exit(1) -+ if ge is not None and value < ge: -+ logging.critical( -+ "the value of %s is not greater than or equal to %s, ai_block_io plug will exit.", -+ key, -+ ge, -+ ) -+ Report.report_pass( -+ f"the value of {key} is not greater than or equal to {ge}, ai_block_io plug will exit." -+ ) -+ exit(1) -+ if lt is not None and value >= lt: -+ logging.critical( -+ "the value of %s is not less than %s, ai_block_io plug will exit.", -+ key, -+ lt, -+ ) -+ Report.report_pass( -+ f"the value of {key} is not less than {lt}, ai_block_io plug will exit." -+ ) -+ exit(1) -+ if le is not None and value > le: -+ logging.critical( -+ "the value of %s is not less than or equal to %s, ai_block_io plug will exit.", -+ key, -+ le, -+ ) -+ Report.report_pass( -+ f"the value of {key} is not less than or equal to {le}, ai_block_io plug will exit." -+ ) -+ exit(1) -+ -+ return value -+ -+ def __read_absolute_threshold(self, items_common: dict): -+ self.__absolute_threshold = self._get_config_value( -+ items_common, -+ "absolute_threshold", -+ float, -+ ConfigParser.DEFAULT_ABSOLUTE_THRESHOLD, -+ gt=0, -+ ) - - def __read__slow_io_detect_frequency(self, items_common: dict): -- try: -- self.__slow_io_detect_frequency = int(items_common.get('slow_io_detect_frequency', -- ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY)) -- if self.__slow_io_detect_frequency < 1 or self.__slow_io_detect_frequency > 10: -- logging.warning( -- f'the slow_io_detect_frequency: {self.__slow_io_detect_frequency} you set is invalid, use default value: {ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY}.') -- self.__slow_io_detect_frequency = ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY -- except ValueError: -- self.__slow_io_detect_frequency = ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY -- logging.warning(f'slow_io_detect_frequency type conversion has error, use default value: {self.__slow_io_detect_frequency}.') -+ self.__slow_io_detect_frequency = self._get_config_value( -+ items_common, -+ "slow_io_detect_frequency", -+ int, -+ ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY, -+ gt=0, -+ le=300, -+ ) - - def __read__disks_to_detect(self, items_common: dict): -- disks_to_detection = items_common.get('disk') -+ disks_to_detection = items_common.get("disk") - if disks_to_detection is None: -- logging.warning(f'config of disk not found, the default value will be used.') -+ logging.warning("config of disk not found, the default value will be used.") - self.__disks_to_detection = None - return -- disk_list = disks_to_detection.split(',') -- if len(disk_list) == 0 or (len(disk_list) == 1 and disk_list[0] == ''): -- logging.warning("you don't specify any disk.") -- self.__disks_to_detection = [] -- return -- if len(disk_list) == 1 and disk_list[0] == 'default': -+ disks_to_detection = disks_to_detection.strip() -+ if not disks_to_detection: -+ logging.critical("the value of disk is empty, ai_block_io plug will exit.") -+ Report.report_pass( -+ "the value of disk is empty, ai_block_io plug will exit." -+ ) -+ exit(1) -+ disk_list = disks_to_detection.split(",") -+ if len(disk_list) == 1 and disk_list[0] == "default": - self.__disks_to_detection = None - return - self.__disks_to_detection = disk_list - - def __read__train_data_duration(self, items_algorithm: dict): -- try: -- self.__train_data_duration = float(items_algorithm.get('train_data_duration', -- ConfigParser.DEFAULT_TRAIN_DATA_DURATION)) -- if self.__train_data_duration <= 0 or self.__train_data_duration > 720: -- logging.warning( -- f'the train_data_duration: {self.__train_data_duration} you set is invalid, use default value: {ConfigParser.DEFAULT_TRAIN_DATA_DURATION}.') -- self.__train_data_duration = ConfigParser.DEFAULT_TRAIN_DATA_DURATION -- except ValueError: -- self.__train_data_duration = ConfigParser.DEFAULT_TRAIN_DATA_DURATION -- logging.warning(f'the train_data_duration type conversion has error, use default value: {self.__train_data_duration}.') -+ self.__train_data_duration = self._get_config_value( -+ items_algorithm, -+ "train_data_duration", -+ float, -+ ConfigParser.DEFAULT_TRAIN_DATA_DURATION, -+ gt=0, -+ le=720, -+ ) - - def __read__train_update_duration(self, items_algorithm: dict): - default_train_update_duration = ConfigParser.DEFAULT_TRAIN_UPDATE_DURATION - if default_train_update_duration > self.__train_data_duration: - default_train_update_duration = self.__train_data_duration / 2 -- -- try: -- self.__train_update_duration = float(items_algorithm.get('train_update_duration', -- ConfigParser.DEFAULT_TRAIN_UPDATE_DURATION)) -- if self.__train_update_duration <= 0 or self.__train_update_duration > self.__train_data_duration: -- logging.warning( -- f'the train_update_duration: {self.__train_update_duration} you set is invalid, use default value: {default_train_update_duration}.') -- self.__train_update_duration = default_train_update_duration -- except ValueError: -- self.__train_update_duration = default_train_update_duration -- logging.warning(f'the train_update_duration type conversion has error, use default value: {self.__train_update_duration}.') -+ self.__train_update_duration = self._get_config_value( -+ items_algorithm, -+ "train_update_duration", -+ float, -+ default_train_update_duration, -+ gt=0, -+ le=self.__train_data_duration, -+ ) - - def __read__algorithm_type_and_parameter(self, items_algorithm: dict): -- algorithm_type = items_algorithm.get('algorithm_type', ConfigParser.DEFAULT_ALGORITHM_TYPE) -+ algorithm_type = items_algorithm.get( -+ "algorithm_type", ConfigParser.DEFAULT_ALGORITHM_TYPE -+ ) - self.__algorithm_type = get_threshold_type_enum(algorithm_type) -+ if self.__algorithm_type is None: -+ logging.critical( -+ "the algorithm_type: %s you set is invalid. ai_block_io plug will exit.", -+ algorithm_type, -+ ) -+ Report.report_pass( -+ f"the algorithm_type: {algorithm_type} you set is invalid. ai_block_io plug will exit." -+ ) -+ exit(1) - - if self.__algorithm_type == ThresholdType.NSigmaThreshold: -- try: -- self.__n_sigma_parameter = float(items_algorithm.get('n_sigma_parameter', -- ConfigParser.DEFAULT_N_SIGMA_PARAMETER)) -- if self.__n_sigma_parameter <= 0 or self.__n_sigma_parameter > 10: -- logging.warning( -- f'the n_sigma_parameter: {self.__n_sigma_parameter} you set is invalid, use default value: {ConfigParser.DEFAULT_N_SIGMA_PARAMETER}.') -- self.__n_sigma_parameter = ConfigParser.DEFAULT_N_SIGMA_PARAMETER -- except ValueError: -- self.__n_sigma_parameter = ConfigParser.DEFAULT_N_SIGMA_PARAMETER -- logging.warning(f'the n_sigma_parameter type conversion has error, use default value: {self.__n_sigma_parameter}.') -+ self.__n_sigma_parameter = self._get_config_value( -+ items_algorithm, -+ "n_sigma_parameter", -+ float, -+ ConfigParser.DEFAULT_N_SIGMA_PARAMETER, -+ gt=0, -+ le=10, -+ ) - elif self.__algorithm_type == ThresholdType.BoxplotThreshold: -- try: -- self.__boxplot_parameter = float(items_algorithm.get('boxplot_parameter', -- ConfigParser.DEFAULT_BOXPLOT_PARAMETER)) -- if self.__boxplot_parameter <= 0 or self.__boxplot_parameter > 10: -- logging.warning( -- f'the boxplot_parameter: {self.__boxplot_parameter} you set is invalid, use default value: {ConfigParser.DEFAULT_BOXPLOT_PARAMETER}.') -- self.__n_sigma_parameter = ConfigParser.DEFAULT_BOXPLOT_PARAMETER -- except ValueError: -- self.__boxplot_parameter = ConfigParser.DEFAULT_BOXPLOT_PARAMETER -- logging.warning(f'the boxplot_parameter type conversion has error, use default value: {self.__boxplot_parameter}.') -+ self.__boxplot_parameter = self._get_config_value( -+ items_algorithm, -+ "boxplot_parameter", -+ float, -+ ConfigParser.DEFAULT_BOXPLOT_PARAMETER, -+ gt=0, -+ le=10, -+ ) -+ -+ def __read__stage(self, items_algorithm: dict): -+ stage_str = items_algorithm.get('stage', ConfigParser.DEFAULT_STAGE) -+ stage_list = stage_str.split(',') -+ if len(stage_list) == 1 and stage_list[0] == '': -+ logging.critical('stage value not allow is empty, exiting...') -+ exit(1) -+ if len(stage_list) == 1 and stage_list[0] == 'default': -+ logging.warning(f'stage will enable default value: {ConfigParser.DEFAULT_STAGE}') -+ self.__stage = ALL_STAGE_LIST -+ return -+ for stage in stage_list: -+ if stage not in ALL_STAGE_LIST: -+ logging.critical(f'stage: {stage} is not valid stage, ai_block_io will exit...') -+ exit(1) -+ dup_stage_list = set(stage_list) -+ if 'bio' not in dup_stage_list: -+ logging.critical('stage must contains bio stage, exiting...') -+ exit(1) -+ self.__stage = dup_stage_list -+ -+ def __read__iotype(self, items_algorithm: dict): -+ iotype_str = items_algorithm.get('iotype', ConfigParser.DEFAULT_IOTYPE) -+ iotype_list = iotype_str.split(',') -+ if len(iotype_list) == 1 and iotype_list[0] == '': -+ logging.critical('iotype value not allow is empty, exiting...') -+ exit(1) -+ if len(iotype_list) == 1 and iotype_list[0] == 'default': -+ logging.warning(f'iotype will enable default value: {ConfigParser.DEFAULT_IOTYPE}') -+ self.__iotype = ALL_IOTPYE_LIST -+ return -+ for iotype in iotype_list: -+ if iotype not in ALL_IOTPYE_LIST: -+ logging.critical(f'iotype: {iotype} is not valid iotype, ai_block_io will exit...') -+ exit(1) -+ dup_iotype_list = set(iotype_list) -+ self.__iotype = dup_iotype_list - - def __read__window_size(self, items_sliding_window: dict): -- try: -- self.__window_size = int(items_sliding_window.get('window_size', -- ConfigParser.DEFAULT_WINDOW_SIZE)) -- if self.__window_size < 1 or self.__window_size > 3600: -- logging.warning( -- f'the window_size: {self.__window_size} you set is invalid, use default value: {ConfigParser.DEFAULT_WINDOW_SIZE}.') -- self.__window_size = ConfigParser.DEFAULT_WINDOW_SIZE -- except ValueError: -- self.__window_size = ConfigParser.DEFAULT_WINDOW_SIZE -- logging.warning(f'window_size type conversion has error, use default value: {self.__window_size}.') -+ self.__window_size = self._get_config_value( -+ items_sliding_window, -+ "window_size", -+ int, -+ ConfigParser.DEFAULT_WINDOW_SIZE, -+ gt=0, -+ le=3600, -+ ) - - def __read__window_minimum_threshold(self, items_sliding_window: dict): - default_window_minimum_threshold = ConfigParser.DEFAULT_WINDOW_MINIMUM_THRESHOLD - if default_window_minimum_threshold > self.__window_size: - default_window_minimum_threshold = self.__window_size / 2 -- try: -- self.__window_minimum_threshold = ( -- int(items_sliding_window.get('window_minimum_threshold', -- ConfigParser.DEFAULT_WINDOW_MINIMUM_THRESHOLD))) -- if self.__window_minimum_threshold < 1 or self.__window_minimum_threshold > self.__window_size: -- logging.warning( -- f'the window_minimum_threshold: {self.__window_minimum_threshold} you set is invalid, use default value: {default_window_minimum_threshold}.') -- self.__window_minimum_threshold = default_window_minimum_threshold -- except ValueError: -- self.__window_minimum_threshold = default_window_minimum_threshold -- logging.warning(f'window_minimum_threshold type conversion has error, use default value: {self.__window_minimum_threshold}.') -+ self.__window_minimum_threshold = self._get_config_value( -+ items_sliding_window, -+ "window_minimum_threshold", -+ int, -+ default_window_minimum_threshold, -+ gt=0, -+ le=self.__window_size, -+ ) - - def read_config_from_file(self): -+ if not os.path.exists(self.__config_file_name): -+ init_log_format(self.__log_level) -+ logging.critical( -+ "config file %s not found, ai_block_io plug will exit.", -+ self.__config_file_name, -+ ) -+ Report.report_pass( -+ f"config file {self.__config_file_name} not found, ai_block_io plug will exit." -+ ) -+ exit(1) -+ - con = configparser.ConfigParser() - try: -- con.read(self.__config_file_name, encoding='utf-8') -+ con.read(self.__config_file_name, encoding="utf-8") - except configparser.Error as e: - init_log_format(self.__log_level) -- logging.critical(f'config file read error: {e}, ai_block_io plug will exit.') -+ logging.critical( -+ f"config file read error: %s, ai_block_io plug will exit.", e -+ ) -+ Report.report_pass( -+ f"config file read error: {e}, ai_block_io plug will exit." -+ ) - exit(1) - -- if con.has_section('common'): -- items_common = dict(con.items('common')) -- self.__log_level = items_common.get('log_level', ConfigParser.DEFAULT_LOG_LEVEL) -+ if con.has_section('log'): -+ items_log = dict(con.items('log')) -+ # 情况一：没有log，则使用默认值 -+ # 情况二：有log，值为空或异常，使用默认值 -+ # 情况三：有log，值正常，则使用该值 -+ self.__log_level = items_log.get('level', ConfigParser.DEFAULT_LOG_LEVEL) - init_log_format(self.__log_level) -+ else: -+ init_log_format(self.__log_level) -+ logging.warning(f"log section parameter not found, it will be set to default value.") -+ -+ if con.has_section("common"): -+ items_common = dict(con.items("common")) - self.__read_absolute_threshold(items_common) - self.__read__slow_io_detect_frequency(items_common) - self.__read__disks_to_detect(items_common) -+ self.__read__stage(items_common) -+ self.__read__iotype(items_common) - else: -- init_log_format(self.__log_level) -- logging.warning("common section parameter not found, it will be set to default value.") -+ logging.warning( -+ "common section parameter not found, it will be set to default value." -+ ) - -- if con.has_section('algorithm'): -- items_algorithm = dict(con.items('algorithm')) -+ if con.has_section("algorithm"): -+ items_algorithm = dict(con.items("algorithm")) - self.__read__train_data_duration(items_algorithm) - self.__read__train_update_duration(items_algorithm) - self.__read__algorithm_type_and_parameter(items_algorithm) - else: -- logging.warning("algorithm section parameter not found, it will be set to default value.") -- -- if con.has_section('sliding_window'): -- items_sliding_window = dict(con.items('sliding_window')) -- sliding_window_type = items_sliding_window.get('sliding_window_type', -- ConfigParser.DEFAULT_SLIDING_WINDOW_TYPE) -- self.__sliding_window_type = get_sliding_window_type_enum(sliding_window_type) -+ logging.warning( -+ "algorithm section parameter not found, it will be set to default value." -+ ) -+ -+ if con.has_section("sliding_window"): -+ items_sliding_window = dict(con.items("sliding_window")) -+ sliding_window_type = items_sliding_window.get( -+ "sliding_window_type", ConfigParser.DEFAULT_SLIDING_WINDOW_TYPE -+ ) -+ self.__sliding_window_type = get_sliding_window_type_enum( -+ sliding_window_type -+ ) - self.__read__window_size(items_sliding_window) - self.__read__window_minimum_threshold(items_sliding_window) - else: -- logging.warning("sliding_window section parameter not found, it will be set to default value.") -+ logging.warning( -+ "sliding_window section parameter not found, it will be set to default value." -+ ) - - self.__print_all_config_value() - -+ def __repr__(self): -+ config_str = { -+ 'log.level': self.__log_level, -+ 'common.absolute_threshold': self.__absolute_threshold, -+ 'common.slow_io_detect_frequency': self.__slow_io_detect_frequency, -+ 'common.disk': self.__disks_to_detection, -+ 'common.stage': self.__stage, -+ 'common.iotype': self.__iotype, -+ 'algorithm.train_data_duration': self.__train_data_duration, -+ 'algorithm.train_update_duration': self.__train_update_duration, -+ 'algorithm.algorithm_type': self.__algorithm_type, -+ 'algorithm.boxplot_parameter': self.__boxplot_parameter, -+ 'algorithm.n_sigma_parameter': self.__n_sigma_parameter, -+ 'sliding_window.sliding_window_type': self.__sliding_window_type, -+ 'sliding_window.window_size': self.__window_size, -+ 'sliding_window.window_minimum_threshold': self.__window_minimum_threshold -+ } -+ return str(config_str) -+ - def __print_all_config_value(self): -- pass -+ logging.info(f"all config is follow:\n {self}") -+ -+ def get_train_data_duration_and_train_update_duration(self): -+ return self.__train_data_duration, self.__train_update_duration - -- def get_slow_io_detect_frequency(self): -+ def get_window_size_and_window_minimum_threshold(self): -+ return self.__window_size, self.__window_minimum_threshold -+ -+ @property -+ def slow_io_detect_frequency(self): - return self.__slow_io_detect_frequency - -- def get_algorithm_type(self): -+ @property -+ def algorithm_type(self): - return self.__algorithm_type - -- def get_sliding_window_type(self): -+ @property -+ def sliding_window_type(self): - return self.__sliding_window_type - -- def get_train_data_duration_and_train_update_duration(self): -- return self.__train_data_duration, self.__train_update_duration -+ @property -+ def train_data_duration(self): -+ return self.__train_data_duration - -- def get_window_size_and_window_minimum_threshold(self): -- return self.__window_size, self.__window_minimum_threshold -+ @property -+ def train_update_duration(self): -+ return self.__train_update_duration -+ -+ @property -+ def window_size(self): -+ return self.__window_size - -- def get_absolute_threshold(self): -+ @property -+ def window_minimum_threshold(self): -+ return self.__window_minimum_threshold -+ -+ @property -+ def absolute_threshold(self): - return self.__absolute_threshold - -- def get_log_level(self): -+ @property -+ def log_level(self): - return self.__log_level - -- def get_disks_to_detection(self): -+ @property -+ def disks_to_detection(self): - return self.__disks_to_detection - -- def get_boxplot_parameter(self): -+ @property -+ def stage(self): -+ return self.__stage -+ -+ @property -+ def iotype(self): -+ return self.__iotype -+ -+ @property -+ def boxplot_parameter(self): - return self.__boxplot_parameter - -- def get_n_sigma_parameter(self): -+ @property -+ def n_sigma_parameter(self): - return self.__n_sigma_parameter -diff --git a/src/python/sentryPlugins/ai_block_io/data_access.py b/src/python/sentryPlugins/ai_block_io/data_access.py -index c7679cd..ed997e6 100644 ---- a/src/python/sentryPlugins/ai_block_io/data_access.py -+++ b/src/python/sentryPlugins/ai_block_io/data_access.py -@@ -41,11 +41,14 @@ def check_collect_valid(period): - try: - data = json.loads(data_raw["message"]) - except Exception as e: -- logging.warning(f"get io data failed, {e}") -+ logging.warning(f"get valid devices failed, occur exception: {e}") -+ return None -+ if not data: -+ logging.warning(f"get valid devices failed, return {data_raw}") - return None - return [k for k in data.keys()] - else: -- logging.warning(f"get io data failed, return {data_raw}") -+ logging.warning(f"get valid devices failed, return {data_raw}") - return None - - -@@ -60,7 +63,7 @@ def _get_raw_data(period, disk_list): - - def _get_io_stage_data(data): - io_stage_data = IOStageData() -- for data_type in ('read', 'write', 'flush', 'discard'): -+ for data_type in ("read", "write", "flush", "discard"): - if data_type in data: - getattr(io_stage_data, data_type).latency = data[data_type][0] - getattr(io_stage_data, data_type).io_dump = data[data_type][1] -@@ -87,7 +90,7 @@ def get_io_data_from_collect_plug(period, disk_list): - getattr(disk_ret, k) - setattr(disk_ret, k, _get_io_stage_data(v)) - except AttributeError: -- logging.debug(f'no attr {k}') -+ logging.debug(f"no attr {k}") - continue - ret[disk] = disk_ret - return ret -diff --git a/src/python/sentryPlugins/ai_block_io/detector.py b/src/python/sentryPlugins/ai_block_io/detector.py -index 0ed282b..e710ddd 100644 ---- a/src/python/sentryPlugins/ai_block_io/detector.py -+++ b/src/python/sentryPlugins/ai_block_io/detector.py -@@ -53,3 +53,28 @@ class Detector: - f' io_type_name: {self._metric_name.get_io_access_type_name()},' - f' metric_name: {self._metric_name.get_metric_name()}, threshold_type: {self._threshold},' - f' sliding_window_type: {self._slidingWindow}') -+ -+ -+class DiskDetector: -+ -+ def __init__(self, disk_name: str): -+ self._disk_name = disk_name -+ self._detector_list = [] -+ -+ def add_detector(self, detector: Detector): -+ self._detector_list.append(detector) -+ -+ def is_slow_io_event(self, io_data_dict_with_disk_name: dict): -+ # 只有bio阶段发生异常，就认为发生了慢IO事件 -+ # todo：根因诊断 -+ for detector in self._detector_list: -+ result = detector.is_slow_io_event(io_data_dict_with_disk_name) -+ if result[0] and detector.get_metric_name().get_stage_name() == 'bio': -+ return result[0], detector.get_metric_name(), result[1], result[2] -+ return False, None, None, None -+ -+ def __repr__(self): -+ msg = f'disk: {self._disk_name}, ' -+ for detector in self._detector_list: -+ msg += f'\n detector: [{detector}]' -+ return msg -diff --git a/src/python/sentryPlugins/ai_block_io/utils.py b/src/python/sentryPlugins/ai_block_io/utils.py -index 8dbba06..0ed37b9 100644 ---- a/src/python/sentryPlugins/ai_block_io/utils.py -+++ b/src/python/sentryPlugins/ai_block_io/utils.py -@@ -25,8 +25,7 @@ def get_threshold_type_enum(algorithm_type: str): - return ThresholdType.BoxplotThreshold - if algorithm_type.lower() == 'n_sigma': - return ThresholdType.NSigmaThreshold -- logging.warning(f"the algorithm type: {algorithm_type} you set is invalid, use default value: boxplot") -- return ThresholdType.BoxplotThreshold -+ return None - - - def get_sliding_window_type_enum(sliding_window_type: str): --- -2.23.0 - diff --git a/avg_block_io-send-alarm-to-xalarmd.patch b/avg_block_io-send-alarm-to-xalarmd.patch deleted file mode 100644 index 3995d08..0000000 --- a/avg_block_io-send-alarm-to-xalarmd.patch +++ /dev/null @@ -1,73 +0,0 @@ -From 7d5ad8f2dd87432b8f46ea5002400ee46cb6756a Mon Sep 17 00:00:00 2001 -From: gaoruoshu -Date: Wed, 9 Oct 2024 14:22:38 +0800 -Subject: [PATCH] avg_block_io send alarm to xalarmd - ---- - config/tasks/avg_block_io.mod | 2 ++ - .../sentryPlugins/avg_block_io/module_conn.py | 23 +++++++++++++++---- - 2 files changed, 21 insertions(+), 4 deletions(-) - -diff --git a/config/tasks/avg_block_io.mod b/config/tasks/avg_block_io.mod -index b9b6f34..bcd063b 100644 ---- a/config/tasks/avg_block_io.mod -+++ b/config/tasks/avg_block_io.mod -@@ -3,3 +3,5 @@ enabled=yes - task_start=/usr/bin/python3 /usr/bin/avg_block_io - task_stop=pkill -f /usr/bin/avg_block_io - type=oneshot -+alarm_id=1002 -+alarm_clear_time=5 -diff --git a/src/python/sentryPlugins/avg_block_io/module_conn.py b/src/python/sentryPlugins/avg_block_io/module_conn.py -index 0da4208..2fc5a83 100644 ---- a/src/python/sentryPlugins/avg_block_io/module_conn.py -+++ b/src/python/sentryPlugins/avg_block_io/module_conn.py -@@ -16,6 +16,7 @@ import time - from .utils import is_abnormal - from sentryCollector.collect_plugin import is_iocollect_valid, get_io_data, Result_Messages - from syssentry.result import ResultLevel, report_result -+from xalarm.sentry_notify import xalarm_report, MINOR_ALM, ALARM_TYPE_OCCUR - - - TASK_NAME = "avg_block_io" -@@ -68,19 +69,33 @@ def process_report_data(disk_name, rw, io_data): - if not is_abnormal((disk_name, 'bio', rw), io_data): - return - -+ msg = {"alarm_source": TASK_NAME, "driver_name": disk_name, "io_type": rw} -+ - ctrl_stage = ['throtl', 'wbt', 'iocost', 'bfq'] - for stage_name in ctrl_stage: - if is_abnormal((disk_name, stage_name, rw), io_data): -- logging.warning("{} - {} - {} report IO press".format(time.ctime(), disk_name, rw)) -+ msg["reason"] = "IO press slow" -+ msg["block_stack"] = f"bio,{stage_name}" -+ logging.warning("{} - {} report IO press slow".format(disk_name, rw)) -+ xalarm_report(1002, MINOR_ALM, ALARM_TYPE_OCCUR, json.dumps(msg)) - return - - if is_abnormal((disk_name, 'rq_driver', rw), io_data): -- logging.warning("{} - {} - {} report driver".format(time.ctime(), disk_name, rw)) -+ msg["reason"] = "driver slow" -+ msg["block_stack"] = "bio,rq_driver" -+ logging.warning("{} - {} report driver slow".format(disk_name, rw)) -+ xalarm_report(1002, MINOR_ALM, ALARM_TYPE_OCCUR, json.dumps(msg)) - return - - kernel_stage = ['gettag', 'plug', 'deadline', 'hctx', 'requeue'] - for stage_name in kernel_stage: - if is_abnormal((disk_name, stage_name, rw), io_data): -- logging.warning("{} - {} - {} report kernel".format(time.ctime(), disk_name, rw)) -+ msg["reason"] = "kernel slow" -+ msg["block_stack"] = f"bio,{stage_name}" -+ logging.warning("{} - {} report kernel slow".format(disk_name, rw)) -+ xalarm_report(1002, MINOR_ALM, ALARM_TYPE_OCCUR, json.dumps(msg)) - return -- logging.warning("{} - {} - {} report IO press".format(time.ctime(), disk_name, rw)) -+ msg["reason"] = "unknown" -+ msg["block_stack"] = "bio" -+ logging.warning("{} - {} report UNKNOWN slow".format(disk_name, rw)) -+ xalarm_report(1002, MINOR_ALM, ALARM_TYPE_OCCUR, json.dumps(msg)) --- -2.33.0 - diff --git a/bugfix-typo.patch b/bugfix-typo.patch deleted file mode 100644 index 946a0cb..0000000 --- a/bugfix-typo.patch +++ /dev/null @@ -1,34 +0,0 @@ -From 7d5ad8f2dd87432b8f46ea5002400ee46cb6756a Mon Sep 17 00:00:00 2001 -From: gaoruoshu -Date: Wed, 9 Oct 2024 14:22:38 +0800 -Subject: [PATCH] bugfix typo - ---- - src/python/sentryPlugins/avg_block_io/avg_block_io.py | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/src/python/sentryPlugins/avg_block_io/avg_block_io.py b/src/python/sentryPlugins/avg_block_io/avg_block_io.py -index b6b3b28..26a60c5 100644 ---- a/src/python/sentryPlugins/avg_block_io/avg_block_io.py -+++ b/src/python/sentryPlugins/avg_block_io/avg_block_io.py -@@ -114,7 +114,7 @@ def read_config_lat_iodump(io_dic, config): - common_param = {} - lat_sec = None - if not config.has_section("latency"): -- logging.warning("Cannot find algorithm section in config file") -+ logging.warning("Cannot find latency section in config file") - else: - lat_sec = config["latency"] - -@@ -122,7 +122,7 @@ def read_config_lat_iodump(io_dic, config): - if not config.has_section("iodump"): - logging.warning("Cannot find iodump section in config file") - else: -- lat_sec = config["iodump"] -+ iodump_sec = config["iodump"] - - if not lat_sec and not iodump_sec: - return common_param --- -2.27.0 - diff --git a/change-alarm-length.patch b/change-alarm-length.patch deleted file mode 100644 index 27c49ed..0000000 --- a/change-alarm-length.patch +++ /dev/null @@ -1,56 +0,0 @@ -From 67439c0040b1fb0614ac009bf53062e9ec2880aa Mon Sep 17 00:00:00 2001 -From: jinsaihang -Date: Wed, 9 Oct 2024 11:55:35 +0800 -Subject: [PATCH 1/2] change alarm length - -Signed-off-by: jinsaihang ---- - src/python/syssentry/sentryctl | 3 +++ - src/python/syssentry/syssentry.py | 3 +++ - 2 files changed, 6 insertions(+) - -diff --git a/src/python/syssentry/sentryctl b/src/python/syssentry/sentryctl -index 675c17a..3de93d0 100644 ---- a/src/python/syssentry/sentryctl -+++ b/src/python/syssentry/sentryctl -@@ -25,6 +25,7 @@ MAX_PARAM_LENGTH = 256 - - RESULT_MSG_DATA_LEN = 4 - CTL_MSG_LEN_LEN = 3 -+ALARM_MSG_DATA_LEN = 6 - DEFAULT_ALARM_TIME_RANGE = 10 - - def status_output_format(res_data): -@@ -173,6 +174,8 @@ if __name__ == '__main__': - request_message = json.dumps(req_msg_struct) - if client_args.cmd_type == 'get_result': - result_message = client_send_and_recv(request_message, RESULT_MSG_DATA_LEN) -+ elif client_args.cmd_type == 'get_alarm': -+ result_message = client_send_and_recv(request_message, ALARM_MSG_DATA_LEN) - else: - result_message = client_send_and_recv(request_message, CTL_MSG_LEN_LEN) - if not result_message: -diff --git a/src/python/syssentry/syssentry.py b/src/python/syssentry/syssentry.py -index c2dee85..ea09095 100644 ---- a/src/python/syssentry/syssentry.py -+++ b/src/python/syssentry/syssentry.py -@@ -56,6 +56,7 @@ CTL_MSG_MAGIC_LEN = 3 - CTL_MSG_LEN_LEN = 3 - CTL_MAGIC = "CTL" - RES_MAGIC = "RES" -+ALARM_MSG_DATA_LEN = 6 - - CTL_LISTEN_QUEUE_LEN = 5 - SERVER_EPOLL_TIMEOUT = 0.3 -@@ -256,6 +257,8 @@ def server_recv(server_socket: socket.socket): - res_head = RES_MAGIC - if cmd_type == "get_result": - res_data_len = str(len(res_data)).zfill(RESULT_MSG_HEAD_LEN - RESULT_MSG_MAGIC_LEN) -+ elif cmd_type == "get_alarm": -+ res_data_len = str(len(res_data)).zfill(ALARM_MSG_DATA_LEN) - else: - res_data_len = str(len(res_data)).zfill(CTL_MSG_MAGIC_LEN) - res_head += res_data_len --- -2.27.0 - diff --git a/change-avg_block_io-config.patch b/change-avg_block_io-config.patch deleted file mode 100644 index 09d8b47..0000000 --- a/change-avg_block_io-config.patch +++ /dev/null @@ -1,55 +0,0 @@ -From aaff413d6954003a3c21af21003c3bc134f940e2 Mon Sep 17 00:00:00 2001 -From: gaoruoshu -Date: Tue, 5 Nov 2024 10:31:10 +0800 -Subject: [PATCH] change avg_block_io config - ---- - config/plugins/avg_block_io.ini | 8 ++++---- - .../src/python/sentryPlugins/avg_block_io/config.py | 8 ++++---- - 2 files changed, 8 insertions(+), 8 deletions(-) - -diff --git a/config/plugins/avg_block_io.ini b/config/plugins/avg_block_io.ini -index 5c4b9b0..3b4ee33 100644 ---- a/config/plugins/avg_block_io.ini -+++ b/config/plugins/avg_block_io.ini -@@ -12,12 +12,12 @@ win_size=30 - win_threshold=6 - - [latency_nvme_ssd] --read_avg_lim=300 --write_avg_lim=300 -+read_avg_lim=10000 -+write_avg_lim=10000 - read_avg_time=3 - write_avg_time=3 --read_tot_lim=500 --write_tot_lim=500 -+read_tot_lim=50000 -+write_tot_lim=50000 - - [latency_sata_ssd] - read_avg_lim=10000 -diff --git a/src/python/sentryPlugins/avg_block_io/config.py b/src/python/sentryPlugins/avg_block_io/config.py -index c8f45ce..c1e8ab1 100644 ---- a/src/python/sentryPlugins/avg_block_io/config.py -+++ b/src/python/sentryPlugins/avg_block_io/config.py -@@ -42,12 +42,12 @@ DEFAULT_PARAM = { - CONF_ALGO_SIZE: 30, - CONF_ALGO_THRE: 6 - }, 'latency_nvme_ssd': { -- 'read_avg_lim': 300, -- 'write_avg_lim': 300, -+ 'read_avg_lim': 10000, -+ 'write_avg_lim': 10000, - 'read_avg_time': 3, - 'write_avg_time': 3, -- 'read_tot_lim': 500, -- 'write_tot_lim': 500, -+ 'read_tot_lim': 50000, -+ 'write_tot_lim': 50000, - }, 'latency_sata_ssd' : { - 'read_avg_lim': 10000, - 'write_avg_lim': 10000, --- -2.39.5 (Apple Git-154) - diff --git a/change-status-of-period-task-and-sort-mod-file.patch b/change-status-of-period-task-and-sort-mod-file.patch deleted file mode 100644 index 12aab0e..0000000 --- a/change-status-of-period-task-and-sort-mod-file.patch +++ /dev/null @@ -1,36 +0,0 @@ -From 8cc13a422ed29e48b0c5b86b2da2a5dc8ad4aa59 Mon Sep 17 00:00:00 2001 -From: zhuofeng -Date: Fri, 13 Dec 2024 11:20:55 +0800 -Subject: [PATCH] change status of period task and sort mod file - ---- - src/python/syssentry/cron_process.py | 1 + - src/python/syssentry/load_mods.py | 1 + - 2 files changed, 2 insertions(+) - -diff --git a/src/python/syssentry/cron_process.py b/src/python/syssentry/cron_process.py -index 50780b3..5543d67 100644 ---- a/src/python/syssentry/cron_process.py -+++ b/src/python/syssentry/cron_process.py -@@ -144,6 +144,7 @@ def period_tasks_handle(): - - if not task.onstart: - logging.debug("period onstart not enabled, task: %s", task.name) -+ task.runtime_status = EXITED_STATUS - continue - - if task.runtime_status == WAITING_STATUS and \ -diff --git a/src/python/syssentry/load_mods.py b/src/python/syssentry/load_mods.py -index 48d7e66..5be5540 100644 ---- a/src/python/syssentry/load_mods.py -+++ b/src/python/syssentry/load_mods.py -@@ -224,6 +224,7 @@ def load_tasks(): - return "failed", "" - - mod_files = os.listdir(TASKS_STORAGE_PATH) -+ mod_files.sort() - for mod_file in mod_files: - logging.debug("find mod, path is %s", mod_file) - if not mod_file.endswith(MOD_FILE_SUFFIX): --- -2.33.0 diff --git a/cpu_utility-and-cpu_patrol-must-be-an-integer.patch b/cpu_utility-and-cpu_patrol-must-be-an-integer.patch deleted file mode 100644 index 3780958..0000000 --- a/cpu_utility-and-cpu_patrol-must-be-an-integer.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 6e98b2e5008ffabfda8d1c10778717f972b54398 Mon Sep 17 00:00:00 2001 -From: jwolf <523083921@qq.com> -Date: Mon, 22 Jul 2024 14:58:27 +0800 -Subject: [PATCH] cpu_utility and cpu_patrol musht be an integer - ---- - src/c/catcli/catlib/cli_param_checker.c | 5 +++-- - 1 file changed, 3 insertions(+), 2 deletions(-) - -diff --git a/src/c/catcli/catlib/cli_param_checker.c b/src/c/catcli/catlib/cli_param_checker.c -index a1aa636..e400428 100644 ---- a/src/c/catcli/catlib/cli_param_checker.c -+++ b/src/c/catcli/catlib/cli_param_checker.c -@@ -2,6 +2,7 @@ - #include - #include - #include -+#include - #include - #include - #include "cli_common.h" -@@ -13,7 +14,7 @@ - void checkset_cpu_usage_percentage(char *getopt_optarg, catcli_request_body *p_request_body, struct option_errs *errs) - { - long cpu_utility = strtol(getopt_optarg, NULL, DECIMAL); -- if (cpu_utility <= 0 || cpu_utility > CPU_USAGE_PERCENTAGE_MAX) { -+ if (cpu_utility <= 0 || cpu_utility > CPU_USAGE_PERCENTAGE_MAX || strchr(getopt_optarg, '.') != NULL) { - strncpy(errs->patrol_module_err, - "\"cpu_utility \" must be an integer greater in the range (0,100],correct \"-u, --cpu_utility\"\n", MAX_ERR_LEN); - } -@@ -68,7 +69,7 @@ void checkset_cpulist(char *getopt_optarg, catcli_request_body *p_request_body, - void checkset_patrol_time(char *getopt_optarg, catcli_request_body *p_request_body, struct option_errs *errs) - { - long second = strtol(getopt_optarg, NULL, DECIMAL); -- if (second <= 0 || second > INT_MAX) { -+ if (second <= 0 || second > INT_MAX || strchr(getopt_optarg, '.') != NULL) { - strncpy(errs->patrol_time_err, - "\"patrol_second\" must be a number in the range of (0,INT_MAX] ,correct \"-t, --patrol_second\"\n", - MAX_ERR_LEN); --- -Gitee diff --git a/diff-disk-type-use-diff-config.patch b/diff-disk-type-use-diff-config.patch deleted file mode 100644 index 70976d3..0000000 --- a/diff-disk-type-use-diff-config.patch +++ /dev/null @@ -1,430 +0,0 @@ -From e7c1b0095e16369fb09ae62ffa3158be5e8893a1 Mon Sep 17 00:00:00 2001 -From: gaoruoshu -Date: Fri, 11 Oct 2024 10:48:35 +0800 -Subject: [PATCH] diff disk type use diff config - ---- - config/plugins/avg_block_io.ini | 26 +++- - src/python/sentryCollector/collect_plugin.py | 6 + - .../avg_block_io/avg_block_io.py | 144 ++++++++---------- - .../sentryPlugins/avg_block_io/module_conn.py | 19 ++- - .../sentryPlugins/avg_block_io/utils.py | 43 ++++++ - 5 files changed, 146 insertions(+), 92 deletions(-) - -diff --git a/config/plugins/avg_block_io.ini b/config/plugins/avg_block_io.ini -index 858db18..5c4b9b0 100644 ---- a/config/plugins/avg_block_io.ini -+++ b/config/plugins/avg_block_io.ini -@@ -11,13 +11,29 @@ period_time=1 - win_size=30 - win_threshold=6 - --[latency] --read_avg_lim=10 --write_avg_lim=10 -+[latency_nvme_ssd] -+read_avg_lim=300 -+write_avg_lim=300 - read_avg_time=3 - write_avg_time=3 --read_tot_lim=50 --write_tot_lim=50 -+read_tot_lim=500 -+write_tot_lim=500 -+ -+[latency_sata_ssd] -+read_avg_lim=10000 -+write_avg_lim=10000 -+read_avg_time=3 -+write_avg_time=3 -+read_tot_lim=50000 -+write_tot_lim=50000 -+ -+[latency_sata_hdd] -+read_avg_lim=15000 -+write_avg_lim=15000 -+read_avg_time=3 -+write_avg_time=3 -+read_tot_lim=50000 -+write_tot_lim=50000 - - [iodump] - read_iodump_lim=0 -diff --git a/src/python/sentryCollector/collect_plugin.py b/src/python/sentryCollector/collect_plugin.py -index 31bf11b..bec405a 100644 ---- a/src/python/sentryCollector/collect_plugin.py -+++ b/src/python/sentryCollector/collect_plugin.py -@@ -79,6 +79,12 @@ class DiskType(): - TYPE_SATA_SSD = 1 - TYPE_SATA_HDD = 2 - -+Disk_Type = { -+ DiskType.TYPE_NVME_SSD: "nvme_ssd", -+ DiskType.TYPE_SATA_SSD: "sata_ssd", -+ DiskType.TYPE_SATA_HDD: "sata_hdd" -+} -+ - def client_send_and_recv(request_data, data_str_len, protocol): - """client socket send and recv message""" - try: -diff --git a/src/python/sentryPlugins/avg_block_io/avg_block_io.py b/src/python/sentryPlugins/avg_block_io/avg_block_io.py -index cf2ded3..fdad995 100644 ---- a/src/python/sentryPlugins/avg_block_io/avg_block_io.py -+++ b/src/python/sentryPlugins/avg_block_io/avg_block_io.py -@@ -14,8 +14,9 @@ import configparser - import time - - from .stage_window import IoWindow, IoDumpWindow --from .module_conn import avg_is_iocollect_valid, avg_get_io_data, report_alarm_fail, process_report_data, sig_handler --from .utils import update_avg_and_check_abnormal, get_log_level -+from .module_conn import avg_is_iocollect_valid, avg_get_io_data, report_alarm_fail, process_report_data, sig_handler, get_disk_type_by_name -+from .utils import update_avg_and_check_abnormal, get_log_level, get_section_value -+from sentryCollector.collect_plugin import Disk_Type - - CONFIG_FILE = "/etc/sysSentry/plugins/avg_block_io.ini" - -@@ -37,44 +38,40 @@ def read_config_common(config): - disk = [] if disk_name == "default" else disk_name.split(",") - except configparser.NoOptionError: - disk = [] -- logging.warning("Unset disk, set to default") -+ logging.warning("Unset common.disk, set to default") - - try: - stage_name = config.get("common", "stage") - stage = [] if stage_name == "default" else stage_name.split(",") - except configparser.NoOptionError: - stage = [] -- logging.warning("Unset stage, set to read,write") -+ logging.warning("Unset common.stage, set to default") - - if len(disk) > 10: -- logging.warning("Too many disks, record only max 10 disks") -+ logging.warning("Too many common.disks, record only max 10 disks") - disk = disk[:10] - - try: - iotype_name = config.get("common", "iotype").split(",") -- iotype_list = [rw.lower() for rw in iotype_name if rw.lower() in ['read', 'write', 'flush', 'discard']] -- err_iotype = [rw.lower() for rw in iotype_name if rw.lower() not in ['read', 'write', 'flush', 'discard']] -+ iotype_list = [rw.lower() for rw in iotype_name if rw.lower() in ['read', 'write']] -+ err_iotype = [rw.lower() for rw in iotype_name if rw.lower() not in ['read', 'write']] - -- if iotype_list in [None, []]: -- iotype_list = ["read", "write"] -- except configparser.NoOptionError: -- iotype = ["read", "write"] -- logging.warning("Unset iotype, set to default") -+ if err_iotype: -+ report_alarm_fail("Invalid common.iotype config") - -- if err_iotype: -- logging.warning("{} in common.iotype are not valid, set iotype={}".format(err_iotype, iotype_list)) -- -+ except configparser.NoOptionError: -+ iotype_list = ["read", "write"] -+ logging.warning("Unset common.iotype, set to read,write") - - try: - period_time = int(config.get("common", "period_time")) - if not (1 <= period_time <= 300): - raise ValueError("Invalid period_time") - except ValueError: -- period_time = 1 -- logging.warning("Invalid period_time, set to 1s") -+ report_alarm_fail("Invalid common.period_time") - except configparser.NoOptionError: - period_time = 1 -- logging.warning("Unset period_time, use 1s as default") -+ logging.warning("Unset common.period_time, use 1s as default") - - return period_time, disk, stage, iotype_list - -@@ -87,76 +84,56 @@ def read_config_algorithm(config): - try: - win_size = int(config.get("algorithm", "win_size")) - if not (1 <= win_size <= 300): -- raise ValueError("Invalid win_size") -+ raise ValueError("Invalid algorithm.win_size") - except ValueError: -- win_size = 30 -- logging.warning("Invalid win_size, set to 30") -+ report_alarm_fail("Invalid algorithm.win_size config") - except configparser.NoOptionError: - win_size = 30 -- logging.warning("Unset win_size, use 30 as default") -+ logging.warning("Unset algorithm.win_size, use 30 as default") - - try: - win_threshold = int(config.get("algorithm", "win_threshold")) - if win_threshold < 1 or win_threshold > 300 or win_threshold > win_size: -- raise ValueError("Invalid win_threshold") -+ raise ValueError("Invalid algorithm.win_threshold") - except ValueError: -- win_threshold = 6 -- logging.warning("Invalid win_threshold, set to 6") -+ report_alarm_fail("Invalid algorithm.win_threshold config") - except configparser.NoOptionError: - win_threshold = 6 -- logging.warning("Unset win_threshold, use 6 as default") -+ logging.warning("Unset algorithm.win_threshold, use 6 as default") - - return win_size, win_threshold - - --def read_config_lat_iodump(io_dic, config): -- """read config file, get [latency] [iodump] section value""" -+def read_config_latency(config): -+ """read config file, get [latency_xxx] section value""" - common_param = {} -- lat_sec = None -- if not config.has_section("latency"): -- logging.warning("Cannot find latency section in config file") -- else: -- lat_sec = config["latency"] -- -- iodump_sec = None -- if not config.has_section("iodump"): -- logging.warning("Cannot find iodump section in config file") -- else: -- iodump_sec = config["iodump"] -- -- if not lat_sec and not iodump_sec: -- return common_param -- -- for io_type in io_dic["iotype_list"]: -- common_param[io_type] = {} -- -- latency_keys = { -- "avg_lim": "{}_avg_lim".format(io_type), -- "avg_time": "{}_avg_time".format(io_type), -- "tot_lim": "{}_tot_lim".format(io_type), -- } -- iodump_key = "{}_iodump_lim".format(io_type) -+ for type_name in Disk_Type: -+ section_name = f"latency_{Disk_Type[type_name]}" -+ if not config.has_section(section_name): -+ report_alarm_fail(f"Cannot find {section_name} section in config file") - -- if iodump_sec and iodump_key in iodump_sec and iodump_sec[iodump_key].isdecimal(): -- common_param[io_type][iodump_key] = int(iodump_sec[iodump_key]) -+ common_param[Disk_Type[type_name]] = get_section_value(section_name, config) -+ return common_param - -- if not lat_sec: -- continue - -- for key_suffix, key_template in latency_keys.items(): -- if key_template in lat_sec and lat_sec[key_template].isdecimal(): -- common_param[io_type][key_template] = int(lat_sec[key_template]) -+def read_config_iodump(config): -+ """read config file, get [iodump] section value""" -+ common_param = {} -+ section_name = "iodump" -+ if not config.has_section(section_name): -+ report_alarm_fail(f"Cannot find {section_name} section in config file") - -- return common_param -+ return get_section_value(section_name, config) - - --def read_config_stage(config, stage, iotype_list): -- """read config file, get [STAGE_NAME] section value""" -+def read_config_stage(config, stage, iotype_list, curr_disk_type): -+ """read config file, get [STAGE_NAME_diskType] section value""" - res = {} -- if not stage in config: -+ section_name = f"{stage}_{curr_disk_type}" -+ if not config.has_section(section_name): - return res - -- for key in config[stage]: -+ for key in config[section_name]: - if config[stage][key].isdecimal(): - res[key] = int(config[stage][key]) - -@@ -171,11 +148,12 @@ def init_io_win(io_dic, config, common_param): - for disk_name in io_dic["disk_list"]: - io_data[disk_name] = {} - io_avg_value[disk_name] = {} -+ curr_disk_type = get_disk_type_by_name(disk_name) - for stage_name in io_dic["stage_list"]: - io_data[disk_name][stage_name] = {} - io_avg_value[disk_name][stage_name] = {} -- # step3. 解析stage配置 -- curr_stage_param = read_config_stage(config, stage_name, iotype_list) -+ # 解析stage配置 -+ curr_stage_param = read_config_stage(config, stage_name, iotype_list, curr_disk_type) - for rw in iotype_list: - io_data[disk_name][stage_name][rw] = {} - io_avg_value[disk_name][stage_name][rw] = [0, 0] -@@ -187,10 +165,10 @@ def init_io_win(io_dic, config, common_param): - iodump_lim_key = "{}_iodump_lim".format(rw) - - # 获取值，优先从 curr_stage_param 获取，如果不存在，则从 common_param 获取 -- avg_lim_value = curr_stage_param.get(avg_lim_key, common_param.get(rw, {}).get(avg_lim_key)) -- avg_time_value = curr_stage_param.get(avg_time_key, common_param.get(rw, {}).get(avg_time_key)) -- tot_lim_value = curr_stage_param.get(tot_lim_key, common_param.get(rw, {}).get(tot_lim_key)) -- iodump_lim_value = curr_stage_param.get(iodump_lim_key, common_param.get(rw, {}).get(iodump_lim_key)) -+ avg_lim_value = curr_stage_param.get(avg_lim_key, common_param.get(curr_disk_type, {}).get(avg_lim_key)) -+ avg_time_value = curr_stage_param.get(avg_time_key, common_param.get(curr_disk_type, {}).get(avg_time_key)) -+ tot_lim_value = curr_stage_param.get(tot_lim_key, common_param.get(curr_disk_type, {}).get(tot_lim_key)) -+ iodump_lim_value = curr_stage_param.get(iodump_lim_key, common_param.get("iodump", {}).get(iodump_lim_key)) - - if avg_lim_value and avg_time_value and tot_lim_value: - io_data[disk_name][stage_name][rw]["latency"] = IoWindow(window_size=io_dic["win_size"], window_threshold=io_dic["win_threshold"], abnormal_multiple=avg_time_value, abnormal_multiple_lim=avg_lim_value, abnormal_time=tot_lim_value) -@@ -217,28 +195,21 @@ def get_valid_disk_stage_list(io_dic, config_disk, config_stage): - stage_list = [key for key in all_stage_set if key in config_stage] - not_in_stage_list = [key for key in config_stage if key not in all_stage_set] - -- if not config_disk: -+ if not_in_stage_list: -+ report_alarm_fail(f"Invalid common.stage_list config, cannot set {not_in_stage_list}") -+ -+ if not config_disk and not not_in_disk_list: - disk_list = [key for key in all_disk_set] - -- if not config_stage: -+ if not config_stage and not not_in_stage_list: - stage_list = [key for key in all_stage_set] - - disk_list = disk_list[:10] if len(disk_list) > 10 else disk_list -- stage_list = stage_list[:15] if len(stage_list) > 15 else stage_list -- -- if config_disk and not disk_list: -- logging.warning("Cannot get valid disk by disk={}, set to default".format(config_disk)) -- disk_list, stage_list = get_valid_disk_stage_list(io_dic, [], config_stage) -- -- if config_stage and not stage_list: -- logging.warning("Cannot get valid stage by stage={}, set to default".format(config_stage)) -- disk_list, stage_list = get_valid_disk_stage_list(io_dic, config_disk, []) - - if not stage_list or not disk_list: - report_alarm_fail("Cannot get valid disk name or stage name.") - - log_invalid_keys(not_in_disk_list, 'disk', config_disk, disk_list) -- log_invalid_keys(not_in_stage_list, 'stage', config_stage, stage_list) - - return disk_list, stage_list - -@@ -310,8 +281,13 @@ def main(): - # step1. 解析公共配置 --- algorithm - io_dic["win_size"], io_dic["win_threshold"] = read_config_algorithm(config) - -- # step2. 循环创建窗口 -- common_param = read_config_lat_iodump(io_dic, config) -+ # step2. 解析公共配置 --- latency_xxx -+ common_param = read_config_latency(config) -+ -+ # step3. 解析公共配置 --- iodump -+ common_param['iodump'] = read_config_iodump(config) -+ -+ # step4. 循环创建窗口 - io_data, io_avg_value = init_io_win(io_dic, config, common_param) - - main_loop(io_dic, io_data, io_avg_value) -diff --git a/src/python/sentryPlugins/avg_block_io/module_conn.py b/src/python/sentryPlugins/avg_block_io/module_conn.py -index 40b3fcc..8d6f429 100644 ---- a/src/python/sentryPlugins/avg_block_io/module_conn.py -+++ b/src/python/sentryPlugins/avg_block_io/module_conn.py -@@ -14,7 +14,7 @@ import sys - import time - - from .utils import is_abnormal, get_win_data, log_slow_win --from sentryCollector.collect_plugin import is_iocollect_valid, get_io_data, Result_Messages -+from sentryCollector.collect_plugin import is_iocollect_valid, get_io_data, Result_Messages, get_disk_type, Disk_Type - from syssentry.result import ResultLevel, report_result - from xalarm.sentry_notify import xalarm_report, MINOR_ALM, ALARM_TYPE_OCCUR - -@@ -51,7 +51,7 @@ def check_result_validation(res, reason): - try: - json_data = json.loads(res['message']) - except json.JSONDecodeError: -- err_msg = "Failed to {}: invalid return message".format(reason) -+ err_msg = f"Failed to {reason}: invalid return message" - report_alarm_fail(err_msg) - - return json_data -@@ -60,7 +60,7 @@ def check_result_validation(res, reason): - def report_alarm_fail(alarm_info): - """report result to xalarmd""" - report_result(TASK_NAME, ResultLevel.FAIL, json.dumps({"msg": alarm_info})) -- logging.error(alarm_info) -+ logging.critical(alarm_info) - sys.exit(1) - - -@@ -114,3 +114,16 @@ def process_report_data(disk_name, rw, io_data): - - log_slow_win(msg, "unknown") - xalarm_report(1002, MINOR_ALM, ALARM_TYPE_OCCUR, json.dumps(msg)) -+ -+ -+def get_disk_type_by_name(disk_name): -+ res = get_disk_type(disk_name) -+ disk_type_str = check_result_validation(get_disk_type(disk_name), f'Invalid disk type {disk_name}') -+ try: -+ curr_disk_type = int(disk_type_str) -+ if curr_disk_type not in Disk_Type: -+ raise ValueError -+ except ValueError: -+ report_alarm_fail(f"Failed to get disk type for {disk_name}") -+ -+ return Disk_Type[curr_disk_type] -\ No newline at end of file -diff --git a/src/python/sentryPlugins/avg_block_io/utils.py b/src/python/sentryPlugins/avg_block_io/utils.py -index 3b7f027..cef1edd 100644 ---- a/src/python/sentryPlugins/avg_block_io/utils.py -+++ b/src/python/sentryPlugins/avg_block_io/utils.py -@@ -26,6 +26,49 @@ LogLevel = { - } - - -+DEFAULT_PARAM = { -+ 'latency_nvme_ssd': { -+ 'read_avg_lim': 300, -+ 'write_avg_lim': 300, -+ 'read_avg_time': 3, -+ 'write_avg_time': 3, -+ 'read_tot_lim': 500, -+ 'write_tot_lim': 500, -+ }, 'latency_sata_ssd' : { -+ 'read_avg_lim': 10000, -+ 'write_avg_lim': 10000, -+ 'read_avg_time': 3, -+ 'write_avg_time': 3, -+ 'read_tot_lim': 50000, -+ 'write_tot_lim': 50000, -+ }, 'latency_sata_hdd' : { -+ 'read_avg_lim': 15000, -+ 'write_avg_lim': 15000, -+ 'read_avg_time': 3, -+ 'write_avg_time': 3, -+ 'read_tot_lim': 50000, -+ 'write_tot_lim': 50000 -+ }, 'iodump': { -+ 'read_iodump_lim': 0, -+ 'write_iodump_lim': 0 -+ } -+} -+ -+ -+def get_section_value(section_name, config): -+ common_param = {} -+ config_sec = config[section_name] -+ for config_key in DEFAULT_PARAM[section_name]: -+ if config_key in config_sec: -+ if not config_sec[config_key].isdecimal(): -+ report_alarm_fail(f"Invalid {section_name}.{config_key} config.") -+ common_param[config_key] = int(config_sec[config_key]) -+ else: -+ logging.warning(f"Unset {section_name}.{config_key} in config file, use {DEFAULT_PARAM[section_name][config_key]} as default") -+ common_param[config_key] = DEFAULT_PARAM[section_name][config_key] -+ return common_param -+ -+ - def get_log_level(filename): - if not os.path.exists(filename): - return logging.INFO --- -2.27.0 diff --git a/enrich-alert-info-about-kernel-stack.patch b/enrich-alert-info-about-kernel-stack.patch deleted file mode 100644 index bf04a6e..0000000 --- a/enrich-alert-info-about-kernel-stack.patch +++ /dev/null @@ -1,29 +0,0 @@ -From 41bf507ca6cbbdf5e646a405de6b8d5b9be4bd28 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?=E8=B4=BA=E6=9C=89=E5=BF=97?= <1037617413@qq.com> -Date: Wed, 16 Oct 2024 17:20:01 +0800 -Subject: [PATCH] enrich alert info about kernel stack - ---- - src/python/sentryPlugins/ai_block_io/detector.py | 6 ++++-- - 1 file changed, 4 insertions(+), 2 deletions(-) - -diff --git a/src/python/sentryPlugins/ai_block_io/detector.py b/src/python/sentryPlugins/ai_block_io/detector.py -index ed8b64a..8536f7a 100644 ---- a/src/python/sentryPlugins/ai_block_io/detector.py -+++ b/src/python/sentryPlugins/ai_block_io/detector.py -@@ -103,8 +103,10 @@ class DiskDetector: - elif len(diagnosis_info["rq_driver"]) != 0: - root_cause = "[Root Cause: disk slow]" - elif len(diagnosis_info["io_stage"]) != 0: -- stage = diagnosis_info["io_stage"][0][1].stage_name -- root_cause = f"[Root Cause: io stage slow, stage: {stage}]" -+ stage_list = [] -+ for io_stage in diagnosis_info["io_stage"]: -+ stage_list.append(io_stage[0].stage_name) -+ root_cause = f"[Root Cause: io stage slow, stage: {stage_list}]" - if root_cause is None: - root_cause = "[Root Cause: high io pressure]" - return True, diagnosis_info["bio"][0][0], diagnosis_info["bio"][0][1], root_cause --- -2.23.0 - diff --git a/feature-add-avg_block_io-plugin.patch b/feature-add-avg_block_io-plugin.patch deleted file mode 100644 index 5477f18..0000000 --- a/feature-add-avg_block_io-plugin.patch +++ /dev/null @@ -1,572 +0,0 @@ -From acb77d6a69aa9269b0f691613bef53efd0c01e53 Mon Sep 17 00:00:00 2001 -From: gaoruoshu -Date: Thu, 12 Sep 2024 11:31:34 +0800 -Subject: [PATCH 2/2] add avg_block_io plugin - ---- - config/plugins/avg_block_io.ini | 21 ++ - config/tasks/avg_block_io.mod | 5 + - src/python/sentryPlugins/__init__.py | 0 - .../sentryPlugins/avg_block_io/__init__.py | 0 - .../avg_block_io/avg_block_io.py | 257 ++++++++++++++++++ - .../sentryPlugins/avg_block_io/module_conn.py | 86 ++++++ - .../avg_block_io/stage_window.py | 47 ++++ - .../sentryPlugins/avg_block_io/utils.py | 86 ++++++ - 8 files changed, 502 insertions(+) - create mode 100644 config/plugins/avg_block_io.ini - create mode 100644 config/tasks/avg_block_io.mod - create mode 100644 src/python/sentryPlugins/__init__.py - create mode 100644 src/python/sentryPlugins/avg_block_io/__init__.py - create mode 100644 src/python/sentryPlugins/avg_block_io/avg_block_io.py - create mode 100644 src/python/sentryPlugins/avg_block_io/module_conn.py - create mode 100644 src/python/sentryPlugins/avg_block_io/stage_window.py - create mode 100644 src/python/sentryPlugins/avg_block_io/utils.py - -diff --git a/config/plugins/avg_block_io.ini b/config/plugins/avg_block_io.ini -new file mode 100644 -index 0000000..bc33dde ---- /dev/null -+++ b/config/plugins/avg_block_io.ini -@@ -0,0 +1,21 @@ -+[common] -+disk=default -+stage=default -+iotype=read,write -+period_time=1 -+ -+[algorithm] -+win_size=30 -+win_threshold=6 -+ -+[latency] -+read_avg_lim=10 -+write_avg_lim=10 -+read_avg_time=3 -+write_avg_time=3 -+read_tot_lim=50 -+write_tot_lim=50 -+ -+[iodump] -+read_iodump_lim=0 -+write_iodump_lim=0 -diff --git a/config/tasks/avg_block_io.mod b/config/tasks/avg_block_io.mod -new file mode 100644 -index 0000000..814c483 ---- /dev/null -+++ b/config/tasks/avg_block_io.mod -@@ -0,0 +1,5 @@ -+[common] -+enabled=yes -+task_start=/usr/bin/python3 /usr/bin/avg_block_io -+task_stop=pkill avg_block_io -+type=oneshot -\ No newline at end of file -diff --git a/src/python/sentryPlugins/__init__.py b/src/python/sentryPlugins/__init__.py -new file mode 100644 -index 0000000..e69de29 -diff --git a/src/python/sentryPlugins/avg_block_io/__init__.py b/src/python/sentryPlugins/avg_block_io/__init__.py -new file mode 100644 -index 0000000..e69de29 -diff --git a/src/python/sentryPlugins/avg_block_io/avg_block_io.py b/src/python/sentryPlugins/avg_block_io/avg_block_io.py -new file mode 100644 -index 0000000..ff2071d ---- /dev/null -+++ b/src/python/sentryPlugins/avg_block_io/avg_block_io.py -@@ -0,0 +1,257 @@ -+# coding: utf-8 -+# Copyright (c) 2024 Huawei Technologies Co., Ltd. -+# sysSentry is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+import logging -+import signal -+import configparser -+import time -+ -+from .stage_window import IoWindow, IoDumpWindow -+from .module_conn import avg_is_iocollect_valid, avg_get_io_data, report_alarm_fail, process_report_data, sig_handler -+from .utils import update_avg_and_check_abnormal -+ -+CONFIG_FILE = "/etc/sysSentry/plugins/avg_block_io.ini" -+ -+def log_invalid_keys(not_in_list, keys_name, config_list, default_list): -+ """print invalid log""" -+ if config_list and default_list: -+ logging.warning("{} in common.{} are not valid, set {}={}".format(not_in_list, keys_name, keys_name, default_list)) -+ elif config_list == ["default"]: -+ logging.warning("Default {} use {}".format(keys_name, default_list)) -+ -+ -+def read_config_common(config): -+ """read config file, get [common] section value""" -+ try: -+ common_sec = config['common'] -+ except configparser.NoSectionError: -+ report_alarm_fail("Cannot find common section in config file") -+ -+ try: -+ period_time = int(common_sec.get("period_time", 1)) -+ if not (1 <= period_time <= 300): -+ raise ValueError("Invalid period_time") -+ except ValueError: -+ period_time = 1 -+ logging.warning("Invalid period_time, set to 1s") -+ -+ disk = common_sec.get('disk').split(",") if common_sec.get('disk') not in [None, 'default'] else [] -+ stage = common_sec.get('stage').split(",") if common_sec.get('stage') not in [None, 'default'] else [] -+ -+ if len(disk) > 10: -+ logging.warning("Too many disks, record only max 10 disks") -+ disk = disk[:10] -+ -+ iotype = common_sec.get('iotype', 'read,write').split(",") -+ iotype_list = [rw.lower() for rw in iotype if rw.lower() in ['read', 'write', 'flush', 'discard']] -+ err_iotype = [rw for rw in iotype if rw.lower() not in ['read', 'write', 'flush', 'discard']] -+ -+ if err_iotype: -+ logging.warning("{} in common.iotype are not valid, set iotype={}".format(err_iotype, iotype_list)) -+ -+ return period_time, disk, stage, iotype_list -+ -+ -+def read_config_algorithm(config): -+ """read config file, get [algorithm] section value""" -+ if not config.has_section("algorithm"): -+ report_alarm_fail("Cannot find algorithm section in config file") -+ -+ try: -+ win_size = int(config.get("algorithm", "win_size")) -+ if not (1 <= win_size <= 300): -+ raise ValueError("Invalid win_size") -+ win_threshold = int(config.get("algorithm", "win_threshold")) -+ if win_threshold < 1 or win_threshold > 300 or win_threshold > win_size: -+ raise ValueError("Invalid win_threshold") -+ except ValueError: -+ report_alarm_fail("Invalid win_threshold or win_size") -+ -+ return win_size, win_threshold -+ -+ -+def read_config_lat_iodump(io_dic, config): -+ """read config file, get [latency] [iodump] section value""" -+ common_param = {} -+ for io_type in io_dic["iotype_list"]: -+ common_param[io_type] = {} -+ -+ latency_keys = { -+ "avg_lim": "{}_avg_lim".format(io_type), -+ "avg_time": "{}_avg_time".format(io_type), -+ "tot_lim": "{}_tot_lim".format(io_type), -+ } -+ iodump_key = "{}_iodump_lim".format(io_type) -+ -+ for key_suffix, key_template in latency_keys.items(): -+ if key_template in config["latency"] and config["latency"][key_template].isdecimal(): -+ common_param[io_type][key_template] = int(config["latency"][key_template]) -+ -+ if iodump_key in config["iodump"] and config["iodump"][iodump_key].isdecimal(): -+ common_param[io_type][iodump_key] = int(config["iodump"][iodump_key]) -+ -+ return common_param -+ -+ -+def read_config_stage(config, stage, iotype_list): -+ """read config file, get [STAGE_NAME] section value""" -+ res = {} -+ if not stage in config: -+ return res -+ -+ for key in config[stage]: -+ if config[stage][key].isdecimal(): -+ res[key] = int(config[stage][key]) -+ -+ return res -+ -+ -+def init_io_win(io_dic, config, common_param): -+ """initialize windows of latency, iodump, and dict of avg_value""" -+ iotype_list = io_dic["iotype_list"] -+ io_data = {} -+ io_avg_value = {} -+ for disk_name in io_dic["disk_list"]: -+ io_data[disk_name] = {} -+ io_avg_value[disk_name] = {} -+ for stage_name in io_dic["stage_list"]: -+ io_data[disk_name][stage_name] = {} -+ io_avg_value[disk_name][stage_name] = {} -+ # step3. 解析stage配置 -+ curr_stage_param = read_config_stage(config, stage_name, iotype_list) -+ for rw in iotype_list: -+ io_data[disk_name][stage_name][rw] = {} -+ io_avg_value[disk_name][stage_name][rw] = [0, 0] -+ -+ # 对每个rw创建latency和iodump窗口 -+ avg_lim_key = "{}_avg_lim".format(rw) -+ avg_time_key = "{}_avg_time".format(rw) -+ tot_lim_key = "{}_tot_lim".format(rw) -+ iodump_lim_key = "{}_iodump_lim".format(rw) -+ -+ # 获取值，优先从 curr_stage_param 获取，如果不存在，则从 common_param 获取 -+ avg_lim_value = curr_stage_param.get(avg_lim_key, common_param.get(rw, {}).get(avg_lim_key)) -+ avg_time_value = curr_stage_param.get(avg_time_key, common_param.get(rw, {}).get(avg_time_key)) -+ tot_lim_value = curr_stage_param.get(tot_lim_key, common_param.get(rw, {}).get(tot_lim_key)) -+ iodump_lim_value = curr_stage_param.get(iodump_lim_key, common_param.get(rw, {}).get(iodump_lim_key)) -+ -+ if avg_lim_value and avg_time_value and tot_lim_value: -+ io_data[disk_name][stage_name][rw]["latency"] = IoWindow(window_size=io_dic["win_size"], window_threshold=io_dic["win_threshold"], abnormal_multiple=avg_time_value, abnormal_multiple_lim=avg_lim_value, abnormal_time=tot_lim_value) -+ -+ if iodump_lim_value is not None: -+ io_data[disk_name][stage_name][rw]["iodump"] = IoDumpWindow(window_size=io_dic["win_size"], window_threshold=io_dic["win_threshold"], abnormal_time=iodump_lim_value) -+ return io_data, io_avg_value -+ -+ -+def get_valid_disk_stage_list(io_dic, config_disk, config_stage): -+ """get disk_list and stage_list by sentryCollector""" -+ json_data = avg_is_iocollect_valid(io_dic, config_disk, config_stage) -+ -+ all_disk_set = json_data.keys() -+ all_stage_set = set() -+ for disk_stage_list in json_data.values(): -+ all_stage_set.update(disk_stage_list) -+ -+ disk_list = [key for key in config_disk if key in all_disk_set] -+ not_in_disk_list = [key for key in config_disk if key not in all_disk_set] -+ -+ stage_list = [key for key in config_stage if key in all_stage_set] -+ not_in_stage_list = [key for key in config_stage if key not in all_stage_set] -+ -+ if not config_disk: -+ disk_list = [key for key in all_disk_set] -+ -+ if not config_stage: -+ stage_list = [key for key in all_stage_set] -+ -+ if config_disk and not disk_list: -+ logging.warning("Cannot get valid disk by disk={}, set to default".format(config_disk)) -+ disk_list, stage_list = get_valid_disk_stage_list(io_dic, [], config_stage) -+ -+ if config_stage and not stage_list: -+ logging.warning("Cannot get valid stage by stage={}, set to default".format(config_stage)) -+ disk_list, stage_list = get_valid_disk_stage_list(io_dic, config_disk, []) -+ -+ if not stage_list or not disk_list: -+ report_alarm_fail("Cannot get valid disk name or stage name.") -+ -+ log_invalid_keys(not_in_disk_list, 'disk', config_disk, disk_list) -+ log_invalid_keys(not_in_stage_list, 'stage', config_stage, stage_list) -+ -+ return disk_list, stage_list -+ -+ -+def main_loop(io_dic, io_data, io_avg_value): -+ """main loop of avg_block_io""" -+ period_time = io_dic["period_time"] -+ disk_list = io_dic["disk_list"] -+ stage_list = io_dic["stage_list"] -+ iotype_list = io_dic["iotype_list"] -+ win_size = io_dic["win_size"] -+ # 开始循环 -+ while True: -+ # 等待x秒 -+ time.sleep(period_time) -+ -+ # 采集模块对接，获取周期数据 -+ curr_period_data = avg_get_io_data(io_dic) -+ -+ # 处理周期数据 -+ reach_size = False -+ for disk_name in disk_list: -+ for stage_name in stage_list: -+ for rw in iotype_list: -+ if disk_name in curr_period_data and stage_name in curr_period_data[disk_name] and rw in curr_period_data[disk_name][stage_name]: -+ io_key = (disk_name, stage_name, rw) -+ reach_size = update_avg_and_check_abnormal(curr_period_data, io_key, win_size, io_avg_value, io_data) -+ -+ # win_size不满时不进行告警判断 -+ if not reach_size: -+ continue -+ -+ # 判断异常窗口、异常场景 -+ for disk_name in disk_list: -+ for rw in iotype_list: -+ process_report_data(disk_name, rw, io_data) -+ -+ -+def main(): -+ """main func""" -+ # 注册停止信号-2/-15 -+ signal.signal(signal.SIGINT, sig_handler) -+ signal.signal(signal.SIGTERM, sig_handler) -+ -+ # 初始化配置读取 -+ config = configparser.ConfigParser(comment_prefixes=('#', ';')) -+ try: -+ config.read(CONFIG_FILE) -+ except configparser.Error: -+ report_alarm_fail("Failed to read config file") -+ -+ io_dic = {} -+ -+ # 读取配置文件 -- common段 -+ io_dic["period_time"], disk, stage, io_dic["iotype_list"] = read_config_common(config) -+ -+ # 采集模块对接，is_iocollect_valid() -+ io_dic["disk_list"], io_dic["stage_list"] = get_valid_disk_stage_list(io_dic, disk, stage) -+ -+ if "bio" not in io_dic["stage_list"]: -+ report_alarm_fail("Cannot run avg_block_io without bio stage") -+ -+ # 初始化窗口 -- config读取，对应is_iocollect_valid返回的结果 -+ # step1. 解析公共配置 --- algorithm -+ io_dic["win_size"], io_dic["win_threshold"] = read_config_algorithm(config) -+ -+ # step2. 循环创建窗口 -+ common_param = read_config_lat_iodump(io_dic, config) -+ io_data, io_avg_value = init_io_win(io_dic, config, common_param) -+ -+ main_loop(io_dic, io_data, io_avg_value) -diff --git a/src/python/sentryPlugins/avg_block_io/module_conn.py b/src/python/sentryPlugins/avg_block_io/module_conn.py -new file mode 100644 -index 0000000..caa0191 ---- /dev/null -+++ b/src/python/sentryPlugins/avg_block_io/module_conn.py -@@ -0,0 +1,86 @@ -+# coding: utf-8 -+# Copyright (c) 2024 Huawei Technologies Co., Ltd. -+# sysSentry is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+import json -+import logging -+import sys -+import time -+ -+from .utils import is_abnormal -+from sentryCollector.collect_plugin import is_iocollect_valid, get_io_data, Result_Messages -+from syssentry.result import ResultLevel, report_result -+ -+ -+TASK_NAME = "avg_block_io" -+ -+def sig_handler(signum, _f): -+ """stop avg_block_io""" -+ report_result(TASK_NAME, ResultLevel.PASS, json.dumps({})) -+ logging.info("Finished avg_block_io plugin running.") -+ sys.exit(0) -+ -+def avg_get_io_data(io_dic): -+ """get_io_data from sentryCollector""" -+ res = get_io_data(io_dic["period_time"], io_dic["disk_list"], io_dic["stage_list"], io_dic["iotype_list"]) -+ return check_result_validation(res, 'get io data') -+ -+ -+def avg_is_iocollect_valid(io_dic, config_disk, config_stage): -+ """is_iocollect_valid from sentryCollector""" -+ res = is_iocollect_valid(io_dic["period_time"], config_disk, config_stage) -+ return check_result_validation(res, 'check config validation') -+ -+ -+def check_result_validation(res, reason): -+ """check validation of result from sentryCollector""" -+ if not 'ret' in res or not 'message' in res: -+ err_msg = "Failed to {}: Cannot connect to sentryCollector.".format(reason) -+ report_alarm_fail(err_msg) -+ if res['ret'] != 0: -+ err_msg = "Failed to {}: {}".format(reason, Result_Messages[res['ret']]) -+ report_alarm_fail(err_msg) -+ -+ try: -+ json_data = json.loads(res['message']) -+ except json.JSONDecodeError: -+ err_msg = "Failed to {}: invalid return message".format(reason) -+ report_alarm_fail(err_msg) -+ -+ return json_data -+ -+ -+def report_alarm_fail(alarm_info): -+ """report result to xalarmd""" -+ report_result(TASK_NAME, ResultLevel.FAIL, json.dumps({"msg": alarm_info})) -+ logging.error(alarm_info) -+ sys.exit(1) -+ -+ -+def process_report_data(disk_name, rw, io_data): -+ """check abnormal window and report to xalarm""" -+ if not is_abnormal((disk_name, 'bio', rw), io_data): -+ return -+ -+ ctrl_stage = ['throtl', 'wbt', 'iocost', 'bfq'] -+ for stage_name in ctrl_stage: -+ if is_abnormal((disk_name, stage_name, rw), io_data): -+ logging.warning("{} - {} - {} report IO press".format(time.ctime(), disk_name, rw)) -+ return -+ -+ if is_abnormal((disk_name, 'rq_driver', rw), io_data): -+ logging.warning("{} - {} - {} report driver".format(time.ctime(), disk_name, rw)) -+ return -+ -+ kernel_stage = ['gettag', 'plug', 'deadline', 'hctx', 'requeue'] -+ for stage_name in kernel_stage: -+ if is_abnormal((disk_name, stage_name, rw), io_data): -+ logging.warning("{} - {} - {} report kernel".format(time.ctime(), disk_name, rw)) -+ return -+ logging.warning("{} - {} - {} report IO press".format(time.ctime(), disk_name, rw)) -diff --git a/src/python/sentryPlugins/avg_block_io/stage_window.py b/src/python/sentryPlugins/avg_block_io/stage_window.py -new file mode 100644 -index 0000000..9b0ce79 ---- /dev/null -+++ b/src/python/sentryPlugins/avg_block_io/stage_window.py -@@ -0,0 +1,47 @@ -+# coding: utf-8 -+# Copyright (c) 2024 Huawei Technologies Co., Ltd. -+# sysSentry is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+ -+class AbnormalWindowBase: -+ def __init__(self, window_size=10, window_threshold=7): -+ self.window_size = window_size -+ self.window_threshold = window_threshold -+ self.abnormal_window = [False] * window_size -+ -+ def append_new_period(self, ab_res, avg_val=0): -+ self.abnormal_window.pop(0) -+ if self.is_abnormal_period(ab_res, avg_val): -+ self.abnormal_window.append(True) -+ else: -+ self.abnormal_window.append(False) -+ -+ def is_abnormal_window(self): -+ return sum(self.abnormal_window) > self.window_threshold -+ -+ -+class IoWindow(AbnormalWindowBase): -+ def __init__(self, window_size=10, window_threshold=7, abnormal_multiple=5, abnormal_multiple_lim=30, abnormal_time=40): -+ super().__init__(window_size, window_threshold) -+ self.abnormal_multiple = abnormal_multiple -+ self.abnormal_multiple_lim = abnormal_multiple_lim -+ self.abnormal_time = abnormal_time -+ -+ def is_abnormal_period(self, value, avg_val): -+ return (value > avg_val * self.abnormal_multiple and value > self.abnormal_multiple_lim) or \ -+ (value > self.abnormal_time) -+ -+ -+class IoDumpWindow(AbnormalWindowBase): -+ def __init__(self, window_size=10, window_threshold=7, abnormal_time=40): -+ super().__init__(window_size, window_threshold) -+ self.abnormal_time = abnormal_time -+ -+ def is_abnormal_period(self, value, avg_val=0): -+ return value > self.abnormal_time -diff --git a/src/python/sentryPlugins/avg_block_io/utils.py b/src/python/sentryPlugins/avg_block_io/utils.py -new file mode 100644 -index 0000000..54ed080 ---- /dev/null -+++ b/src/python/sentryPlugins/avg_block_io/utils.py -@@ -0,0 +1,86 @@ -+# coding: utf-8 -+# Copyright (c) 2024 Huawei Technologies Co., Ltd. -+# sysSentry is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+AVG_VALUE = 0 -+AVG_COUNT = 1 -+ -+ -+def get_nested_value(data, keys): -+ """get data from nested dict""" -+ for key in keys: -+ if key in data: -+ data = data[key] -+ else: -+ return None -+ return data -+ -+ -+def set_nested_value(data, keys, value): -+ """set data to nested dict""" -+ for key in keys[:-1]: -+ if key in data: -+ data = data[key] -+ else: -+ return False -+ data[keys[-1]] = value -+ return True -+ -+ -+def is_abnormal(io_key, io_data): -+ """check if latency and iodump win abnormal""" -+ for key in ['latency', 'iodump']: -+ all_keys = get_nested_value(io_data, io_key) -+ if all_keys and key in all_keys: -+ win = get_nested_value(io_data, io_key + (key,)) -+ if win and win.is_abnormal_window(): -+ return True -+ return False -+ -+ -+def update_io_avg(old_avg, period_value, win_size): -+ """update average of latency window""" -+ if old_avg[AVG_COUNT] < win_size: -+ new_avg_count = old_avg[AVG_COUNT] + 1 -+ new_avg_value = (old_avg[AVG_VALUE] * old_avg[AVG_COUNT] + period_value[0]) / new_avg_count -+ else: -+ new_avg_count = old_avg[AVG_COUNT] -+ new_avg_value = (old_avg[AVG_VALUE] * (old_avg[AVG_COUNT] - 1) + period_value[0]) / new_avg_count -+ return [new_avg_value, new_avg_count] -+ -+ -+def update_io_data(old_avg, period_value, win_size, io_data, io_key): -+ """update data of latency and iodump window""" -+ all_wins = get_nested_value(io_data, io_key) -+ if all_wins and "latency" in all_wins: -+ io_data[io_key[0]][io_key[1]][io_key[2]]["latency"].append_new_period(period_value[0], old_avg[AVG_VALUE]) -+ if all_wins and "iodump" in all_wins: -+ io_data[io_key[0]][io_key[1]][io_key[2]]["iodump"].append_new_period(period_value[1]) -+ -+ -+def update_avg_and_check_abnormal(data, io_key, win_size, io_avg_value, io_data): -+ """update avg and check abonrmal, return true if win_size full""" -+ period_value = get_nested_value(data, io_key) -+ old_avg = get_nested_value(io_avg_value, io_key) -+ -+ # 更新avg数据 -+ if old_avg[AVG_COUNT] < win_size: -+ set_nested_value(io_avg_value, io_key, update_io_avg(old_avg, period_value, win_size)) -+ return False -+ -+ # 更新win数据 -- 判断异常周期 -+ update_io_data(old_avg, period_value, win_size, io_data, io_key) -+ all_wins = get_nested_value(io_data, io_key) -+ if all_wins and 'latency' not in all_wins: -+ return True -+ period = get_nested_value(io_data, io_key + ("latency",)) -+ if period and period.is_abnormal_period(period_value[0], old_avg[AVG_VALUE]): -+ return True -+ set_nested_value(io_avg_value, io_key, update_io_avg(old_avg, period_value, win_size)) -+ return True --- -2.33.0 - diff --git a/fix-ai_block_io-root-cause-bug.patch b/fix-ai_block_io-root-cause-bug.patch deleted file mode 100644 index f6de787..0000000 --- a/fix-ai_block_io-root-cause-bug.patch +++ /dev/null @@ -1,33 +0,0 @@ -From ac9ce326dee20edde2451946e34ea9a13bd8c338 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?=E8=B4=BA=E6=9C=89=E5=BF=97?= <1037617413@qq.com> -Date: Wed, 16 Oct 2024 11:50:46 +0800 -Subject: [PATCH] fix ai_block_io root cause bug - ---- - src/python/sentryPlugins/ai_block_io/detector.py | 8 ++++---- - 1 file changed, 4 insertions(+), 4 deletions(-) - -diff --git a/src/python/sentryPlugins/ai_block_io/detector.py b/src/python/sentryPlugins/ai_block_io/detector.py -index 5b21714..ed8b64a 100644 ---- a/src/python/sentryPlugins/ai_block_io/detector.py -+++ b/src/python/sentryPlugins/ai_block_io/detector.py -@@ -101,12 +101,12 @@ class DiskDetector: - if len(diagnosis_info["bio"]) == 0: - return False, None, None, None - elif len(diagnosis_info["rq_driver"]) != 0: -- root_cause = "[Root Cause：disk slow]" -+ root_cause = "[Root Cause: disk slow]" - elif len(diagnosis_info["io_stage"]) != 0: -- stage = diagnosis_info["io_stage"][0][1].get_stage_name() -- root_cause = f"[Root Cause：io stage slow, stage: {stage}]" -+ stage = diagnosis_info["io_stage"][0][1].stage_name -+ root_cause = f"[Root Cause: io stage slow, stage: {stage}]" - if root_cause is None: -- root_cause = "[Root Cause：high io pressure]" -+ root_cause = "[Root Cause: high io pressure]" - return True, diagnosis_info["bio"][0][0], diagnosis_info["bio"][0][1], root_cause - - def __repr__(self): --- -2.23.0 - diff --git a/fix-ai_block_io-some-issues.patch b/fix-ai_block_io-some-issues.patch deleted file mode 100644 index d80cbe8..0000000 --- a/fix-ai_block_io-some-issues.patch +++ /dev/null @@ -1,832 +0,0 @@ -From 35ba8fe8e241c5e3508c5dadc82a777065a5cc4d Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?=E8=B4=BA=E6=9C=89=E5=BF=97?= <1037617413@qq.com> -Date: Mon, 30 Sep 2024 00:15:29 +0800 -Subject: [PATCH] fix ai_block_io some issues - ---- - ..._slow_io_detection.ini => ai_block_io.ini} | 6 +- - config/tasks/ai_block_io.mod | 5 + - .../tasks/ai_threshold_slow_io_detection.mod | 5 - - ...ow_io_detection.py => test_ai_block_io.py} | 0 - .../README.md | 0 - .../__init__.py | 0 - .../ai_block_io.py} | 57 ++-- - .../alarm_report.py | 2 +- - .../ai_block_io/config_parser.py | 256 ++++++++++++++++++ - .../data_access.py | 3 + - .../detector.py | 17 +- - .../io_data.py | 0 - .../sliding_window.py | 0 - .../threshold.py | 13 +- - .../utils.py | 15 +- - .../config_parser.py | 141 ---------- - src/python/setup.py | 2 +- - 17 files changed, 336 insertions(+), 186 deletions(-) - rename config/plugins/{ai_threshold_slow_io_detection.ini => ai_block_io.ini} (66%) - create mode 100644 config/tasks/ai_block_io.mod - delete mode 100644 config/tasks/ai_threshold_slow_io_detection.mod - rename selftest/test/{test_ai_threshold_slow_io_detection.py => test_ai_block_io.py} (100%) - rename src/python/sentryPlugins/{ai_threshold_slow_io_detection => ai_block_io}/README.md (100%) - rename src/python/sentryPlugins/{ai_threshold_slow_io_detection => ai_block_io}/__init__.py (100%) - rename src/python/sentryPlugins/{ai_threshold_slow_io_detection/slow_io_detection.py => ai_block_io/ai_block_io.py} (66%) - rename src/python/sentryPlugins/{ai_threshold_slow_io_detection => ai_block_io}/alarm_report.py (98%) - create mode 100644 src/python/sentryPlugins/ai_block_io/config_parser.py - rename src/python/sentryPlugins/{ai_threshold_slow_io_detection => ai_block_io}/data_access.py (99%) - rename src/python/sentryPlugins/{ai_threshold_slow_io_detection => ai_block_io}/detector.py (77%) - rename src/python/sentryPlugins/{ai_threshold_slow_io_detection => ai_block_io}/io_data.py (100%) - rename src/python/sentryPlugins/{ai_threshold_slow_io_detection => ai_block_io}/sliding_window.py (100%) - rename src/python/sentryPlugins/{ai_threshold_slow_io_detection => ai_block_io}/threshold.py (92%) - rename src/python/sentryPlugins/{ai_threshold_slow_io_detection => ai_block_io}/utils.py (86%) - delete mode 100644 src/python/sentryPlugins/ai_threshold_slow_io_detection/config_parser.py - -diff --git a/config/plugins/ai_threshold_slow_io_detection.ini b/config/plugins/ai_block_io.ini -similarity index 66% -rename from config/plugins/ai_threshold_slow_io_detection.ini -rename to config/plugins/ai_block_io.ini -index 44eb928..01ce266 100644 ---- a/config/plugins/ai_threshold_slow_io_detection.ini -+++ b/config/plugins/ai_block_io.ini -@@ -4,9 +4,9 @@ slow_io_detect_frequency=1 - log_level=info - - [algorithm] --train_data_duration=0.1 --train_update_duration=0.02 --algorithm_type=n_sigma -+train_data_duration=24 -+train_update_duration=2 -+algorithm_type=boxplot - boxplot_parameter=1.5 - n_sigma_parameter=3 - -diff --git a/config/tasks/ai_block_io.mod b/config/tasks/ai_block_io.mod -new file mode 100644 -index 0000000..1971d7d ---- /dev/null -+++ b/config/tasks/ai_block_io.mod -@@ -0,0 +1,5 @@ -+[common] -+enabled=yes -+task_start=/usr/bin/python3 /usr/bin/ai_block_io -+task_stop=pkill -f /usr/bin/ai_block_io -+type=oneshot -\ No newline at end of file -diff --git a/config/tasks/ai_threshold_slow_io_detection.mod b/config/tasks/ai_threshold_slow_io_detection.mod -deleted file mode 100644 -index 2729f72..0000000 ---- a/config/tasks/ai_threshold_slow_io_detection.mod -+++ /dev/null -@@ -1,5 +0,0 @@ --[common] --enabled=yes --task_start=/usr/bin/python3 /usr/bin/ai_threshold_slow_io_detection --task_stop=pkill -f /usr/bin/ai_threshold_slow_io_detection --type=oneshot -\ No newline at end of file -diff --git a/selftest/test/test_ai_threshold_slow_io_detection.py b/selftest/test/test_ai_block_io.py -similarity index 100% -rename from selftest/test/test_ai_threshold_slow_io_detection.py -rename to selftest/test/test_ai_block_io.py -diff --git a/src/python/sentryPlugins/ai_threshold_slow_io_detection/README.md b/src/python/sentryPlugins/ai_block_io/README.md -similarity index 100% -rename from src/python/sentryPlugins/ai_threshold_slow_io_detection/README.md -rename to src/python/sentryPlugins/ai_block_io/README.md -diff --git a/src/python/sentryPlugins/ai_threshold_slow_io_detection/__init__.py b/src/python/sentryPlugins/ai_block_io/__init__.py -similarity index 100% -rename from src/python/sentryPlugins/ai_threshold_slow_io_detection/__init__.py -rename to src/python/sentryPlugins/ai_block_io/__init__.py -diff --git a/src/python/sentryPlugins/ai_threshold_slow_io_detection/slow_io_detection.py b/src/python/sentryPlugins/ai_block_io/ai_block_io.py -similarity index 66% -rename from src/python/sentryPlugins/ai_threshold_slow_io_detection/slow_io_detection.py -rename to src/python/sentryPlugins/ai_block_io/ai_block_io.py -index 43cf770..31b8a97 100644 ---- a/src/python/sentryPlugins/ai_threshold_slow_io_detection/slow_io_detection.py -+++ b/src/python/sentryPlugins/ai_block_io/ai_block_io.py -@@ -23,7 +23,7 @@ from .data_access import get_io_data_from_collect_plug, check_collect_valid - from .io_data import MetricName - from .alarm_report import AlarmReport - --CONFIG_FILE = "/etc/sysSentry/plugins/ai_threshold_slow_io_detection.ini" -+CONFIG_FILE = "/etc/sysSentry/plugins/ai_block_io.ini" - - - def sig_handler(signum, frame): -@@ -40,34 +40,48 @@ class SlowIODetection: - - def __init__(self, config_parser: ConfigParser): - self._config_parser = config_parser -- self.__set_log_format() - self.__init_detector_name_list() - self.__init_detector() - -- def __set_log_format(self): -- log_format = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s" -- log_level = get_log_level(self._config_parser.get_log_level()) -- logging.basicConfig(level=log_level, format=log_format) -- - def __init_detector_name_list(self): - self._disk_list = check_collect_valid(self._config_parser.get_slow_io_detect_frequency()) -- for disk in self._disk_list: -- self._detector_name_list.append(MetricName(disk, "bio", "read", "latency")) -- self._detector_name_list.append(MetricName(disk, "bio", "write", "latency")) -+ disks_to_detection: list = self._config_parser.get_disks_to_detection() -+ # 情况1：None，则启用所有磁盘检测 -+ # 情况2：is not None and len = 0，则不启动任何磁盘检测 -+ # 情况3：len ！= 0，则取交集 -+ if disks_to_detection is None: -+ for disk in self._disk_list: -+ self._detector_name_list.append(MetricName(disk, "bio", "read", "latency")) -+ self._detector_name_list.append(MetricName(disk, "bio", "write", "latency")) -+ elif len(disks_to_detection) == 0: -+ logging.warning('please attention: conf file not specify any disk to detection, ' -+ 'so it will not start ai block io.') -+ else: -+ disks_name_to_detection = [] -+ for disk_name_to_detection in disks_to_detection: -+ disks_name_to_detection.append(disk_name_to_detection.get_disk_name()) -+ disk_intersection = [disk for disk in self._disk_list if disk in disks_name_to_detection] -+ for disk in disk_intersection: -+ self._detector_name_list.append(MetricName(disk, "bio", "read", "latency")) -+ self._detector_name_list.append(MetricName(disk, "bio", "write", "latency")) -+ logging.info(f'start to detection follow disk and it\'s metric: {self._detector_name_list}') - - def __init_detector(self): - train_data_duration, train_update_duration = (self._config_parser. - get_train_data_duration_and_train_update_duration()) - slow_io_detection_frequency = self._config_parser.get_slow_io_detect_frequency() -- threshold_type = get_threshold_type_enum(self._config_parser.get_algorithm_type()) -+ threshold_type = self._config_parser.get_algorithm_type() - data_queue_size, update_size = get_data_queue_size_and_update_size(train_data_duration, - train_update_duration, - slow_io_detection_frequency) -- sliding_window_type = get_sliding_window_type_enum(self._config_parser.get_sliding_window_type()) -+ sliding_window_type = self._config_parser.get_sliding_window_type() - window_size, window_threshold = self._config_parser.get_window_size_and_window_minimum_threshold() - - for detector_name in self._detector_name_list: -- threshold = ThresholdFactory().get_threshold(threshold_type, data_queue_size=data_queue_size, -+ threshold = ThresholdFactory().get_threshold(threshold_type, -+ boxplot_parameter=self._config_parser.get_boxplot_parameter(), -+ n_sigma_paramter=self._config_parser.get_n_sigma_parameter(), -+ data_queue_size=data_queue_size, - data_queue_update_size=update_size) - sliding_window = SlidingWindowFactory().get_sliding_window(sliding_window_type, queue_length=window_size, - threshold=window_threshold) -@@ -89,6 +103,7 @@ class SlowIODetection: - logging.debug(f'step1. Get io data: {str(io_data_dict_with_disk_name)}') - if io_data_dict_with_disk_name is None: - continue -+ - # Step2：慢IO检测 - logging.debug('step2. Start to detection slow io event.') - slow_io_event_list = [] -@@ -103,13 +118,14 @@ class SlowIODetection: - for slow_io_event in slow_io_event_list: - metric_name: MetricName = slow_io_event[0] - result = slow_io_event[1] -- AlarmReport.report_major_alm(f"disk {metric_name.get_disk_name()} has slow io event." -- f"stage: {metric_name.get_metric_name()}," -- f"type: {metric_name.get_io_access_type_name()}," -- f"metric: {metric_name.get_metric_name()}," -- f"current window: {result[1]}," -- f"threshold: {result[2]}") -- logging.error(f"slow io event happen: {str(slow_io_event)}") -+ alarm_content = (f"disk {metric_name.get_disk_name()} has slow io event. " -+ f"stage is: {metric_name.get_stage_name()}, " -+ f"io access type is: {metric_name.get_io_access_type_name()}, " -+ f"metric is: {metric_name.get_metric_name()}, " -+ f"current window is: {result[1]}, " -+ f"threshold is: {result[2]}") -+ AlarmReport.report_major_alm(alarm_content) -+ logging.warning(alarm_content) - - # Step4：等待检测时间 - logging.debug('step4. Wait to start next slow io event detection loop.') -@@ -120,6 +136,7 @@ def main(): - # Step1：注册消息处理函数 - signal.signal(signal.SIGINT, sig_handler) - signal.signal(signal.SIGTERM, sig_handler) -+ - # Step2：断点恢复 - # todo: - -diff --git a/src/python/sentryPlugins/ai_threshold_slow_io_detection/alarm_report.py b/src/python/sentryPlugins/ai_block_io/alarm_report.py -similarity index 98% -rename from src/python/sentryPlugins/ai_threshold_slow_io_detection/alarm_report.py -rename to src/python/sentryPlugins/ai_block_io/alarm_report.py -index 3f4f34e..230c8cd 100644 ---- a/src/python/sentryPlugins/ai_threshold_slow_io_detection/alarm_report.py -+++ b/src/python/sentryPlugins/ai_block_io/alarm_report.py -@@ -15,7 +15,7 @@ import json - - - class AlarmReport: -- TASK_NAME = "SLOW_IO_DETECTION" -+ TASK_NAME = "ai_block_io" - - @staticmethod - def report_pass(info: str): -diff --git a/src/python/sentryPlugins/ai_block_io/config_parser.py b/src/python/sentryPlugins/ai_block_io/config_parser.py -new file mode 100644 -index 0000000..632391d ---- /dev/null -+++ b/src/python/sentryPlugins/ai_block_io/config_parser.py -@@ -0,0 +1,256 @@ -+# coding: utf-8 -+# Copyright (c) 2024 Huawei Technologies Co., Ltd. -+# sysSentry is licensed under the Mulan PSL v2. -+# You can use this software according to the terms and conditions of the Mulan PSL v2. -+# You may obtain a copy of Mulan PSL v2 at: -+# http://license.coscl.org.cn/MulanPSL2 -+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -+# PURPOSE. -+# See the Mulan PSL v2 for more details. -+ -+import configparser -+import json -+import logging -+ -+from .io_data import MetricName -+from .threshold import ThresholdType -+from .utils import get_threshold_type_enum, get_sliding_window_type_enum, get_log_level -+ -+LOG_FORMAT = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s" -+ -+ -+def init_log_format(log_level: str): -+ logging.basicConfig(level=get_log_level(log_level), format=LOG_FORMAT) -+ -+ -+class ConfigParser: -+ DEFAULT_ABSOLUTE_THRESHOLD = 40 -+ DEFAULT_SLOW_IO_DETECTION_FREQUENCY = 1 -+ DEFAULT_LOG_LEVEL = 'info' -+ -+ DEFAULT_ALGORITHM_TYPE = 'boxplot' -+ DEFAULT_TRAIN_DATA_DURATION = 24 -+ DEFAULT_TRAIN_UPDATE_DURATION = 2 -+ DEFAULT_BOXPLOT_PARAMETER = 1.5 -+ DEFAULT_N_SIGMA_PARAMETER = 3 -+ -+ DEFAULT_SLIDING_WINDOW_TYPE = 'not_continuous' -+ DEFAULT_WINDOW_SIZE = 30 -+ DEFAULT_WINDOW_MINIMUM_THRESHOLD = 6 -+ -+ def __init__(self, config_file_name): -+ self.__absolute_threshold = ConfigParser.DEFAULT_ABSOLUTE_THRESHOLD -+ self.__slow_io_detect_frequency = ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY -+ self.__log_level = ConfigParser.DEFAULT_LOG_LEVEL -+ self.__disks_to_detection: list = [] -+ -+ self.__algorithm_type = ConfigParser.DEFAULT_ALGORITHM_TYPE -+ self.__train_data_duration = ConfigParser.DEFAULT_TRAIN_UPDATE_DURATION -+ self.__train_update_duration = ConfigParser.DEFAULT_TRAIN_UPDATE_DURATION -+ self.__boxplot_parameter = ConfigParser.DEFAULT_BOXPLOT_PARAMETER -+ self.__n_sigma_parameter = ConfigParser.DEFAULT_N_SIGMA_PARAMETER -+ -+ self.__sliding_window_type = ConfigParser.DEFAULT_SLIDING_WINDOW_TYPE -+ self.__window_size = ConfigParser.DEFAULT_WINDOW_SIZE -+ self.__window_minimum_threshold = ConfigParser.DEFAULT_WINDOW_MINIMUM_THRESHOLD -+ -+ self.__config_file_name = config_file_name -+ -+ def __read_absolute_threshold(self, items_common: dict): -+ try: -+ self.__absolute_threshold = float(items_common.get('absolute_threshold', -+ ConfigParser.DEFAULT_ABSOLUTE_THRESHOLD)) -+ if self.__absolute_threshold <= 0: -+ logging.warning( -+ f'the_absolute_threshold: {self.__absolute_threshold} you set is invalid, use default value: {ConfigParser.DEFAULT_ABSOLUTE_THRESHOLD}.') -+ self.__absolute_threshold = ConfigParser.DEFAULT_ABSOLUTE_THRESHOLD -+ except ValueError: -+ self.__absolute_threshold = ConfigParser.DEFAULT_ABSOLUTE_THRESHOLD -+ logging.warning( -+ f'the_absolute_threshold type conversion has error, use default value: {self.__absolute_threshold}.') -+ -+ def __read__slow_io_detect_frequency(self, items_common: dict): -+ try: -+ self.__slow_io_detect_frequency = int(items_common.get('slow_io_detect_frequency', -+ ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY)) -+ if self.__slow_io_detect_frequency < 1 or self.__slow_io_detect_frequency > 10: -+ logging.warning( -+ f'the slow_io_detect_frequency: {self.__slow_io_detect_frequency} you set is invalid, use default value: {ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY}.') -+ self.__slow_io_detect_frequency = ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY -+ except ValueError: -+ self.__slow_io_detect_frequency = ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY -+ logging.warning(f'slow_io_detect_frequency type conversion has error, use default value: {self.__slow_io_detect_frequency}.') -+ -+ def __read__disks_to_detect(self, items_common: dict): -+ disks_to_detection = items_common.get('disks_to_detect') -+ if disks_to_detection is None: -+ logging.warning(f'config of disks_to_detect not found, the default value be used.') -+ self.__disks_to_detection = None -+ return -+ try: -+ disks_to_detection_list = json.loads(disks_to_detection) -+ for disk_to_detection in disks_to_detection_list: -+ disk_name = disk_to_detection.get('disk_name', None) -+ stage_name = disk_to_detection.get('stage_name', None) -+ io_access_type_name = disk_to_detection.get('io_access_type_name', None) -+ metric_name = disk_to_detection.get('metric_name', None) -+ if not (disk_name is None or stage_name is None or io_access_type_name is None or metric_name is None): -+ metric_name_object = MetricName(disk_name, stage_name, io_access_type_name, metric_name) -+ self.__disks_to_detection.append(metric_name_object) -+ else: -+ logging.warning(f'config of disks_to_detect\'s some part has some error: {disk_to_detection}, it will be ignored.') -+ except json.decoder.JSONDecodeError as e: -+ logging.warning(f'config of disks_to_detect is error: {e}, it will be ignored and default value be used.') -+ self.__disks_to_detection = None -+ -+ def __read__train_data_duration(self, items_algorithm: dict): -+ try: -+ self.__train_data_duration = float(items_algorithm.get('train_data_duration', -+ ConfigParser.DEFAULT_TRAIN_DATA_DURATION)) -+ if self.__train_data_duration <= 0 or self.__train_data_duration > 720: -+ logging.warning( -+ f'the train_data_duration: {self.__train_data_duration} you set is invalid, use default value: {ConfigParser.DEFAULT_TRAIN_DATA_DURATION}.') -+ self.__train_data_duration = ConfigParser.DEFAULT_TRAIN_DATA_DURATION -+ except ValueError: -+ self.__train_data_duration = ConfigParser.DEFAULT_TRAIN_DATA_DURATION -+ logging.warning(f'the train_data_duration type conversion has error, use default value: {self.__train_data_duration}.') -+ -+ def __read__train_update_duration(self, items_algorithm: dict): -+ default_train_update_duration = ConfigParser.DEFAULT_TRAIN_UPDATE_DURATION -+ if default_train_update_duration > self.__train_data_duration: -+ default_train_update_duration = self.__train_data_duration / 2 -+ -+ try: -+ self.__train_update_duration = float(items_algorithm.get('train_update_duration', -+ ConfigParser.DEFAULT_TRAIN_UPDATE_DURATION)) -+ if self.__train_update_duration <= 0 or self.__train_update_duration > self.__train_data_duration: -+ logging.warning( -+ f'the train_update_duration: {self.__train_update_duration} you set is invalid, use default value: {default_train_update_duration}.') -+ self.__train_update_duration = default_train_update_duration -+ except ValueError: -+ self.__train_update_duration = default_train_update_duration -+ logging.warning(f'the train_update_duration type conversion has error, use default value: {self.__train_update_duration}.') -+ -+ def __read__algorithm_type_and_parameter(self, items_algorithm: dict): -+ algorithm_type = items_algorithm.get('algorithm_type', ConfigParser.DEFAULT_ALGORITHM_TYPE) -+ self.__algorithm_type = get_threshold_type_enum(algorithm_type) -+ -+ if self.__algorithm_type == ThresholdType.NSigmaThreshold: -+ try: -+ self.__n_sigma_parameter = float(items_algorithm.get('n_sigma_parameter', -+ ConfigParser.DEFAULT_N_SIGMA_PARAMETER)) -+ if self.__n_sigma_parameter <= 0 or self.__n_sigma_parameter > 10: -+ logging.warning( -+ f'the n_sigma_parameter: {self.__n_sigma_parameter} you set is invalid, use default value: {ConfigParser.DEFAULT_N_SIGMA_PARAMETER}.') -+ self.__n_sigma_parameter = ConfigParser.DEFAULT_N_SIGMA_PARAMETER -+ except ValueError: -+ self.__n_sigma_parameter = ConfigParser.DEFAULT_N_SIGMA_PARAMETER -+ logging.warning(f'the n_sigma_parameter type conversion has error, use default value: {self.__n_sigma_parameter}.') -+ elif self.__algorithm_type == ThresholdType.BoxplotThreshold: -+ try: -+ self.__boxplot_parameter = float(items_algorithm.get('boxplot_parameter', -+ ConfigParser.DEFAULT_BOXPLOT_PARAMETER)) -+ if self.__boxplot_parameter <= 0 or self.__boxplot_parameter > 10: -+ logging.warning( -+ f'the boxplot_parameter: {self.__boxplot_parameter} you set is invalid, use default value: {ConfigParser.DEFAULT_BOXPLOT_PARAMETER}.') -+ self.__n_sigma_parameter = ConfigParser.DEFAULT_BOXPLOT_PARAMETER -+ except ValueError: -+ self.__boxplot_parameter = ConfigParser.DEFAULT_BOXPLOT_PARAMETER -+ logging.warning(f'the boxplot_parameter type conversion has error, use default value: {self.__boxplot_parameter}.') -+ -+ def __read__window_size(self, items_sliding_window: dict): -+ try: -+ self.__window_size = int(items_sliding_window.get('window_size', -+ ConfigParser.DEFAULT_WINDOW_SIZE)) -+ if self.__window_size < 1 or self.__window_size > 3600: -+ logging.warning( -+ f'the window_size: {self.__window_size} you set is invalid, use default value: {ConfigParser.DEFAULT_WINDOW_SIZE}.') -+ self.__window_size = ConfigParser.DEFAULT_WINDOW_SIZE -+ except ValueError: -+ self.__window_size = ConfigParser.DEFAULT_WINDOW_SIZE -+ logging.warning(f'window_size type conversion has error, use default value: {self.__window_size}.') -+ -+ def __read__window_minimum_threshold(self, items_sliding_window: dict): -+ default_window_minimum_threshold = ConfigParser.DEFAULT_WINDOW_MINIMUM_THRESHOLD -+ if default_window_minimum_threshold > self.__window_size: -+ default_window_minimum_threshold = self.__window_size / 2 -+ try: -+ self.__window_minimum_threshold = ( -+ int(items_sliding_window.get('window_minimum_threshold', -+ ConfigParser.DEFAULT_WINDOW_MINIMUM_THRESHOLD))) -+ if self.__window_minimum_threshold < 1 or self.__window_minimum_threshold > self.__window_size: -+ logging.warning( -+ f'the window_minimum_threshold: {self.__window_minimum_threshold} you set is invalid, use default value: {default_window_minimum_threshold}.') -+ self.__window_minimum_threshold = default_window_minimum_threshold -+ except ValueError: -+ self.__window_minimum_threshold = default_window_minimum_threshold -+ logging.warning(f'window_minimum_threshold type conversion has error, use default value: {self.__window_minimum_threshold}.') -+ -+ def read_config_from_file(self): -+ con = configparser.ConfigParser() -+ con.read(self.__config_file_name, encoding='utf-8') -+ -+ if con.has_section('common'): -+ items_common = dict(con.items('common')) -+ self.__log_level = items_common.get('log_level', ConfigParser.DEFAULT_LOG_LEVEL) -+ init_log_format(self.__log_level) -+ self.__read_absolute_threshold(items_common) -+ self.__read__slow_io_detect_frequency(items_common) -+ self.__read__disks_to_detect(items_common) -+ else: -+ init_log_format(self.__log_level) -+ logging.warning("common section parameter not found, it will be set to default value.") -+ -+ if con.has_section('algorithm'): -+ items_algorithm = dict(con.items('algorithm')) -+ self.__read__train_data_duration(items_algorithm) -+ self.__read__train_update_duration(items_algorithm) -+ self.__read__algorithm_type_and_parameter(items_algorithm) -+ else: -+ logging.warning("algorithm section parameter not found, it will be set to default value.") -+ -+ if con.has_section('sliding_window'): -+ items_sliding_window = dict(con.items('sliding_window')) -+ sliding_window_type = items_sliding_window.get('sliding_window_type', -+ ConfigParser.DEFAULT_SLIDING_WINDOW_TYPE) -+ self.__sliding_window_type = get_sliding_window_type_enum(sliding_window_type) -+ self.__read__window_size(items_sliding_window) -+ self.__read__window_minimum_threshold(items_sliding_window) -+ else: -+ logging.warning("sliding_window section parameter not found, it will be set to default value.") -+ -+ self.__print_all_config_value() -+ -+ def __print_all_config_value(self): -+ pass -+ -+ def get_slow_io_detect_frequency(self): -+ return self.__slow_io_detect_frequency -+ -+ def get_algorithm_type(self): -+ return self.__algorithm_type -+ -+ def get_sliding_window_type(self): -+ return self.__sliding_window_type -+ -+ def get_train_data_duration_and_train_update_duration(self): -+ return self.__train_data_duration, self.__train_update_duration -+ -+ def get_window_size_and_window_minimum_threshold(self): -+ return self.__window_size, self.__window_minimum_threshold -+ -+ def get_absolute_threshold(self): -+ return self.__absolute_threshold -+ -+ def get_log_level(self): -+ return self.__log_level -+ -+ def get_disks_to_detection(self): -+ return self.__disks_to_detection -+ -+ def get_boxplot_parameter(self): -+ return self.__boxplot_parameter -+ -+ def get_n_sigma_parameter(self): -+ return self.__n_sigma_parameter -diff --git a/src/python/sentryPlugins/ai_threshold_slow_io_detection/data_access.py b/src/python/sentryPlugins/ai_block_io/data_access.py -similarity index 99% -rename from src/python/sentryPlugins/ai_threshold_slow_io_detection/data_access.py -rename to src/python/sentryPlugins/ai_block_io/data_access.py -index d9f3460..01c5315 100644 ---- a/src/python/sentryPlugins/ai_threshold_slow_io_detection/data_access.py -+++ b/src/python/sentryPlugins/ai_block_io/data_access.py -@@ -17,6 +17,8 @@ from sentryCollector.collect_plugin import ( - get_io_data, - is_iocollect_valid, - ) -+ -+ - from .io_data import IOStageData, IOData - - COLLECT_STAGES = [ -@@ -32,6 +34,7 @@ COLLECT_STAGES = [ - "iocost", - ] - -+ - def check_collect_valid(period): - data_raw = is_iocollect_valid(period) - if data_raw["ret"] == 0: -diff --git a/src/python/sentryPlugins/ai_threshold_slow_io_detection/detector.py b/src/python/sentryPlugins/ai_block_io/detector.py -similarity index 77% -rename from src/python/sentryPlugins/ai_threshold_slow_io_detection/detector.py -rename to src/python/sentryPlugins/ai_block_io/detector.py -index eda9825..bcf62cb 100644 ---- a/src/python/sentryPlugins/ai_threshold_slow_io_detection/detector.py -+++ b/src/python/sentryPlugins/ai_block_io/detector.py -@@ -26,19 +26,26 @@ class Detector: - self._threshold = threshold - self._slidingWindow = sliding_window - self._threshold.attach_observer(self._slidingWindow) -+ self._count = 0 - - def get_metric_name(self): - return self._metric_name - - def is_slow_io_event(self, io_data_dict_with_disk_name: dict): -- logging.debug(f'Enter Detector: {self}') -+ self._count += 1 -+ if self._count % 15 == 0: -+ self._count = 0 -+ logging.info(f"({self._metric_name}) 's latest threshold is: {self._threshold.get_threshold()}.") -+ logging.debug(f'enter Detector: {self}') - metric_value = get_metric_value_from_io_data_dict_by_metric_name(io_data_dict_with_disk_name, self._metric_name) -- if metric_value > 1e-6: -- logging.debug(f'Input metric value: {str(metric_value)}') -- self._threshold.push_latest_data_to_queue(metric_value) -+ if metric_value is None: -+ logging.debug('not found metric value, so return None.') -+ return False, None, None -+ logging.debug(f'input metric value: {str(metric_value)}') -+ self._threshold.push_latest_data_to_queue(metric_value) - detection_result = self._slidingWindow.is_slow_io_event(metric_value) - logging.debug(f'Detection result: {str(detection_result)}') -- logging.debug(f'Exit Detector: {self}') -+ logging.debug(f'exit Detector: {self}') - return detection_result - - def __repr__(self): -diff --git a/src/python/sentryPlugins/ai_threshold_slow_io_detection/io_data.py b/src/python/sentryPlugins/ai_block_io/io_data.py -similarity index 100% -rename from src/python/sentryPlugins/ai_threshold_slow_io_detection/io_data.py -rename to src/python/sentryPlugins/ai_block_io/io_data.py -diff --git a/src/python/sentryPlugins/ai_threshold_slow_io_detection/sliding_window.py b/src/python/sentryPlugins/ai_block_io/sliding_window.py -similarity index 100% -rename from src/python/sentryPlugins/ai_threshold_slow_io_detection/sliding_window.py -rename to src/python/sentryPlugins/ai_block_io/sliding_window.py -diff --git a/src/python/sentryPlugins/ai_threshold_slow_io_detection/threshold.py b/src/python/sentryPlugins/ai_block_io/threshold.py -similarity index 92% -rename from src/python/sentryPlugins/ai_threshold_slow_io_detection/threshold.py -rename to src/python/sentryPlugins/ai_block_io/threshold.py -index 9e1ca7b..ff85d85 100644 ---- a/src/python/sentryPlugins/ai_threshold_slow_io_detection/threshold.py -+++ b/src/python/sentryPlugins/ai_block_io/threshold.py -@@ -79,9 +79,9 @@ class AbsoluteThreshold(Threshold): - - - class BoxplotThreshold(Threshold): -- def __init__(self, parameter: float = 1.5, data_queue_size: int = 10000, data_queue_update_size: int = 1000): -+ def __init__(self, boxplot_parameter: float = 1.5, data_queue_size: int = 10000, data_queue_update_size: int = 1000, **kwargs): - super().__init__(data_queue_size, data_queue_update_size) -- self.parameter = parameter -+ self.parameter = boxplot_parameter - - def _update_threshold(self): - data = list(self.data_queue.queue) -@@ -94,6 +94,8 @@ class BoxplotThreshold(Threshold): - self.notify_observer() - - def push_latest_data_to_queue(self, data): -+ if data < 1e-6: -+ return - try: - self.data_queue.put(data, block=False) - except queue.Full: -@@ -111,9 +113,9 @@ class BoxplotThreshold(Threshold): - - - class NSigmaThreshold(Threshold): -- def __init__(self, parameter: float = 2.0, data_queue_size: int = 10000, data_queue_update_size: int = 1000): -+ def __init__(self, n_sigma_parameter: float = 3.0, data_queue_size: int = 10000, data_queue_update_size: int = 1000, **kwargs): - super().__init__(data_queue_size, data_queue_update_size) -- self.parameter = parameter -+ self.parameter = n_sigma_parameter - - def _update_threshold(self): - data = list(self.data_queue.queue) -@@ -125,6 +127,8 @@ class NSigmaThreshold(Threshold): - self.notify_observer() - - def push_latest_data_to_queue(self, data): -+ if data < 1e-6: -+ return - try: - self.data_queue.put(data, block=False) - except queue.Full: -@@ -157,4 +161,3 @@ class ThresholdFactory: - return NSigmaThreshold(*args, **kwargs) - else: - raise ValueError(f"Invalid threshold type: {threshold_type}") -- -diff --git a/src/python/sentryPlugins/ai_threshold_slow_io_detection/utils.py b/src/python/sentryPlugins/ai_block_io/utils.py -similarity index 86% -rename from src/python/sentryPlugins/ai_threshold_slow_io_detection/utils.py -rename to src/python/sentryPlugins/ai_block_io/utils.py -index f66e5ed..8dbba06 100644 ---- a/src/python/sentryPlugins/ai_threshold_slow_io_detection/utils.py -+++ b/src/python/sentryPlugins/ai_block_io/utils.py -@@ -8,13 +8,16 @@ - # IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR - # PURPOSE. - # See the Mulan PSL v2 for more details. -+ - import logging - from dataclasses import asdict - -+ - from .threshold import ThresholdType - from .sliding_window import SlidingWindowType - from .io_data import MetricName, IOData - -+ - def get_threshold_type_enum(algorithm_type: str): - if algorithm_type.lower() == 'absolute': - return ThresholdType.AbsoluteThreshold -@@ -22,7 +25,7 @@ def get_threshold_type_enum(algorithm_type: str): - return ThresholdType.BoxplotThreshold - if algorithm_type.lower() == 'n_sigma': - return ThresholdType.NSigmaThreshold -- logging.info('not found correct algorithm type, use default: boxplot.') -+ logging.warning(f"the algorithm type: {algorithm_type} you set is invalid, use default value: boxplot") - return ThresholdType.BoxplotThreshold - - -@@ -33,7 +36,7 @@ def get_sliding_window_type_enum(sliding_window_type: str): - return SlidingWindowType.ContinuousSlidingWindow - if sliding_window_type.lower() == 'median': - return SlidingWindowType.MedianSlidingWindow -- logging.info('not found correct sliding window type, use default: not_continuous.') -+ logging.warning(f"the sliding window type: {sliding_window_type} you set is invalid, use default value: not_continuous") - return SlidingWindowType.NotContinuousSlidingWindow - - -@@ -62,6 +65,8 @@ def get_log_level(log_level: str): - return logging.INFO - elif log_level.lower() == 'warning': - return logging.WARNING -- elif log_level.lower() == 'fatal': -- return logging.FATAL -- return None -+ elif log_level.lower() == 'error': -+ return logging.ERROR -+ elif log_level.lower() == 'critical': -+ return logging.CRITICAL -+ return logging.INFO -diff --git a/src/python/sentryPlugins/ai_threshold_slow_io_detection/config_parser.py b/src/python/sentryPlugins/ai_threshold_slow_io_detection/config_parser.py -deleted file mode 100644 -index cd4e6f1..0000000 ---- a/src/python/sentryPlugins/ai_threshold_slow_io_detection/config_parser.py -+++ /dev/null -@@ -1,141 +0,0 @@ --# coding: utf-8 --# Copyright (c) 2024 Huawei Technologies Co., Ltd. --# sysSentry is licensed under the Mulan PSL v2. --# You can use this software according to the terms and conditions of the Mulan PSL v2. --# You may obtain a copy of Mulan PSL v2 at: --# http://license.coscl.org.cn/MulanPSL2 --# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR --# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR --# PURPOSE. --# See the Mulan PSL v2 for more details. -- --import configparser --import logging -- -- --class ConfigParser: -- -- DEFAULT_ABSOLUTE_THRESHOLD = 40 -- DEFAULT_SLOW_IO_DETECTION_FREQUENCY = 1 -- DEFAULT_LOG_LEVEL = 'info' -- DEFAULT_TRAIN_DATA_DURATION = 24 -- DEFAULT_TRAIN_UPDATE_DURATION = 2 -- DEFAULT_ALGORITHM_TYPE = 'boxplot' -- DEFAULT_N_SIGMA_PARAMETER = 3 -- DEFAULT_BOXPLOT_PARAMETER = 1.5 -- DEFAULT_SLIDING_WINDOW_TYPE = 'not_continuous' -- DEFAULT_WINDOW_SIZE = 30 -- DEFAULT_WINDOW_MINIMUM_THRESHOLD = 6 -- -- def __init__(self, config_file_name): -- self.__boxplot_parameter = None -- self.__window_minimum_threshold = None -- self.__window_size = None -- self.__sliding_window_type = None -- self.__n_sigma_parameter = None -- self.__algorithm_type = None -- self.__train_update_duration = None -- self.__log_level = None -- self.__slow_io_detect_frequency = None -- self.__absolute_threshold = None -- self.__train_data_duration = None -- self.__config_file_name = config_file_name -- -- def read_config_from_file(self): -- -- con = configparser.ConfigParser() -- con.read(self.__config_file_name, encoding='utf-8') -- -- items_common = dict(con.items('common')) -- items_algorithm = dict(con.items('algorithm')) -- items_sliding_window = dict(con.items('sliding_window')) -- -- try: -- self.__absolute_threshold = int(items_common.get('absolute_threshold', -- ConfigParser.DEFAULT_ABSOLUTE_THRESHOLD)) -- except ValueError: -- self.__absolute_threshold = ConfigParser.DEFAULT_ABSOLUTE_THRESHOLD -- logging.warning('absolute threshold type conversion has error, use default value.') -- -- try: -- self.__slow_io_detect_frequency = int(items_common.get('slow_io_detect_frequency', -- ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY)) -- except ValueError: -- self.__slow_io_detect_frequency = ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY -- logging.warning('slow_io_detect_frequency type conversion has error, use default value.') -- -- self.__log_level = items_common.get('log_level', ConfigParser.DEFAULT_LOG_LEVEL) -- -- try: -- self.__train_data_duration = float(items_algorithm.get('train_data_duration', -- ConfigParser.DEFAULT_TRAIN_DATA_DURATION)) -- except ValueError: -- self.__train_data_duration = ConfigParser.DEFAULT_TRAIN_DATA_DURATION -- logging.warning('train_data_duration type conversion has error, use default value.') -- -- try: -- self.__train_update_duration = float(items_algorithm.get('train_update_duration', -- ConfigParser.DEFAULT_TRAIN_UPDATE_DURATION)) -- except ValueError: -- self.__train_update_duration = ConfigParser.DEFAULT_TRAIN_UPDATE_DURATION -- logging.warning('train_update_duration type conversion has error, use default value.') -- -- try: -- self.__algorithm_type = items_algorithm.get('algorithm_type', ConfigParser.DEFAULT_ALGORITHM_TYPE) -- except ValueError: -- self.__algorithm_type = ConfigParser.DEFAULT_ALGORITHM_TYPE -- logging.warning('algorithmType type conversion has error, use default value.') -- -- if self.__algorithm_type == 'n_sigma': -- try: -- self.__n_sigma_parameter = float(items_algorithm.get('n_sigma_parameter', -- ConfigParser.DEFAULT_N_SIGMA_PARAMETER)) -- except ValueError: -- self.__n_sigma_parameter = ConfigParser.DEFAULT_N_SIGMA_PARAMETER -- logging.warning('n_sigma_parameter type conversion has error, use default value.') -- elif self.__algorithm_type == 'boxplot': -- try: -- self.__boxplot_parameter = float(items_algorithm.get('boxplot_parameter', -- ConfigParser.DEFAULT_BOXPLOT_PARAMETER)) -- except ValueError: -- self.__boxplot_parameter = ConfigParser.DEFAULT_BOXPLOT_PARAMETER -- logging.warning('boxplot_parameter type conversion has error, use default value.') -- -- self.__sliding_window_type = items_sliding_window.get('sliding_window_type', -- ConfigParser.DEFAULT_SLIDING_WINDOW_TYPE) -- -- try: -- self.__window_size = int(items_sliding_window.get('window_size', -- ConfigParser.DEFAULT_WINDOW_SIZE)) -- except ValueError: -- self.__window_size = ConfigParser.DEFAULT_WINDOW_SIZE -- logging.warning('window_size type conversion has error, use default value.') -- -- try: -- self.__window_minimum_threshold = ( -- int(items_sliding_window.get('window_minimum_threshold', -- ConfigParser.DEFAULT_WINDOW_MINIMUM_THRESHOLD))) -- except ValueError: -- self.__window_minimum_threshold = ConfigParser.DEFAULT_WINDOW_MINIMUM_THRESHOLD -- logging.warning('window_minimum_threshold type conversion has error, use default value.') -- -- def get_slow_io_detect_frequency(self): -- return self.__slow_io_detect_frequency -- -- def get_algorithm_type(self): -- return self.__algorithm_type -- -- def get_sliding_window_type(self): -- return self.__sliding_window_type -- -- def get_train_data_duration_and_train_update_duration(self): -- return self.__train_data_duration, self.__train_update_duration -- -- def get_window_size_and_window_minimum_threshold(self): -- return self.__window_size, self.__window_minimum_threshold -- -- def get_absolute_threshold(self): -- return self.__absolute_threshold -- -- def get_log_level(self): -- return self.__log_level -diff --git a/src/python/setup.py b/src/python/setup.py -index dac6481..9e26a10 100644 ---- a/src/python/setup.py -+++ b/src/python/setup.py -@@ -34,7 +34,7 @@ setup( - 'xalarmd=xalarm.xalarm_daemon:alarm_process_create', - 'sentryCollector=sentryCollector.collectd:main', - 'avg_block_io=sentryPlugins.avg_block_io.avg_block_io:main', -- 'ai_threshold_slow_io_detection=sentryPlugins.ai_threshold_slow_io_detection.slow_io_detection:main' -+ 'ai_block_io=sentryPlugins.ai_block_io.ai_block_io:main' - ] - }, - ) --- -2.23.0 - diff --git a/fix-alarm_info-newline-break-error.patch b/fix-alarm_info-newline-break-error.patch deleted file mode 100644 index ba3d3b1..0000000 --- a/fix-alarm_info-newline-break-error.patch +++ /dev/null @@ -1,48 +0,0 @@ -From fe1bb401c1f77860616e74c1dbf5fe6aa862b17d Mon Sep 17 00:00:00 2001 -From: jinsaihang -Date: Sat, 26 Oct 2024 07:18:16 +0000 -Subject: [PATCH] fix alarm_info newline break error - -Signed-off-by: jinsaihang ---- - sysSentry-1.0.2/src/python/syssentry/alarm.py | 23 +++++++++++++++++++ - 1 file changed, 23 insertions(+) - -diff --git a/src/python/syssentry/alarm.py b/src/python/syssentry/alarm.py -index 2575307..b35a126 100644 ---- a/src/python/syssentry/alarm.py -+++ b/src/python/syssentry/alarm.py -@@ -180,7 +180,30 @@ def get_alarm_result(task_name: str, time_range: int, detailed: bool) -> List[Di - if 'details' in alarm_info: - alarm_info.pop('details', None) - alarm.pop('msg1', None) -+ -+ # dump each {key,value} of details in one line -+ if 'details' in alarm_info and isinstance(alarm_info['details'], dict): -+ for key in alarm_info['details']: -+ alarm_info['details'][key] = json.dumps(alarm_info['details'][key], indent=None) -+ - alarm['alarm_info'] = alarm_info -+ alarm_list = [alarm for alarm in alarm_list if 'alarm_source' in alarm['alarm_info'] and alarm['alarm_info']['alarm_source'] == task_name] -+ -+ alarm_level_mapping = { -+ 1: 'MINOR_ALM', -+ 2: 'MAJOR_ALM', -+ 3: 'CRITICAL_ALM' -+ } -+ -+ alarm_type_mapping = { -+ 1: 'ALARM_TYPE_OCCUR', -+ 2: 'ALARM_TYPE_RECOVER' -+ } -+ -+ for alarm in alarm_list: -+ alarm['alarm_level'] = alarm_level_mapping.get(alarm['alarm_level'], 'UNKNOWN_LEVEL') -+ alarm['alarm_type'] = alarm_type_mapping.get(alarm['alarm_type'], 'UNKNOWN_TYPE') - return alarm_list -+ - finally: - alarm_list_lock.release() --- -2.27.0 - diff --git a/fix-bug-step-2-about-collect-module-and-avg-block-io.patch b/fix-bug-step-2-about-collect-module-and-avg-block-io.patch deleted file mode 100644 index 6b80cb9..0000000 --- a/fix-bug-step-2-about-collect-module-and-avg-block-io.patch +++ /dev/null @@ -1,323 +0,0 @@ -From e6eb39799b3ca15fb385c572863417ea26bdfa66 Mon Sep 17 00:00:00 2001 -From: zhuofeng -Date: Wed, 25 Sep 2024 11:03:29 +0800 -Subject: [PATCH] fix-bug-step-2-about-collect-module-and-avg-block-io - ---- - src/python/sentryCollector/collect_config.py | 11 ++- - src/python/sentryCollector/collect_io.py | 25 ++--- - src/python/sentryCollector/collect_plugin.py | 6 +- - src/python/sentryCollector/collect_server.py | 1 - - src/python/sentryCollector/collectd.py | 4 +- - .../avg_block_io/avg_block_io.py | 92 ++++++++++++++----- - 6 files changed, 96 insertions(+), 43 deletions(-) - -diff --git a/src/python/sentryCollector/collect_config.py b/src/python/sentryCollector/collect_config.py -index b6cc75c..0fdd9f0 100644 ---- a/src/python/sentryCollector/collect_config.py -+++ b/src/python/sentryCollector/collect_config.py -@@ -49,14 +49,14 @@ class CollectConfig: - self.config = configparser.ConfigParser() - self.config.read(self.filename) - except configparser.Error: -- logging.error("collectd configure file read failed") -+ logging.error("collect configure file read failed") - return - - try: - common_config = self.config[CONF_COMMON] -- modules_str = common_config[CONF_MODULES] -+ modules_str = common_config[CONF_MODULES].lower() - # remove space -- modules_list = modules_str.replace(" ", "").split(',') -+ modules_list = set(modules_str.replace(" ", "").split(',')) - except KeyError as e: - logging.error("read config data failed, %s", e) - return -@@ -98,7 +98,7 @@ class CollectConfig: - CONF_IO, CONF_IO_MAX_SAVE, CONF_IO_MAX_SAVE_DEFAULT) - result_io_config[CONF_IO_MAX_SAVE] = CONF_IO_MAX_SAVE_DEFAULT - # disk -- disk = io_map_value.get(CONF_IO_DISK) -+ disk = io_map_value.get(CONF_IO_DISK).lower() - if disk: - disk_str = disk.replace(" ", "") - pattern = r'^[a-zA-Z0-9-_,]+$' -@@ -106,12 +106,13 @@ class CollectConfig: - logging.warning("module_name = %s section, field = %s is incorrect, use default %s", - CONF_IO, CONF_IO_DISK, CONF_IO_DISK_DEFAULT) - disk_str = CONF_IO_DISK_DEFAULT -+ disk_str = ",".join(set(disk_str.split(','))) - result_io_config[CONF_IO_DISK] = disk_str - else: - logging.warning("module_name = %s section, field = %s is incorrect, use default %s", - CONF_IO, CONF_IO_DISK, CONF_IO_DISK_DEFAULT) - result_io_config[CONF_IO_DISK] = CONF_IO_DISK_DEFAULT -- logging.info("config get_io_config: %s", result_io_config) -+ logging.debug("config get_io_config: %s", result_io_config) - return result_io_config - - def get_common_config(self): -diff --git a/src/python/sentryCollector/collect_io.py b/src/python/sentryCollector/collect_io.py -index 104b734..9c8dae7 100644 ---- a/src/python/sentryCollector/collect_io.py -+++ b/src/python/sentryCollector/collect_io.py -@@ -177,10 +177,8 @@ class CollectIo(): - - def is_kernel_avaliable(self): - base_path = '/sys/kernel/debug/block' -+ all_disk = [] - for disk_name in os.listdir(base_path): -- if not self.loop_all and disk_name not in self.disk_list: -- continue -- - disk_path = os.path.join(base_path, disk_name) - blk_io_hierarchy_path = os.path.join(disk_path, 'blk_io_hierarchy') - -@@ -190,12 +188,18 @@ class CollectIo(): - - for file_name in os.listdir(blk_io_hierarchy_path): - file_path = os.path.join(blk_io_hierarchy_path, file_name) -- - if file_name == 'stats': -- stage_list = self.extract_first_column(file_path) -- self.disk_map_stage[disk_name] = stage_list -- self.window_value[disk_name] = {} -- IO_GLOBAL_DATA[disk_name] = {} -+ all_disk.append(disk_name) -+ -+ for disk_name in self.disk_list: -+ if not self.loop_all and disk_name not in all_disk: -+ logging.warning("the %s disk not exist!", disk_name) -+ continue -+ stats_file = '/sys/kernel/debug/block/{}/blk_io_hierarchy/stats'.format(disk_name) -+ stage_list = self.extract_first_column(stats_file) -+ self.disk_map_stage[disk_name] = stage_list -+ self.window_value[disk_name] = {} -+ IO_GLOBAL_DATA[disk_name] = {} - - return len(IO_GLOBAL_DATA) != 0 - -@@ -203,7 +207,7 @@ class CollectIo(): - logging.info("collect io thread start") - - if not self.is_kernel_avaliable() or len(self.disk_map_stage) == 0: -- logging.warning("no disks meet the requirements. collect io thread exits") -+ logging.warning("no disks meet the requirements. collect io thread exit") - return - - for disk_name, stage_list in self.disk_map_stage.items(): -@@ -239,5 +243,4 @@ class CollectIo(): - - # set stop event, notify thread exit - def stop_thread(self): -- logging.debug("collect io thread is preparing to exit") -- self.stop_event.set() -+ self.stop_event.set() -diff --git a/src/python/sentryCollector/collect_plugin.py b/src/python/sentryCollector/collect_plugin.py -index 9132473..1faa5e3 100644 ---- a/src/python/sentryCollector/collect_plugin.py -+++ b/src/python/sentryCollector/collect_plugin.py -@@ -10,7 +10,7 @@ - # See the Mulan PSL v2 for more details. - - """ --collcet plugin -+collect plugin - """ - import json - import socket -@@ -75,7 +75,7 @@ def client_send_and_recv(request_data, data_str_len, protocol): - try: - client_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - except socket.error: -- print("collect_plugin: client creat socket error") -+ print("collect_plugin: client create socket error") - return None - - try: -@@ -128,7 +128,7 @@ def client_send_and_recv(request_data, data_str_len, protocol): - def validate_parameters(param, len_limit, char_limit): - ret = ResultMessage.RESULT_SUCCEED - if not param: -- print("parm is invalid") -+ print("param is invalid") - ret = ResultMessage.RESULT_NOT_PARAM - return [False, ret] - -diff --git a/src/python/sentryCollector/collect_server.py b/src/python/sentryCollector/collect_server.py -index bab4e56..11d1af0 100644 ---- a/src/python/sentryCollector/collect_server.py -+++ b/src/python/sentryCollector/collect_server.py -@@ -281,5 +281,4 @@ class CollectServer(): - pass - - def stop_thread(self): -- logging.debug("collect listen thread is preparing to exit") - self.stop_event.set() -diff --git a/src/python/sentryCollector/collectd.py b/src/python/sentryCollector/collectd.py -index 3a836df..d9d8862 100644 ---- a/src/python/sentryCollector/collectd.py -+++ b/src/python/sentryCollector/collectd.py -@@ -79,7 +79,7 @@ def main(): - for info in module_list: - class_name = Module_Map_Class.get(info) - if not class_name: -- logging.info("%s correspond to class is not exists", info) -+ logging.info("%s correspond to class is not exist", info) - continue - cn = class_name(module_config) - collect_thread = threading.Thread(target=cn.main_loop) -@@ -94,4 +94,4 @@ def main(): - finally: - pass - -- logging.info("All threads have finished. Main thread is exiting.") -\ No newline at end of file -+ logging.info("all threads have finished. main thread exit.") -\ No newline at end of file -diff --git a/src/python/sentryPlugins/avg_block_io/avg_block_io.py b/src/python/sentryPlugins/avg_block_io/avg_block_io.py -index 73f0b22..ac35be2 100644 ---- a/src/python/sentryPlugins/avg_block_io/avg_block_io.py -+++ b/src/python/sentryPlugins/avg_block_io/avg_block_io.py -@@ -28,33 +28,53 @@ def log_invalid_keys(not_in_list, keys_name, config_list, default_list): - - - def read_config_common(config): -- """read config file, get [common] section value""" -- try: -- common_sec = config['common'] -- except configparser.NoSectionError: -+ """read config file, get [common] section value""" -+ if not config.has_section("common"): - report_alarm_fail("Cannot find common section in config file") - - try: -- period_time = int(common_sec.get("period_time", 1)) -- if not (1 <= period_time <= 300): -- raise ValueError("Invalid period_time") -- except ValueError: -- period_time = 1 -- logging.warning("Invalid period_time, set to 1s") -+ disk_name = config.get("common", "disk") -+ disk = [] if disk_name == "default" else disk_name.split(",") -+ except configparser.NoOptionError: -+ disk = [] -+ logging.warning("Unset disk, set to default") - -- disk = common_sec.get('disk').split(",") if common_sec.get('disk') not in [None, 'default'] else [] -- stage = common_sec.get('stage').split(",") if common_sec.get('stage') not in [None, 'default'] else [] -+ try: -+ stage_name = config.get("common", "stage") -+ stage = [] if stage_name == "default" else stage_name.split(",") -+ except configparser.NoOptionError: -+ stage = [] -+ logging.warning("Unset stage, set to read,write") - - if len(disk) > 10: - logging.warning("Too many disks, record only max 10 disks") - disk = disk[:10] - -- iotype = common_sec.get('iotype', 'read,write').split(",") -- iotype_list = [rw.lower() for rw in iotype if rw.lower() in ['read', 'write', 'flush', 'discard']] -- err_iotype = [rw for rw in iotype if rw.lower() not in ['read', 'write', 'flush', 'discard']] -+ try: -+ iotype_name = config.get("common", "iotype").split(",") -+ iotype_list = [rw.lower() for rw in iotype_name if rw.lower() in ['read', 'write', 'flush', 'discard']] -+ err_iotype = [rw.lower() for rw in iotype_name if rw.lower() not in ['read', 'write', 'flush', 'discard']] -+ -+ if iotype_list in [None, []]: -+ iotype_list = ["read", "write"] -+ except configparser.NoOptionError: -+ iotype = ["read", "write"] -+ logging.warning("Unset iotype, set to default") - - if err_iotype: - logging.warning("{} in common.iotype are not valid, set iotype={}".format(err_iotype, iotype_list)) -+ -+ -+ try: -+ period_time = int(config.get("common", "period_time")) -+ if not (1 <= period_time <= 300): -+ raise ValueError("Invalid period_time") -+ except ValueError: -+ period_time = 1 -+ logging.warning("Invalid period_time, set to 1s") -+ except configparser.NoOptionError: -+ period_time = 1 -+ logging.warning("Unset period_time, use 1s as default") - - return period_time, disk, stage, iotype_list - -@@ -68,11 +88,23 @@ def read_config_algorithm(config): - win_size = int(config.get("algorithm", "win_size")) - if not (1 <= win_size <= 300): - raise ValueError("Invalid win_size") -+ except ValueError: -+ win_size = 30 -+ logging.warning("Invalid win_size, set to 30") -+ except configparser.NoOptionError: -+ win_size = 30 -+ logging.warning("Unset win_size, use 30 as default") -+ -+ try: - win_threshold = int(config.get("algorithm", "win_threshold")) - if win_threshold < 1 or win_threshold > 300 or win_threshold > win_size: - raise ValueError("Invalid win_threshold") - except ValueError: -- report_alarm_fail("Invalid win_threshold or win_size") -+ win_threshold = 6 -+ logging.warning("Invalid win_threshold, set to 6") -+ except configparser.NoOptionError: -+ win_threshold = 6 -+ logging.warning("Unset win_threshold, use 6 as default") - - return win_size, win_threshold - -@@ -80,6 +112,21 @@ def read_config_algorithm(config): - def read_config_lat_iodump(io_dic, config): - """read config file, get [latency] [iodump] section value""" - common_param = {} -+ lat_sec = None -+ if not config.has_section("latency"): -+ logging.warning("Cannot find algorithm section in config file") -+ else: -+ lat_sec = config["latency"] -+ -+ iodump_sec = None -+ if not config.has_section("iodump"): -+ logging.warning("Cannot find iodump section in config file") -+ else: -+ lat_sec = config["iodump"] -+ -+ if not lat_sec and not iodump_sec: -+ return common_param -+ - for io_type in io_dic["iotype_list"]: - common_param[io_type] = {} - -@@ -90,13 +137,16 @@ def read_config_lat_iodump(io_dic, config): - } - iodump_key = "{}_iodump_lim".format(io_type) - -+ if iodump_sec and iodump_key in iodump_sec and iodump_sec[iodump_key].isdecimal(): -+ common_param[io_type][iodump_key] = int(iodump_sec[iodump_key]) -+ -+ if not lat_sec: -+ continue -+ - for key_suffix, key_template in latency_keys.items(): -- if key_template in config["latency"] and config["latency"][key_template].isdecimal(): -- common_param[io_type][key_template] = int(config["latency"][key_template]) -+ if key_template in lat_sec and lat_sec[key_template].isdecimal(): -+ common_param[io_type][key_template] = int(lat_sec[key_template]) - -- if iodump_key in config["iodump"] and config["iodump"][iodump_key].isdecimal(): -- common_param[io_type][iodump_key] = int(config["iodump"][iodump_key]) -- - return common_param - - --- -2.33.0 - diff --git a/fix-config-relative-some-issues.patch b/fix-config-relative-some-issues.patch deleted file mode 100644 index dbc0815..0000000 --- a/fix-config-relative-some-issues.patch +++ /dev/null @@ -1,243 +0,0 @@ -From c9f62e01f09a56743ccc3e470f273875ab22ac5f Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?=E8=B4=BA=E6=9C=89=E5=BF=97?= <1037617413@qq.com> -Date: Wed, 9 Oct 2024 16:19:52 +0800 -Subject: [PATCH] fix config relative some issues - ---- - .../sentryPlugins/ai_block_io/README.md | 1 - - .../sentryPlugins/ai_block_io/ai_block_io.py | 21 +++++----- - .../ai_block_io/config_parser.py | 42 +++++++++---------- - .../sentryPlugins/ai_block_io/detector.py | 2 +- - .../ai_block_io/sliding_window.py | 8 ++-- - .../sentryPlugins/ai_block_io/threshold.py | 6 +-- - 6 files changed, 39 insertions(+), 41 deletions(-) - -diff --git a/src/python/sentryPlugins/ai_block_io/README.md b/src/python/sentryPlugins/ai_block_io/README.md -index f9b8388..95c1111 100644 ---- a/src/python/sentryPlugins/ai_block_io/README.md -+++ b/src/python/sentryPlugins/ai_block_io/README.md -@@ -1,2 +1 @@ - # slow_io_detection -- -diff --git a/src/python/sentryPlugins/ai_block_io/ai_block_io.py b/src/python/sentryPlugins/ai_block_io/ai_block_io.py -index 31b8a97..3b00ef3 100644 ---- a/src/python/sentryPlugins/ai_block_io/ai_block_io.py -+++ b/src/python/sentryPlugins/ai_block_io/ai_block_io.py -@@ -16,8 +16,7 @@ import logging - from .detector import Detector - from .threshold import ThresholdFactory, AbsoluteThreshold - from .sliding_window import SlidingWindowFactory --from .utils import (get_threshold_type_enum, get_sliding_window_type_enum, get_data_queue_size_and_update_size, -- get_log_level) -+from .utils import get_data_queue_size_and_update_size - from .config_parser import ConfigParser - from .data_access import get_io_data_from_collect_plug, check_collect_valid - from .io_data import MetricName -@@ -45,25 +44,25 @@ class SlowIODetection: - - def __init_detector_name_list(self): - self._disk_list = check_collect_valid(self._config_parser.get_slow_io_detect_frequency()) -+ logging.info(f"ai_block_io plug has found disks: {self._disk_list}") - disks_to_detection: list = self._config_parser.get_disks_to_detection() - # 情况1：None，则启用所有磁盘检测 - # 情况2：is not None and len = 0，则不启动任何磁盘检测 - # 情况3：len ！= 0，则取交集 - if disks_to_detection is None: -+ logging.warning("you not specify any disk or use default, so ai_block_io will enable all available disk.") - for disk in self._disk_list: - self._detector_name_list.append(MetricName(disk, "bio", "read", "latency")) - self._detector_name_list.append(MetricName(disk, "bio", "write", "latency")) - elif len(disks_to_detection) == 0: -- logging.warning('please attention: conf file not specify any disk to detection, ' -- 'so it will not start ai block io.') -+ logging.warning('please attention: conf file not specify any disk to detection, so it will not start ai block io.') - else: -- disks_name_to_detection = [] -- for disk_name_to_detection in disks_to_detection: -- disks_name_to_detection.append(disk_name_to_detection.get_disk_name()) -- disk_intersection = [disk for disk in self._disk_list if disk in disks_name_to_detection] -- for disk in disk_intersection: -- self._detector_name_list.append(MetricName(disk, "bio", "read", "latency")) -- self._detector_name_list.append(MetricName(disk, "bio", "write", "latency")) -+ for disk_to_detection in disks_to_detection: -+ if disk_to_detection in self._disk_list: -+ self._detector_name_list.append(MetricName(disk_to_detection, "bio", "read", "latency")) -+ self._detector_name_list.append(MetricName(disk_to_detection, "bio", "write", "latency")) -+ else: -+ logging.warning(f"disk：[{disk_to_detection}] not in available disk list, so it will be ignored.") - logging.info(f'start to detection follow disk and it\'s metric: {self._detector_name_list}') - - def __init_detector(self): -diff --git a/src/python/sentryPlugins/ai_block_io/config_parser.py b/src/python/sentryPlugins/ai_block_io/config_parser.py -index 632391d..354c122 100644 ---- a/src/python/sentryPlugins/ai_block_io/config_parser.py -+++ b/src/python/sentryPlugins/ai_block_io/config_parser.py -@@ -10,18 +10,19 @@ - # See the Mulan PSL v2 for more details. - - import configparser --import json - import logging - --from .io_data import MetricName - from .threshold import ThresholdType - from .utils import get_threshold_type_enum, get_sliding_window_type_enum, get_log_level - -+ - LOG_FORMAT = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s" - - - def init_log_format(log_level: str): -- logging.basicConfig(level=get_log_level(log_level), format=LOG_FORMAT) -+ logging.basicConfig(level=get_log_level(log_level.lower()), format=LOG_FORMAT) -+ if log_level.lower() not in ('info', 'warning', 'error', 'debug'): -+ logging.warning(f'the log_level: {log_level} you set is invalid, use default value: info.') - - - class ConfigParser: -@@ -43,7 +44,7 @@ class ConfigParser: - self.__absolute_threshold = ConfigParser.DEFAULT_ABSOLUTE_THRESHOLD - self.__slow_io_detect_frequency = ConfigParser.DEFAULT_SLOW_IO_DETECTION_FREQUENCY - self.__log_level = ConfigParser.DEFAULT_LOG_LEVEL -- self.__disks_to_detection: list = [] -+ self.__disks_to_detection = None - - self.__algorithm_type = ConfigParser.DEFAULT_ALGORITHM_TYPE - self.__train_data_duration = ConfigParser.DEFAULT_TRAIN_UPDATE_DURATION -@@ -83,26 +84,20 @@ class ConfigParser: - logging.warning(f'slow_io_detect_frequency type conversion has error, use default value: {self.__slow_io_detect_frequency}.') - - def __read__disks_to_detect(self, items_common: dict): -- disks_to_detection = items_common.get('disks_to_detect') -+ disks_to_detection = items_common.get('disk') - if disks_to_detection is None: -- logging.warning(f'config of disks_to_detect not found, the default value be used.') -+ logging.warning(f'config of disk not found, the default value will be used.') - self.__disks_to_detection = None - return -- try: -- disks_to_detection_list = json.loads(disks_to_detection) -- for disk_to_detection in disks_to_detection_list: -- disk_name = disk_to_detection.get('disk_name', None) -- stage_name = disk_to_detection.get('stage_name', None) -- io_access_type_name = disk_to_detection.get('io_access_type_name', None) -- metric_name = disk_to_detection.get('metric_name', None) -- if not (disk_name is None or stage_name is None or io_access_type_name is None or metric_name is None): -- metric_name_object = MetricName(disk_name, stage_name, io_access_type_name, metric_name) -- self.__disks_to_detection.append(metric_name_object) -- else: -- logging.warning(f'config of disks_to_detect\'s some part has some error: {disk_to_detection}, it will be ignored.') -- except json.decoder.JSONDecodeError as e: -- logging.warning(f'config of disks_to_detect is error: {e}, it will be ignored and default value be used.') -+ disk_list = disks_to_detection.split(',') -+ if len(disk_list) == 0 or (len(disk_list) == 1 and disk_list[0] == ''): -+ logging.warning("you don't specify any disk.") -+ self.__disks_to_detection = [] -+ return -+ if len(disk_list) == 1 and disk_list[0] == 'default': - self.__disks_to_detection = None -+ return -+ self.__disks_to_detection = disk_list - - def __read__train_data_duration(self, items_algorithm: dict): - try: -@@ -189,7 +184,12 @@ class ConfigParser: - - def read_config_from_file(self): - con = configparser.ConfigParser() -- con.read(self.__config_file_name, encoding='utf-8') -+ try: -+ con.read(self.__config_file_name, encoding='utf-8') -+ except configparser.Error as e: -+ init_log_format(self.__log_level) -+ logging.critical(f'config file read error: {e}, ai_block_io plug will exit.') -+ exit(1) - - if con.has_section('common'): - items_common = dict(con.items('common')) -diff --git a/src/python/sentryPlugins/ai_block_io/detector.py b/src/python/sentryPlugins/ai_block_io/detector.py -index bcf62cb..a48144f 100644 ---- a/src/python/sentryPlugins/ai_block_io/detector.py -+++ b/src/python/sentryPlugins/ai_block_io/detector.py -@@ -50,6 +50,6 @@ class Detector: - - def __repr__(self): - return (f'disk_name: {self._metric_name.get_disk_name()}, stage_name: {self._metric_name.get_stage_name()},' -- f' access_type_name: {self._metric_name.get_io_access_type_name()},' -+ f' io_type_name: {self._metric_name.get_io_access_type_name()},' - f' metric_name: {self._metric_name.get_metric_name()}, threshold_type: {self._threshold},' - f' sliding_window_type: {self._slidingWindow}') -diff --git a/src/python/sentryPlugins/ai_block_io/sliding_window.py b/src/python/sentryPlugins/ai_block_io/sliding_window.py -index d395d48..89191e5 100644 ---- a/src/python/sentryPlugins/ai_block_io/sliding_window.py -+++ b/src/python/sentryPlugins/ai_block_io/sliding_window.py -@@ -52,7 +52,7 @@ class SlidingWindow: - return False, None, None - - def __repr__(self): -- return "SlidingWindow" -+ return "[SlidingWindow]" - - - class NotContinuousSlidingWindow(SlidingWindow): -@@ -65,7 +65,7 @@ class NotContinuousSlidingWindow(SlidingWindow): - return False, self._io_data_queue, self._ai_threshold - - def __repr__(self): -- return "NotContinuousSlidingWindow" -+ return f"[NotContinuousSlidingWindow, window size: {self._queue_length}, threshold: {self._queue_threshold}]" - - - class ContinuousSlidingWindow(SlidingWindow): -@@ -84,7 +84,7 @@ class ContinuousSlidingWindow(SlidingWindow): - return False, self._io_data_queue, self._ai_threshold - - def __repr__(self): -- return "ContinuousSlidingWindow" -+ return f"[ContinuousSlidingWindow, window size: {self._queue_length}, threshold: {self._queue_threshold}]" - - - class MedianSlidingWindow(SlidingWindow): -@@ -98,7 +98,7 @@ class MedianSlidingWindow(SlidingWindow): - return False, self._io_data_queue, self._ai_threshold - - def __repr__(self): -- return "MedianSlidingWindow" -+ return f"[MedianSlidingWindow, window size: {self._queue_length}]" - - - class SlidingWindowFactory: -diff --git a/src/python/sentryPlugins/ai_block_io/threshold.py b/src/python/sentryPlugins/ai_block_io/threshold.py -index ff85d85..3b7a5a8 100644 ---- a/src/python/sentryPlugins/ai_block_io/threshold.py -+++ b/src/python/sentryPlugins/ai_block_io/threshold.py -@@ -75,7 +75,7 @@ class AbsoluteThreshold(Threshold): - pass - - def __repr__(self): -- return "AbsoluteThreshold" -+ return "[AbsoluteThreshold]" - - - class BoxplotThreshold(Threshold): -@@ -109,7 +109,7 @@ class BoxplotThreshold(Threshold): - self.new_data_size = 0 - - def __repr__(self): -- return "BoxplotThreshold" -+ return f"[BoxplotThreshold, param is: {self.parameter}]" - - - class NSigmaThreshold(Threshold): -@@ -142,7 +142,7 @@ class NSigmaThreshold(Threshold): - self.new_data_size = 0 - - def __repr__(self): -- return "NSigmaThreshold" -+ return f"[NSigmaThreshold, param is: {self.parameter}]" - - - class ThresholdType(Enum): --- -2.23.0 - diff --git a/fix-configparser.InterpolationSyntaxError.patch b/fix-configparser.InterpolationSyntaxError.patch deleted file mode 100644 index 8dfd67b..0000000 --- a/fix-configparser.InterpolationSyntaxError.patch +++ /dev/null @@ -1,37 +0,0 @@ -From 65ceade489c4018c3f315104d70be0550a28d9d9 Mon Sep 17 00:00:00 2001 -From: shixuantong -Date: Wed, 11 Sep 2024 10:23:41 +0800 -Subject: [PATCH] fix configparser.InterpolationSyntaxError - ---- - src/python/syssentry/sentry_config.py | 8 ++++++-- - 1 file changed, 6 insertions(+), 2 deletions(-) - -diff --git a/src/python/syssentry/sentry_config.py b/src/python/syssentry/sentry_config.py -index 01f3df8..a0e7b79 100644 ---- a/src/python/syssentry/sentry_config.py -+++ b/src/python/syssentry/sentry_config.py -@@ -103,14 +103,18 @@ class CpuPluginsParamsConfig: - """read config file""" - config_param_section_args = {} - if os.path.exists(self.config_file): -- self.config.read(self.config_file) - try: -+ self.config.read(self.config_file) - config_param_section_args = dict(self.config[self.param_section_name]) -- except (ValueError, KeyError): -+ except (ValueError, KeyError, configparser.InterpolationSyntaxError): - config_param_section_args = {} -+ logging.error("Failed to parse cpu_sentry.ini!") - return config_param_section_args - - def join_cpu_start_cmd(self, cpu_param_dict: dict) -> str: -+ if not cpu_param_dict: -+ return "" -+ - cpu_list = cpu_param_dict.get("cpu_list", "default") - if cpu_list == "default": - cpu_list = CpuPluginsParamsConfig.get_cpu_info() --- -2.27.0 - diff --git a/fix-error-handling.patch b/fix-error-handling.patch deleted file mode 100644 index faadc7a..0000000 --- a/fix-error-handling.patch +++ /dev/null @@ -1,25 +0,0 @@ -From 370b22b032dce9290eebca1cf8d48bd155164b6a Mon Sep 17 00:00:00 2001 -From: shixuantong -Date: Wed, 24 Jul 2024 17:53:58 +0800 -Subject: [PATCH] fix error handling - ---- - src/python/syssentry/cpu_sentry.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/src/python/syssentry/cpu_sentry.py b/src/python/syssentry/cpu_sentry.py -index 3c4d58d..d0bafa8 100644 ---- a/src/python/syssentry/cpu_sentry.py -+++ b/src/python/syssentry/cpu_sentry.py -@@ -87,7 +87,7 @@ class CpuSentry: - } - - def handle_cpu_output(self, stdout: str): -- if "" in stdout: -+ if "ERROR" in stdout: - self.send_result["result"] = ResultLevel.FAIL - self.send_result["details"]["code"] = 1004 - self.send_result["details"]["msg"] = stdout.split("\n")[0] --- -2.27.0 - diff --git a/fix-excessive-CPU-usage.patch b/fix-excessive-CPU-usage.patch deleted file mode 100644 index b72ed52..0000000 --- a/fix-excessive-CPU-usage.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 815537382fc0d5164fe57b0d984ca4a1ed8254ea Mon Sep 17 00:00:00 2001 -From: jinsaihang -Date: Thu, 31 Oct 2024 16:00:50 +0800 -Subject: [PATCH] excessive CPU usage - -Signed-off-by: jinsaihang ---- - sysSentry-1.0.2/src/python/xalarm/xalarm_transfer.py | 3 --- - 1 file changed, 3 deletions(-) - -diff --git a/src/python/xalarm/xalarm_transfer.py b/src/python/xalarm/xalarm_transfer.py -index b072007..4bebe5d 100644 ---- a/src/python/xalarm/xalarm_transfer.py -+++ b/src/python/xalarm/xalarm_transfer.py -@@ -62,7 +62,6 @@ def cleanup_closed_connections(server_sock, epoll, fd_to_socket): - to_remove.append(fileno) - - for fileno in to_remove: -- epoll.unregister(fileno) - fd_to_socket[fileno].close() - del fd_to_socket[fileno] - logging.info(f"cleaned up connection {fileno} for client lost connection.") -@@ -97,7 +96,6 @@ def wait_for_connection(server_sock, epoll, fd_to_socket, thread_should_stop): - logging.info(f"connection reach max num of {MAX_CONNECTION_NUM}, closed current connection!") - connection.close() - continue -- epoll.register(connection.fileno(), select.EPOLLOUT) - fd_to_socket[connection.fileno()] = connection - except socket.error as e: - logging.debug(f"socket error, reason is {e}") -@@ -122,7 +120,6 @@ def transmit_alarm(server_sock, epoll, fd_to_socket, bin_data): - except (BrokenPipeError, ConnectionResetError): - to_remove.append(fileno) - for fileno in to_remove: -- epoll.unregister(fileno) - fd_to_socket[fileno].close() - del fd_to_socket[fileno] - logging.info(f"cleaned up connection {fileno} for client lost connection.") --- -2.27.0 - diff --git a/fix-frequency-param-check-bug.patch b/fix-frequency-param-check-bug.patch deleted file mode 100644 index 06d4b4a..0000000 --- a/fix-frequency-param-check-bug.patch +++ /dev/null @@ -1,70 +0,0 @@ -From a06ad0c944b093a71f49cc9fccd5097c1493ca5e Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?=E8=B4=BA=E6=9C=89=E5=BF=97?= <1037617413@qq.com> -Date: Mon, 21 Oct 2024 17:31:32 +0800 -Subject: [PATCH] fix frequency param check bug - ---- - .../sentryPlugins/ai_block_io/config_parser.py | 13 +++++++++++-- - .../sentryPlugins/ai_block_io/data_access.py | 14 ++++++++++++++ - 2 files changed, 25 insertions(+), 2 deletions(-) - -diff --git a/src/python/sentryPlugins/ai_block_io/config_parser.py b/src/python/sentryPlugins/ai_block_io/config_parser.py -index 447eccd..274a31e 100644 ---- a/src/python/sentryPlugins/ai_block_io/config_parser.py -+++ b/src/python/sentryPlugins/ai_block_io/config_parser.py -@@ -16,6 +16,7 @@ import logging - from .alarm_report import Report - from .threshold import ThresholdType - from .utils import get_threshold_type_enum, get_sliding_window_type_enum, get_log_level -+from .data_access import check_detect_frequency_is_valid - - - LOG_FORMAT = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s" -@@ -165,9 +166,17 @@ class ConfigParser: - "slow_io_detect_frequency", - int, - self.DEFAULT_CONF["common"]["slow_io_detect_frequency"], -- gt=0, -- le=300, -+ gt=0 - ) -+ frequency = self._conf["common"]["slow_io_detect_frequency"] -+ ret = check_detect_frequency_is_valid(frequency) -+ if ret is None: -+ log = f"slow io detect frequency: {frequency} is valid, "\ -+ f"Check whether the value range is too large or is not an "\ -+ f"integer multiple of period_time.. exiting..." -+ Report.report_pass(log) -+ logging.critical(log) -+ exit(1) - - def _read_disks_to_detect(self, items_common: dict): - disks_to_detection = items_common.get("disk") -diff --git a/src/python/sentryPlugins/ai_block_io/data_access.py b/src/python/sentryPlugins/ai_block_io/data_access.py -index 1bc5ed8..e4869d5 100644 ---- a/src/python/sentryPlugins/ai_block_io/data_access.py -+++ b/src/python/sentryPlugins/ai_block_io/data_access.py -@@ -53,6 +53,20 @@ def check_collect_valid(period): - return None - - -+def check_detect_frequency_is_valid(period): -+ data_raw = is_iocollect_valid(period) -+ if data_raw["ret"] == 0: -+ try: -+ data = json.loads(data_raw["message"]) -+ except Exception as e: -+ return None -+ if not data: -+ return None -+ return [k for k in data.keys()] -+ else: -+ return None -+ -+ - def _get_raw_data(period, disk_list): - return get_io_data( - period, --- -2.23.0 - diff --git a/fix-get_alarm-error.patch b/fix-get_alarm-error.patch deleted file mode 100644 index 19d8dcc..0000000 --- a/fix-get_alarm-error.patch +++ /dev/null @@ -1,36 +0,0 @@ -From 8f28a40ffd7dc7aa969a7bfc0a170ed0c8f03bce Mon Sep 17 00:00:00 2001 -From: jinsaihang -Date: Tue, 22 Oct 2024 20:28:59 +0800 -Subject: [PATCH] fix get_alarm error - -Signed-off-by: jinsaihang ---- - sysSentry-1.0.2/src/python/syssentry/alarm.py | 5 +++-- - 1 file changed, 3 insertions(+), 2 deletions(-) - -diff --git a/src/python/syssentry/alarm.py b/src/python/syssentry/alarm.py -index c3f2ee1..2575307 100644 ---- a/src/python/syssentry/alarm.py -+++ b/src/python/syssentry/alarm.py -@@ -139,8 +139,6 @@ def get_alarm_result(task_name: str, time_range: int, detailed: bool) -> List[Di - return [] - alarm_id = task_alarm_id_dict[task_name] - clear_time = alarm_id_clear_time_dict[alarm_id] -- if clear_time < int(time_range): -- return [] - if alarm_id not in alarm_list_dict: - logging.debug("alarm_id does not exist") - return [] -@@ -154,6 +152,9 @@ def get_alarm_result(task_name: str, time_range: int, detailed: bool) -> List[Di - if timestamp - (xalarm_gettime(alarm_list[i])) / MILLISECONDS_UNIT_SECONDS > time_range: - stop_index = i - break -+ if timestamp - (xalarm_gettime(alarm_list[i])) / MILLISECONDS_UNIT_SECONDS > clear_time: -+ stop_index = i -+ break - if stop_index >= 0: - alarm_list = alarm_list[:stop_index] - logging.debug(f"get_alarm_result: final alarm_list of {alarm_id} has {len(alarm_list)} elements") --- -2.27.0 - diff --git a/fix-hbm-online-repair-notice-and-efi-create.patch b/fix-hbm-online-repair-notice-and-efi-create.patch deleted file mode 100644 index 9b0fa99..0000000 --- a/fix-hbm-online-repair-notice-and-efi-create.patch +++ /dev/null @@ -1,508 +0,0 @@ -From 85d6dae9d7c6148f2699ef7da7d2d784043a2ee1 Mon Sep 17 00:00:00 2001 -From: luckky -Date: Wed, 30 Oct 2024 10:41:11 +0800 -Subject: [PATCH] fix hbm online repair notice and efi create - ---- - src/c/hbm_online_repair/hbm_online_repair.c | 5 +- - .../non-standard-hbm-repair.c | 194 +++++++++--------- - .../non-standard-hbm-repair.h | 2 +- - src/c/hbm_online_repair/ras-events.c | 1 - - .../ras-non-standard-handler.c | 33 +-- - .../ras-non-standard-handler.h | 1 + - 6 files changed, 116 insertions(+), 120 deletions(-) - -diff --git a/src/c/hbm_online_repair/hbm_online_repair.c b/src/c/hbm_online_repair/hbm_online_repair.c -index 3ace206..b3b2742 100644 ---- a/src/c/hbm_online_repair/hbm_online_repair.c -+++ b/src/c/hbm_online_repair/hbm_online_repair.c -@@ -127,10 +127,7 @@ int main(int argc, char *argv[]) - return -1; - } - -- ret = init_all_flash(); -- if (ret < 0) { -- log(LOG_ERROR, "flash writer init failed\n"); -- } -+ get_flash_total_size(); - - handle_ras_events(ras); - -diff --git a/src/c/hbm_online_repair/non-standard-hbm-repair.c b/src/c/hbm_online_repair/non-standard-hbm-repair.c -index b175e14..f26d8ae 100644 ---- a/src/c/hbm_online_repair/non-standard-hbm-repair.c -+++ b/src/c/hbm_online_repair/non-standard-hbm-repair.c -@@ -15,7 +15,7 @@ - #include "non-standard-hbm-repair.h" - - extern int page_isolation_threshold; --size_t total_size = 0; -+size_t flash_total_size = 0; - struct hisi_common_error_section { - uint32_t val_bits; - uint8_t version; -@@ -122,28 +122,58 @@ static void parse_fault_addr_info(struct fault_addr_info* info_struct, unsigned - info_struct->crc8 = (uint32_t)fault_addr; - } - --static bool variable_existed(char *name, char *guid) -+static bool is_variable_existing(char *name, char *guid) - { -+ char filename[PATH_MAX]; -+ snprintf(filename, PATH_MAX - 1, "%s/%s-%s", EFIVARFS_PATH, name, guid); -+ -+ return access(filename, F_OK | R_OK) == 0; -+} -+ -+static size_t get_var_size(char *name, char *guid) { - char filename[PATH_MAX]; - int fd; -+ struct stat stat; - - snprintf(filename, PATH_MAX - 1, "%s/%s-%s", EFIVARFS_PATH, name, guid); - - // open var file - fd = open(filename, O_RDONLY); - if (fd < 0) { -- log(LOG_WARNING, "open file %s failed\n", filename); -- return false; -+ log(LOG_WARNING, "open %s failed\n", filename); -+ goto err; -+ } -+ // read stat -+ if (fstat(fd, &stat) != 0) { -+ log(LOG_WARNING, "fstat %s failed\n", filename); -+ goto err; - } - close(fd); -- return true; -+ return stat.st_size; -+err: -+ if (fd >= 0) -+ close(fd); -+ return (size_t)-1; - } - --static uint32_t read_variable_attribute(char *name, char *guid) { -+void get_flash_total_size() { -+ for (int i = 0; i < FLASH_ENTRY_NUM; i++) { -+ if (is_variable_existing(flash_names[i], flash_guids[i])) { -+ flash_total_size += get_var_size(flash_names[i], flash_guids[i]); -+ } -+ } -+ // check total entry size -+ log(LOG_DEBUG, "current fault info total size: %luKB, flash max threshold: %uKB\n", -+ flash_total_size / KB_SIZE, MAX_VAR_SIZE / KB_SIZE); -+ if (flash_total_size > MAX_VAR_SIZE) { -+ log(LOG_WARNING, "fault info storage %zu reach threshold, cannot save new record\n", flash_total_size); -+ } -+} -+ -+static int read_variable_attribute(char *name, char *guid, uint32_t *attribute) { - char filename[PATH_MAX]; - int fd; - size_t readsize; -- uint32_t attribute = (uint32_t)-1; - - snprintf(filename, PATH_MAX - 1, "%s/%s-%s", EFIVARFS_PATH, name, guid); - -@@ -151,17 +181,18 @@ static uint32_t read_variable_attribute(char *name, char *guid) { - fd = open(filename, O_RDONLY); - if (fd < 0) { - log(LOG_ERROR, "open %s failed\n", filename); -- return attribute; -+ return -1; - } - - // read attributes from first 4 bytes -- readsize = read(fd, &attribute, sizeof(uint32_t)); -+ readsize = read(fd, attribute, sizeof(uint32_t)); - if (readsize != sizeof(uint32_t)) { - log(LOG_ERROR, "read attribute of %s failed\n", filename); -+ return -1; - } - - close(fd); -- return attribute; -+ return 0; - } - - static int efivarfs_set_mutable(char *name, char *guid, bool mutable) -@@ -205,8 +236,8 @@ err: - return -1; - } - --static int write_variable(char *name, char *guid, void *value, unsigned long size, uint32_t attribute) { -- int fd, mode; -+static int write_variable(char *name, char *guid, void *value, unsigned long size, uint32_t attribute, bool is_existing) { -+ int fd = -1, mode; - size_t writesize; - void *buffer; - unsigned long total; -@@ -225,16 +256,13 @@ static int write_variable(char *name, char *guid, void *value, unsigned long siz - memcpy(buffer + sizeof(uint32_t), value, size); - - // change attr -- if (efivarfs_set_mutable(name, guid, 1) != 0) { -+ if (is_existing && efivarfs_set_mutable(name, guid, 1) != 0) { - log(LOG_ERROR, "set mutable for %s failed\n", filename); - goto err; - } - - mode = O_WRONLY; -- if (attribute & EFI_VARIABLE_APPEND_WRITE) -- mode |= O_APPEND; -- else -- mode |= O_CREAT; -+ mode |= is_existing ? O_APPEND : O_CREAT; - - // open var file - fd = open(filename, mode, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); -@@ -252,7 +280,7 @@ static int write_variable(char *name, char *guid, void *value, unsigned long siz - - close(fd); - free(buffer); -- if (efivarfs_set_mutable(name, guid, 0) != 0) { -+ if (is_existing && efivarfs_set_mutable(name, guid, 0) != 0) { - log(LOG_ERROR, "set immutable for %s failed\n", filename); - } - return 0; -@@ -261,86 +289,21 @@ err: - close(fd); - if (buffer) - free(buffer); -- if (efivarfs_set_mutable(name, guid, 0) != 0) { -+ if (is_existing && efivarfs_set_mutable(name, guid, 0) != 0) { - log(LOG_ERROR, "set immutable for %s failed\n", filename); - } - return -1; - } - --static int append_variable(char *name, char *guid, void *data, unsigned long size) { -- // prepare append attribute -- uint32_t attribute = read_variable_attribute(name, guid); -- if (attribute == (uint32_t)-1) { -- log(LOG_ERROR, "read %s-%s attribute failed\n", name, guid); -- return -1; -- } -- attribute |= EFI_VARIABLE_APPEND_WRITE; -- -- return write_variable(name, guid, data, size, attribute); --} -- --static size_t get_var_size(char *name, char *guid) { -- char filename[PATH_MAX]; -- int fd; -- struct stat stat; -- -- snprintf(filename, PATH_MAX - 1, "%s/%s-%s", EFIVARFS_PATH, name, guid); -- -- // open var file -- fd = open(filename, O_RDONLY); -- if (fd < 0) { -- log(LOG_WARNING, "open %s failed\n", filename); -- goto err; -- } -- // read stat -- if (fstat(fd, &stat) != 0) { -- log(LOG_WARNING, "fstat %s failed\n", filename); -- goto err; -- } -- close(fd); -- return stat.st_size; --err: -- if (fd >= 0) -- close(fd); -- return (size_t)-1; --} -- --int init_all_flash() { -- for (int i = 0; i < FLASH_ENTRY_NUM; i++) { -- // check existed entry -- if (variable_existed(flash_names[i], flash_guids[i])) { -- total_size += get_var_size(flash_names[i], flash_guids[i]); -- continue; -- } -- // create new entry -- uint32_t attribute = EFI_VARIABLE_NON_VOLATILE | -- EFI_VARIABLE_BOOTSERVICE_ACCESS | -- EFI_VARIABLE_RUNTIME_ACCESS; -- char *data = ""; -- unsigned long size = 1; -- int ret = write_variable(flash_names[i], flash_guids[i], data, size, attribute); -- if (ret) { -- log(LOG_ERROR, "init %s-%s failed, fault info storage funtion not enabled\n", flash_names[i], flash_guids[i]); -- return -1; -- } -- total_size += sizeof(uint32_t) + 1; -- } -- // check total entry size -- log(LOG_DEBUG, "current fault info total size: %luKB, flash max threshold: %uKB\n", -- total_size / KB_SIZE, MAX_VAR_SIZE / KB_SIZE); -- if (total_size > MAX_VAR_SIZE) { -- log(LOG_ERROR, "fault info storage reach threshold, cannot save new record\n"); -- } -- return 0; --} -- - static int write_fault_info_to_flash(const struct hisi_common_error_section *err) { - int ret, guid_index; - uint32_t reg_size; - uint64_t fault_addr; -+ bool is_existing; -+ uint32_t attribute = -1; - - // check flash usage threshold -- if (total_size + sizeof(uint64_t) > MAX_VAR_SIZE) { -+ if (flash_total_size + sizeof(uint64_t) > MAX_VAR_SIZE) { - log(LOG_WARNING, "fault info storage reach threshold, cannot save new record into flash\n"); - return -1; - } -@@ -359,14 +322,29 @@ static int write_fault_info_to_flash(const struct hisi_common_error_section *err - log(LOG_ERROR, "invalid fault info\n"); - return -1; - } -+ -+ // judge if the efivar is existing to set the attribute -+ is_existing = is_variable_existing(flash_names[guid_index], flash_guids[guid_index]); -+ attribute = EFI_VARIABLE_NON_VOLATILE | -+ EFI_VARIABLE_BOOTSERVICE_ACCESS | -+ EFI_VARIABLE_RUNTIME_ACCESS; -+ if (is_existing) { -+ ret = read_variable_attribute(flash_names[guid_index], flash_guids[guid_index], &attribute); -+ if (ret < 0) { -+ log(LOG_ERROR, "read variable %s-%s attribute failed, stop writing\n", flash_names[guid_index], flash_guids[guid_index]); -+ return -1; -+ } -+ attribute |= EFI_VARIABLE_APPEND_WRITE; -+ } -+ - // record physical addr in flash -- ret = append_variable(flash_names[guid_index], flash_guids[guid_index], &fault_addr, sizeof(uint64_t)); -+ ret = write_variable(flash_names[guid_index], flash_guids[guid_index], &fault_addr, sizeof(uint64_t), attribute, is_existing); - if (ret < 0) { -- log(LOG_ERROR, "append to %s-%s failed\n", flash_names[guid_index], flash_guids[guid_index]); -+ log(LOG_ERROR, "write to %s-%s failed\n", flash_names[guid_index], flash_guids[guid_index]); - return -1; - } -- total_size += sizeof(uint64_t); -- log(LOG_INFO, "write hbm fault info to flash success\n"); -+ flash_total_size += sizeof(uint64_t); -+ log(LOG_INFO, "write hbm fault info to flash %s-%s success\n", flash_names[guid_index], flash_guids[guid_index]); - return 0; - } - -@@ -421,7 +399,7 @@ static int get_hardware_corrupted_size() - return hardware_corrupted_size; - } - --static uint8_t get_repair_result_code(int ret) -+static uint8_t get_repair_failed_result_code(int ret) - { - if (ret == -ENOSPC) { - return REPAIR_FAILED_NO_RESOURCE; -@@ -582,11 +560,11 @@ static int hbmc_hbm_page_isolate(const struct hisi_common_error_section *err) - static int hbmc_hbm_after_repair(bool is_acls, const int repair_ret, const unsigned long long paddr) - { - int ret; -- if (repair_ret < 0) { -+ if (repair_ret <= 0) { - log(LOG_WARNING, "HBM %s: Keep page (0x%llx) offline\n", is_acls ? "ACLS" : "SPPR", paddr); - /* not much we can do about errors here */ - (void)write_file("/sys/kernel/page_eject", "remove_page", paddr); -- return get_repair_result_code(repair_ret); -+ return get_repair_failed_result_code(repair_ret); - } - - ret = write_file("/sys/kernel/page_eject", "online_page", paddr); -@@ -615,9 +593,13 @@ static uint8_t hbmc_hbm_repair(const struct hisi_common_error_section *err, char - err->reg_array[HBM_REPAIR_REQ_TYPE] & HBM_PSUE_ACLS; - - ret = write_file(path, is_acls ? "acls_query" : "sppr_query", paddr); -- if (ret < 0) { -- notice_BMC(err, get_repair_result_code(ret)); -- log(LOG_WARNING, "HBM: Address 0x%llx is not supported to %s repair\n", paddr, is_acls ? "ACLS" : "SPPR"); -+ -+ /* Only positive num means the error is supported to repair */ -+ if (ret <= 0) { -+ if (ret != -ENXIO) { -+ notice_BMC(err, get_repair_failed_result_code(ret)); -+ log(LOG_WARNING, "HBM: Address 0x%llx is not supported to %s repair\n", paddr, is_acls ? "ACLS" : "SPPR"); -+ } - return ret; - } - -@@ -642,8 +624,9 @@ static uint8_t hbmc_hbm_repair(const struct hisi_common_error_section *err, char - all_online_success = false; - } - } -- if (ret < 0) { -- notice_BMC(err, get_repair_result_code(ret)); -+ /* The ret is from the acls/sppr repair, and only positive num means the error is repaired successfully */ -+ if (ret <= 0) { -+ notice_BMC(err, get_repair_failed_result_code(ret)); - return ret; - } else if (all_online_success) { - notice_BMC(err, ISOLATE_REPAIR_ONLINE_SUCCESS); -@@ -698,7 +681,7 @@ static void hbm_repair_handler(const struct hisi_common_error_section *err) - struct dirent *dent; - DIR *dir; - int ret; -- bool find_device = false, find_hbm_mem = false; -+ bool find_device = false, find_hbm_mem = false, addr_in_hbm_device = false; - - ret = hbmc_hbm_page_isolate(err); - if (ret < 0) { -@@ -723,10 +706,13 @@ static void hbm_repair_handler(const struct hisi_common_error_section *err) - if (hbmc_get_memory_type(path) == HBM_HBM_MEMORY) { - find_hbm_mem = true; - ret = hbmc_hbm_repair(err, path); -- if (ret != -ENXIO) -+ if (ret != -ENXIO) { -+ addr_in_hbm_device = true; - break; -+ } - } - } -+ - if (!find_device) { - log(LOG_ERROR, "Repair driver is not loaded, skip error, error_type is %u\n", - err->reg_array[HBM_REPAIR_REQ_TYPE] & HBM_ERROR_MASK); -@@ -735,6 +721,10 @@ static void hbm_repair_handler(const struct hisi_common_error_section *err) - log(LOG_ERROR, "No HBM device memory type found, skip error, error_type is %u\n", - err->reg_array[HBM_REPAIR_REQ_TYPE] & HBM_ERROR_MASK); - notice_BMC(err, REPAIR_FAILED_OTHER_REASON); -+ } else if (!addr_in_hbm_device) { -+ log(LOG_ERROR, "Err addr is not in device, skip error, error_type is %u\n", -+ err->reg_array[HBM_REPAIR_REQ_TYPE] & HBM_ERROR_MASK); -+ notice_BMC(err, REPAIR_FAILED_INVALID_PARAM); - } - - closedir(dir); -@@ -769,7 +759,7 @@ static bool hbm_repair_validate(const struct hisi_common_error_section *err) - (err->reg_array_size == HBM_CACHE_ARRAY_SIZE); - - if (!(is_acls_valid || is_sppr_valid || is_cache_mode)) { -- log(LOG_DEBUG, "err type (%u) is unknown or address array length (%u) is invalid\n", -+ log(LOG_WARNING, "err type (%u) is unknown or address array length (%u) is invalid\n", - hbm_repair_reg_type, err->reg_array_size); - return false; - } -diff --git a/src/c/hbm_online_repair/non-standard-hbm-repair.h b/src/c/hbm_online_repair/non-standard-hbm-repair.h -index 7e8e448..ecb04fe 100644 ---- a/src/c/hbm_online_repair/non-standard-hbm-repair.h -+++ b/src/c/hbm_online_repair/non-standard-hbm-repair.h -@@ -84,6 +84,6 @@ - #define FLASH_ENTRY_NUM 8 - #define KB_SIZE 1024 - --extern int init_all_flash(); -+extern void get_flash_total_size(); - - #endif -diff --git a/src/c/hbm_online_repair/ras-events.c b/src/c/hbm_online_repair/ras-events.c -index 0b12329..4d281ad 100644 ---- a/src/c/hbm_online_repair/ras-events.c -+++ b/src/c/hbm_online_repair/ras-events.c -@@ -348,7 +348,6 @@ static int read_ras_event_all_cpus(struct pcpu_data *pdata, - "Error on CPU %i\n", i); - warnonce[i]++; - } -- continue; - } - if (!(fds[i].revents & POLLIN)) { - count_nready++; -diff --git a/src/c/hbm_online_repair/ras-non-standard-handler.c b/src/c/hbm_online_repair/ras-non-standard-handler.c -index 1d1fd04..48ffa70 100644 ---- a/src/c/hbm_online_repair/ras-non-standard-handler.c -+++ b/src/c/hbm_online_repair/ras-non-standard-handler.c -@@ -7,17 +7,21 @@ - #include "ras-non-standard-handler.h" - #include "logger.h" - --static char *uuid_le(const char *uu) -+static int uuid_le(const char *uu, char* uuid) - { -- static char uuid[sizeof("xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx")]; - if (!uu) { - log(LOG_ERROR, "uuid_le failed: uu is empty"); -- return uuid; -+ return -1; - } - size_t uu_len = strlen(uu); -- if (uu_len < SECTION_TYPE_UUID_LEN) { -- log(LOG_ERROR, "uuid_le failed: uu is too short"); -- return uuid; -+ if (uu_len != SECTION_TYPE_UUID_LEN) { -+ log(LOG_ERROR, "uuid_le failed: uu len is incorrect"); -+ return -1; -+ } -+ size_t uuid_len = strlen(uuid); -+ if (uuid_len != strlen(UUID_STR_TYPE)) { -+ log(LOG_ERROR, "uuid_le failed: uuid len is incorrect"); -+ return -1; - } - - char *p = uuid; -@@ -38,7 +42,7 @@ static char *uuid_le(const char *uu) - - *p = 0; - -- return uuid; -+ return 0; - } - - int ras_non_standard_event_handler(struct trace_seq *s, -@@ -52,15 +56,20 @@ int ras_non_standard_event_handler(struct trace_seq *s, - ev.sec_type = tep_get_field_raw(s, event, "sec_type", - record, &len, 1); - if(!ev.sec_type) { -- log(LOG_WARNING, "get event section type failed"); -+ log(LOG_WARNING, "get event section type failed\n"); - return -1; - } - - trace_seq_printf(s, "\n"); -- trace_seq_printf(s, "sec_type: %s\n", uuid_le(ev.sec_type)); -+ char uuid[sizeof(UUID_STR_TYPE)] = UUID_STR_TYPE; -+ if (uuid_le(ev.sec_type, uuid) < 0) { -+ log(LOG_WARNING, "get uuid failed\n"); -+ return -1; -+ } -+ trace_seq_printf(s, "sec_type: %s\n", uuid); - - if (tep_get_field_val(s, event, "len", record, &val, 1) < 0) { -- log(LOG_WARNING, "tep get field val failed"); -+ log(LOG_WARNING, "tep get field val failed\n"); - return -1; - } - -@@ -69,11 +78,11 @@ int ras_non_standard_event_handler(struct trace_seq *s, - - ev.error = tep_get_field_raw(s, event, "buf", record, &len, 1); - if(!ev.error || ev.length != len) { -- log(LOG_WARNING, "get event error failed"); -+ log(LOG_WARNING, "get event error failed\n"); - return -1; - } - -- if (strcmp(uuid_le(ev.sec_type), HISI_COMMON_SECTION_TYPE_UUID) == 0) { -+ if (strcmp(uuid, HISI_COMMON_SECTION_TYPE_UUID) == 0) { - decode_hisi_common_section(&ev); - } - -diff --git a/src/c/hbm_online_repair/ras-non-standard-handler.h b/src/c/hbm_online_repair/ras-non-standard-handler.h -index 0272dc1..15a37ee 100644 ---- a/src/c/hbm_online_repair/ras-non-standard-handler.h -+++ b/src/c/hbm_online_repair/ras-non-standard-handler.h -@@ -7,6 +7,7 @@ - #define BIT(nr) (1UL << (nr)) - - #define SECTION_TYPE_UUID_LEN 16 -+#define UUID_STR_TYPE "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" - #define HISI_COMMON_SECTION_TYPE_UUID "c8b328a8-9917-4af6-9a13-2e08ab2e7586" - - struct ras_non_standard_event { --- -2.43.0 - diff --git a/fix-io_dump-for-collect-module.patch b/fix-io_dump-for-collect-module.patch deleted file mode 100644 index 452ba0a..0000000 --- a/fix-io_dump-for-collect-module.patch +++ /dev/null @@ -1,25 +0,0 @@ -From 6307a1ff4068a541658e3312ca938c6fdd9a5c1a Mon Sep 17 00:00:00 2001 -From: zhuofeng -Date: Sat, 12 Oct 2024 14:51:51 +0800 -Subject: [PATCH] fix io_dump for collect module - ---- - src/python/sentryCollector/collect_io.py | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/src/python/sentryCollector/collect_io.py b/src/python/sentryCollector/collect_io.py -index d734734..11c9d9a 100644 ---- a/src/python/sentryCollector/collect_io.py -+++ b/src/python/sentryCollector/collect_io.py -@@ -154,7 +154,7 @@ class CollectIo(): - try: - with open(io_dump_file, 'r') as file: - for line in file: -- count += line.count('.op=' + Io_Category[category]) -+ count += line.count('.op=' + Io_Category[category].upper()) - if count > 0: - logging.info(f"io_dump info : {disk_name}, {stage}, {category}, {count}") - except FileNotFoundError: --- -2.33.0 - diff --git a/fix-python-3.7-not-support-list-bool-type.patch b/fix-python-3.7-not-support-list-bool-type.patch deleted file mode 100644 index 6214cda..0000000 --- a/fix-python-3.7-not-support-list-bool-type.patch +++ /dev/null @@ -1,53 +0,0 @@ -From 878bcf61467bfd9d015a8089a8367f4333ba76f6 Mon Sep 17 00:00:00 2001 -From: PshySimon -Date: Wed, 9 Oct 2024 10:20:34 +0800 -Subject: [PATCH] fix python 3.7 not support list[bool] type - ---- - src/python/xalarm/register_xalarm.py | 8 ++++---- - 1 file changed, 4 insertions(+), 4 deletions(-) - -diff --git a/src/python/xalarm/register_xalarm.py b/src/python/xalarm/register_xalarm.py -index e58343d..6756b1b 100644 ---- a/src/python/xalarm/register_xalarm.py -+++ b/src/python/xalarm/register_xalarm.py -@@ -26,7 +26,7 @@ ALARM_REGISTER_INFO = None - - - class AlarmRegister: -- def __init__(self, id_filter: list[bool], callback: callable): -+ def __init__(self, id_filter: list, callback: callable): - self.id_filter = id_filter - self.callback = callback - self.socket = self.create_unix_socket() -@@ -49,7 +49,7 @@ class AlarmRegister: - return False - return True - -- def set_id_filter(self, id_filter: list[bool]) -> bool: -+ def set_id_filter(self, id_filter: list) -> bool: - if (len(id_filter) > MAX_NUM_OF_ALARM_ID): - sys.stderr.write("set_id_filter: invalid param id_filter\n") - return False -@@ -118,7 +118,7 @@ class AlarmRegister: - self.socket.close() - - --def xalarm_register(callback: callable, id_filter: list[bool]) -> int: -+def xalarm_register(callback: callable, id_filter: list) -> int: - global ALARM_REGISTER_INFO - - if ALARM_REGISTER_INFO is not None: -@@ -148,7 +148,7 @@ def xalarm_unregister(clientId: int) -> None: - ALARM_REGISTER_INFO = None - - --def xalarm_upgrade(clientId: int, id_filter: list[bool]) -> None: -+def xalarm_upgrade(clientId: int, id_filter: list) -> None: - global ALARM_REGISTER_INFO - if clientId < 0: - sys.stderr.write("xalarm_unregister: invalid client\n") --- -2.27.0 - - diff --git a/fix-result-when-process-output-is-None.patch b/fix-result-when-process-output-is-None.patch deleted file mode 100644 index 9d22700..0000000 --- a/fix-result-when-process-output-is-None.patch +++ /dev/null @@ -1,36 +0,0 @@ -From e8e4fa5fd9e78508567782e17b7b1cb6ace3ef0d Mon Sep 17 00:00:00 2001 -From: shixuantong -Date: Fri, 26 Jul 2024 15:59:42 +0800 -Subject: [PATCH] fix result when process output is None - ---- - src/python/syssentry/cpu_sentry.py | 8 ++++++++ - 1 file changed, 8 insertions(+) - -diff --git a/src/python/syssentry/cpu_sentry.py b/src/python/syssentry/cpu_sentry.py -index d0bafa8..9287e2f 100644 ---- a/src/python/syssentry/cpu_sentry.py -+++ b/src/python/syssentry/cpu_sentry.py -@@ -87,11 +87,19 @@ class CpuSentry: - } - - def handle_cpu_output(self, stdout: str): -+ if not stdout: -+ logging.error("%s process output is None, it may be killed!", LOW_LEVEL_INSPECT_CMD) -+ self.send_result["result"] = ResultLevel.FAIL -+ self.send_result["details"]["code"] = 1005 -+ self.send_result["details"]["msg"] = "cpu_sentry task is killed!" -+ return -+ - if "ERROR" in stdout: - self.send_result["result"] = ResultLevel.FAIL - self.send_result["details"]["code"] = 1004 - self.send_result["details"]["msg"] = stdout.split("\n")[0] - return -+ - out_split = stdout.split("\n") - isolated_cores_number = 0 - found_fault_cores_list = [] --- -2.27.0 - diff --git a/fix-some-about-collect-module-and-avg-block-io.patch b/fix-some-about-collect-module-and-avg-block-io.patch deleted file mode 100644 index a32e5c7..0000000 --- a/fix-some-about-collect-module-and-avg-block-io.patch +++ /dev/null @@ -1,226 +0,0 @@ -From dea58a559f3dbad3dbce3b681639ee89c20b1cee Mon Sep 17 00:00:00 2001 -From: zhuofeng -Date: Fri, 20 Sep 2024 14:35:39 +0800 -Subject: [PATCH] fix some about collect module and avg block io - ---- - config/tasks/avg_block_io.mod | 4 ++-- - src/python/sentryCollector/collect_io.py | 18 +++++++++++------- - src/python/sentryCollector/collect_plugin.py | 17 ++++++++--------- - src/python/sentryCollector/collect_server.py | 6 +++--- - src/python/sentryCollector/collectd.py | 2 -- - .../sentryPlugins/avg_block_io/avg_block_io.py | 13 ++++++++++--- - 6 files changed, 34 insertions(+), 26 deletions(-) - -diff --git a/config/tasks/avg_block_io.mod b/config/tasks/avg_block_io.mod -index 814c483..b9b6f34 100644 ---- a/config/tasks/avg_block_io.mod -+++ b/config/tasks/avg_block_io.mod -@@ -1,5 +1,5 @@ - [common] - enabled=yes - task_start=/usr/bin/python3 /usr/bin/avg_block_io --task_stop=pkill avg_block_io --type=oneshot -\ No newline at end of file -+task_stop=pkill -f /usr/bin/avg_block_io -+type=oneshot -diff --git a/src/python/sentryCollector/collect_io.py b/src/python/sentryCollector/collect_io.py -index b826dc4..104b734 100644 ---- a/src/python/sentryCollector/collect_io.py -+++ b/src/python/sentryCollector/collect_io.py -@@ -175,8 +175,7 @@ class CollectIo(): - - threading.Timer(self.period_time, self.task_loop).start() - -- def main_loop(self): -- logging.info("collect io thread start") -+ def is_kernel_avaliable(self): - base_path = '/sys/kernel/debug/block' - for disk_name in os.listdir(base_path): - if not self.loop_all and disk_name not in self.disk_list: -@@ -198,8 +197,13 @@ class CollectIo(): - self.window_value[disk_name] = {} - IO_GLOBAL_DATA[disk_name] = {} - -- if len(self.disk_map_stage) == 0: -- logging.warning("no disks meet the requirements. the thread exits") -+ return len(IO_GLOBAL_DATA) != 0 -+ -+ def main_loop(self): -+ logging.info("collect io thread start") -+ -+ if not self.is_kernel_avaliable() or len(self.disk_map_stage) == 0: -+ logging.warning("no disks meet the requirements. collect io thread exits") - return - - for disk_name, stage_list in self.disk_map_stage.items(): -@@ -213,7 +217,7 @@ class CollectIo(): - start_time = time.time() - - if self.stop_event.is_set(): -- logging.info("collect io thread exit") -+ logging.debug("collect io thread exit") - return - - for disk_name, stage_list in self.disk_map_stage.items(): -@@ -227,7 +231,7 @@ class CollectIo(): - continue - while sleep_time > 1: - if self.stop_event.is_set(): -- logging.info("collect io thread exit") -+ logging.debug("collect io thread exit") - return - time.sleep(1) - sleep_time -= 1 -@@ -235,5 +239,5 @@ class CollectIo(): - - # set stop event, notify thread exit - def stop_thread(self): -- logging.info("collect io thread is preparing to exit") -+ logging.debug("collect io thread is preparing to exit") - self.stop_event.set() -diff --git a/src/python/sentryCollector/collect_plugin.py b/src/python/sentryCollector/collect_plugin.py -index 49ce0a8..9132473 100644 ---- a/src/python/sentryCollector/collect_plugin.py -+++ b/src/python/sentryCollector/collect_plugin.py -@@ -142,22 +142,21 @@ def validate_parameters(param, len_limit, char_limit): - ret = ResultMessage.RESULT_INVALID_LENGTH - return [False, ret] - -- if len(param) > len_limit: -- print(f"{param} length more than {len_limit}") -- ret = ResultMessage.RESULT_EXCEED_LIMIT -- return [False, ret] -- - pattern = r'^[a-zA-Z0-9_-]+$' - for info in param: -- if len(info) > char_limit: -- print(f"{info} length more than {char_limit}") -- ret = ResultMessage.RESULT_EXCEED_LIMIT -- return [False, ret] - if not re.match(pattern, info): - print(f"{info} is invalid char") - ret = ResultMessage.RESULT_INVALID_CHAR - return [False, ret] - -+ # length of len_limit is exceeded, keep len_limit -+ if len(param) > len_limit: -+ print(f"{param} length more than {len_limit}, keep the first {len_limit}") -+ param[:] = param[0:len_limit] -+ -+ # only keep elements under the char_limit length -+ param[:] = [elem for elem in param if len(elem) <= char_limit] -+ - return [True, ret] - - def is_iocollect_valid(period, disk_list=None, stage=None): -diff --git a/src/python/sentryCollector/collect_server.py b/src/python/sentryCollector/collect_server.py -index fa49781..bab4e56 100644 ---- a/src/python/sentryCollector/collect_server.py -+++ b/src/python/sentryCollector/collect_server.py -@@ -256,7 +256,7 @@ class CollectServer(): - - def server_loop(self): - """main loop""" -- logging.info("collect server thread start") -+ logging.info("collect listen thread start") - server_fd = self.server_fd_create() - if not server_fd: - return -@@ -267,7 +267,7 @@ class CollectServer(): - logging.debug("start server_loop loop") - while True: - if self.stop_event.is_set(): -- logging.info("collect server thread exit") -+ logging.debug("collect listen thread exit") - server_fd = None - return - try: -@@ -281,5 +281,5 @@ class CollectServer(): - pass - - def stop_thread(self): -- logging.info("collect server thread is preparing to exit") -+ logging.debug("collect listen thread is preparing to exit") - self.stop_event.set() -diff --git a/src/python/sentryCollector/collectd.py b/src/python/sentryCollector/collectd.py -index b77c642..3a836df 100644 ---- a/src/python/sentryCollector/collectd.py -+++ b/src/python/sentryCollector/collectd.py -@@ -49,7 +49,6 @@ def sig_handler(signum, _f): - Thread_List[i][0].stop_thread() - - remove_sock_file() -- sys.exit(0) - - def main(): - """main -@@ -64,7 +63,6 @@ def main(): - try: - signal.signal(signal.SIGINT, sig_handler) - signal.signal(signal.SIGTERM, sig_handler) -- signal.signal(signal.SIGHUP, sig_handler) - - logging.info("finish main parse_args") - -diff --git a/src/python/sentryPlugins/avg_block_io/avg_block_io.py b/src/python/sentryPlugins/avg_block_io/avg_block_io.py -index ff2071d..73f0b22 100644 ---- a/src/python/sentryPlugins/avg_block_io/avg_block_io.py -+++ b/src/python/sentryPlugins/avg_block_io/avg_block_io.py -@@ -21,7 +21,7 @@ CONFIG_FILE = "/etc/sysSentry/plugins/avg_block_io.ini" - - def log_invalid_keys(not_in_list, keys_name, config_list, default_list): - """print invalid log""" -- if config_list and default_list: -+ if config_list and not_in_list: - logging.warning("{} in common.{} are not valid, set {}={}".format(not_in_list, keys_name, keys_name, default_list)) - elif config_list == ["default"]: - logging.warning("Default {} use {}".format(keys_name, default_list)) -@@ -144,9 +144,11 @@ def init_io_win(io_dic, config, common_param): - - if avg_lim_value and avg_time_value and tot_lim_value: - io_data[disk_name][stage_name][rw]["latency"] = IoWindow(window_size=io_dic["win_size"], window_threshold=io_dic["win_threshold"], abnormal_multiple=avg_time_value, abnormal_multiple_lim=avg_lim_value, abnormal_time=tot_lim_value) -+ logging.debug("Successfully create {}-{}-{} latency window".format(disk_name, stage_name, rw)) - - if iodump_lim_value is not None: - io_data[disk_name][stage_name][rw]["iodump"] = IoDumpWindow(window_size=io_dic["win_size"], window_threshold=io_dic["win_threshold"], abnormal_time=iodump_lim_value) -+ logging.debug("Successfully create {}-{}-{} iodump window".format(disk_name, stage_name, rw)) - return io_data, io_avg_value - - -@@ -159,10 +161,10 @@ def get_valid_disk_stage_list(io_dic, config_disk, config_stage): - for disk_stage_list in json_data.values(): - all_stage_set.update(disk_stage_list) - -- disk_list = [key for key in config_disk if key in all_disk_set] -+ disk_list = [key for key in all_disk_set if key in config_disk] - not_in_disk_list = [key for key in config_disk if key not in all_disk_set] - -- stage_list = [key for key in config_stage if key in all_stage_set] -+ stage_list = [key for key in all_stage_set if key in config_stage] - not_in_stage_list = [key for key in config_stage if key not in all_stage_set] - - if not config_disk: -@@ -171,6 +173,9 @@ def get_valid_disk_stage_list(io_dic, config_disk, config_stage): - if not config_stage: - stage_list = [key for key in all_stage_set] - -+ disk_list = disk_list[:10] if len(disk_list) > 10 else disk_list -+ stage_list = stage_list[:15] if len(stage_list) > 15 else stage_list -+ - if config_disk and not disk_list: - logging.warning("Cannot get valid disk by disk={}, set to default".format(config_disk)) - disk_list, stage_list = get_valid_disk_stage_list(io_dic, [], config_stage) -@@ -228,6 +233,8 @@ def main(): - signal.signal(signal.SIGINT, sig_handler) - signal.signal(signal.SIGTERM, sig_handler) - -+ logging.basicConfig(level=logging.INFO) -+ - # 初始化配置读取 - config = configparser.ConfigParser(comment_prefixes=('#', ';')) - try: --- -2.33.0 - diff --git a/fix-syssentry-fails-to-be-started-when-cpu_sentry-is.patch b/fix-syssentry-fails-to-be-started-when-cpu_sentry-is.patch deleted file mode 100644 index 675277f..0000000 --- a/fix-syssentry-fails-to-be-started-when-cpu_sentry-is.patch +++ /dev/null @@ -1,56 +0,0 @@ -From 497b3124f017ce4ae99b34261c4fd5dd2a358f5b Mon Sep 17 00:00:00 2001 -From: zhuofeng -Date: Sat, 14 Sep 2024 09:28:00 +0800 -Subject: [PATCH] fix syssentry fails to be started when cpu_sentry is not - installed - ---- - src/python/syssentry/syssentry.py | 11 ++++++----- - 1 file changed, 6 insertions(+), 5 deletions(-) - -diff --git a/src/python/syssentry/syssentry.py b/src/python/syssentry/syssentry.py -index f93956e..776971f 100644 ---- a/src/python/syssentry/syssentry.py -+++ b/src/python/syssentry/syssentry.py -@@ -43,7 +43,6 @@ try: - from .cpu_alarm import cpu_alarm_recv - except ImportError: - CPU_EXIST = False -- logging.debug("Cannot find cpu sentry mod") - - - INSPECTOR = None -@@ -563,20 +562,21 @@ def main(): - if not os.path.exists(SENTRY_RUN_DIR): - os.mkdir(SENTRY_RUN_DIR) - os.chmod(SENTRY_RUN_DIR, mode=SENTRY_RUN_DIR_PERM) -- if not chk_and_set_pidfile(): -- logging.error("get pid file lock failed, exist") -- sys.exit(17) - - logging.basicConfig(filename=SYSSENTRY_LOG_FILE, level=logging.INFO) - os.chmod(SYSSENTRY_LOG_FILE, 0o600) - -+ if not chk_and_set_pidfile(): -+ logging.error("get pid file lock failed, exist") -+ sys.exit(17) -+ - try: - signal.signal(signal.SIGINT, sig_handler) - signal.signal(signal.SIGTERM, sig_handler) - signal.signal(signal.SIGHUP, sig_handler) - signal.signal(signal.SIGCHLD, sigchld_handler) - -- logging.debug("finish main parse_args") -+ logging.info("finish main parse_args") - - _ = SentryConfig.init_param() - TasksMap.init_task_map() -@@ -587,3 +587,4 @@ def main(): - logging.error('%s', traceback.format_exc()) - finally: - release_pidfile() -+ --- -2.33.0 - diff --git a/fix-test_ai_block_io-fail.patch b/fix-test_ai_block_io-fail.patch deleted file mode 100644 index a83d535..0000000 --- a/fix-test_ai_block_io-fail.patch +++ /dev/null @@ -1,91 +0,0 @@ -From 874daf9627c74aa31f1063c250b5477b2eb322e8 Mon Sep 17 00:00:00 2001 -From: shixuantong -Date: Sat, 28 Dec 2024 11:31:23 +0800 -Subject: [PATCH] fix test_ai_block_io fail - ---- - selftest/test/test_ai_block_io.py | 26 +++++++++++++------------- - 1 file changed, 13 insertions(+), 13 deletions(-) - -diff --git a/selftest/test/test_ai_block_io.py b/selftest/test/test_ai_block_io.py -index c36fef5..58ab096 100644 ---- a/selftest/test/test_ai_block_io.py -+++ b/selftest/test/test_ai_block_io.py -@@ -12,9 +12,9 @@ - import unittest - import numpy as np - --from sentryPlugins.ai_threshold_slow_io_detection.threshold import AbsoluteThreshold, BoxplotThreshold, NSigmaThreshold --from sentryPlugins.ai_threshold_slow_io_detection.sliding_window import (NotContinuousSlidingWindow, -- ContinuousSlidingWindow, MedianSlidingWindow) -+from sentryPlugins.ai_block_io.threshold import AbsoluteThreshold, BoxplotThreshold, NSigmaThreshold -+from sentryPlugins.ai_block_io.sliding_window import (NotContinuousSlidingWindow, -+ ContinuousSlidingWindow, MedianSlidingWindow) - - - def _get_boxplot_threshold(data_list: list, parameter): -@@ -98,11 +98,11 @@ class Test(unittest.TestCase): - for data in data_list1: - boxplot_threshold.push_latest_data_to_queue(data) - result = not_continuous.is_slow_io_event(data) -- self.assertFalse(result[0]) -+ self.assertFalse(result[0][0]) - self.assertEqual(23.75, boxplot_threshold.get_threshold()) - boxplot_threshold.push_latest_data_to_queue(24) - result = not_continuous.is_slow_io_event(24) -- self.assertFalse(result[0]) -+ self.assertFalse(result[0][0]) - boxplot_threshold.push_latest_data_to_queue(25) - result = not_continuous.is_slow_io_event(25) - self.assertTrue(result[0]) -@@ -110,7 +110,7 @@ class Test(unittest.TestCase): - for data in data_list2: - boxplot_threshold.push_latest_data_to_queue(data) - result = not_continuous.is_slow_io_event(data) -- self.assertFalse(result[0]) -+ self.assertFalse(result[0][0]) - self.assertEqual(25.625, boxplot_threshold.get_threshold()) - - def test_continuous_sliding_window(self): -@@ -121,14 +121,14 @@ class Test(unittest.TestCase): - for data in data_list: - boxplot_threshold.push_latest_data_to_queue(data) - result = continuous.is_slow_io_event(data) -- self.assertFalse(result[0]) -+ self.assertFalse(result[0][0]) - self.assertEqual(23.75, boxplot_threshold.get_threshold()) - # 没有三个异常点 -- self.assertFalse(continuous.is_slow_io_event(25)[0]) -+ self.assertFalse(continuous.is_slow_io_event(25)[0][0]) - # 不连续的三个异常点 -- self.assertFalse(continuous.is_slow_io_event(25)[0]) -+ self.assertFalse(continuous.is_slow_io_event(25)[0][0]) - # 连续的三个异常点 -- self.assertTrue(continuous.is_slow_io_event(25)[0]) -+ self.assertTrue(continuous.is_slow_io_event(25)[0][0]) - - def test_median_sliding_window(self): - median = MedianSlidingWindow(5, 3) -@@ -137,7 +137,7 @@ class Test(unittest.TestCase): - absolute_threshold.set_threshold(24.5) - data_list = [24, 24, 24, 25, 25] - for data in data_list: -- self.assertFalse(median.is_slow_io_event(data)[0]) -+ self.assertFalse(median.is_slow_io_event(data)[0][0]) - self.assertTrue(median.is_slow_io_event(25)[0]) - - def test_parse_collect_data(self): -@@ -147,8 +147,8 @@ class Test(unittest.TestCase): - "flush": [9.0, 10.0, 11.0, 12.0], - "discard": [13.0, 14.0, 15.0, 16.0], - } -- from io_data import BaseData -- from data_access import _get_io_stage_data -+ from sentryPlugins.ai_block_io.io_data import BaseData -+ from sentryPlugins.ai_block_io.data_access import _get_io_stage_data - - io_data = _get_io_stage_data(collect) - self.assertEqual( --- -2.27.0 - diff --git a/fix-uint8-bug-and-change-isolation-default-value.patch b/fix-uint8-bug-and-change-isolation-default-value.patch deleted file mode 100644 index 959acf7..0000000 --- a/fix-uint8-bug-and-change-isolation-default-value.patch +++ /dev/null @@ -1,61 +0,0 @@ -From 00ea35472d50faea89c881eb45b6d9d11f6b6632 Mon Sep 17 00:00:00 2001 -From: luckky -Date: Fri, 1 Nov 2024 15:09:57 +0800 -Subject: [PATCH] fix uint8 bug and change isolation default value - ---- - src/c/hbm_online_repair/hbm_online_repair.env | 2 +- - src/c/hbm_online_repair/non-standard-hbm-repair.c | 8 ++++---- - 2 files changed, 5 insertions(+), 5 deletions(-) - -diff --git a/src/c/hbm_online_repair/hbm_online_repair.env b/src/c/hbm_online_repair/hbm_online_repair.env -index de56079..7166c8d 100644 ---- a/src/c/hbm_online_repair/hbm_online_repair.env -+++ b/src/c/hbm_online_repair/hbm_online_repair.env -@@ -1,2 +1,2 @@ - HBM_ONLINE_REPAIR_LOG_LEVEL=1 --PAGE_ISOLATION_THRESHOLD=128 -+PAGE_ISOLATION_THRESHOLD=3355443 -diff --git a/src/c/hbm_online_repair/non-standard-hbm-repair.c b/src/c/hbm_online_repair/non-standard-hbm-repair.c -index f26d8ae..b8dde7a 100644 ---- a/src/c/hbm_online_repair/non-standard-hbm-repair.c -+++ b/src/c/hbm_online_repair/non-standard-hbm-repair.c -@@ -359,7 +359,7 @@ static int write_file(char *path, const char *name, unsigned long long value) - - fd = open(fname, O_WRONLY); - if (fd < 0) { -- log(LOG_WARNING, "HBM ACLS: Cannot to open '%s': %s\n", -+ log(LOG_WARNING, "HBM: Cannot to open '%s': %s\n", - fname, strerror(errno)); - return -errno; - } -@@ -367,7 +367,7 @@ static int write_file(char *path, const char *name, unsigned long long value) - snprintf(buf, sizeof(buf), "0x%llx\n", value); - ret = write(fd, buf, strlen(buf)); - if (ret <= 0) -- log(LOG_WARNING, "HBM ACLS: Failed to set %s (0x%llx): %s\n", -+ log(LOG_WARNING, "HBM: Failed to set %s (0x%llx): %s\n", - fname, value, strerror(errno)); - - close(fd); -@@ -557,7 +557,7 @@ static int hbmc_hbm_page_isolate(const struct hisi_common_error_section *err) - return ret < 0 ? ret : 0; - } - --static int hbmc_hbm_after_repair(bool is_acls, const int repair_ret, const unsigned long long paddr) -+static uint8_t hbmc_hbm_after_repair(bool is_acls, const int repair_ret, const unsigned long long paddr) - { - int ret; - if (repair_ret <= 0) { -@@ -577,7 +577,7 @@ static int hbmc_hbm_after_repair(bool is_acls, const int repair_ret, const unsig - } - } - --static uint8_t hbmc_hbm_repair(const struct hisi_common_error_section *err, char *path) -+static int hbmc_hbm_repair(const struct hisi_common_error_section *err, char *path) - { - unsigned long long paddr; - int ret; --- -2.43.0 - diff --git a/fix-version-in-setup.py.patch b/fix-version-in-setup.py.patch deleted file mode 100644 index 42816db..0000000 --- a/fix-version-in-setup.py.patch +++ /dev/null @@ -1,25 +0,0 @@ -From 7baf2815f515c54bc33f41f495ec7c26988b5c44 Mon Sep 17 00:00:00 2001 -From: shixuantong -Date: Tue, 11 Jun 2024 16:47:46 +0800 -Subject: [PATCH] fix version in setup.py - ---- - src/python/setup.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/src/python/setup.py b/src/python/setup.py -index 21dbe9f..f96a96e 100644 ---- a/src/python/setup.py -+++ b/src/python/setup.py -@@ -17,7 +17,7 @@ from setuptools import setup, find_packages - - setup( - name="syssentry", -- version="1.0.1", -+ version="1.0.2", - description="System inspection framework tool set", - packages=find_packages(), - include_package_data=True, --- -2.27.0 - diff --git a/fix-word-error.patch b/fix-word-error.patch deleted file mode 100644 index 1e7de89..0000000 --- a/fix-word-error.patch +++ /dev/null @@ -1,53 +0,0 @@ -From 5be0d121c6fde185d323dc4bcf3026e2c3ee8757 Mon Sep 17 00:00:00 2001 -From: jinsaihang -Date: Mon, 14 Oct 2024 11:30:58 +0800 -Subject: [PATCH] fix word error - -Signed-off-by: jinsaihang ---- - sysSentry-1.0.2/src/python/syssentry/alarm.py | 10 +++++----- - 1 file changed, 5 insertions(+), 5 deletions(-) - -diff --git a/src/python/syssentry/alarm.py b/src/python/syssentry/alarm.py -index d012901..bff527c 100644 ---- a/src/python/syssentry/alarm.py -+++ b/src/python/syssentry/alarm.py -@@ -49,7 +49,7 @@ MAX_ALARM_ID = (MIN_ALARM_ID + MAX_NUM_OF_ALARM_ID - 1) - def update_alarm_list(alarm_info: Xalarm): - alarm_id = xalarm_getid(alarm_info) - if alarm_id < MIN_ALARM_ID or alarm_id > MAX_ALARM_ID: -- logging.warnning(f"Invalid alarm_id {alarm_id}") -+ logging.warning(f"Invalid alarm_id {alarm_id}") - return - timestamp = xalarm_gettime(alarm_info) - if not timestamp: -@@ -97,14 +97,14 @@ def alarm_register(): - task = TasksMap.tasks_dict[task_type][task_name] - alarm_id = task.alarm_id - if not check_alarm_id_if_number(alarm_id): -- logging.warnning(f"Invalid alarm_id {alarm_id}: ignore {task_name} alarm") -+ logging.warning(f"Invalid alarm_id {alarm_id}: ignore {task_name} alarm") - continue - if alarm_id < MIN_ALARM_ID or alarm_id > MAX_ALARM_ID: -- logging.warnning(f"Invalid alarm_id {alarm_id}: ignore {task_name} alarm") -+ logging.warning(f"Invalid alarm_id {alarm_id}: ignore {task_name} alarm") - continue - alarm_clear_time = task.alarm_clear_time - if not check_alarm_clear_time_if_positive_integer(alarm_clear_time): -- logging.warnning(f"Invalid alarm_clear_time {alarm_clear_time}: ignore {task_name} alarm") -+ logging.warning(f"Invalid alarm_clear_time {alarm_clear_time}: ignore {task_name} alarm") - continue - try: - alarm_clear_time = int(alarm_clear_time) -@@ -113,7 +113,7 @@ def alarm_register(): - if alarm_clear_time > sys.maxsize: - raise ValueError("Exceeds maximum value for int") - except (ValueError, OverflowError, TypeError) as e: -- logging.warnning(f"Invalid alarm_clear_time {alarm_clear_time}: ignore {task_name} alarm") -+ logging.warning(f"Invalid alarm_clear_time {alarm_clear_time}: ignore {task_name} alarm") - continue - alarm_list_dict[alarm_id] = [] - task_alarm_id_dict[task_name] = alarm_id --- -2.27.0 - diff --git a/fix-write-file-return-code-bug.patch b/fix-write-file-return-code-bug.patch deleted file mode 100644 index f059224..0000000 --- a/fix-write-file-return-code-bug.patch +++ /dev/null @@ -1,69 +0,0 @@ -From cea094acea79b88e6458cfa264a03c51f08c72fc Mon Sep 17 00:00:00 2001 -From: luckky -Date: Mon, 4 Nov 2024 20:18:05 +0800 -Subject: [PATCH] fix write file return code bug -Set the return code 0 to -EINVAL to unify the processing of return code. - ---- - .../hbm_online_repair/non-standard-hbm-repair.c | 17 ++++++++++------- - 1 file changed, 10 insertions(+), 7 deletions(-) - -diff --git a/src/c/hbm_online_repair/non-standard-hbm-repair.c b/src/c/hbm_online_repair/non-standard-hbm-repair.c -index b8dde7a..97cb9a7 100644 ---- a/src/c/hbm_online_repair/non-standard-hbm-repair.c -+++ b/src/c/hbm_online_repair/non-standard-hbm-repair.c -@@ -112,7 +112,7 @@ static void parse_fault_addr_info(struct fault_addr_info* info_struct, unsigned - info_struct->row_id = fault_addr & FAULT_ADDR_ROW_ID_MASK; - fault_addr >>= FAULT_ADDR_ROW_ID_LEN; - info_struct->column_id = fault_addr & FAULT_ADDR_COLUMN_ID_MASK; -- fault_addr >>= FAULT_ADDR_CHANNEL_ID_LEN; -+ fault_addr >>= FAULT_ADDR_COLUMN_ID_LEN; - info_struct->error_type = fault_addr & FAULT_ADDR_ERROR_TYPE_MASK; - fault_addr >>= FAULT_ADDR_ERROR_TYPE_LEN; - info_struct->repair_type = fault_addr & FAULT_ADDR_REPAIR_TYPE_MASK; -@@ -371,7 +371,12 @@ static int write_file(char *path, const char *name, unsigned long long value) - fname, value, strerror(errno)); - - close(fd); -- return ret > 0 ? 0 : -errno; -+ if (ret == 0) { -+ ret = -EINVAL; -+ } else if (ret < 0) { -+ ret = -errno; -+ } -+ return ret; - } - - static int get_hardware_corrupted_size() -@@ -560,7 +565,7 @@ static int hbmc_hbm_page_isolate(const struct hisi_common_error_section *err) - static uint8_t hbmc_hbm_after_repair(bool is_acls, const int repair_ret, const unsigned long long paddr) - { - int ret; -- if (repair_ret <= 0) { -+ if (repair_ret < 0) { - log(LOG_WARNING, "HBM %s: Keep page (0x%llx) offline\n", is_acls ? "ACLS" : "SPPR", paddr); - /* not much we can do about errors here */ - (void)write_file("/sys/kernel/page_eject", "remove_page", paddr); -@@ -594,8 +599,7 @@ static int hbmc_hbm_repair(const struct hisi_common_error_section *err, char *pa - - ret = write_file(path, is_acls ? "acls_query" : "sppr_query", paddr); - -- /* Only positive num means the error is supported to repair */ -- if (ret <= 0) { -+ if (ret < 0) { - if (ret != -ENXIO) { - notice_BMC(err, get_repair_failed_result_code(ret)); - log(LOG_WARNING, "HBM: Address 0x%llx is not supported to %s repair\n", paddr, is_acls ? "ACLS" : "SPPR"); -@@ -624,8 +628,7 @@ static int hbmc_hbm_repair(const struct hisi_common_error_section *err, char *pa - all_online_success = false; - } - } -- /* The ret is from the acls/sppr repair, and only positive num means the error is repaired successfully */ -- if (ret <= 0) { -+ if (ret < 0) { - notice_BMC(err, get_repair_failed_result_code(ret)); - return ret; - } else if (all_online_success) { --- -2.43.0 - diff --git a/fix-xalarm-non-uniform-log-formatting.patch b/fix-xalarm-non-uniform-log-formatting.patch deleted file mode 100644 index 34f579b..0000000 --- a/fix-xalarm-non-uniform-log-formatting.patch +++ /dev/null @@ -1,60 +0,0 @@ -From 3eba5dcde10e05e7badc99852f76488e667d56e6 Mon Sep 17 00:00:00 2001 -From: caixiaomeng -Date: Mon, 21 Oct 2024 11:57:37 +0800 -Subject: [PATCH] fix xalarm non-uniform log formatting - ---- - src/python/xalarm/sentry_notify.py | 11 ++++++----- - 1 file changed, 6 insertions(+), 5 deletions(-) - -diff --git a/src/python/xalarm/sentry_notify.py b/src/python/xalarm/sentry_notify.py -index 5838473..ffe4147 100644 ---- a/src/python/xalarm/sentry_notify.py -+++ b/src/python/xalarm/sentry_notify.py -@@ -2,6 +2,7 @@ import os - import sys - import time - import socket -+import logging - from struct import error as StructParseError - - from .xalarm_api import alarm_stu2bin, Xalarm -@@ -27,21 +28,21 @@ ALARM_SOCKET_PERMISSION = 0o700 - - def check_params(alarm_id, alarm_level, alarm_type, puc_paras) -> bool: - if not os.path.exists(DIR_XALARM): -- sys.stderr.write(f"check_params: {DIR_XALARM} not exist, failed\n") -+ logging.error(f"check_params: {DIR_XALARM} not exist, failed") - return False - - if not os.path.exists(PATH_REPORT_ALARM): -- sys.stderr.write(f"check_params: {PATH_REPORT_ALARM} not exist, failed\n") -+ logging.error(f"check_params: {PATH_REPORT_ALARM} not exist, failed") - return False - - if (alarm_id < MIN_ALARM_ID or alarm_id > MAX_ALARM_ID or - alarm_level < MINOR_ALM or alarm_level > CRITICAL_ALM or - alarm_type < ALARM_TYPE_OCCUR or alarm_type > ALARM_TYPE_RECOVER): -- sys.stderr.write("check_params: alarm info invalid\n") -+ logging.error("check_params: alarm info invalid") - return False - - if len(puc_paras) >= MAX_PUC_PARAS_LEN: -- sys.stderr.write(f"check_params: alarm msg should be less than {MAX_PUC_PARAS_LEN}\n") -+ logging.error(f"check_params: alarm msg should be less than {MAX_PUC_PARAS_LEN}") - return False - - return True -@@ -61,7 +62,7 @@ def xalarm_report(alarm_id, alarm_level, alarm_type, puc_paras) -> bool: - - sock.sendto(alarm_stu2bin(alarm_info), PATH_REPORT_ALARM) - except (FileNotFoundError, StructParseError, socket.error, OSError, UnicodeError) as e: -- sys.stderr.write(f"check_params: error occurs when sending msg.{e}\n") -+ logging.error(f"error occurs when sending msg.") - return False - finally: - sock.close() --- -2.27.0 - - diff --git a/fix-xalarm_Report-function-not-refuse-alarm-msg-exce.patch b/fix-xalarm_Report-function-not-refuse-alarm-msg-exce.patch deleted file mode 100644 index 1bf5c3b..0000000 --- a/fix-xalarm_Report-function-not-refuse-alarm-msg-exce.patch +++ /dev/null @@ -1,76 +0,0 @@ -From f6a26ea0759f36ebcaebe05d4d24c7234a110c63 Mon Sep 17 00:00:00 2001 -From: caixiaomeng -Date: Fri, 11 Oct 2024 12:12:53 +0800 -Subject: [PATCH] fix xalarm_Report function not refuse alarm msg exceeds - maximum - ---- - src/libso/xalarm/register_xalarm.c | 5 +++++ - src/python/xalarm/register_xalarm.py | 6 +++--- - src/python/xalarm/sentry_notify.py | 4 ++-- - 3 files changed, 10 insertions(+), 5 deletions(-) - -diff --git a/src/libso/xalarm/register_xalarm.c b/src/libso/xalarm/register_xalarm.c -index 5aff2bc..952a28b 100644 ---- a/src/libso/xalarm/register_xalarm.c -+++ b/src/libso/xalarm/register_xalarm.c -@@ -339,6 +339,11 @@ int xalarm_Report(unsigned short usAlarmId, unsigned char ucAlarmLevel, - return -1; - } - -+ if (pucParas == NULL || (int)strlen(pucParas) > MAX_PARAS_LEN) { -+ fprintf(stderr, "%s: alarm info invalid\n", __func__); -+ return -1; -+ } -+ - if (memset(&info, 0, sizeof(struct alarm_info)) == NULL) { - fprintf(stderr, "%s: memset info failed, ret: %d\n", __func__, ret); - return -1; -diff --git a/src/python/xalarm/register_xalarm.py b/src/python/xalarm/register_xalarm.py -index edd9994..39623bd 100644 ---- a/src/python/xalarm/register_xalarm.py -+++ b/src/python/xalarm/register_xalarm.py -@@ -45,7 +45,7 @@ class AlarmRegister: - return False - - if self.socket is None: -- sys.stderr.write("check_params: scoket create failed\n") -+ sys.stderr.write("check_params: socket create failed\n") - return False - return True - -@@ -151,10 +151,10 @@ def xalarm_unregister(clientId: int) -> None: - def xalarm_upgrade(clientId: int, id_filter: list) -> None: - global ALARM_REGISTER_INFO - if clientId < 0: -- sys.stderr.write("xalarm_unregister: invalid client\n") -+ sys.stderr.write("xalarm_upgrade: invalid client\n") - return - if ALARM_REGISTER_INFO is None: -- sys.stderr.write("xalarm_unregister: alarm has not registered\n") -+ sys.stderr.write("xalarm_upgrade: alarm has not registered\n") - return - ALARM_REGISTER_INFO.id_filter = id_filter - -diff --git a/src/python/xalarm/sentry_notify.py b/src/python/xalarm/sentry_notify.py -index c763a24..5838473 100644 ---- a/src/python/xalarm/sentry_notify.py -+++ b/src/python/xalarm/sentry_notify.py -@@ -27,11 +27,11 @@ ALARM_SOCKET_PERMISSION = 0o700 - - def check_params(alarm_id, alarm_level, alarm_type, puc_paras) -> bool: - if not os.path.exists(DIR_XALARM): -- sys.stderr.write(f"check_params: {DIR_XALARM} not exist, failed") -+ sys.stderr.write(f"check_params: {DIR_XALARM} not exist, failed\n") - return False - - if not os.path.exists(PATH_REPORT_ALARM): -- sys.stderr.write(f"check_params: {PATH_REPORT_ALARM} not exist, failed") -+ sys.stderr.write(f"check_params: {PATH_REPORT_ALARM} not exist, failed\n") - return False - - if (alarm_id < MIN_ALARM_ID or alarm_id > MAX_ALARM_ID or --- -2.27.0 - - diff --git a/fix-xalarm_upgrade-not-return-val-and-fail-when-thre.patch b/fix-xalarm_upgrade-not-return-val-and-fail-when-thre.patch deleted file mode 100644 index 5b1f231..0000000 --- a/fix-xalarm_upgrade-not-return-val-and-fail-when-thre.patch +++ /dev/null @@ -1,71 +0,0 @@ -From 624efd60495403743fc251b7d689d920841e44c8 Mon Sep 17 00:00:00 2001 -From: caixiaomeng -Date: Fri, 11 Oct 2024 17:54:04 +0800 -Subject: [PATCH] fix xalarm_upgrade not return val and fail when thread - stopped - ---- - src/libso/xalarm/register_xalarm.c | 11 ++++++++++- - src/python/xalarm/register_xalarm.py | 10 +++++++--- - 2 files changed, 17 insertions(+), 4 deletions(-) - -diff --git a/src/libso/xalarm/register_xalarm.c b/src/libso/xalarm/register_xalarm.c -index 952a28b..6768242 100644 ---- a/src/libso/xalarm/register_xalarm.c -+++ b/src/libso/xalarm/register_xalarm.c -@@ -156,7 +156,11 @@ static void *alarm_recv(void *arg) - continue; - } - printf("recv error len:%d errno:%d\n", recvlen, errno); -- } -+ } else if (recvlen == 0) { -+ printf("connection closed by xalarmd, maybe connections reach max num or service stopped.\n"); -+ g_register_info.thread_should_stop = 1; -+ break; -+ } - } - return NULL; - } -@@ -211,6 +215,11 @@ bool xalarm_Upgrade(struct alarm_subscription_info id_filter, int client_id) - printf("%s: invalid args\n", __func__); - return false; - } -+ -+ if (g_register_info.thread_should_stop) { -+ printf("%s: upgrade failed, alarm thread has stopped\n", __func__); -+ return false; -+ } - set_alarm_id(id_filter); - - return true; -diff --git a/src/python/xalarm/register_xalarm.py b/src/python/xalarm/register_xalarm.py -index 39623bd..2a6dabf 100644 ---- a/src/python/xalarm/register_xalarm.py -+++ b/src/python/xalarm/register_xalarm.py -@@ -148,15 +148,19 @@ def xalarm_unregister(clientId: int) -> None: - ALARM_REGISTER_INFO = None - - --def xalarm_upgrade(clientId: int, id_filter: list) -> None: -+def xalarm_upgrade(id_filter: list, clientId: int) -> bool: - global ALARM_REGISTER_INFO - if clientId < 0: - sys.stderr.write("xalarm_upgrade: invalid client\n") -- return -+ return False - if ALARM_REGISTER_INFO is None: - sys.stderr.write("xalarm_upgrade: alarm has not registered\n") -- return -+ return False -+ if ALARM_REGISTER_INFO.thread_should_stop: -+ sys.stderr.write("xalarm_upgrade: upgrade failed, alarm thread has stopped\n") -+ return False - ALARM_REGISTER_INFO.id_filter = id_filter -+ return True - - - def xalarm_getid(alarm_info: Xalarm) -> int: --- -2.27.0 - - diff --git a/get_alarm-d-abnomal-display.patch b/get_alarm-d-abnomal-display.patch deleted file mode 100644 index 8a7924a..0000000 --- a/get_alarm-d-abnomal-display.patch +++ /dev/null @@ -1,26 +0,0 @@ -From 132334913c4afebefd6afa835f790fa8a5fbf123 Mon Sep 17 00:00:00 2001 -From: jinsaihang -Date: Mon, 28 Oct 2024 09:22:53 +0800 -Subject: [PATCH] get_alarm -d abnomal display - -Signed-off-by: jinsaihang ---- - sysSentry-1.0.2/src/python/syssentry/alarm.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/src/python/syssentry/alarm.py b/src/python/syssentry/alarm.py -index b35a126..e5cc313 100644 ---- a/src/python/syssentry/alarm.py -+++ b/src/python/syssentry/alarm.py -@@ -184,7 +184,7 @@ def get_alarm_result(task_name: str, time_range: int, detailed: bool) -> List[Di - # dump each {key,value} of details in one line - if 'details' in alarm_info and isinstance(alarm_info['details'], dict): - for key in alarm_info['details']: -- alarm_info['details'][key] = json.dumps(alarm_info['details'][key], indent=None) -+ alarm_info['details'][key] = str(alarm_info['details'][key]) - - alarm['alarm_info'] = alarm_info - alarm_list = [alarm for alarm in alarm_list if 'alarm_source' in alarm['alarm_info'] and alarm['alarm_info']['alarm_source'] == task_name] --- -2.27.0 - diff --git a/get_io_data-failed-wont-stop-avg_block_io-and-del-di.patch b/get_io_data-failed-wont-stop-avg_block_io-and-del-di.patch deleted file mode 100644 index ec2aaf2..0000000 --- a/get_io_data-failed-wont-stop-avg_block_io-and-del-di.patch +++ /dev/null @@ -1,168 +0,0 @@ -From b21607fcec4b290bc78c9f6c4a26db1a2df32a66 Mon Sep 17 00:00:00 2001 -From: gaoruoshu -Date: Tue, 15 Oct 2024 21:21:10 +0800 -Subject: [PATCH] get_io_data failed wont stop avg_block_io and del disk not - support - ---- - src/python/sentryCollector/collect_plugin.py | 14 ++++----- - .../avg_block_io/avg_block_io.py | 9 ++++-- - .../sentryPlugins/avg_block_io/module_conn.py | 31 +++++++++++++------ - 3 files changed, 35 insertions(+), 19 deletions(-) - -diff --git a/src/python/sentryCollector/collect_plugin.py b/src/python/sentryCollector/collect_plugin.py -index bec405a..53dddec 100644 ---- a/src/python/sentryCollector/collect_plugin.py -+++ b/src/python/sentryCollector/collect_plugin.py -@@ -90,14 +90,14 @@ def client_send_and_recv(request_data, data_str_len, protocol): - try: - client_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - except socket.error: -- logging.error("collect_plugin: client create socket error") -+ logging.debug("collect_plugin: client create socket error") - return None - - try: - client_socket.connect(COLLECT_SOCKET_PATH) - except OSError: - client_socket.close() -- logging.error("collect_plugin: client connect error") -+ logging.debug("collect_plugin: client connect error") - return None - - req_data_len = len(request_data) -@@ -109,23 +109,23 @@ def client_send_and_recv(request_data, data_str_len, protocol): - res_data = res_data.decode() - except (OSError, UnicodeError): - client_socket.close() -- logging.error("collect_plugin: client communicate error") -+ logging.debug("collect_plugin: client communicate error") - return None - - res_magic = res_data[:CLT_MSG_MAGIC_LEN] - if res_magic != "RES": -- logging.error("res msg format error") -+ logging.debug("res msg format error") - return None - - protocol_str = res_data[CLT_MSG_MAGIC_LEN:CLT_MSG_MAGIC_LEN+CLT_MSG_PRO_LEN] - try: - protocol_id = int(protocol_str) - except ValueError: -- logging.error("recv msg protocol id is invalid %s", protocol_str) -+ logging.debug("recv msg protocol id is invalid %s", protocol_str) - return None - - if protocol_id >= ClientProtocol.PRO_END: -- logging.error("protocol id is invalid") -+ logging.debug("protocol id is invalid") - return None - - try: -@@ -134,7 +134,7 @@ def client_send_and_recv(request_data, data_str_len, protocol): - res_msg_data = res_msg_data.decode() - return res_msg_data - except (OSError, ValueError, UnicodeError): -- logging.error("collect_plugin: client recv res msg error") -+ logging.debug("collect_plugin: client recv res msg error") - finally: - client_socket.close() - -diff --git a/src/python/sentryPlugins/avg_block_io/avg_block_io.py b/src/python/sentryPlugins/avg_block_io/avg_block_io.py -index cd47919..899d517 100644 ---- a/src/python/sentryPlugins/avg_block_io/avg_block_io.py -+++ b/src/python/sentryPlugins/avg_block_io/avg_block_io.py -@@ -15,7 +15,7 @@ import time - - from .config import read_config_log, read_config_common, read_config_algorithm, read_config_latency, read_config_iodump, read_config_stage - from .stage_window import IoWindow, IoDumpWindow --from .module_conn import avg_is_iocollect_valid, avg_get_io_data, report_alarm_fail, process_report_data, sig_handler, get_disk_type_by_name -+from .module_conn import avg_is_iocollect_valid, avg_get_io_data, report_alarm_fail, process_report_data, sig_handler, get_disk_type_by_name, check_disk_list_validation - from .utils import update_avg_and_check_abnormal - - CONFIG_FILE = "/etc/sysSentry/plugins/avg_block_io.ini" -@@ -79,6 +79,8 @@ def get_valid_disk_stage_list(io_dic, config_disk, config_stage): - if not disk_list: - report_alarm_fail("Cannot get valid disk name") - -+ disk_list = check_disk_list_validation(disk_list) -+ - disk_list = disk_list[:10] if len(disk_list) > 10 else disk_list - - if not config_disk: -@@ -117,7 +119,10 @@ def main_loop(io_dic, io_data, io_avg_value): - time.sleep(period_time) - - # 采集模块对接，获取周期数据 -- curr_period_data = avg_get_io_data(io_dic) -+ is_success, curr_period_data = avg_get_io_data(io_dic) -+ if not is_success: -+ logging.error(f"{curr_period_data['msg']}") -+ continue - - # 处理周期数据 - reach_size = False -diff --git a/src/python/sentryPlugins/avg_block_io/module_conn.py b/src/python/sentryPlugins/avg_block_io/module_conn.py -index cbdaad4..a67ef45 100644 ---- a/src/python/sentryPlugins/avg_block_io/module_conn.py -+++ b/src/python/sentryPlugins/avg_block_io/module_conn.py -@@ -40,25 +40,25 @@ def avg_is_iocollect_valid(io_dic, config_disk, config_stage): - logging.debug(f"send to sentryCollector is_iocollect_valid: period={io_dic['period_time']}, " - f"disk={config_disk}, stage={config_stage}") - res = is_iocollect_valid(io_dic["period_time"], config_disk, config_stage) -- return check_result_validation(res, 'check config validation') -+ is_success, data = check_result_validation(res, 'check config validation') -+ if not is_success: -+ report_alarm_fail(f"{data['msg']}") -+ return data - - - def check_result_validation(res, reason): - """check validation of result from sentryCollector""" - if not 'ret' in res or not 'message' in res: -- err_msg = "Failed to {}: Cannot connect to sentryCollector.".format(reason) -- report_alarm_fail(err_msg) -+ return False, {'msg': f"Failed to {reason}: Cannot connect to sentryCollector"} - if res['ret'] != 0: -- err_msg = "Failed to {}: {}".format(reason, Result_Messages[res['ret']]) -- report_alarm_fail(err_msg) -+ return False, {'msg': f"Failed to {reason}: {Result_Messages[res['ret']]}"} - - try: - json_data = json.loads(res['message']) - except json.JSONDecodeError: -- err_msg = f"Failed to {reason}: invalid return message" -- report_alarm_fail(err_msg) -+ return False, {'msg': f"Failed to {reason}: invalid return message"} - -- return json_data -+ return True, json_data - - - def report_alarm_fail(alarm_info): -@@ -120,10 +120,21 @@ def process_report_data(disk_name, rw, io_data): - xalarm_report(1002, MINOR_ALM, ALARM_TYPE_OCCUR, json.dumps(msg)) - - -+def check_disk_list_validation(disk_list): -+ valid_disk_list = [] -+ for disk_name in disk_list: -+ is_success, _ = check_result_validation(get_disk_type(disk_name), "") -+ if not is_success: -+ continue -+ valid_disk_list.append(disk_name) -+ return valid_disk_list -+ -+ - def get_disk_type_by_name(disk_name): - logging.debug(f"send to sentryCollector get_disk_type: disk_name={disk_name}") -- res = get_disk_type(disk_name) -- disk_type_str = check_result_validation(get_disk_type(disk_name), f'Invalid disk type {disk_name}') -+ is_success, disk_type_str = check_result_validation(get_disk_type(disk_name), f'Invalid disk type {disk_name}') -+ if not is_success: -+ report_alarm_fail(f"{disk_type_str['msg']}") - try: - curr_disk_type = int(disk_type_str) - if curr_disk_type not in Disk_Type: --- -2.27.0 diff --git a/hbm_online_repair-add-unload-driver.patch b/hbm_online_repair-add-unload-driver.patch deleted file mode 100644 index dd7a269..0000000 --- a/hbm_online_repair-add-unload-driver.patch +++ /dev/null @@ -1,107 +0,0 @@ -From 74f18b0e1fd4f99fa7d1d95e08894b408dcafe51 Mon Sep 17 00:00:00 2001 -From: luckky -Date: Wed, 18 Dec 2024 14:31:04 +0800 -Subject: [PATCH] hbm_online_repair add unload driver - ---- - src/c/hbm_online_repair/hbm_online_repair.c | 47 +++++++++++++-------- - 1 file changed, 29 insertions(+), 18 deletions(-) - -diff --git a/src/c/hbm_online_repair/hbm_online_repair.c b/src/c/hbm_online_repair/hbm_online_repair.c -index 00c9c0b..6783485 100644 ---- a/src/c/hbm_online_repair/hbm_online_repair.c -+++ b/src/c/hbm_online_repair/hbm_online_repair.c -@@ -11,6 +11,8 @@ - #define DEFAULT_LOG_LEVEL LOG_INFO - #define DEFAULT_PAGE_ISOLATION_THRESHOLD 3355443 - -+#define DRIVER_COMMAND_LEN 32 -+ - int global_level_setting; - int page_isolation_threshold; - -@@ -57,25 +59,31 @@ int execute_command(const char *command) - return -1; - } - -- ret = WEXITSTATUS(ret); -+ ret = -WEXITSTATUS(ret); - log(LOG_DEBUG, "command %s exited with status: %d\n", command, ret); - return ret; - } - --int load_required_driver(void) -+int handle_driver(char* driver_name, bool load) - { - int ret; -- ret = execute_command("modprobe hisi_mem_ras 2>&1"); -- if (ret < 0) { -- log(LOG_ERROR, "load repair driver failed\n"); -- return ret; -- } -- ret = execute_command("modprobe page_eject 2>&1"); -- if (ret < 0) { -- log(LOG_ERROR, "load page driver failed\n"); -+ char command[DRIVER_COMMAND_LEN]; -+ -+ snprintf(command, DRIVER_COMMAND_LEN, "%s %s 2>&1", load ? "modprobe" : "rmmod", driver_name); -+ ret = execute_command(command); -+ log(ret < 0 ? LOG_ERROR : LOG_DEBUG, "%s %s %s\n", load ? "load" : "unload", driver_name, ret < 0 ? "failed" : "success"); -+ return ret; -+} -+ -+int handle_all_drivers(bool load) -+{ -+ int ret; -+ -+ ret = handle_driver("hisi_mem_ras", load); -+ if (ret < 0) - return ret; -- } -- log(LOG_INFO, "load required driver success\n"); -+ -+ ret = handle_driver("page_eject", load); - return ret; - } - -@@ -116,21 +124,21 @@ int main(int argc, char *argv[]) - - hbm_param_init(); - -- ret = load_required_driver(); -+ ret = handle_all_drivers(true); - if (ret < 0) { -- log(LOG_DEBUG, "load required driver failed\n"); - return ret; - } - - struct ras_events *ras = init_trace_instance(); -- if (!ras) -- return -1; -+ if (!ras) { -+ ret = -1; -+ goto err_unload; -+ } - - ret = toggle_ras_event(ras->tracing, "ras", "non_standard_event", 1); - if (ret < 0) { - log(LOG_WARNING, "unable to enable ras non_standard_event.\n"); -- free(ras); -- return -1; -+ goto err_free; - } - - get_flash_total_size(); -@@ -142,6 +150,9 @@ int main(int argc, char *argv[]) - log(LOG_WARNING, "unable to disable ras non_standard_event.\n"); - } - -+err_free: - free(ras); -+err_unload: -+ handle_all_drivers(false); - return ret; - } --- -2.43.0 - diff --git a/listen-thread-of-collect-module-exits-occasionally.patch b/listen-thread-of-collect-module-exits-occasionally.patch deleted file mode 100644 index b667ea0..0000000 --- a/listen-thread-of-collect-module-exits-occasionally.patch +++ /dev/null @@ -1,104 +0,0 @@ -From 2135b4e41666d99922eda79e9ee04bbc2b557fea Mon Sep 17 00:00:00 2001 -From: zhuofeng -Date: Wed, 16 Oct 2024 12:13:21 +0800 -Subject: [PATCH] listen thread of collect module exits occasionally - ---- - src/python/sentryCollector/collect_io.py | 4 +--- - src/python/sentryCollector/collect_server.py | 18 ++++++++---------- - 2 files changed, 9 insertions(+), 13 deletions(-) - -diff --git a/src/python/sentryCollector/collect_io.py b/src/python/sentryCollector/collect_io.py -index 5fe1efc..de308b3 100644 ---- a/src/python/sentryCollector/collect_io.py -+++ b/src/python/sentryCollector/collect_io.py -@@ -231,9 +231,7 @@ class CollectIo(): - if self.get_blk_io_hierarchy(disk_name, stage_list) < 0: - continue - self.append_period_lat(disk_name, stage_list) -- -- logging.debug(f"no-lock collect data : {IO_GLOBAL_DATA}") -- -+ - elapsed_time = time.time() - start_time - sleep_time = self.period_time - elapsed_time - if sleep_time < 0: -diff --git a/src/python/sentryCollector/collect_server.py b/src/python/sentryCollector/collect_server.py -index 11d1af0..ad3ac0e 100644 ---- a/src/python/sentryCollector/collect_server.py -+++ b/src/python/sentryCollector/collect_server.py -@@ -64,7 +64,7 @@ class CollectServer(): - self.io_global_data = IO_GLOBAL_DATA - - if len(IO_CONFIG_DATA) == 0: -- logging.error("the collect thread is not started, the data is invalid. ") -+ logging.error("the collect thread is not started, the data is invalid.") - return json.dumps(result_rev) - - period_time = IO_CONFIG_DATA[0] -@@ -75,7 +75,7 @@ class CollectServer(): - stage_list = json.loads(data_struct['stage']) - - if (period < period_time) or (period > period_time * max_save) or (period % period_time): -- logging.error("is_iocollect_valid: period time: %d is invalid", period) -+ logging.error("is_iocollect_valid: period time is invalid, user period: %d, config period_time: %d", period, period_time) - return json.dumps(result_rev) - - for disk_name, stage_info in self.io_global_data.items(): -@@ -96,7 +96,7 @@ class CollectServer(): - self.io_global_data = IO_GLOBAL_DATA - - if len(IO_CONFIG_DATA) == 0: -- logging.error("the collect thread is not started, the data is invalid. ") -+ logging.error("the collect thread is not started, the data is invalid.") - return json.dumps(result_rev) - period_time = IO_CONFIG_DATA[0] - max_save = IO_CONFIG_DATA[1] -@@ -107,11 +107,11 @@ class CollectServer(): - iotype_list = json.loads(data_struct['iotype']) - - if (period < period_time) or (period > period_time * max_save) or (period % period_time): -- logging.error("get_io_data: period time: %d is invalid", period) -+ logging.error("get_io_data: period time is invalid, user period: %d, config period_time: %d", period, period_time) - return json.dumps(result_rev) - - collect_index = period // period_time - 1 -- logging.debug("period: %d, collect_index: %d", period, collect_index) -+ logging.debug("user period: %d, config period_time: %d, collect_index: %d", period, period_time, collect_index) - - for disk_name, stage_info in self.io_global_data.items(): - if disk_name not in disk_list: -@@ -124,7 +124,7 @@ class CollectServer(): - for iotype_name, iotype_info in iotype_info.items(): - if iotype_name not in iotype_list: - continue -- if len(iotype_info) < collect_index: -+ if len(iotype_info) - 1 < collect_index: - continue - result_rev[disk_name][stage_name][iotype_name] = iotype_info[collect_index] - -@@ -250,10 +250,8 @@ class CollectServer(): - except socket.error: - logging.error("server fd create failed") - server_fd = None -- - return server_fd - -- - def server_loop(self): - """main loop""" - logging.info("collect listen thread start") -@@ -277,8 +275,8 @@ class CollectServer(): - self.server_recv(server_fd) - else: - continue -- except socket.error: -- pass -+ except Exception: -+ logging.error('collect listen exception : %s', traceback.format_exc()) - - def stop_thread(self): - self.stop_event.set() --- -2.33.0 - diff --git a/make-debug-msg-clear.patch b/make-debug-msg-clear.patch deleted file mode 100644 index 540750d..0000000 --- a/make-debug-msg-clear.patch +++ /dev/null @@ -1,69 +0,0 @@ -From edbe32637a939d0788bcbde9211a61cfded436bf Mon Sep 17 00:00:00 2001 -From: luckky -Date: Tue, 5 Nov 2024 17:22:27 +0800 -Subject: [PATCH] make debug msg clear -1. Change the page_isolation_threshold default value for 128(kb) to 3355443(kb) -to synchronize the modification of the .mod file. -2. Add specific command info in debug message to make debug message clear. -3. Update the commit of the log level and format of syssentry. -4. Change the interval 180 to 10 to short the restart time. - ---- - config/tasks/hbm_online_repair.mod | 2 +- - .../src/c/hbm_online_repair/hbm_online_repair.c | 8 ++++---- - 2 files changed, 5 insertions(+), 5 deletions(-) - -diff --git a/config/tasks/hbm_online_repair.mod b/config/tasks/hbm_online_repair.mod -index 77dd73e..4dcef43 100644 ---- a/config/tasks/hbm_online_repair.mod -+++ b/config/tasks/hbm_online_repair.mod -@@ -3,7 +3,7 @@ enabled=yes - task_start=/usr/bin/hbm_online_repair - task_stop=kill $pid - type=period --interval=180 -+interval=10 - onstart=yes - env_file=/etc/sysconfig/hbm_online_repair.env - conflict=up -\ No newline at end of file -diff --git a/src/c/hbm_online_repair/hbm_online_repair.c b/src/c/hbm_online_repair/hbm_online_repair.c -index b3b2742..943f201 100644 ---- a/src/c/hbm_online_repair/hbm_online_repair.c -+++ b/src/c/hbm_online_repair/hbm_online_repair.c -@@ -9,7 +9,7 @@ - #include "non-standard-hbm-repair.h" - - #define DEFAULT_LOG_LEVEL LOG_INFO --#define DEFAULT_PAGE_ISOLATION_THRESHOLD 128 -+#define DEFAULT_PAGE_ISOLATION_THRESHOLD 3355443 - - int global_level_setting; - int page_isolation_threshold; -@@ -44,7 +44,7 @@ int execute_command(const char *command) - } - - fgets(buffer, sizeof(buffer), fp); -- log(LOG_DEBUG, "output of command is: %s\n", buffer); -+ log(LOG_DEBUG, "output of command %s is: %s\n", command, buffer); - - ret = pclose(fp); - if (ret < 0) { -@@ -53,12 +53,12 @@ int execute_command(const char *command) - } - - if (!WIFEXITED(ret)) { -- log(LOG_ERROR, "command did not terminate normally\n"); -+ log(LOG_ERROR, "command %s did not terminate normally\n", command); - return -1; - } - - ret = WEXITSTATUS(ret); -- log(LOG_DEBUG, "command exited with status: %d\n", ret); -+ log(LOG_DEBUG, "command %s exited with status: %d\n", command, ret); - return ret; - } - --- -2.43.0 - diff --git a/modify-abnormal-stack-when-the-disk-field-is-not-con.patch b/modify-abnormal-stack-when-the-disk-field-is-not-con.patch deleted file mode 100644 index 5b0084b..0000000 --- a/modify-abnormal-stack-when-the-disk-field-is-not-con.patch +++ /dev/null @@ -1,28 +0,0 @@ -From b5794ef43f768d7ea9bbbac450deaabbdcff4997 Mon Sep 17 00:00:00 2001 -From: zhuofeng -Date: Sat, 12 Oct 2024 17:57:01 +0800 -Subject: [PATCH] modify abnormal stack when the disk field is not configured - ---- - src/python/sentryCollector/collect_config.py | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/src/python/sentryCollector/collect_config.py b/src/python/sentryCollector/collect_config.py -index 5aa38ec..7ca9898 100644 ---- a/src/python/sentryCollector/collect_config.py -+++ b/src/python/sentryCollector/collect_config.py -@@ -127,9 +127,9 @@ class CollectConfig: - CONF_IO, CONF_IO_MAX_SAVE, CONF_IO_MAX_SAVE_DEFAULT) - result_io_config[CONF_IO_MAX_SAVE] = CONF_IO_MAX_SAVE_DEFAULT - # disk -- disk = io_map_value.get(CONF_IO_DISK).lower() -+ disk = io_map_value.get(CONF_IO_DISK) - if disk: -- disk_str = disk.replace(" ", "") -+ disk_str = disk.lower().replace(" ", "") - pattern = r'^[a-zA-Z0-9-_,]+$' - if not re.match(pattern, disk_str): - logging.warning("module_name = %s section, field = %s is incorrect, use default %s", --- -2.33.0 - diff --git a/modify-logrotate-rule.patch b/modify-logrotate-rule.patch deleted file mode 100644 index be0ddd2..0000000 --- a/modify-logrotate-rule.patch +++ /dev/null @@ -1,27 +0,0 @@ -From 0d3323d13797f3f9d3124e3938787d2573bf249d Mon Sep 17 00:00:00 2001 -From: zhangnan -Date: Mon, 28 Oct 2024 17:32:49 +0800 -Subject: [PATCH] modify logrotate rule - ---- - config/logrotate | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/config/logrotate b/config/logrotate -index f54e7b3..e855118 100644 ---- a/config/logrotate -+++ b/config/logrotate -@@ -1,8 +1,9 @@ - /var/log/sysSentry/*.log { -- nocompress -+ compress - missingok - notifempty - copytruncate - rotate 2 - size +4096k -+ hourly - } --- -2.33.0 - diff --git a/optimize-log-printing.patch b/optimize-log-printing.patch deleted file mode 100644 index 591ae9f..0000000 --- a/optimize-log-printing.patch +++ /dev/null @@ -1,125 +0,0 @@ -From 91c37cec1639c79b2b5ddcd6b173b4d7aa0ce9db Mon Sep 17 00:00:00 2001 -From: jinsaihang -Date: Wed, 16 Oct 2024 14:51:24 +0800 -Subject: [PATCH] optimize log printing - -Signed-off-by: jinsaihang ---- - src/python/syssentry/alarm.py | 53 ++++++++++++++++--------------- - src/python/syssentry/load_mods.py | 15 +++++---- - 2 files changed, 35 insertions(+), 33 deletions(-) - -diff --git a/src/python/syssentry/alarm.py b/src/python/syssentry/alarm.py -index bff527c..c3f2ee1 100644 ---- a/src/python/syssentry/alarm.py -+++ b/src/python/syssentry/alarm.py -@@ -76,16 +76,26 @@ def update_alarm_list(alarm_info: Xalarm): - finally: - alarm_list_lock.release() - --def check_alarm_id_if_number(alarm_id): -- if isinstance(alarm_id, int): -- return True -- else: -+def validate_alarm_id(alarm_id): -+ if alarm_id is None: -+ return False -+ try: -+ alarm_id = int(alarm_id) -+ if MIN_ALARM_ID <= alarm_id <= MAX_ALARM_ID: -+ return True -+ else: -+ return False -+ except ValueError: - return False - --def check_alarm_clear_time_if_positive_integer(alarm_clear_time): -- if isinstance(alarm_clear_time, int) and alarm_clear_time > 0: -- return True -- else: -+def validate_alarm_clear_time(alarm_clear_time): -+ try: -+ alarm_clear_time = int(alarm_clear_time) -+ if alarm_clear_time > 0 and alarm_clear_time <= sys.maxsize: -+ return True -+ else: -+ return False -+ except ValueError: - return False - - def alarm_register(): -@@ -93,34 +103,25 @@ def alarm_register(): - # 初始化告警ID映射字典、告警老化时间字典 - for task_type in TasksMap.tasks_dict: - for task_name in TasksMap.tasks_dict[task_type]: -- logging.info(f"alarm_register: {task_name} is registered") - task = TasksMap.tasks_dict[task_type][task_name] -- alarm_id = task.alarm_id -- if not check_alarm_id_if_number(alarm_id): -- logging.warning(f"Invalid alarm_id {alarm_id}: ignore {task_name} alarm") -+ if not validate_alarm_id(task.alarm_id): -+ logging.warning(f"Invalid alarm_id {task.alarm_id}: ignore {task_name} alarm") - continue -- if alarm_id < MIN_ALARM_ID or alarm_id > MAX_ALARM_ID: -- logging.warning(f"Invalid alarm_id {alarm_id}: ignore {task_name} alarm") -+ if not validate_alarm_clear_time(task.alarm_clear_time): -+ logging.warning(f"Invalid alarm_clear_time {task.alarm_clear_time}: ignore {task_name} alarm") - continue -+ task.alarm_id = int(task.alarm_id) -+ task.alarm_clear_time = int(task.alarm_clear_time) -+ alarm_id = task.alarm_id - alarm_clear_time = task.alarm_clear_time -- if not check_alarm_clear_time_if_positive_integer(alarm_clear_time): -- logging.warning(f"Invalid alarm_clear_time {alarm_clear_time}: ignore {task_name} alarm") -- continue -- try: -- alarm_clear_time = int(alarm_clear_time) -- if alarm_clear_time <= 0: -- raise ValueError("Not a positive integer") -- if alarm_clear_time > sys.maxsize: -- raise ValueError("Exceeds maximum value for int") -- except (ValueError, OverflowError, TypeError) as e: -- logging.warning(f"Invalid alarm_clear_time {alarm_clear_time}: ignore {task_name} alarm") -- continue -+ - alarm_list_dict[alarm_id] = [] - task_alarm_id_dict[task_name] = alarm_id - if alarm_id not in alarm_id_clear_time_dict: - alarm_id_clear_time_dict[alarm_id] = alarm_clear_time - else: - alarm_id_clear_time_dict[alarm_id] = max(alarm_clear_time, alarm_id_clear_time_dict[alarm_id]) -+ logging.info(f"alarm_register: {task_name} is registered") - # 注册告警回调 - id_filter = [True] * 128 - clientId = xalarm_register(update_alarm_list, id_filter) -diff --git a/src/python/syssentry/load_mods.py b/src/python/syssentry/load_mods.py -index f74f165..78db446 100644 ---- a/src/python/syssentry/load_mods.py -+++ b/src/python/syssentry/load_mods.py -@@ -198,15 +198,16 @@ def parse_mod_conf(mod_name, mod_conf): - task.load_enabled = is_enabled - - try: -- task.alarm_id = int(mod_conf.get(CONF_TASK, CONF_ALARM_ID)) -- task.alarm_clear_time = int(mod_conf.get(CONF_TASK, CONF_ALARM_CLEAR_TIME)) -- if not (MIN_ALARM_ID <= task.alarm_id <= MAX_ALARM_ID): -- raise ValueError("Invalid alarm_id") -- except ValueError: - task.alarm_id = mod_conf.get(CONF_TASK, CONF_ALARM_ID) -- task.alarm_clear_time = mod_conf.get(CONF_TASK, CONF_ALARM_CLEAR_TIME) - except configparser.NoOptionError: -- logging.warning("Unset alarm_clear_time, use 15s as default") -+ task.alarm_id = None -+ logging.warning(f"{mod_name} alarm_id not set, alarm_id is None") -+ -+ if task.alarm_id is not None: -+ try: -+ task.alarm_clear_time = mod_conf.get(CONF_TASK, CONF_ALARM_CLEAR_TIME) -+ except configparser.NoOptionError: -+ logging.warning(f"{mod_name} not set alarm_clear_time, use 15s as default") - - if CONF_ONSTART in mod_conf.options(CONF_TASK): - is_onstart = (mod_conf.get(CONF_TASK, CONF_ONSTART) == 'yes') --- -2.27.0 - diff --git a/optimize-the-handing-of-cat-cli-error-msg-in-cpu_sentry.patch b/optimize-the-handing-of-cat-cli-error-msg-in-cpu_sentry.patch deleted file mode 100644 index 7bacd26..0000000 --- a/optimize-the-handing-of-cat-cli-error-msg-in-cpu_sentry.patch +++ /dev/null @@ -1,77 +0,0 @@ -From cb3d0ea18eed3d48f2753f878d9726f58fe616b1 Mon Sep 17 00:00:00 2001 -From: shixuantong -Date: Sat, 21 Sep 2024 09:53:42 +0800 -Subject: [PATCH] optimize the handing of cat-cli error msg in cpu_sentry - ---- - src/python/syssentry/cpu_sentry.py | 36 +++++++++++++++++------------- - 1 file changed, 21 insertions(+), 15 deletions(-) - -diff --git a/src/python/syssentry/cpu_sentry.py b/src/python/syssentry/cpu_sentry.py -index 99af127..582d4b3 100644 ---- a/src/python/syssentry/cpu_sentry.py -+++ b/src/python/syssentry/cpu_sentry.py -@@ -26,6 +26,8 @@ CPU_SENTRY_PARAM_CONFIG = "/etc/sysSentry/plugins/cpu_sentry.ini" - # Inspection commands running at the bottom layer - LOW_LEVEL_INSPECT_CMD = "cat-cli" - -+# max length of msg in details -+DETAILS_LOG_MSG_MAX_LEN = 255 - - class CpuSentry: - """ -@@ -94,22 +96,10 @@ class CpuSentry: - self.send_result["details"]["msg"] = "cpu_sentry task is killed!" - return - -- if "ERROR" in stdout: -- self.send_result["result"] = ResultLevel.FAIL -- self.send_result["details"]["code"] = 1004 -- -- # Remove ANSI escape sequences -- error_info = stdout.split("\n")[0] -- if error_info.startswith("\u001b"): -- ansi_escape = r'\x1b\[([0-9]+)(;[0-9]+)*([A-Za-z])' -- error_info = re.sub(ansi_escape, '', error_info) -- -- self.send_result["details"]["msg"] = error_info -- return -- - out_split = stdout.split("\n") -- isolated_cores_number = 0 -+ isolated_cores_number = -1 - found_fault_cores_list = [] -+ error_msg_list = [] - for out_line_i in out_split: - if "handle_patrol_result: Found fault cores" in out_line_i: - cores_number_tmp = out_line_i.split("Found fault cores:")[1] -@@ -121,9 +111,25 @@ class CpuSentry: - elif out_line_i.startswith(''): - self.send_result["details"]["isolated_cpu_list"] = out_line_i.split(':')[1] - break -+ elif "ERROR" in out_line_i: -+ logging.error("[cat-cli error] - %s\n", out_line_i) -+ error_msg_list.append(out_line_i) - - found_fault_cores_number = len(set(found_fault_cores_list)) -- if found_fault_cores_number == 0: -+ if isolated_cores_number == -1: -+ self.send_result["result"] = ResultLevel.FAIL -+ self.send_result["details"]["code"] = 1004 -+ -+ send_error_msg = "" -+ # Remove ANSI escape sequences -+ for error_info in error_msg_list: -+ if error_info.startswith("\u001b"): -+ ansi_escape = r'\x1b\[([0-9]+)(;[0-9]+)*([A-Za-z])' -+ error_info = re.sub(ansi_escape, '', error_info) -+ if len(send_error_msg) + len(error_info) < DETAILS_LOG_MSG_MAX_LEN: -+ send_error_msg += error_info -+ self.send_result["details"]["msg"] = send_error_msg -+ elif found_fault_cores_number == 0: - self.send_result["details"]["code"] = 0 - self.send_result["result"] = ResultLevel.PASS - elif 0 in found_fault_cores_list: --- -2.27.0 - diff --git a/over-threshold-should-be-warn-level-log-in-cat-cli.patch b/over-threshold-should-be-warn-level-log-in-cat-cli.patch deleted file mode 100644 index 53a2739..0000000 --- a/over-threshold-should-be-warn-level-log-in-cat-cli.patch +++ /dev/null @@ -1,25 +0,0 @@ -From 3dda5f68db38b63b1e45a28558a9fcd341c1f945 Mon Sep 17 00:00:00 2001 -From: jwolf <523083921@qq.com> -Date: Fri, 20 Sep 2024 15:59:40 +0800 -Subject: [PATCH] should be warn-level log - ---- - src/c/catcli/catlib/plugin/cpu_patrol/cpu_patrol_result.c | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/src/c/catcli/catlib/plugin/cpu_patrol/cpu_patrol_result.c b/src/c/catcli/catlib/plugin/cpu_patrol/cpu_patrol_result.c -index 9f8d80c..f4f3172 100644 ---- a/src/c/catcli/catlib/plugin/cpu_patrol/cpu_patrol_result.c -+++ b/src/c/catcli/catlib/plugin/cpu_patrol/cpu_patrol_result.c -@@ -23,7 +23,7 @@ static cat_return_t insert_core_to_list(core_list_st *core_list, int coreid) - return CAT_OK; - } - if ((core_list->current_nums == MAX_ISOLATE_CORES_PER_PATROL) || (coreid < 0)) { -- CAT_LOG_E("Insert error, core id(%d)", coreid); -+ CAT_LOG_W("Too many cores need to isolate,do not isolate core(%d)", coreid); - return CAT_ERR; - } - --- -2.27.0 - diff --git a/param-must-be-integer.patch b/param-must-be-integer.patch deleted file mode 100644 index d9b7ac1..0000000 --- a/param-must-be-integer.patch +++ /dev/null @@ -1,23 +0,0 @@ -From 34febf57060060d1f8262941af49e3beeb1f7f5d Mon Sep 17 00:00:00 2001 -From: jwolf <523083921@qq.com> -Date: Fri, 30 Aug 2024 16:59:56 +0800 -Subject: [PATCH] param must be integer - ---- - src/c/catcli/catlib/cli_param_checker.c | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/src/c/catcli/catlib/cli_param_checker.c b/src/c/catcli/catlib/cli_param_checker.c -index 5b38402..71edf17 100644 ---- a/src/c/catcli/catlib/cli_param_checker.c -+++ b/src/c/catcli/catlib/cli_param_checker.c -@@ -17,6 +17,7 @@ void checkset_cpu_usage_percentage(char *getopt_optarg, catcli_request_body *p_r - if (cpu_utility <= 0 || cpu_utility > CPU_USAGE_PERCENTAGE_MAX || strchr(getopt_optarg, '.') != NULL) { - strncpy(errs->patrol_module_err, - "\"cpu_utility \" must be an integer greater in the range (0,100],correct \"-u, --cpu_utility\"\n", MAX_ERR_LEN); -+ p_request_body->cpu_utility = 0; - } else { - p_request_body->cpu_utility = (int)cpu_utility; - } --- -Gitee diff --git a/precise-alarm-query-time.patch b/precise-alarm-query-time.patch deleted file mode 100644 index f69a2d4..0000000 --- a/precise-alarm-query-time.patch +++ /dev/null @@ -1,91 +0,0 @@ -From 7fa9e80531bb3d4fa587e5fb7a99e3af59feda7e Mon Sep 17 00:00:00 2001 -From: jinsaihang -Date: Sat, 12 Oct 2024 16:51:37 +0800 -Subject: [PATCH] precise alarm query time - -Signed-off-by: jinsaihang ---- - sysSentry-1.0.2/src/python/syssentry/alarm.py | 25 +++++++++++++++++-- - .../src/python/syssentry/load_mods.py | 3 ++- - 2 files changed, 25 insertions(+), 3 deletions(-) - -diff --git a/src/python/syssentry/alarm.py b/src/python/syssentry/alarm.py -index 43c1065..d012901 100644 ---- a/src/python/syssentry/alarm.py -+++ b/src/python/syssentry/alarm.py -@@ -76,6 +76,18 @@ def update_alarm_list(alarm_info: Xalarm): - finally: - alarm_list_lock.release() - -+def check_alarm_id_if_number(alarm_id): -+ if isinstance(alarm_id, int): -+ return True -+ else: -+ return False -+ -+def check_alarm_clear_time_if_positive_integer(alarm_clear_time): -+ if isinstance(alarm_clear_time, int) and alarm_clear_time > 0: -+ return True -+ else: -+ return False -+ - def alarm_register(): - logging.debug(f"alarm_register: enter") - # 初始化告警ID映射字典、告警老化时间字典 -@@ -84,10 +96,16 @@ def alarm_register(): - logging.info(f"alarm_register: {task_name} is registered") - task = TasksMap.tasks_dict[task_type][task_name] - alarm_id = task.alarm_id -+ if not check_alarm_id_if_number(alarm_id): -+ logging.warnning(f"Invalid alarm_id {alarm_id}: ignore {task_name} alarm") -+ continue - if alarm_id < MIN_ALARM_ID or alarm_id > MAX_ALARM_ID: - logging.warnning(f"Invalid alarm_id {alarm_id}: ignore {task_name} alarm") - continue - alarm_clear_time = task.alarm_clear_time -+ if not check_alarm_clear_time_if_positive_integer(alarm_clear_time): -+ logging.warnning(f"Invalid alarm_clear_time {alarm_clear_time}: ignore {task_name} alarm") -+ continue - try: - alarm_clear_time = int(alarm_clear_time) - if alarm_clear_time <= 0: -@@ -119,6 +137,9 @@ def get_alarm_result(task_name: str, time_range: int, detailed: bool) -> List[Di - logging.debug("task_name does not exist") - return [] - alarm_id = task_alarm_id_dict[task_name] -+ clear_time = alarm_id_clear_time_dict[alarm_id] -+ if clear_time < int(time_range): -+ return [] - if alarm_id not in alarm_list_dict: - logging.debug("alarm_id does not exist") - return [] -@@ -126,10 +147,10 @@ def get_alarm_result(task_name: str, time_range: int, detailed: bool) -> List[Di - logging.debug(f"get_alarm_result: alarm_list of {alarm_id} has {len(alarm_list)} elements") - # clear alarm_info older than clear time threshold - stop_index = -1 -- timestamp = int(datetime.now().timestamp()) -+ timestamp = datetime.now().timestamp() - for i in range(len(alarm_list)): - logging.debug(f"timestamp, alarm_list[{i}].timestamp: {timestamp}, {xalarm_gettime(alarm_list[i])}") -- if timestamp - (xalarm_gettime(alarm_list[i])) / MILLISECONDS_UNIT_SECONDS > int(time_range): -+ if timestamp - (xalarm_gettime(alarm_list[i])) / MILLISECONDS_UNIT_SECONDS > time_range: - stop_index = i - break - if stop_index >= 0: -diff --git a/src/python/syssentry/load_mods.py b/src/python/syssentry/load_mods.py -index 7daf17d..f74f165 100644 ---- a/src/python/syssentry/load_mods.py -+++ b/src/python/syssentry/load_mods.py -@@ -203,7 +203,8 @@ def parse_mod_conf(mod_name, mod_conf): - if not (MIN_ALARM_ID <= task.alarm_id <= MAX_ALARM_ID): - raise ValueError("Invalid alarm_id") - except ValueError: -- logging.warning("Invalid alarm_id") -+ task.alarm_id = mod_conf.get(CONF_TASK, CONF_ALARM_ID) -+ task.alarm_clear_time = mod_conf.get(CONF_TASK, CONF_ALARM_CLEAR_TIME) - except configparser.NoOptionError: - logging.warning("Unset alarm_clear_time, use 15s as default") - --- -2.27.0 - diff --git a/refactor-config.py-and-bugfix-uncorrect-slow-io-repo.patch b/refactor-config.py-and-bugfix-uncorrect-slow-io-repo.patch deleted file mode 100644 index a0be948..0000000 --- a/refactor-config.py-and-bugfix-uncorrect-slow-io-repo.patch +++ /dev/null @@ -1,566 +0,0 @@ -From d5cb115a97e27c8270e8fb385fb3914af9ba3c34 Mon Sep 17 00:00:00 2001 -From: gaoruoshu -Date: Tue, 15 Oct 2024 10:00:07 +0000 -Subject: [PATCH] refactor config.py and bugfix uncorrect slow io report - -Signed-off-by: gaoruoshu ---- - .../avg_block_io/avg_block_io.py | 155 ++----------- - .../sentryPlugins/avg_block_io/config.py | 208 ++++++++++++++++++ - .../sentryPlugins/avg_block_io/module_conn.py | 9 +- - .../sentryPlugins/avg_block_io/utils.py | 72 ------ - 4 files changed, 238 insertions(+), 206 deletions(-) - create mode 100644 src/python/sentryPlugins/avg_block_io/config.py - -diff --git a/src/python/sentryPlugins/avg_block_io/avg_block_io.py b/src/python/sentryPlugins/avg_block_io/avg_block_io.py -index f3ade09..cd47919 100644 ---- a/src/python/sentryPlugins/avg_block_io/avg_block_io.py -+++ b/src/python/sentryPlugins/avg_block_io/avg_block_io.py -@@ -13,132 +13,13 @@ import signal - import configparser - import time - -+from .config import read_config_log, read_config_common, read_config_algorithm, read_config_latency, read_config_iodump, read_config_stage - from .stage_window import IoWindow, IoDumpWindow - from .module_conn import avg_is_iocollect_valid, avg_get_io_data, report_alarm_fail, process_report_data, sig_handler, get_disk_type_by_name --from .utils import update_avg_and_check_abnormal, get_log_level, get_section_value --from sentryCollector.collect_plugin import Disk_Type -+from .utils import update_avg_and_check_abnormal - - CONFIG_FILE = "/etc/sysSentry/plugins/avg_block_io.ini" - --def log_invalid_keys(not_in_list, keys_name, config_list, default_list): -- """print invalid log""" -- if config_list and not_in_list: -- logging.warning("{} in common.{} are not valid, set {}={}".format(not_in_list, keys_name, keys_name, default_list)) -- elif config_list == ["default"]: -- logging.warning("Default {} use {}".format(keys_name, default_list)) -- -- --def read_config_common(config): -- """read config file, get [common] section value""" -- if not config.has_section("common"): -- report_alarm_fail("Cannot find common section in config file") -- -- try: -- disk_name = config.get("common", "disk") -- disk = [] if disk_name == "default" else disk_name.split(",") -- except configparser.NoOptionError: -- disk = [] -- logging.warning("Unset common.disk, set to default") -- -- try: -- stage_name = config.get("common", "stage") -- stage = [] if stage_name == "default" else stage_name.split(",") -- except configparser.NoOptionError: -- stage = [] -- logging.warning("Unset common.stage, set to default") -- -- if len(disk) > 10: -- logging.warning("Too many common.disks, record only max 10 disks") -- disk = disk[:10] -- -- try: -- iotype_name = config.get("common", "iotype").split(",") -- iotype_list = [rw.lower() for rw in iotype_name if rw.lower() in ['read', 'write']] -- err_iotype = [rw.lower() for rw in iotype_name if rw.lower() not in ['read', 'write']] -- -- if err_iotype: -- report_alarm_fail("Invalid common.iotype config") -- -- except configparser.NoOptionError: -- iotype_list = ["read", "write"] -- logging.warning("Unset common.iotype, set to read,write") -- -- try: -- period_time = int(config.get("common", "period_time")) -- if not (1 <= period_time <= 300): -- raise ValueError("Invalid period_time") -- except ValueError: -- report_alarm_fail("Invalid common.period_time") -- except configparser.NoOptionError: -- period_time = 1 -- logging.warning("Unset common.period_time, use 1s as default") -- -- return period_time, disk, stage, iotype_list -- -- --def read_config_algorithm(config): -- """read config file, get [algorithm] section value""" -- if not config.has_section("algorithm"): -- report_alarm_fail("Cannot find algorithm section in config file") -- -- try: -- win_size = int(config.get("algorithm", "win_size")) -- if not (1 <= win_size <= 300): -- raise ValueError("Invalid algorithm.win_size") -- except ValueError: -- report_alarm_fail("Invalid algorithm.win_size config") -- except configparser.NoOptionError: -- win_size = 30 -- logging.warning("Unset algorithm.win_size, use 30 as default") -- -- try: -- win_threshold = int(config.get("algorithm", "win_threshold")) -- if win_threshold < 1 or win_threshold > 300 or win_threshold > win_size: -- raise ValueError("Invalid algorithm.win_threshold") -- except ValueError: -- report_alarm_fail("Invalid algorithm.win_threshold config") -- except configparser.NoOptionError: -- win_threshold = 6 -- logging.warning("Unset algorithm.win_threshold, use 6 as default") -- -- return win_size, win_threshold -- -- --def read_config_latency(config): -- """read config file, get [latency_xxx] section value""" -- common_param = {} -- for type_name in Disk_Type: -- section_name = f"latency_{Disk_Type[type_name]}" -- if not config.has_section(section_name): -- report_alarm_fail(f"Cannot find {section_name} section in config file") -- -- common_param[Disk_Type[type_name]] = get_section_value(section_name, config) -- return common_param -- -- --def read_config_iodump(config): -- """read config file, get [iodump] section value""" -- common_param = {} -- section_name = "iodump" -- if not config.has_section(section_name): -- report_alarm_fail(f"Cannot find {section_name} section in config file") -- -- return get_section_value(section_name, config) -- -- --def read_config_stage(config, stage, iotype_list, curr_disk_type): -- """read config file, get [STAGE_NAME_diskType] section value""" -- res = {} -- section_name = f"{stage}_{curr_disk_type}" -- if not config.has_section(section_name): -- return res -- -- for key in config[section_name]: -- if config[stage][key].isdecimal(): -- res[key] = int(config[stage][key]) -- -- return res -- - - def init_io_win(io_dic, config, common_param): - """initialize windows of latency, iodump, and dict of avg_value""" -@@ -192,24 +73,33 @@ def get_valid_disk_stage_list(io_dic, config_disk, config_stage): - disk_list = [key for key in all_disk_set if key in config_disk] - not_in_disk_list = [key for key in config_disk if key not in all_disk_set] - -+ if not config_disk and not not_in_disk_list: -+ disk_list = [key for key in all_disk_set] -+ -+ if not disk_list: -+ report_alarm_fail("Cannot get valid disk name") -+ -+ disk_list = disk_list[:10] if len(disk_list) > 10 else disk_list -+ -+ if not config_disk: -+ logging.info(f"Default common.disk using disk={disk_list}") -+ elif sorted(disk_list) != sorted(config_disk): -+ logging.warning(f"Set common.disk to {disk_list}") -+ - stage_list = [key for key in all_stage_set if key in config_stage] - not_in_stage_list = [key for key in config_stage if key not in all_stage_set] - - if not_in_stage_list: - report_alarm_fail(f"Invalid common.stage_list config, cannot set {not_in_stage_list}") - -- if not config_disk and not not_in_disk_list: -- disk_list = [key for key in all_disk_set] -- -- if not config_stage and not not_in_stage_list: -+ if not config_stage: - stage_list = [key for key in all_stage_set] - -- disk_list = disk_list[:10] if len(disk_list) > 10 else disk_list -- -- if not stage_list or not disk_list: -- report_alarm_fail("Cannot get valid disk name or stage name.") -+ if not stage_list: -+ report_alarm_fail("Cannot get valid stage name.") - -- log_invalid_keys(not_in_disk_list, 'disk', config_disk, disk_list) -+ if not config_stage: -+ logging.info(f"Default common.stage using stage={stage_list}") - - return disk_list, stage_list - -@@ -254,9 +144,8 @@ def main(): - signal.signal(signal.SIGINT, sig_handler) - signal.signal(signal.SIGTERM, sig_handler) - -- log_level = get_log_level(CONFIG_FILE) -+ log_level = read_config_log(CONFIG_FILE) - log_format = "%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s" -- - logging.basicConfig(level=log_level, format=log_format) - - # 初始化配置读取 -@@ -274,6 +163,8 @@ def main(): - # 采集模块对接，is_iocollect_valid() - io_dic["disk_list"], io_dic["stage_list"] = get_valid_disk_stage_list(io_dic, disk, stage) - -+ logging.debug(f"disk={io_dic['disk_list']}, stage={io_dic['stage_list']}") -+ - if "bio" not in io_dic["stage_list"]: - report_alarm_fail("Cannot run avg_block_io without bio stage") - -diff --git a/src/python/sentryPlugins/avg_block_io/config.py b/src/python/sentryPlugins/avg_block_io/config.py -new file mode 100644 -index 0000000..c8f45ce ---- /dev/null -+++ b/src/python/sentryPlugins/avg_block_io/config.py -@@ -0,0 +1,208 @@ -+import configparser -+import logging -+import os -+ -+from .module_conn import report_alarm_fail -+from sentryCollector.collect_plugin import Disk_Type -+ -+ -+CONF_LOG = 'log' -+CONF_LOG_LEVEL = 'level' -+LogLevel = { -+ "debug": logging.DEBUG, -+ "info": logging.INFO, -+ "warning": logging.WARNING, -+ "error": logging.ERROR, -+ "critical": logging.CRITICAL -+} -+ -+CONF_COMMON = 'common' -+CONF_COMMON_DISK = 'disk' -+CONF_COMMON_STAGE = 'stage' -+CONF_COMMON_IOTYPE = 'iotype' -+CONF_COMMON_PER_TIME = 'period_time' -+ -+CONF_ALGO = 'algorithm' -+CONF_ALGO_SIZE = 'win_size' -+CONF_ALGO_THRE = 'win_threshold' -+ -+CONF_LATENCY = 'latency_{}' -+CONF_IODUMP = 'iodump' -+ -+ -+DEFAULT_PARAM = { -+ CONF_LOG: { -+ CONF_LOG_LEVEL: 'info' -+ }, CONF_COMMON: { -+ CONF_COMMON_DISK: 'default', -+ CONF_COMMON_STAGE: 'default', -+ CONF_COMMON_IOTYPE: 'read,write', -+ CONF_COMMON_PER_TIME: 1 -+ }, CONF_ALGO: { -+ CONF_ALGO_SIZE: 30, -+ CONF_ALGO_THRE: 6 -+ }, 'latency_nvme_ssd': { -+ 'read_avg_lim': 300, -+ 'write_avg_lim': 300, -+ 'read_avg_time': 3, -+ 'write_avg_time': 3, -+ 'read_tot_lim': 500, -+ 'write_tot_lim': 500, -+ }, 'latency_sata_ssd' : { -+ 'read_avg_lim': 10000, -+ 'write_avg_lim': 10000, -+ 'read_avg_time': 3, -+ 'write_avg_time': 3, -+ 'read_tot_lim': 50000, -+ 'write_tot_lim': 50000, -+ }, 'latency_sata_hdd' : { -+ 'read_avg_lim': 15000, -+ 'write_avg_lim': 15000, -+ 'read_avg_time': 3, -+ 'write_avg_time': 3, -+ 'read_tot_lim': 50000, -+ 'write_tot_lim': 50000 -+ }, CONF_IODUMP: { -+ 'read_iodump_lim': 0, -+ 'write_iodump_lim': 0 -+ } -+} -+ -+ -+def get_section_value(section_name, config): -+ common_param = {} -+ config_sec = config[section_name] -+ for config_key in DEFAULT_PARAM[section_name]: -+ if config_key in config_sec: -+ if not config_sec[config_key].isdecimal(): -+ report_alarm_fail(f"Invalid {section_name}.{config_key} config.") -+ common_param[config_key] = int(config_sec[config_key]) -+ else: -+ common_param[config_key] = DEFAULT_PARAM[section_name][config_key] -+ logging.warning(f"Unset {section_name}.{config_key} in config file, use {common_param[config_key]} as default") -+ return common_param -+ -+ -+def read_config_log(filename): -+ """read config file, get [log] section value""" -+ default_log_level = DEFAULT_PARAM[CONF_LOG][CONF_LOG_LEVEL] -+ if not os.path.exists(filename): -+ return LogLevel.get(default_log_level) -+ -+ config = configparser.ConfigParser() -+ config.read(filename) -+ -+ log_level = config.get(CONF_LOG, CONF_LOG_LEVEL, fallback=default_log_level) -+ if log_level.lower() in LogLevel: -+ return LogLevel.get(log_level.lower()) -+ return LogLevel.get(default_log_level) -+ -+ -+def read_config_common(config): -+ """read config file, get [common] section value""" -+ if not config.has_section(CONF_COMMON): -+ report_alarm_fail(f"Cannot find {CONF_COMMON} section in config file") -+ -+ try: -+ disk_name = config.get(CONF_COMMON, CONF_COMMON_DISK).lower() -+ disk = [] if disk_name == "default" else disk_name.split(",") -+ except configparser.NoOptionError: -+ disk = [] -+ logging.warning(f"Unset {CONF_COMMON}.{CONF_COMMON_DISK}, set to default") -+ -+ try: -+ stage_name = config.get(CONF_COMMON, CONF_COMMON_STAGE).lower() -+ stage = [] if stage_name == "default" else stage_name.split(",") -+ except configparser.NoOptionError: -+ stage = [] -+ logging.warning(f"Unset {CONF_COMMON}.{CONF_COMMON_STAGE}, set to default") -+ -+ if len(disk) > 10: -+ logging.warning(f"Too many {CONF_COMMON}.disks, record only max 10 disks") -+ disk = disk[:10] -+ -+ try: -+ iotype_name = config.get(CONF_COMMON, CONF_COMMON_IOTYPE).lower().split(",") -+ iotype_list = [rw.lower() for rw in iotype_name if rw.lower() in ['read', 'write']] -+ err_iotype = [rw.lower() for rw in iotype_name if rw.lower() not in ['read', 'write']] -+ -+ if err_iotype: -+ report_alarm_fail(f"Invalid {CONF_COMMON}.{CONF_COMMON_IOTYPE} config") -+ -+ except configparser.NoOptionError: -+ iotype_list = DEFAULT_PARAM[CONF_COMMON][CONF_COMMON_IOTYPE] -+ logging.warning(f"Unset {CONF_COMMON}.{CONF_COMMON_IOTYPE}, use {iotupe_list} as default") -+ -+ try: -+ period_time = int(config.get(CONF_COMMON, CONF_COMMON_PER_TIME)) -+ if not (1 <= period_time <= 300): -+ raise ValueError("Invalid period_time") -+ except ValueError: -+ report_alarm_fail(f"Invalid {CONF_COMMON}.{CONF_COMMON_PER_TIME}") -+ except configparser.NoOptionError: -+ period_time = DEFAULT_PARAM[CONF_COMMON][CONF_COMMON_PER_TIME] -+ logging.warning(f"Unset {CONF_COMMON}.{CONF_COMMON_PER_TIME}, use {period_time} as default") -+ -+ return period_time, disk, stage, iotype_list -+ -+ -+def read_config_algorithm(config): -+ """read config file, get [algorithm] section value""" -+ if not config.has_section(CONF_ALGO): -+ report_alarm_fail(f"Cannot find {CONF_ALGO} section in config file") -+ -+ try: -+ win_size = int(config.get(CONF_ALGO, CONF_ALGO_SIZE)) -+ if not (1 <= win_size <= 300): -+ raise ValueError(f"Invalid {CONF_ALGO}.{CONF_ALGO_SIZE}") -+ except ValueError: -+ report_alarm_fail(f"Invalid {CONF_ALGO}.{CONF_ALGO_SIZE} config") -+ except configparser.NoOptionError: -+ win_size = DEFAULT_PARAM[CONF_ALGO][CONF_ALGO_SIZE] -+ logging.warning(f"Unset {CONF_ALGO}.{CONF_ALGO_SIZE}, use {win_size} as default") -+ -+ try: -+ win_threshold = int(config.get(CONF_ALGO, CONF_ALGO_THRE)) -+ if win_threshold < 1 or win_threshold > 300 or win_threshold > win_size: -+ raise ValueError(f"Invalid {CONF_ALGO}.{CONF_ALGO_THRE}") -+ except ValueError: -+ report_alarm_fail(f"Invalid {CONF_ALGO}.{CONF_ALGO_THRE} config") -+ except configparser.NoOptionError: -+ win_threshold = DEFAULT_PARAM[CONF_ALGO]['win_threshold'] -+ logging.warning(f"Unset {CONF_ALGO}.{CONF_ALGO_THRE}, use {win_threshold} as default") -+ -+ return win_size, win_threshold -+ -+ -+def read_config_latency(config): -+ """read config file, get [latency_xxx] section value""" -+ common_param = {} -+ for type_name in Disk_Type: -+ section_name = CONF_LATENCY.format(Disk_Type[type_name]) -+ if not config.has_section(section_name): -+ report_alarm_fail(f"Cannot find {section_name} section in config file") -+ -+ common_param[Disk_Type[type_name]] = get_section_value(section_name, config) -+ return common_param -+ -+ -+def read_config_iodump(config): -+ """read config file, get [iodump] section value""" -+ if not config.has_section(CONF_IODUMP): -+ report_alarm_fail(f"Cannot find {CONF_IODUMP} section in config file") -+ -+ return get_section_value(CONF_IODUMP, config) -+ -+ -+def read_config_stage(config, stage, iotype_list, curr_disk_type): -+ """read config file, get [STAGE_NAME_diskType] section value""" -+ res = {} -+ section_name = f"{stage}_{curr_disk_type}" -+ if not config.has_section(section_name): -+ return res -+ -+ for key in config[section_name]: -+ if config[stage][key].isdecimal(): -+ res[key] = int(config[stage][key]) -+ -+ return res -diff --git a/src/python/sentryPlugins/avg_block_io/module_conn.py b/src/python/sentryPlugins/avg_block_io/module_conn.py -index 8d6f429..cbdaad4 100644 ---- a/src/python/sentryPlugins/avg_block_io/module_conn.py -+++ b/src/python/sentryPlugins/avg_block_io/module_conn.py -@@ -29,12 +29,16 @@ def sig_handler(signum, _f): - - def avg_get_io_data(io_dic): - """get_io_data from sentryCollector""" -+ logging.debug(f"send to sentryCollector get_io_data: period={io_dic['period_time']}, " -+ f"disk={io_dic['disk_list']}, stage={io_dic['stage_list']}, iotype={io_dic['iotype_list']}") - res = get_io_data(io_dic["period_time"], io_dic["disk_list"], io_dic["stage_list"], io_dic["iotype_list"]) - return check_result_validation(res, 'get io data') - - - def avg_is_iocollect_valid(io_dic, config_disk, config_stage): - """is_iocollect_valid from sentryCollector""" -+ logging.debug(f"send to sentryCollector is_iocollect_valid: period={io_dic['period_time']}, " -+ f"disk={config_disk}, stage={config_stage}") - res = is_iocollect_valid(io_dic["period_time"], config_disk, config_stage) - return check_result_validation(res, 'check config validation') - -@@ -79,7 +83,7 @@ def process_report_data(disk_name, rw, io_data): - # io press - ctrl_stage = ['throtl', 'wbt', 'iocost', 'bfq'] - for stage_name in ctrl_stage: -- abnormal, abnormal_list = is_abnormal((disk_name, 'bio', rw), io_data) -+ abnormal, abnormal_list = is_abnormal((disk_name, stage_name, rw), io_data) - if not abnormal: - continue - msg["reason"] = "IO press" -@@ -117,6 +121,7 @@ def process_report_data(disk_name, rw, io_data): - - - def get_disk_type_by_name(disk_name): -+ logging.debug(f"send to sentryCollector get_disk_type: disk_name={disk_name}") - res = get_disk_type(disk_name) - disk_type_str = check_result_validation(get_disk_type(disk_name), f'Invalid disk type {disk_name}') - try: -@@ -126,4 +131,4 @@ def get_disk_type_by_name(disk_name): - except ValueError: - report_alarm_fail(f"Failed to get disk type for {disk_name}") - -- return Disk_Type[curr_disk_type] -\ No newline at end of file -+ return Disk_Type[curr_disk_type] -diff --git a/src/python/sentryPlugins/avg_block_io/utils.py b/src/python/sentryPlugins/avg_block_io/utils.py -index c381c07..1bfd4e8 100644 ---- a/src/python/sentryPlugins/avg_block_io/utils.py -+++ b/src/python/sentryPlugins/avg_block_io/utils.py -@@ -8,84 +8,12 @@ - # IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR - # PURPOSE. - # See the Mulan PSL v2 for more details. --import configparser - import logging - import os - - AVG_VALUE = 0 - AVG_COUNT = 1 - --CONF_LOG = 'log' --CONF_LOG_LEVEL = 'level' --LogLevel = { -- "debug": logging.DEBUG, -- "info": logging.INFO, -- "warning": logging.WARNING, -- "error": logging.ERROR, -- "critical": logging.CRITICAL --} -- -- --DEFAULT_PARAM = { -- 'latency_nvme_ssd': { -- 'read_avg_lim': 300, -- 'write_avg_lim': 300, -- 'read_avg_time': 3, -- 'write_avg_time': 3, -- 'read_tot_lim': 500, -- 'write_tot_lim': 500, -- }, 'latency_sata_ssd' : { -- 'read_avg_lim': 10000, -- 'write_avg_lim': 10000, -- 'read_avg_time': 3, -- 'write_avg_time': 3, -- 'read_tot_lim': 50000, -- 'write_tot_lim': 50000, -- }, 'latency_sata_hdd' : { -- 'read_avg_lim': 15000, -- 'write_avg_lim': 15000, -- 'read_avg_time': 3, -- 'write_avg_time': 3, -- 'read_tot_lim': 50000, -- 'write_tot_lim': 50000 -- }, 'iodump': { -- 'read_iodump_lim': 0, -- 'write_iodump_lim': 0 -- } --} -- -- --def get_section_value(section_name, config): -- common_param = {} -- config_sec = config[section_name] -- for config_key in DEFAULT_PARAM[section_name]: -- if config_key in config_sec: -- if not config_sec[config_key].isdecimal(): -- report_alarm_fail(f"Invalid {section_name}.{config_key} config.") -- common_param[config_key] = int(config_sec[config_key]) -- else: -- logging.warning(f"Unset {section_name}.{config_key} in config file, use {DEFAULT_PARAM[section_name][config_key]} as default") -- common_param[config_key] = DEFAULT_PARAM[section_name][config_key] -- return common_param -- -- --def get_log_level(filename): -- if not os.path.exists(filename): -- return logging.INFO -- -- try: -- config = configparser.ConfigParser() -- config.read(filename) -- if not config.has_option(CONF_LOG, CONF_LOG_LEVEL): -- return logging.INFO -- log_level = config.get(CONF_LOG, CONF_LOG_LEVEL) -- -- if log_level.lower() in LogLevel: -- return LogLevel.get(log_level.lower()) -- return logging.INFO -- except configparser.Error: -- return logging.INFO -- - - def get_nested_value(data, keys): - """get data from nested dict""" --- -2.27.0 diff --git a/set-logrotate.patch b/set-logrotate.patch deleted file mode 100644 index 47c507a..0000000 --- a/set-logrotate.patch +++ /dev/null @@ -1,92 +0,0 @@ -From d74076f4b772822de4f5bee1c8a778dd6b1771d2 Mon Sep 17 00:00:00 2001 -From: shixuantong -Date: Wed, 11 Dec 2024 15:25:33 +0800 -Subject: [PATCH] set logrotate - ---- - config/logrotate | 9 --------- - config/logrotate-sysSentry.conf | 35 +++++++++++++++++++++++++++++++++ - src/sh/logrotate-sysSentry.cron | 13 ++++++++++++ - 3 files changed, 48 insertions(+), 9 deletions(-) - delete mode 100644 config/logrotate - create mode 100644 config/logrotate-sysSentry.conf - create mode 100644 src/sh/logrotate-sysSentry.cron - -diff --git a/config/logrotate b/config/logrotate -deleted file mode 100644 -index 3dc77f5..0000000 ---- a/config/logrotate -+++ /dev/null -@@ -1,9 +0,0 @@ --/var/log/sysSentry/*.log { -- compress -- missingok -- notifempty -- copytruncate -- rotate 2 -- size +4096k -- hourly --} -diff --git a/config/logrotate-sysSentry.conf b/config/logrotate-sysSentry.conf -new file mode 100644 -index 0000000..cf5f994 ---- /dev/null -+++ b/config/logrotate-sysSentry.conf -@@ -0,0 +1,35 @@ -+# keep 4 hours worth of backlogs -+rotate 4 -+ -+# create new (empty) log files after rotating old ones -+create -+ -+# compress log files -+compress -+ -+# if a log file does not exist, go no to the next one without an error msg -+missingok -+ -+# do not rotate the log if it is empty -+notifempty -+ -+copytruncate -+ -+# ignore any following matches of a log file. -+# Note that order is significant, it will not overwrite and take the first match. -+# require logrotate >= 3.21.0 -+ignoreduplicates -+ -+/var/log/sysSentry/sysSentry.log { -+ rotate 8 -+ size +4096k -+} -+ -+/var/log/sysSentry/cpu_sentry.log { -+ rotate 2 -+ size +2048k -+} -+ -+/var/log/sysSentry/*.log { -+ size +4096k -+} -diff --git a/src/sh/logrotate-sysSentry.cron b/src/sh/logrotate-sysSentry.cron -new file mode 100644 -index 0000000..64d02f9 ---- /dev/null -+++ b/src/sh/logrotate-sysSentry.cron -@@ -0,0 +1,13 @@ -+#!/bin/sh -+ -+TMPF=`mktemp /tmp/logrotate-sysSentry.XXXXXXXXX` -+ -+/usr/sbin/logrotate /etc/logrotate-sysSentry.conf -v --log=$TMPF -s /var/lib/logrotate-syssentry/logrotate.status -+EXITVALUE=$? -+if [ $EXITVALUE != 0 ]; then -+ /bin/logger -t logrotate "ALERT exited abnormally with [$EXITVALUE], for details, see /var/log/sysSentry/logrotate.log" -+ /bin/logger -t logrotate -f $TMPF -+fi -+rm -rf $TMPF -+rm -rf /var/lib/logrotate-syssentry/logrotate.status -+exit $EXITVALUE --- -2.27.0 - diff --git a/setting-parameters-must-be-integer.patch b/setting-parameters-must-be-integer.patch deleted file mode 100644 index 20e11f7..0000000 --- a/setting-parameters-must-be-integer.patch +++ /dev/null @@ -1,38 +0,0 @@ -From 4abad77067557234d938de3914094c80181030c1 Mon Sep 17 00:00:00 2001 -From: jwolf <523083921@qq.com> -Date: Fri, 30 Aug 2024 14:30:46 +0800 -Subject: [PATCH] must be integer - ---- - c/catcli/catlib/cli_param_checker.c | 6 ++++-- - 1 file changed, 4 insertions(+), 2 deletions(-) - -diff --git a/src/c/catcli/catlib/cli_param_checker.c b/src/c/catcli/catlib/cli_param_checker.c -index e400428..5b38402 100644 ---- a/src/c/catcli/catlib/cli_param_checker.c -+++ b/src/c/catcli/catlib/cli_param_checker.c -@@ -17,8 +17,9 @@ void checkset_cpu_usage_percentage(char *getopt_optarg, catcli_request_body *p_r - if (cpu_utility <= 0 || cpu_utility > CPU_USAGE_PERCENTAGE_MAX || strchr(getopt_optarg, '.') != NULL) { - strncpy(errs->patrol_module_err, - "\"cpu_utility \" must be an integer greater in the range (0,100],correct \"-u, --cpu_utility\"\n", MAX_ERR_LEN); -+ } else { -+ p_request_body->cpu_utility = (int)cpu_utility; - } -- p_request_body->cpu_utility = (int)cpu_utility; - } - - void checkset_cpulist(char *getopt_optarg, catcli_request_body *p_request_body, struct option_errs *errs) -@@ -73,8 +74,9 @@ void checkset_patrol_time(char *getopt_optarg, catcli_request_body *p_request_bo - strncpy(errs->patrol_time_err, - "\"patrol_second\" must be a number in the range of (0,INT_MAX] ,correct \"-t, --patrol_second\"\n", - MAX_ERR_LEN); -+ } else { -+ p_request_body->patrol_second = (int)second; - } -- p_request_body->patrol_second = (int)second; - } - - void checkset_patrol_type(char *getopt_optarg, catcli_request_body *p_request_body, struct option_errs *errs) --- -2.27.0 - diff --git a/split-cpu_sentry-and-syssentry.patch b/split-cpu_sentry-and-syssentry.patch deleted file mode 100644 index 7b2dc7e..0000000 --- a/split-cpu_sentry-and-syssentry.patch +++ /dev/null @@ -1,155 +0,0 @@ -From 3f6e4d12618597b5aab6b0633f1bda800526ea54 Mon Sep 17 00:00:00 2001 -From: gaoruoshu -Date: Wed, 14 Aug 2024 21:10:20 +0800 -Subject: [PATCH] split cpu_sentry and syssentry - ---- - src/python/syssentry/cpu_alarm.py | 42 +++++++++++++++++++++++++ - src/python/syssentry/syssentry.py | 52 ++++++------------------------- - 2 files changed, 52 insertions(+), 42 deletions(-) - -diff --git a/src/python/syssentry/cpu_alarm.py b/src/python/syssentry/cpu_alarm.py -index d972c42..0b1642b 100644 ---- a/src/python/syssentry/cpu_alarm.py -+++ b/src/python/syssentry/cpu_alarm.py -@@ -1,6 +1,7 @@ - import re - import math - import logging -+import socket - from enum import Enum - - from .utils import execute_command -@@ -15,6 +16,12 @@ BINARY = 2 - MIN_DATA_LEN = 0 - MAX_DATA_LEN = 999 - -+PARAM_REP_LEN = 3 -+PARAM_TYPE_LEN = 1 -+PARAM_MODULE_LEN = 1 -+PARAM_TRANS_TO_LEN = 2 -+PARAM_DATA_LEN = 3 -+ - - class Type(Enum): - CE = 0x00 -@@ -207,3 +214,38 @@ def check_fixed_param(data, expect): - raise ValueError("expected str param is not valid") - return data - raise NotImplementedError("unexpected param type") -+ -+ -+def cpu_alarm_recv(server_socket: socket.socket): -+ try: -+ client_socket, _ = server_socket.accept() -+ logging.debug("cpu alarm fd listen ok") -+ -+ data = client_socket.recv(PARAM_REP_LEN) -+ check_fixed_param(data, "REP") -+ -+ data = client_socket.recv(PARAM_TYPE_LEN) -+ _type = check_fixed_param(data, Type) -+ -+ data = client_socket.recv(PARAM_MODULE_LEN) -+ module = check_fixed_param(data, Module) -+ -+ data = client_socket.recv(PARAM_TRANS_TO_LEN) -+ trans_to = check_fixed_param(data, TransTo) -+ -+ data = client_socket.recv(PARAM_DATA_LEN) -+ data_len = check_fixed_param(data, (MIN_DATA_LEN, MAX_DATA_LEN)) -+ -+ data = client_socket.recv(data_len) -+ -+ command, event_type, socket_id, core_id = parser_cpu_alarm_info(data) -+ except socket.error: -+ logging.error("socket error") -+ return -+ except (ValueError, OSError, UnicodeError, TypeError, NotImplementedError): -+ logging.error("server recv cpu alarm msg failed!") -+ client_socket.close() -+ return -+ -+ upload_bmc(_type, module, command, event_type, socket_id, core_id) -+ -diff --git a/src/python/syssentry/syssentry.py b/src/python/syssentry/syssentry.py -index 3d5cb8d..f93956e 100644 ---- a/src/python/syssentry/syssentry.py -+++ b/src/python/syssentry/syssentry.py -@@ -36,8 +36,15 @@ from .heartbeat import (heartbeat_timeout_chk, heartbeat_fd_create, - from .result import RESULT_MSG_HEAD_LEN, RESULT_MSG_MAGIC_LEN, RESULT_MAGIC - from .result import RESULT_LEVEL_ERR_MSG_DICT, ResultLevel - from .utils import get_current_time_string --from .cpu_alarm import (upload_bmc, check_fixed_param, parser_cpu_alarm_info, -- Type, Module, TransTo, MIN_DATA_LEN, MAX_DATA_LEN) -+ -+ -+CPU_EXIST = True -+try: -+ from .cpu_alarm import cpu_alarm_recv -+except ImportError: -+ CPU_EXIST = False -+ logging.debug("Cannot find cpu sentry mod") -+ - - INSPECTOR = None - -@@ -76,45 +83,6 @@ PID_FILE_FLOCK = None - RESULT_SOCKET_PATH = "/var/run/sysSentry/result.sock" - - CPU_ALARM_SOCKET_PATH = "/var/run/sysSentry/report.sock" --PARAM_REP_LEN = 3 --PARAM_TYPE_LEN = 1 --PARAM_MODULE_LEN = 1 --PARAM_TRANS_TO_LEN = 2 --PARAM_DATA_LEN = 3 -- -- --def cpu_alarm_recv(server_socket: socket.socket): -- try: -- client_socket, _ = server_socket.accept() -- logging.debug("cpu alarm fd listen ok") -- -- data = client_socket.recv(PARAM_REP_LEN) -- check_fixed_param(data, "REP") -- -- data = client_socket.recv(PARAM_TYPE_LEN) -- _type = check_fixed_param(data, Type) -- -- data = client_socket.recv(PARAM_MODULE_LEN) -- module = check_fixed_param(data, Module) -- -- data = client_socket.recv(PARAM_TRANS_TO_LEN) -- trans_to = check_fixed_param(data, TransTo) -- -- data = client_socket.recv(PARAM_DATA_LEN) -- data_len = check_fixed_param(data, (MIN_DATA_LEN, MAX_DATA_LEN)) -- -- data = client_socket.recv(data_len) -- -- command, event_type, socket_id, core_id = parser_cpu_alarm_info(data) -- except socket.error: -- logging.error("socket error") -- return -- except (ValueError, OSError, UnicodeError, TypeError, NotImplementedError): -- logging.error("server recv cpu alarm msg failed!") -- client_socket.close() -- return -- -- upload_bmc(_type, module, command, event_type, socket_id, core_id) - - - def msg_data_process(msg_data): -@@ -480,7 +448,7 @@ def main_loop(): - server_result_recv(server_result_fd) - elif event_fd == heartbeat_fd.fileno(): - heartbeat_recv(heartbeat_fd) -- elif event_fd == cpu_alarm_fd.fileno(): -+ elif CPU_EXIST and event_fd == cpu_alarm_fd.fileno(): - cpu_alarm_recv(cpu_alarm_fd) - else: - continue --- -2.33.0 - - diff --git a/sysSentry-1.0.2.tar.gz b/sysSentry-1.0.2.tar.gz deleted file mode 100644 index abb6fd18b9697ae71350686a3b0bf9a233d7d184..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 317440 zcmeFadwX2Pkw4DAp6AJXIE>9o0%@cfT^vNjuCQgS@5-`&k{rTg{GE(vj-*+SW@hK& z%MzZ)Br%wav9sY8a6-I5;F}GYYzP<|!+Y8FjO@Si7Ji?q>h9C$GB;g}*+@@<=bY1B zU0q#WRb5?OT|HY}bc?mh*`Z`&B#|0^U>p8M@V9q-`~mtGf5CtGU2<${}luEVGN9#qW?^e<8)6Z&P{IjLva(-oN*?bNPQPQ(37VkI^_6^3|F%<>cJuOubNxl{2+UsesXk zO32~;#0ZzF*YbsY?JQm-M@C{jk$R`_{YQZo^8ZYxkg2RC$Qsx@&K>9f=(vCWC&wpJ zJM(|@gzo~vvE_WB=2ni!^11X{rg~E5b#f?~O#N_Yc6UL*?|G3H^S=gG*dpui0oL=q z+5ck`$-O)F-@Rb1w&P#PRC5`3tyD~`m2w-$rjz_nrIG>pzc)28wqyTooZ{WrJeDo3 zVStav++yZL!Ocybb*nL&{b}-x)TV~()yi=7M7}s|n86uZ)+I{isgwCa!P!&J=VG<9 zWp}DnbgQeSTC7x*`UK7`o=QW_xKqP!EjtX3Q%(j0F-9*6`D|^fUhW?CoswHXfwuGC zTK#QC0CbxF$}< zN-LF8EmL#10>O^+e=M~A8%x6fzvKVe3hK9qF?=dh8OCUnT9%EV;fE7=cAkyxDwbdz zlq+tv8r!v&uU7NLmC{LiUCS@KYvtP6*sg4;e707p7qeiM*e+oiCl%XO&HtP0JTf-& z=)}p`b35zE_G9o}el^3c_WzV=2H$_#&DOTY0n};!kEDY7e_~_=`#(GLfBQM(m>TKh zvDqU0K&LW=snP9kb*F{xCCC&dH{XZk)6FdH&d%+_u$A1f% zp!e|~AHkNuPW;C`V1V}HPfU=Oy`Xf(UBL#STS;e%xpci~9^Erpw_MA^NmyNtM!19i zPmPZU^nYq%@7^8zKbn|Z{C4+)!?e0Tk*Ti6c02oE&R6m)t2L)TJ22>^MpC0g_|Mp& zbFiK{?dF{&H@liZvBaP=SIZ@0v1+Ma$+}LwkUvo!cF(ZUSY3_#-{T{gBk=NcCSMD` zh4ZkiUb{8t5jPgw-Tdn-cdotRaoqYlzieE&(frM|zkU5`+7V8xRr#_u1%BFMpCWz89?EIWze?5dkRNsA+y62pvBItLHAeuG$vobLsu2|qKS zm26fQ)bA5nXUNpnFP))+v=FT2HR!A7z4q6{J2XRK;Fm#D(f%V%Az%)DOb zO8!nKhE*pGQYq*0#~nk0)Z_n)T#*K+C7NhiKiam&s#aV0bsl+c)TcthD) z%~Tx#WI^KQo{3vTjXOzal;H^*rQLsZ_WYwW;>2|Yr*;}x$78#=R2ab)=$V2RnMQi) z3)f?m6$J+!;UXuWbMjSZB%ykHv>4T2te1Q5v(;wiR3={_=e*TwX<4BXkNJsV^%3zI zTiHv1+unZGiyeBp%g3xcg={>E1)x-YZLOS!dYy`^9}sT49Z~;^>l$BPZN7hQ{m(ZWzk7x3 zfXlyJ|LEfFn>QOTUtNFsi{`oKEsYk^DY&d6OoryS{ar*iN;G{G2li8)2{R4egZKX%$m@LVy=z~p~drA5HvZGYq8O@~3)sGM7+*8BFdZF-`Q(JY5 zj31e_!tF!8(&l4T5rhiT-&Tw(!Rf$~2c~D|qFFM8L`$}`n(?tFL$Jp=1agD?YBktu zz@vGVDlX?0(+Os&@0JDa)qvl)+I4yr;Tz7eW6n^~iT|`*bu#rcKhriAgqm=OdUML5 zFF%bt$B#ee2UQ)8EzIzNs4ERz>3OrJG7?#e+T6O$^7&60bjyu1hj$L3NQkHwHyC4th+;D^ zv*Pw({Gp#6?7s;d_X^m5sj<WDp$IhZ2{XdOMzq)(w`Q{fFn$KVJd4Mkc zy!pmwjmsCBe|e*M?xW`KuipLRtG91{*tqm^dW@@&x6T>UQ z#IQ24)G}|FCxh+y@huH4KaVNsbJ&=-et(TsPs8(7(=YwCFon0!%hSS%66NCoI^q?n z*0Wg%hA!PXG9bn!;;7on{M_xxNQ+NToO2j8aUbP#!T@Zl7{883P53C1a3iIhIAW5V zI*BT075+@;bf!{7_(HrFhe$p*N$|Stl}1oJWU4b!R4cfr+`?pUV7X35$h174O!b5> zRp|&fnwCAaqx6PuN<4Uhp}z9RcB9Q6Oq@|Fx>I}lPs3|kaDHUDa>Tt8%@TVz#1f?L zIy7gk-*jnbkB6ueC(aZfl=65gHz@puHSP5^d=I&G=i2kPuU`~l)?mTLobG`{-_d9V zF0Kd?IFC5VmUMWO7O*K$A}}>tP6^$kHsr3lIkpJ*JnLcj+#r^ey6$sj+r#TTjsa7G zl=~GmJ(RVv;Fe33nsyuKi_0Y^Uqp05UYy3-LD14?9Gs4_DE83YI3XbuQqEpy=c3VROSs;O

4nqQ;fp4jNpYC$plcm)6vXL&< z|Cj>E4@3JuokKk$Y0zQ)KfZS&c>bS5LY$rW&q#7^>C5kA{m;IoF86<$SN_n+lAG8ukfpA_x7!C zu{5=d$(y^c|K`p$_DqD}r~<$J9b6RTfY1PAyYHUAxPI%?=9@3xzWD{BQSRKj^|!At z-2V2{=IdXnO)cK(`rFqpweweK#o%+yVq#0TeO}q0Wlo=T`kpOU^2M67C-q$4Pg$u( zLNFfGP7oMGy_~}t0N(FTOAxtlC!&k}H3ma25O`ue_pLZU3@%(KtG9RHAvPXg{l(N4 zEXc3x=!1ugAs4V=4NJI*CpbDZhFdX}eEW0etp}>zHXTE1wI!lO?xc$Opd>6gj=Zb5 z*;C3K>9h+*JC01a`wla`KX;!YR&fiZ43_2;^?0A5#>rXQZ2E2!H)$cFBE4|TH;I)q zHh_lQW$ZTd7LbuN`o)m!0O2txZ3fg1R~T$+3&2|L7($d!B}eV4(MCMk-`hCOSSXY# zle`?tmR5>54V80{X{Wx1iCx{)yv<}XTOd_7fgd5T{gl(hawx8l9h=(!10P}uVStlS zRP0>|mEB6dlru3NTdPE9)fP|~+azMxJ%gY?Qg(<)%;lGt-3oZkS+11UoKgWo4W8aX zINe#o$#OP|f=28)_v<*f-lisUZ)&HS0MMUj4u$r(#a@ao#Iu67m%T!I@0u#;?d!Jt z*UTz3GcNC(-^qYpvZ|yRdHu*Z$z&#kFc1u)eD1t-l^7@aA`aeQ1p#=3X{ZX{Q6f zM(ICet+r01_uR(X?zu6&qIQ2|&vmTJy+!477Qy7yGACWfh2qQB)SjA_=pZwAc2$akeMAJn=%HO=z4fCi?JL)h$V~!AGL_6)bsyg@5v=Ml zm~IU978zbFmMvteRh&Do*7o5TrTH)^cKSaF;)%6P^ob79Vtur3Oa_$Bqy9%S(x|p%j&f{+Jp% z)B593a!`Sfu6(Bb=fU=$NA%|roaJwgK7ExwMkA3NYS)3*-&%i34vh|u5B-n|fB?X3 zd~jlL@8AzxwLdy&m2Fij$>5T$VW7pqWNI)uft&`Rrsd0Ea_?Xgr5+tjjkE+bG|{?e zXU;^9_HkhwS#cu6zIf1y371QiwG8;NR6ONYaG<%wY}rreFmTMgCEpHA8cxOhKuu?# zo->IyU=6?lXKE@q7kZmgJeAEnG82o}23^c58i6GoPK=yTNuE$`K07qw4v=*eU4L-2 zEqHKT`}*L|i7tw?ZN%_+n|=smw=FQvyx+PAv3={WLrDk$%(1b-(ea^Cnnm4|m6C1f zYrsCnFk6=#qH%8hSxcyvNEzAG8U}<3&&bv#+M*ycwsmEaRh?O3kYqwzAdht}(H6vG zK%{MfGiUxWS8ZE@UTmsm*{eS;FF=`@*oX@KI?+}v-gW4vOa zycJbk2oYfv^@wER=On7|sJpz%zRFpbpb9*Sa!L_*kY+^mRk$_(NVM*)LF?gsKR za4J+@{~DD{W+DCc6Zk>rOxxxM+kSOgevOALjpGIZSQWq-25(w07Jq_^6g~AC-r}3G zKxl|PQ-W}2w=$df{e36uxE%x;A`@iz1CgS8I-N!DDI3j{wIU!+<(8(rMlEh4u=E$Q zRN63PN0wJ}hcZr?e%+TTRyMOd+xEH;0J73*eg0Qgb`<|B$7gzu0`9Q>PmSyi`G5B! zCEQN@@3xQr0}tH){$B~=#U`(JD}_YRQu?SKGNwCRy3L6Z zAbhX>|KwQE|35K-E1`Du{}zsO zD^T~p{+CT&d;j;&w=dp(<6Bq+zWraMiW8QcOpQ;(sPt zGS6{#B1lPI$vrX^$I<;tJjP*~ZT5;WV?>dvn0OuHX={@<8^4gRq~z3&XFGK(p~J0C zh$z*c%@irW=yD!fmkcE5#94=8G=e*+yfQR9#BqKM5G1JyLfTIiHB_N@CV(}QLrgYy zWb|%NkBRimc6o}lOK2xq#VG4NVMf&o?MF&je#QUsnKTw%Q_+OAVbyxdjAxIo|7h<0 zwi8VuPVF~|h3Y)YlG!-Yfao^**z7$E*uTFNTCGIu_u_p|`hV5MvgL$}?W&%90G;&z z-m&2Ne+(A@@96(~!9e%9{+HDVr{Nj#lEE66K;zqA-1!nozCOSE&g&Eg&e>rlPOO~^ zz=9gerxS4@mYE}TG1h>WWoHw?RFpl2HE6_?YVT0!F{i9(yl^U`j@?x1wTlEloj@V& zkEdYTX<$PO7-@1B5oLz)d`?}@KyGeIKO+My~|+~zzl z5$M=W`tp~C?3`B2v)K+sCo2`Ac9EqtqEDN}C^}Q|_r@OE4j)Yfl{f0AF|w_-|6d(h z;5{C-f7z4z@09;@97*;9@qc5ZBe?%*C;!LxTL9mQ{@=*?uRGUnHZHwsC(Q7y0KN^T zox(@Lh`^Fan40f5Z1X8`B6E^uAI)OA0mjZ=hpA&uUPYh&FahTwe*p4fac&YAE#_1j zEy#MbOjIxYW(lt7DdBv+6F-L70qD`=&lK~>+)K`ZDF)j2H6^dz=9(mRk6Ny!i9s4n4^1o_;_?1@G-V#uZWrXvb9voI2;J1 zs1=2s$fDNrOy&{qFsk_n@0p!4wC(h<);FFYDV8WoC@4#ksnMQf1$>Hx)%Q?Z6p;at z`_HqT&uMlP5B@q9cYO!r7aY8_%th#fzpF9SYLG;B*Mn$Rg0H&u!h__RoG-PlgifO! z&{b_M6*5P9QO(g-iT_OM_M4bHE!@Wx=wX}`V?TtnG~{5QVm zG#1BenL_J8pB56UgZxj8j)m_385!L<|J#C^Tl@Muk^i)&?>_&-C+{PPZ}YFuQ{G=r z^xM|chX~YCal>~!s$Dr@^!6KZW?K06n51U(o(D_d4)HwEyB(}w_&_vW=G;2fLc11@ z_3@sOAc(UZebQ9lasRfOoIvt*PKOFhRlAMWdsO~9z0-qZnV?i4yy)T{gb^29oK?X^ zKKWWFgmr4q4)_*IrITrr@K*Q|cMi8H9;2(LhtRzdM-J8aZcp@4$Am0*9C{kXjSN54 z!&QOpwZiuxTv^8>2X<^ppW%9S=1v}VWAIq7THfI&RNv_K*VYQt?;*6F^grbgO-shq zUW|Ya`hRpR8M^;@WMb$1&pqL$?}7g3Bu~P1_wxGZKxSaACA=xn-aP0rPKRVXE1ziH zV$Hm0UD`#GB`i?+EX-5WvrD)`88zAkdl+j&eD)%Wz0P-M1sL79mH2mwwYCDRrduQ3 z@|aYwJ+PbTEy9>Mz&H5XgzBLKD%<&Pm`OOcyP~c{4?*6JPARVHWSi0cPuIM(9oJf? zc!atmiAr!kbQ(ALsmUV;$z(T&fLLA)3QUg#-ii~utsl4oUIdeI(^$xfMc)!RYLBj* zZf!{3XR++cGW6`ljT(v86xy*AeRf4p_Me>0=*9HowmR5i4PeP z>?9ldH=N}IxVUrtF+`fVMTSqGP1iOz0|-55ywC>g#H2NSy>k8 zd%3>T+sVnjB~#a8{{&65dEUPME6rN@K~r(hja=jE;q1 zl%osb$a!oK4*y>Z!NkME){`Dz_zj0?gl&MTHS(P`joI|ZtFM2ft0e69vXNG;PQQj`WVsXnm_WzWp&d2@jopff(_;x>i0HWPaEL%Ekk41Y#NXGe^)q!@tR&Drx-;U)P^5V^W zbvKT2^qv*tVmWv>lZwyox%O=NkdNv(k&Fs;&QfEfq~{tH?C+_6l%p1ov`nWTkq}QI z8RZD|QD>`%wzYeXX#PDos?sjV=Lx2+xNXlVDj{<^^+wWNu;fr z?QONS!v0@U@H^vNL=`z7hR*miCPd54|1ED@>VL@~F}xN1B>lg4e4PCN_!<0{ zKKtYUljD>U!WrL+#yi2NlZtVi2P&mftzFqp-`mx_kH2*qe@H;w?C7qQ64?#+f(Cd3 zXFTX%{009_jwi2OK^%7 zOUO}^$1Sk?N{K;du9i#SvQ@g7btE;GaHi=VRH}wN4K6MrKLrOif!ps^vz2_gmM;}2 zNr&bn%_=?($9Ct7*+M<%IzOt`a`{qX^>P0z@)h~s)r;`r=KSxJaH_Z~dzOtP{#eLV)@E}^&v%sy~B*<{(E1 z3_5^^9&mBJr|=lehu)^%eNnoT{KpjAtoUbn-$wqAl5OBS|Cii5ij?p>`QPsWhit$6 z*FyTc&;g5cGc!lhOS6X{7xE~{8M?#oQ+}mk6NWKAh;mvaesWhy4nv7i<`lfR7R{oKYe|0${Dd`d$|Iov)mt-^*miUFSVa3 z##PPdAOsz0oURE!k;$F}=t?~cMMk@>~hWC*C9mJds4tqP*}x!*2V zlLsF|RqH2=tts4ve9e;X8l`=Nq*V;<$5m>IVfm{mp1?24tai*y)RE&r$4vgDLPdiQ zmrwf2r#hBT`O1$5%i}gnwC@)(x=%Dk+gM>intGO!pSmJ2w55c(i$A&*l1fLP4hp&djI8iD(Bv}!0 zP>(lwxRV0}=S!Abn*L?u&!M~r3k!JfHB4*h7FuXy z1#kIalU|X1bF(w^OMd@v%KLn1R_eiLyy%R2?>)#*U@bqd*5&>J#zWv$^VS;dnD9RM z7Xc0s{6Whl{tsf$g2woh#Z|;2Uw!tk^5rGf(6Jb?rLoSZ#fvm- zkwy_YNN`PiK(W{P8lm7yMh$w0#l9KvuGi^;E`il+L5#P0sh5r6)pl@o9;^_#e3nAG z8E;4y^zw_C*|%C%V8aVl`9MV01IMdEEs5-vs#nxos>(aeqFwT9c-U!N`g!w>&;IuH zJ9Kd)j@Qtrn00d+Z}h4(O`aQZa-@^yS8C8nBofC3oN*BRXv(pr6c+9tHU@Y?h&7_- z8Fk8{k$$Zg)~x|}eg;nh`eZ9y4PI@l39n*=#5I*z<`8l|ABESHC2Sva$Z z+V#4PU+nV9J457XQQE))LmnCrw|p2AJ@#71NuS7-fFo%S<$R6=!9X-t`luM){RSC5 zxa4xs>>r+i)cdeA!4EmgnII*MH-M)ynF<~4Kk_?latEk`{g+Jb4V?d=8=-dYf9lba z+gS1K-~ZvosHm%Z5&)o$zWp-?rjO1orHw$b_F}>tDW;Td2X^b&_HUv?59=-O`2&aH z!uU!0$n?VWB3YRvPyD;<*jqQ&nk5iJ*z02L9lsU?Pi#f>&2#y3<3@@Fy^LBY8Q*Ba zC(*dF7HCmo$8-&n2VL)Fx5-nL3`CsMMu2@kciT5MN4K+9OZkuWS9MFG|K$F^9^?NI zFt`){x1HmzVYeTD9D^&@Hkm-zN8%41a2{HU495scHBO3@Ev2*=K! z85tctHafZ%1@5Wog?Zd_L*N`PH?kH6>sBfyPcATS9^7OkTxVus;qU^Z*&1%nJyBmV z_#WfzC~IpO4yMPKIEBx~Id#f$=O6w-<)4V^g|mO=iK9;{qR{GJa4b5Iwp);aqQm^> zgZr9SHu`Ue_5a93GCcorCD=~<=SB(ML%q$;e_x&lEC0jcx&2u6E*+#q^QRGaXsKlT zK3X$F#5SNKgjrnk$QjhgK)_$Ek|{go5+zr|SBJ76Ku#<<*ta>v1je*jq?f0BGJ&gkdl20(}TpMn4g>i^M^o%}BwAJiUd zZg&0~bDvlxon6bNk*+L zqD*0G&zSW#i-c`hN60lJQjpc|)ShQ2CWl7EkXHbvWSYqnyh!xSpv*GGc|)`7 zSm=W*==o~LBf6oTZw8GJ+L9qSZn6b>%@C4=wa80)Qr*cR+YSSiflwPWTsf zhS#w1tRqo{0wE^zILGEO#W^;MY0eoF&S3;hJ01nqD8{FdEeMLk>MEOfU=(^O9O3K4 zKep&X@w@B8gB$5Y53*&a=>1QD_WIu#berl(2mOy+Z^83l6B9f4KWvKRU02?${`cAc z;x0nkWAdB%?QQ7WdJ(V4M3{>A_doJTxXi$S&w)rrI|V{e;H+!pytohc^q)}61W4Ow zpO{`em|i@5bYb7jvDESB;sfzXN4=e!ou8RMd@OlfjwTs%C3-~=Y=ymo{*F8sa4=?Z zFMTo4Sz26pprUBu^HEY%Gp$PM6|bA3&ajmTV{*XLhuu2z5;u#_P%+-RdBCL^a4KdVzL(A`vu0D0<*U zo3*l4jab~;1HF8NwCTkI6s8!CxNj51EI#sjwEbjT3GNiwZ057(1uJ2*UdgsXdj)#j zx0KPsWJ8a(nrxn9RSgYYG98$nAtRW%3HGn%BxUeFi-MSI_KGB6^>1R@g{i@q?%_r^ zGv7iKSkA!vvTY>jC||-I6@)g`Q=f?QjRMJ>uC;sTf-UVoO2pr}dRHaL{_Ac3e~jb5 zcJlvpMcz#nYCHa%(M{6NQ@+ZLHq^oXA4yIG?|<1lF}8F5Z=>Yysh&>uAG6Z=;|!X) zEk%<0;aR^#KK?Ra>0*6Bu>6!tnoLK*#Z@;k5{t;1&zU`Y!f>bXc2c18{3mAEMDi~n z-)a7jPXx~YCim{#|Gu4rZo&RR>|UGz`d2dtEI}M^KYwlWD#+T zEBtm}#YIfOq@);2CWn$Z=)|YRa1tt%Hq3r!9S!piO$APU{lLpo!KqNpITaoJIz)er z=?R<$De@zqt_KqMYZY}Lu1*Qh9J{&z&}JN>`?u19vj%emKU>a!BE;sm{urYU^FKL( z!~XvKPvfI|ck(}NU^IKLvPJ&C#4yJT%bFBy6T>g0PjeQZUi5_l&>j;OOjB_rCaM=| zlukb$BUYnr_O#j_hp*7QBEJ}(W9n0y%2lWO2V(TWdy3Jn9XYjd>Jct3Bmm4g1Y_74 zqF4g{oqp!wXMoy33_Ezc#49v~C>(?%@MmWa*XIN=h|Zb94u5r3y?h+5B8B>Ml6g25 z+r=)KJ^vWk_uECXf)9zzQlru( zYF1%QW!)TwD$wU_2JcNTV!N=AY)u6cVN3(wo{I@+7HRySMai53m>7*o?|s~iLz>Bs z!GN>HH+IMV2@YDbizq7d`&W1zV*3JKFzb?FaP#*xvD-_@C|vaT6sstN)KI{Gasx z*#(68fC&ivs6`mVTB!;V%EZYsGnCH}Gb2`21 z7Rpfa&uSHQlrqs`Q*XpMd}L-}dI|BjhmR~Fr_dq=;&HT|{xIp7SG#aAC$qAHz*|@b z(Cq~XTvcJ$&yVQ$7(iQm|L5X%CW zE0&>_(PjV?=T34G&d~m!P^QR5=$pCp6UY)l)J-}=PYAMHPg>hW*_M25sxHDAPNdtK z1~lEMA&PcNO6>Ab7Mmv&x=alb%|KQPh{P$&K*1AULBwrRueeL8*e*Wyu_tBV?6SV| zjo2_MC&psCcCDQRcR534(B3No@}s3)GQfckEl|lDRLCOoK0RY+U%_f6hIB`_{MXuYXKGzWucM z`d5uhFE=i{(75!g#-+DfA*B%i+t(KWw0Y%ErcVIiVmn0%-GhzieK( z3_#LS*Bj^m(tPv9=GFJ{{_eXkG%jEGpL0LA+ce;H zo4D}m_Vsh7Id9=5K$|{L34+zx{3V2-?4Kqw(qQZr}Pf5cfA8_$ci+FI~QU;{%~HkZxYN z0a6nSoClj+0Ritk|9T5}fsXJ6B)5ef>uB&9Cm9dliS;P`LTpN5X}~ zZTuTOxqbaLwBI=Y=f>q%&3l6#)%xUv#s|NoH&TU%uX*JqDGc}kVppdYU;InsvtNym zJhcAq=grqXM1i|+(GLRh@*C?PevIM69Juq>U*3K3S6uyz+c)0?%-c7=#AIz;xUQ+0 zOgPQgF5JFxLB?FhNOM*5+NImq{}^DP=I_61UIZHlr;32V>}-B^{q|Sqo98|PHbA0r z{^Rw_zpx%J-ue71`E>WUzgvIhPmPPem&e=JFG83!|NVozub;yk8zUkU=KUqi_zzpn zb!toIIuEy*Zm0{$cRsx#B7w%^@*i~nQ@a1Rum8Gn;s4!F{f9sjT7&x_d+vPnOUN0a z&fVXDW!{8vvazOqe)BGo>B1Lx&;6dr@zLwem);7-vsTDVeq)+~g~ ztLGb+|I=<-FvnC|=N5@3fB1RxcUNdupm#t2SP%;co4_P)TtSvEpvpQ!cR~7o`}!vW z!Hf>jBQbxAMdhVm-TCX6(jD{;J-T!4r8}Q~4MtLZxpWb4018sFap@)fmEOqsO5OFF zpHtnduQp!(7??nT(YSyCm|})pgye8BG`li66}?7aK2M1^C9dZ!x_-GR35pX_4KYA?|ktW*b`%BzEbJ; zKWo1I8Vm$LBp&?s{N1Y&74d{r1gQzzpDL zo6j*%n>Q{iqv6(nV5%}nuQgx#h(?p&+ zKv4@C05c1NR~t88$82p}_)J8&@HH()=>JB;)Kkx4-$; zn|bejc=zq|_^Jj8BH`0lv525PT!$pUn0=@hBtR*Nk{F^df$p8F(B0>yoRUP$dh7r1 z#`x>&#=P`8Zu6p%%G0r(Q%UhnKew*0;{76z<1!oTp25Y-SdnIdJ-xi$NLr0Kt8J}k7_st#M zkMteR696!Oc*&WYJv6(7mX;3VaefG?LKg6O<`D8T4?2ft7WN%P6Vp%3&dn}8O`r}S zPrw2EJMAF*-qP&8qqwZkIdXL22<{a~$h=;3iT-{n;<8_$4(Ppb>sCx za}3dq8k}B6_)H$-P_;Yay;c2Scyys?Z58n-kZH9_iE@BrsMG^OR-6-O2dRtnR74bL=M}!D3g=M&hgYeb8t!n-Ex!))xu$e^cixv&!1w^DZLjNc%!bRl;-pM$ihl+I=fwX;LGRSr+ZQgI0PXXE-fNIoR0 zu2kGpNbp9}z%ZZYNrJ4!56XJ7N5Xk%!DW#VQq!AUZJyg)*%M+XXw7is0Qsel1Tz z7kCi=DeSU^Odj@139~6jlfhzquo=TMqszm1T4tfMT(9Bys}kI&^P*>pF5SvGT@ zZVtrtgDWf?LI^{8BW#Ri2AaYAI*~7mQd2t2G)iL2=q009&ouu(D3qVa6B63*;0PgT$m zGqz$$U#M^rZa@Ryw0L9sRUygI2R&2;NT(Gc60UorJDEkoXDg*D`b+~UBZ!<95P%0 z?mA{?j~YY_)o1jw%HgOy32K<9MG;r-lzPSXQwCZAdLUb@UsIMng^0TOG7d|m8p;d z#vDL{b0k=e@S4kKPQb3Joz-;2ipg7nP0#Ol_8p#ITA0NuiEdb%#s8LYFZYbMGTZPX zEJ4ub4?DOr9qHB<4{r}Vg_rq=_lr9XsPz} z;iIw&!=i8h^r7h|XBMr+-C)$}j@E~W5SrPCbsgYy1Qb6k9+}xUizr-fbbfK>|2qoA z=!F7|e$Ot_lJUU8;X~B&f^&2cSH}?yy)Xny^v*{`6rg9*FGI*d>i(hWC8U}|=bi#- zVM8uD`=$W`OI~z!nt#kYPaT}ychH%ec@o$F*5QSxvAPA_oLQ`Q_y2g7iyh!*u>M?} z)yvca&f-y2R2>lZfbVTCQ(UPd;>{a$thJTngqGb;EzANb9_}X&L`2Dk}`sB>~ zz8Q@7(o-`)#%?R34UD^8!rw9+06(6dof87gpciO@N>h@*MeYekXa6im{4r4umQItH zniho%WwDu+H9>-Wsu+!9=nhRDY=PZ4R09G90tzcW+`vNeCsT#k&yhkiOKPY!Pt**u z2%06B`J@ce;Z}rxn9dQ0m+==M-jz^Iqnv~0zr9;r8xJBVx zn+6k%|5MiEcOL(VlwdpOKQ=wOodL8Of4S5pmr(rB8Bm)N?c+a3Mp6@@`yVFu;{K13 zHZZsBQ=P){9p`~csnqs*>rUUd4551q=-5$s+mFYr`#Nz=j|{$(%edrpCF6$gB*fwj z-WXUe^a?v(^x$eRs#zqjPQ?^|m%|IPa?kN}0 z|GhUw@_%RkZ$JBl=l|49z2H_3W1sgdRv>E*W@TA?(9Qv*Z@`)~V#0g#9gKtg?X}8T zB@K`-W|-sv=Zx8NXnJ=3$n5^uKVvO(sFZW3U}uw$H%1;K2o9u?8KP6M@CEBva5YUO zN2+KxBp;@Bq{C)8#LV-dWMU+d8eW*0-hXIoj(r%^I?Vs%-ehq8k0wWU?td`!+s+@m zv8t^u@`QB|akzf-V$8fih&}`d@CY-+T|j zI8^@Dwe_oiLQs%`^VJs=l=K#&p5Ix2??Wkh=iAE&Bc%fbzj~X(>o32+p+pqF4D{}N z^9`c88ZTZ#7`*^#e*Jpm(%XbK+Pd|t+c#dpdx``F4i{e(n23bC``h13C_AE;5Ch7Q zLA!T5pajJ=e)`g#_s^S7#D4nIpCS?MYV0h0)^J(S`BG;H8(oH`%9O%EM)^fGVljeu z<5hz9?SI}q_nw3hA_}*0@vT@aN$)RTLujr|anlrFLo8#5huju0wM<(yaK10v(n52% zriUX$DWL4-_jsVL$p`>QtzN|qKDTdv&;)6(yt4lC7fqb5xqbtgG04B+hl|3knxs#< zP3~q2J(0eNN&wQ*(E6uWFRHf8{*UjGQYYtU1iG~#jY zpac+is9GvJ;hK#LZ#7=L(c}=%#+U!qc=`8tZr!37F=lu{fC5i%{TjR~)UoiP3xtKj z?;%3%Kf(xvY6!_A5JBWFS{4Y>?A`Mh*Kd7lwT4)iQYIIu3u13nMi}l<@}G`mZxmLm zdL;jQ%>OV#_y6qVe`rO~t$QCe{<37-IyUHi{KrP={Zm!UCkIqg zz0A4QY$ac&$k(a(qBtV4zSfS&WpYGvusxwb$f$a{;VB;){R7gJil@}*r4hxKnOedH z3i25dSvbbAoCiWv$5bS~lYTu;eb)0*XE!S~}H~Dj?%)lkQmy>IV z)Ua^lI;Nmi#k`BGp}YoD@^3=^!SK-j(J+t2wqMOdmBP?k)Pn{z?AcSDQ)sF!=(G80o zNB8uA(IK8{m8^I&%X03rlTOp&+H|@fhr*U|gb!BqBsNrV3WO<8!$mIkGjA~YUzEz~ z8I~m9^P*a&7OHhL-an8qtq^<*wED?V&F@l~4>nVM*ztwsga(Y@AN<78B-M_w__!2$ zMpwGKS>_jjTq~o3+z)6#`G*1D{o~+=2uoN(mvB6%t|IS=X4P7$%=EH*N{9-j)9T_J zKtDhikr|-4Tn#r@BBFXq`o$C`V2NrO2Re`$mivtVAOWZ@d~7IroO6HfDd%%>|1o`k z85%Z5D}j<8EEwwkTHee{94VFEV!sB1lR~u|&elxXC1#1uvxO3p@h|79yf*S+tCK=F zKVPZf@(zPP^+V0~1m8c|AE(YTT8qONTeGk=5pznx)vc09&psOW!ysL7+Yq2 zF55zWT0ADiV$rFZLW+M6K-|R&EBR4uHeycqAj?pEeOg>MsCF z@O&gN`j6q%%QzL6bJMCg$y*i!=(Hg=+U2rX<}Sl8RdXJka{B7!K7XqiAw5a6t8Vrr z-J#CKE&5T@suw8kuTS4(ltly#I&p?W2S=q@3Tbk4-fid_FSmG#uekNYW0Yx|7qUgZ z+kMjWSEH=t3fMa&SVF=8vy=~COgStHR6{F_!=Tfb>GS3!b*fZNWLLrbB4}|m!HKE*>fkq# zN(rzy9k0Zk6~*QwUkI!sbU?bA-i5{{yNa6^Xi$x*)JkBhQS^LH)RC5dkn*6JE#()N z_9H9H#^ii>KxA-v#z)fI^q4RVSKr?86ydpny+gC z31GJ@&6X`jvNWM63{N(|a|xvq(Bcpn`6>q(6|-)?W&kH&N-W`A)a>D>Dmb#_RtES4 zDT!KDB*D5o=a(-u5WerWc;Rltq(EPpBgwol{X{LY>JvHlM17?{F8yL%58_g#hjKPS z)ok$@DA8o~8^oqPEhobl2|ABEBmQ17U^WX)Y4m~A0$FOy5bAu8-ID2Q#Ml8$>Xw@& zV?nfljG$*?ou4eYZJ*)1E2 z2G#12At9DkUKctL1W5XeKR)G%?AKPf^N390h;CTsv0^ZrRfJNCHrx|+D^M3*uxFMx zNG61^jTrYja_5(dT|4D9{(&%f{Ac1h>GZMdBm9K(<(g>#1h8gjL$N6v-#i(@U@HEv^*z!z641 z=qtvYSiqwN^((S{E*n^<;u1N(V}Z4>_;d)6{^PnWbc-dhsJQM3tqS{X=3xc!pz_?b z!`@5|ZZ$YLsEw!zLhmCciJ?%?b>*iR7i%+7HsGJ$sj|Hk=x+Z92I&bLp|5UW^9Sg+ z-~SoG^&f%ze^VoSckch#-f`E^clLjF_J1fZEnCFCBn1-xLk>=w`Z)WGbG$hucO{Z!L?iu_CSNJkA@*hZY4e8l zzL-ns89tJefVOjxy9M7_(`rMF>cf^=!ar=G5h#Xv;w;1Q0J4p08C0Ga~}$eJqk_Yu1nob_BNDyq@_; zB_Hn@sg_&P%6V3g<#QV*+j5-f<^k%Ngo;!?67uem#?4y}(Ca9&VSvD3aXmiFMgU9i z)DZU0gH*%gzs~$bHlSt{k4vL?ib~R--&$oqTd54g+VtBj+0Z(nlL%YTzB%rpOW-v! znktwxOodE=ZkohtI}kJAU2oBd-%R(n5|$eM3`N$4KJ@%nZv=XtH;+Lu6G$bx;@N9g z4R>&RK#e$ek|*&Q>?8W8Flr(pt4k4FR3fbiXB}@W?ulK588p=mFyeEZI+iUO#JV zcSMsO_i1=~A-jU)yg2DnHc;YXuI-H%#Ubdu^ChipiJq1_1eHCqhFl>Af(MialZPBb zqv5x5g;3e<1SOR;jskknf4W!P1ReI)fB*eB; zzrE#}`B*}zB6pI$K~=z_Xt|oUSFPBZrA^vk#G?)F`dasvr>R5+_G(v{eX6SV5XegBW+aoECQZG}L_D#UMK}h> z9xdn3xO9LDC!8qNbOo8u(xx7VZncj4Sb|+M=p5qA7$6&^_g^aE#VlBOdTCmEe`xwA z<_Swx+%%80%1cOQJcZQcMFc()quXS}hY9o2kP&Z3&07$gfP3$Vw0MdGRy>BG5%SX; zJYXe5Z6HfAau}zP#rf!rsy!ZCoLN9-;PlLq!?-7YY4*?z-5-Sg@x&;yrf^Uln_HY8 z7TDs7kd+vla};lff^xR0kC!|+&BSwpAB1Z7P(t{ZevfIx)>fZ4rSwPdja5Wg&MqQv zR1ujH_#B=axT#*Dzlv13MdTH>5l^2gam0nfUJC#~(m&ZYAB+La{+S=A=Z}IqB!$r9 zzTupEYPeW06#8NdhY_7@-uK~n_S1_>OwApBGJOE?#WV)Pr!tjcENq62_2>P9lW7G5 zH8WWz{O$IH5(hwctqx{H7+|H^%OwWWO{ubadI1G6Dlf^sNlQ`;`^*)bbWBUP5kt-0XQbSkT5|b9!m1Oo0D(6bl7n%ExXG|gecB`5<`3nbZkz}y=;uL;Z~Bn)cPhRDqxTCsV-YX?S8 zv6@CA={CI)$=pLzn!mhei1O(I#hG}z+$)DcXNLuuG}v5G`1nSfzyF^_0&r zPgPqm17wl=uQ=<2D?=n%tuC&8tW0$HcKV2Q`*@E8D;%pAEz#kD*HEz{ zfMvD|*jejMRAJIEUN!{f+aCgzpMgs;zRbe|CWSG>mfeMPeRDlg^Fse1+$Asq6krvZHPb$-o zN$heMczA$ASmPtVxFJE!CXP)Quh-g-#}gRX8QLAwK#%je9Sdm47tD^p5CA5x1w}A| zF-7M&u&Pw^_F0@fsl~}~g+nn;#Nh|H9Pv{;wyKVvPQSa7aEd0+fyxy@29i3C!$hEJ9yseO2({oX3S}igpXo2I z&%>ZDQWVHbhq4vWPeWJo{{#EebWKs8I5jB0BbFdT(pOop5W)}*u8RdZ>&V z(v;~=G6(1)^hfe&f=DCHkMK!;S)&_^pG9+U#3;+^X7^$heU0T{+d*=K@}>T4g;l~>wKSt$W%ic;Oztmd{Wx>wt7aj z1M|7hbPTg)EmI%~?_zaS}*S*16Y0CFdtZ45nu!5F(gUfjrWiO zeT;{~#huK0lKbX>&oQwPx1z!-;8UbtsM*tLrTqN82s(;1 z+De{!<9o`XKDH;EuIzbqxFaO3R%)y0HJ#PeT@zBlDA6yRAK8ZzdqPtaR$PGXkqnJ4 zwI4mY8Z&QHX~_7katVnNqc>(WSZ~r8wxR|L>i{5WVcUlgqPIP9AYZtDito6qwxYNC zE+_{&iTX0++(`{h857#G0N^V7L*!Vbg|<4rE5gC2N_fL!K@JesJIxj+7l$&^6rnfi z8w@nng|86mmNAE8F3?19Xu#lK6dj97#P4gCLu~E+@dXy6|*S0+R{L?MWu563MoRz z>-{a;5?HJ=Wt17cr$QC1I7vxKiB$k?&pFHH+_zfdA&UmCj9yK+yteN5hmBaOK#zkG z_HZ!L0+DMWos47<2}YfGlTMkEvfWB-?IW+j9wVm|wEWYro~93^kIv8j1i>J(F-aS; z$V7$ffDUo=2)z#Un3&Sq;I+C;c%pCxc|K3M{lRiFJ*kY> zFs_oyg$q&{`ZEbj0-y$+K*|yipzMq?$Y4N%*C#zRN|j=02oFByDfOXA15HY0#jT2{ zJhGiw7M7J_C?DTa_YBq;MQ@*hp5;EYf@MfBC=$JS?ohZrbu*EB7K(< z)JGC#_Ygn&nzbcoQ(QwmJ4-c;Hj!53)EAkYQ(d~nzHG4Hp7o+DhMSQbS>Yp8iXT)( zZ7(y?eH~@=IAXAj)EzGGVQ@4Yf6&||Bd12oH<7VZ3k!|+!v#Phk-!i}BP_DPuW;

7=5hw)F+GO|DmzoN8kN8s2BN-?SOH`PB!VT+I;2Z6P++o){Vc2mlwo7GH2(a?g%bSc3`TT~UogZ_;56fWd5ov3W6 zk6X6L2{-OD#ClohsF~Zw-VS!7&8o)`QW;TnY$Z*{R;(qDIm)6P;|e}7pe;O(t+nQf zEhNoXPyESZKt?nI(cmlNHHZLfkoTaQ1|@-?rCjZH&FT%wdeH~_F|s|oKsRj#JJA*y z+mr<|_G2V6c7bk?v5^lU(i5hTxP^j{OwJP}_>Cg6{tBivGr&Cq@O8S^)@=&6Y1yY$)hb?&jC(UiI(Ql>N?lIJ>kXJs zQj_rEXw|AdzH6vSw1Mv{e7ZwGQ%(BAKE6P}_|Wzg;^i!WUb9k~MYM#+(hRVsvFUHP z3RN}y;_a<1TUoRp{^fA^{3P@|Dd5Z$0xlq@3a6pW3Zjh7t zw3UAtPGmkX!-Tbl zMEuL_+UANpM3JN7btQD@ka&;&ydv#9zM&3!(?}SYWA<@eHoIELsdSOhG&nw6$GN|> zPGAZoB`-9-=(LeO?7I@l41!n{p~;kG8W$(v76^`_M<(!Uf-+vKcd7|ia3RW6<>=+2-XRtjyY-OdIWUFn*veW=7^LP!c(&cBtMMmmLad@ z)ukrrZ#V7e)UdUKm}R7Sc24JOtNf~efR0yFzAYbGR0pka02=L?^6e0k3pCCf5d5WG z-c5o)f8k@kc7mJuQ*uX1AaIT<`a{j>jsEKZQiu|O7!6I4C38X(CW;6&&Rf;pNZhDj z(fHAq4!u=wk1+eg9%TJ5S3F?R&l-g@rqi@g*?;*Z#iEgmzyl(p(?1!6c|{RHFtimP zOvgz%_8{U#V5wK>W%U@Pu)udxMM71JNCwU4`xA?^PtMLS!3s8yOE~>*al2Zj5Y=eF zBy|Z^1xxhcPj1TylG#TE<(vr}k}$QruA4$UEq#TKnaan#^15}j>w&XM%1;6&ac@+f z{-|)m%Go4uy=h2-CzO7iD+VuL(KR7dPf`#7Lq_!w$GiDNv4gUoJPy#KuDouZtpv2#JaS4!YJdiXEG6O=1zUyf%h^G=ODubd z)_q)5+ek_+gB!bwWI(yKleh*v80%oVhXQi3h<|&SGsd}7{+FCsLx`);Xv1*SzW_~$ z5}<%*F)IJ+AZcCXwerTzuy%5(TRBNh#7#q$HU!u@dEl&&^A zf8ely_c|OpkZ-(qOda1EL5WxSCqnjW!Ebdz1ahbbiDwo%-B&qy^hl32_8pwt&sA)j zR#DO(7-d4jV#`-o5#A3;4*i716mlX|HJ4Ha%I&_0qq&u{LVMf|2D_DYnWZ6@v`*o# zBw{Jmi&cPAY|#6v^^%%Z9zvNGYKqOAp=0$pRnTgZ(Qz3%TcJ|hB;@1)OzNx-84bDY zLyN`x=c-zh0ixaguh^)-7~rtL1`t1W^esH*6tqf!P zVOXyH9YvlJtPEjHTgEu~Bw-SR&WUdi0*D zaf)afGYhErU)sVHpH255i77N!XogeQ{7}EJ{6NYL3Uc^YUNo!4HwYqy0?Y@~S(hXa zQlIXtbrqt1r5m%jTN=}e5~Sz6P2qz{&J1C03#z=>6ineW3ufzy(gsmYvST>KTkDfs zR#gz7XX<(U8HD7G1%}_VS+aQUz#l(M22w!VewyZkT;N;59AFP17$da9^JYIZ-ZT3A1fIhz(*U1EN0qJ zh8V##i$(%LOhRaL?#NFP<~>52(W0igJ3x+E&#do>iRlYEpnv*LWT_F@WaM0%5m{TS zBk2>ejC3<9n5^n}tj6}(mzP#MtP1;o-3AQ6c^49~Ntt*@bvqLqp~rllEuWvfRKe2Z)6bnqZ>Y z)C#Sg{hic7=8Ta~9>g$EY0eyyhT8xd4p|*&iJw+_gY8W^L9T7j#B^n)PKm#caFR_5 z@mrblvaeu`pL46(N}inoQ+gGl4P~QYM8`0e%3(@kF_nf7sJ0ola*?z;AzQ z$~rDXSG{uMbisrCN@Q~Iq?-5G!7r4j5HP3~;Fig~mcy2->o>XMT_QwxVpnTZefp%H z+G7?JNQWa`1O&f9gRzpY{VKO6GBr-MBKSbhpWy zNhDUc)cD(#vfZ^y#u_Fdg*vw!X|IgwylG$R`rG%)cZUOoCAMyzzM6LFY67IQEWo>q zOTUH9$Y2KoJO`DSz*t*zi8+?Nig5gKF$f8(WJ2>l3k+`0lkB8zNlJ<#2sEFSGpmyh z-4ny{P1s^qxw7K%=NLX4#VuRliDlWbZJOYs9(*6x?BXOBba!o1!doM!+BV6Tw&?6% zb}xEtk=D1}wMiT!T)P#0_4u#*)`gF^qID%Ex}vp|?Y31rdP=DY)~q;sQm`G}aGl8B z+)%Ouzr&E}V}6%2jIih;hkF3~oI&4-*FVwjaAO%HiYTIlEGnsu=^6z-_(29o!5)ODp7;+Hv?4l_#0WgJC=$>tcIiZ# zLe^t2g_u;+t8229^({9+T%#)JwNF)h3OrW_H7dZx+u;dioq}@U{@G9I)&Krom&epw z=l+*_!Wui9Jv;Zmw7dU> zS4Non4&UlYu@Am&9q$U4z+MgfZ@7O&j1|3qQ^DmDayh723|(B@5_vNVrH@}sFU}wU zZB~}!OIQ&hx#0BTKjS^_vY38yMqkom;w|}Hr z!V=d&F9`Qa&HwrE5$h{f6&^mgP+{&c@=;N9R9TtAQb6?v0fRXGF+E3x;NrrWnhb8N z;4UGT)xy#fGt*0HyO0-v-?I4hzNOj2bnQr=S2&ZU=nX$|{;t9PusU~m-h5RT#jCnA z^FP)XUw~YvIQ&a}BCO|T_c67I(+fzuiSfa;0}}DIc=W)5*`E+z3GhK5e~@?T-iWbS zObrI2!=|5PZbtF`_*s`yNym#N_`MZoZA!?d1hEZLz^2=u;MoiQGDllI|o$}1saD%%Yc$#=0 zkrzMNRovf`1&>Q+c)rzw#{fOXTXOza`u^`TrytoP8LhE;ArqfcT?4fQvRP+W`&C)% z8r6<{CW2bsZ98CwK0XpK1oYkzs9moW+L}K&o=$F6CNOQWRV1`hnNn1rl1i)?m_ulC zDL8ts%3@>s+J=pR9*Y+(_K3u4o0n=HGvd_R=GCElDxgv!q~o7!tCWwHw4uaPYLBmv zA?T&<$Z{b=yEw#4fhMJOg__B`>8q#k4G9WwNLpY^P07B89A2=Y%4idgKBSpDqX&f) z$QF+X>!#Mj;b2ex8RN1dT+L7^Ew$0QP+_gBdUOqw2E5dA5UDrEHC%dNao!GEn9$;? z^t2ca;pS(62T!X|Fe8%B5=nhqgpmbN$Dm+M7Wo<0*=nOo?Gt&sLmnXl>}ndttQ32| z%A#xS`ge${(X&2{+yac{bz+sTyya|63qP9cFnTSBggx3&6 zO2Xy!PaSGPmg@psNa8}97(s9?x&KRm)`>h5BjWH@oEFAeMHdL)spCUAw~!I@f!J@; zayMxBbu;tWwq`s)RPjvg11|X$#QyTm8?sFTNh2IIOR1P*=?HeWP{yevcZiC`hpkyx zC%L>9ydhOo@;(Xid{e~x2+Z0}jX=M9o)^AOSZwLx>%*l3l`ui9;%z6!);8R3XwWM@ zjpAXZ{D~1G!9h^Mr&&3KfJQ&H@e6&aCBdl z*&u5(tr9v|4ICdr5*Z^!jdn7GsHh@SRMnR1n(77cME@`jxDSO677`WxQjoe&)bkk6 zZdc?T4kZ;cCkoh-!}7(@3odt$PRNZ&Yj0)84DeKr9hqPnbobYcB$M$<-GX0)1`;@+GG>1+;WatI?#IhGG3tjKK&1Br}%`rp!wzLwf)I@4e7AR0H`981y| z^b&R45y*NoBM}}rj8pg}x6i%d?Ey~;o4_PZ*i7J2p8)Ks-;c ziXo-?R_2`}pDi zXYWhfyhKqBO_(Z!a0B{@JFx*|&@yMmYRe?`y=5Bx=J=dElO+$i`-- zTogb#CJJl`OXGTW#~xQbpsp~ryupf!hnbBg)j*H5`@SP7_&XFw{1Q!IDFm6I%1O~2b8Z9k}-ONFeav8Q= zoN#**T^1O@vXP5dg$;YU4b*CMibC1NtHM7ca^?IxE@l@OWgBc5dTz*{c6%mYLn^+F zu*v@?*|5t16H-og5n^6S_O^qtxLr;aw|rh3j#Th3OUX9p?h!qsbKU9g<62YN>0w{V z%^d+pJT2J`iiUBHS9(rfDJ$>AV5hP%k|cBSv{+D2Mf4qsYQm6g*g+q4qw6wHn1oDMj>uFgzoP_hXFCITnYZOoK7?H16e4Ew|$5UU$9;M~nDKT!_ zJxfb+LD?bDH|d`FlAu#Xq-?tWgArc;T z2D~e4b`|%&t&5wCDYbwq>-e>t6wDSjPXGjdE^ltGpBOo?|LtV|+gx35!4VX*>nwOp zgmXV7Ad4#{k;mK{nEKlD<*l_vah7!tU^1*GvS`bSi_%(Xg#;IBA~5Uw5m)NpKQ zok1x$1J29xE4by@Y%c2Er*z1s*uMJ56XQ%tU*BwrA7z!VnrrJ@al(5$rc9Pj$1qMC zu<1ZhMqjnG5rS&+(S=AE%Su#11Cw~nLj3rq{5$_76arY4!3woA{$(nS`p2#n|6sVi zzJkD+s2-Fa)DQUr^=8tpG>|ql~M#zmU#rRF7!Av)H@?GmTM3%DIchzSVa37BEbtYOh(%+z(G@;8qz-8F>&KM&PV z)L}_Ka)XB8jo1>Pi9?`2bd=O_XdbFW%4Y{9#v={Du@Nm4amI4)plTN}%M4PEi4l6E zzdg)z1B}pf!7NKW=Bsxu(P5I1Rqt#Mq$9CH0(goF2U_`b@huS|sBK)?Bx)}-#qD3H zWrO-P1S7>cQ;~94HZ~B&j9~rrK+7lm>~)ywc)}PAZU>}l2g+Fh&(5w~ovt8tYyxu0 zgSawV8;cG3es6`BY9mXHCm>3+`5loL$W2FD%Q!Z!LFFGN4in0AQdB**8}HOoWq>xO zteDdQM(sqM5M9OTaU)5&hy?VOc>Yma(6?`(Pek!GgJrhJ58B{Ka9I>v99tZtk_9k6 z23%n|x!xWNL+5LLtPTFTFd8_OzI02tO*TV zG&Dv5Xg=PIrme{<_@$OKC$F0A)(T?L=x)W8TvG69 zKai(&2@!MZ1uJVtgQLklOkT2t-!v=Qh{zPQQn{x#J3BLbpX^Xp$~R0{@+iFEFoB3z zStL3QpPHCDvrA2fsmCrrMGCyIiLueqBfVE6&R9z!b(|YiI*_MiB`)7GMqiihw6wA> z77&js`*t146)xgT0^?v`Lb!Wlt*U{eNdH{uVP zV00prOczj~lB%T}nV`^U+72g5fl-L-o2!hA5{f%a6`Dl|x(j1P9#GO){?Uc86GJ66 z-7cZ&_6q%}ike1v0PaL~;m-DB+$SPz6ZWuKaAf?HqaD!;WP|IC`{X-Gdl3nPLM;sf z7^JdsrLnk$fKV;C3Oc$_NKfoK8LqsMh^He8IfYs2Khe)gC;=xf8l0Q@@0psLIWsX| zJ2^BtGg}*C9_su0qjjw)D_gtK3Jf5cv>*#B?op&q1J2XGYuj=PxSz%O@nhz81Vw7K zScEGl+7GZAie+3giGe7@lM_1y&ce##*%8x@$QI%Tj5KSgA87}(jV>6zL3#F?|b z_v(0VWsl?Fi@+WD?H~JnDrl>KX`GUwhCx`Y}`0=6* zx4cBO@fb1`x&o1;B!+)pv>D8vdOEPDZuGP0RXmXVu8ni)Xn#7Q!AJ(tT!N}d_rvBfK*{&jgW^u>NR z@um$!*kJc6d#Y3^zu?htT`C){xe!wYG>I5qW%2wvFQ$ zO^BN?hiAY=X(m$V0#@}UBO}6GKn^|`FAmPz> zWKc~rF*YYcD0FKB3Rl2KgGgST*cFISaf1fU5ZGWAiQss=N`bz(u?n9ZYe4kS!yW|@ zoRI4!d>&?qPo=rsqTt4&FJK(LxT#tl1u9Y{Mr{f&<$<9~n=~txL$BzycpV_BMB{4? zDnkv?1EMJHg7@jE+>tBR6NMCx!d)Bbz$p)V1QXz1TktG%%Jh$6O4FG)g|woGn)%Fl zDSP85-y?r_B0rWdkf3WsCNL~Pv553{HSs*TnGUDql%+Y=V1F5`0K(LLK(Xo9cE z@G9Azd7AMaO2sI>Af_1RZp4*`u%I102VeQu-G+U!Dh_t`Kj7mHggK) z9?2e@nnr@O8s67tv)aSwkbVa5W@#62_S~7N8eUFKPo6n{a%%c?c0Zb!o|(_iOr4#A zYA~Ch!O!U-)P2G6L6lb%}yd$@BGC5Q)i~;A37HlHs7fhOx7*0bZ-`HEP)HCZ9!+x0haG_i7$L~fjIdS#KhhM+dotifK-|Cu#JH2@`X9T{ zKV`o0{rNiq$u$^>k?B!|opiv01~`!wD7V=7)?A9rN?4Vj1PS^`|8?z}MD4Jv?ebVl zk5fL)`>(TlfYK)cAm)z?7YH#YSoVh( zeo6#oI50WTStCV#2lTKYEX8OyAcy@N#*^{5L9$~{0A)Q#q|y}#*N6^qPcX#Dj||32 zR9PvTv;vblY~z!X{x(x1H(QQbwQ*^T|WrKtL-K~XlH-(z&d?YvnZ9gd*^bKzmcVdY-a`-S-wg?-coK#G1zhkm(8 z*dZ*Sdh)!DvwK z{Gc(|7|4e1lOT*?-unABLL-UhN!%S%x2c@w$;$}sVg}A)yNFmXjZMbt!q%ilVvJ(n zL>_I7Qx$JZtw!6=7*}wyDQ+6g<>n&K-pE?a&9VU_QMS6eaHYArwaS|*%os$qsgpr# zryxMpx(>Zwfd8xHgJ5)2m1HNfY>WdL8yXvRe+S3M2FH&K9zHsB_-+rHQ#MIG)EZ*7 zu_4yc+FI*NsH4A!#z*mfX#9)5i5PKY%5IuESP;e>O1nYAz7Q<4@;6l&2Ufp)f;NH! zUmpDOz@hl}=fmA3`YJ}ybiGF;%1+8UwP%soKwi;B$;s6kwTnGRIGZg+@lcPpAzSq6 zLbj|WPX!Yoj3RHC#v8wgWkg+aC=q{A!7F+oMs=xhR4UMPgN?dfxw$kLE_7g0mD2@Q9Inh)QbVX>v_eZ%9v_l-G$f`59z4H| zlVD0S|KG<#(xB5ubMtD4iHk~b+F;<4t_<1yLMwZmwW#&I&RJBorx}aN=jJPV!`Z?R zdzz)F+Ghy6qraz~&W2XA(JM@?kqxcD?jsvQ23!2Qg?}rSy*%uKyoBj?^zX|Ig9GWyddM9nkKPW%rw zr^OS!Q=goj!X?VY8QR$5;Zb^VDi((PQQbz1>m1%-%*{+bSeutrGpJA*d17H>1efel zpg`lNtcJ@9tH!)UGK7qc#xKrJoSvGrH^;Ijhpk38FFwxncq~K%>NVV<&F{e(kWf85_=uqs=SKbJ42|6^PWOgKA#YWo0(b2}_Zg(qaoi`B)2J``&$g0N! zZIqyKO!HiTX4p?uej~9rj?mOgRq=f%yxLhfpSTg)kzW)Wpk+qd0RcY=L(NNn)p=An zslujkO=EQ-m`vqwE-VavV`Av9AG5!sLtm^vcF44HhJ(CTPHJ0JV$wwk{tdb)TeLXV za>+Bo0~Q;&#U*-UxO_WpT3>n;hXY=2T1YUGMO|zb!4Ctsi#!NHmcFULC(K_?suz3& z?PUd3F20IP4Y9^ z>3gDQcU>rh>4iyemdajTg&eL)?*bGkA}M%ZRdt~vr z5C07O6^J12z^84EcfpRtk)Ye7iBt9S(^C&)k?C)86(p*e ztxcTOUThZZwUBkU;Q@lh%@yQ&m!z=>BaYl4j@L=$&P%g1@KoHcVPmY8u)dg7U?h?T zWlc~pyto46=ja+HM>Ck5JoMVF<|FD1IPXf3myHQ$LDdGMh&>1l2+BcTtUy`=!{2~n zeZ>tW7M+G%RftnwY8qf0e$Asb8F1QV^b0Y8>2QWoO1UmLWup*T2%LjH&=ss;5amLQ zY}b&BjF)5>T8d8YSXC~Q%G~+MN!SC$K?}+oD0w=(1SKi~_P1bUG5lbum6R7>aMxj= zL{`YnXCn90vfL{SBq|FwWhZZe->oA!4>k~tlQ9x#k8X($N_3m@^*_fCj~(cL zZk3LwTRT1Ie-7+F|AO^De(8%Ah~jXP|5qK(oStX1x$5J71znKsD6WtSK`s+8BT%}N zvxOUjKM~X=Y|Gw2d^J#6I(s|+fPflTbe4^0(PN({p8fFl@y`AB=c`y z-~0pj_n^MXw6^EZe zgjPf3uk^XvI|_n$7S%K4a0)~$*yTx7CPhzqDRG8M4u$K`CEOQhCyb>ojk2N2sT_!) z13`iWoMIz5`G}Qf9%{S`y|VE{e&qvxpbud;3OK6Z@FEfXhUcbE(_J)e!VWN~Bk$bOy@1|Bs-%&jAq5<}? z(B}DrX%@m$rEKbu`OF9C4+R2k`9h-lq+6?RTP#6*1Chu|Z16cc?^rri77d832#3Y4 z%il1xi9)~vVykbo7=56M(~~?%_`@c1OOl#kZ>i-Rig3!TuP$7IwHZRGS=9?C$OU<% z@{CI3$$)@EdGHM>Ms;(*Q*I54IxftTK%m{31&JLGe6xY*ata%#Q^1ZO>5Lq#1yDWk zPL-~(CriLJhC$Qr!u@313V>02Fk0$3!Z5g4A>r&}#1paDJZCsJKKtp)78ZaU6MbiO z-9w=oRegj0N#=Y*Cmr<~O&mhqgeQ1N&N}LpneSp40a+XC4D3jt4KD=^L2lZ8BMj$? z+C}^Pc&A7}azu!!NuYyG8_vmYw%y$B6DmiR$mfJWU&yT#$S&p_oOj8mzyf%J2mPLSQ*#)R zEuvZpB;jyETV~!Z?rR1}jIM`(b2d$Acsqvi0qb;PaPXBWbA)dTPcNl6U)7^NUIx1= zEgXae9C+f`-{=Jh*LcbBh3)vExpC`OG|!ET`yj0frHqUJ9#tTEF|)jh;L{$+3aE#= zGRoh;5cHOGMR)jI^X#Jc#pYzE-i23(9u=tAIfX%pOdJ%NVZ`0R>>`8_Oa@uxQY3C+ zH09R_dxzK)zL!|DRA_d9+B_x_D`B-f4+f8jn1OBd4o@m%gUfV~$BC(&+)G}}p?QdZHm4d0eJM z&z^LCv5Ba+`Gy6rdlpH60Jc`#c#;mwcYe)QEB#_Dixsaudk|19kQG+w~FjLoPe?(V;rX1tr$^uN!$p7j3` zpMQV4e`2b;{(tny(ecr|{(tQ7;RF5u?Vz$A^#2F;pMSynf3el04IP7vF_i~ybz}2S z+d#Z>f9s;Dm{iw}6I4SbK2`3!-%zkZ4*W)IjjC3BgY)?2f@wV;Too~>KvYfk=?~s zYsE{^lb0JJ@MRB~uj>M0;i@@hjRG(ZR2t#07J~b^kpV-P3pba`l_xJZ7cXZ|VPGy& zB}7LQf=IZDL}R@459dq+S)DK=c{<7fDju1t&8$OxU3CG;M3L%8fKp-w%3ys+1Z44h zmzy~5p#mcp4m*Hd15~X`SP%9K7@3+9!*s9NegHInidykcBRW(cxZnb7r%36?8L6?J z4PAjJPQUZSiJP#fjs--~5h-#ua#a*^;?%;aceDyg1XGSPZGu|S=!udMivF!@%Eol-oQ^o?5GGZ5nNxHYPwYfZWcVwm=Xe6xXuppvE?%NZpc1xf4kP$_5 zi0)`&+A|lu-PvsuzEa)>6wehP&uH``R$O| zZe<@m$^Ud&Tv%DySl!Q9ouqA*Sy^Z4EHWOY0e3E z%+v!#WZqdv9)sCN$=he5bp%yAHRaq~_;Pw;mIcTA z;M7q+IWa$BjgJZ-(St93@r&TsnoVtPnrh+em6Blb3?$n2#%Ar++#Mvum|4vpOdDM zUaIny7XE`8)yy^@UTy2tuE}4y2M+=cxHR7mYXktu9r7i-j~J>C?PLtrLTCs;B1c07 zjxZz|MM2`k3Qxh)<(2_s%pHlWhyww|8CBP5qQIyp_yQbwmL%0>Rsbqhvt?uVD2-dT zK4kuNds)8q;r@Q6#-2u2Jt9-%j3_G#WPp8m7>LLjit!3X%)#DTQ=6M zlct@#2;^U=jBK?xMxf^%S!ttwMJ*)IAYSJ9caT(NG@cZq6e$J>9G-|A>>gO^Jo(8S z!{-N~f|0fr_Fl~K*FMBh`QOCY6HEGscIdP*d=@U0h-EolGl`rdBoi~k z4{aaGp@uJQw6@l(qmaNQuT6Z&qEsoS9Z_XR^A0)AKCUx{+CB@OR{@I~nxtitHgkq74RvIQ#*qRXQEG8&JenD3&=hr;P>A5wTLm_U` zJrJid2lwe{zOWpiIg96p3k37~4W#h&WZ=dgDh|3OT8}GBg$OC3Jo=|LAUxngjF(*C zXK{m8Eq>IwBIYo(FS-qN-VKD8-4m4vte!~E&7byY^MmXD6`qF6_CXGL(vg z)9gBvA^~X*YC>qO@X9Q75m7~$%DZUX6Afx)L~7{T3R|Vly5m#$<@}HYN$uuEq@lZ% zPg4gRhK^G8P!2;|fhPS(q-w^Fg~-1)>Liffk>Q)Lp*z+Ko8P*=Q=yHWVBj8D9XCn9 z7=5TvZYW_3mD{uv(oj`^M^$Y&LVwpq=@C%Yf-DLyyC~19n*SfV%zELa_ z*4f3zr6#g))A<=BytVH63#LupQS6llLa%&;3ZW4pCDHX3iVkTj(rl0?R7TxeUuj|g zxwyJmwetXO-Eq|7s6?9)o zRbP`$R)Ng`Y(#-!;4$8G6x4g^(Qz0;(cZh(R(k8)_gOJ3`NyLZ?9nPC}SHoj&=t%6T;>1$NOeJvSL%)27o`;Hx^t(%askjJ^?TAs{4G1?ef|wJ#_>D8}{X4*rTn;u?K7>aF=W(yL%uz1T}|;#5Ds! z)#PJ*tBz&&*mem=fD5TkBbpiwr_Bq6afk z*hI{-0D|L{Uh)82u`wd`^$Fph(#T_Mj0zriW-KIrKf$q8=OHFN`1Q{hauhBgvqO%+q4{Cf|nYC7s-T-{t)B<4Ny8Wo)N;CvGgy6nV z+y$`bh$g`r2S=dhi8_z4&WnOaq#lu`Y>!|y&=CKmcE)Hn3Iw7(Lg>dP69>z%&h<48 z#@u+haM<2}?7{*n64D0Pi1;FW{;UGN!!*f~THZ8ZW;u)HU{Q~tF_HJtMrT{M?_3sz zguUbrSv8bVz?a`ObQ8^vjiXF`=Z5ZEj;if46j5!en5Vn74Fr#pfcc$aFPQ?s5kbId z^7zJnuF&ok*|!cl9MDqeguwO=7!Q=aK*i=|s+FRqTbol1Ld~exjU`!w)k<{e&?ZR% zAf1l)5(h7C4+eU={}bDNNMHNA{@d05Z|wMZ{{HXi@uLU!f474Ida(aH@c;h{w*TYO zZ$oFe4sRV~Ns?4hQGz085|{)9FgdHQZ)UnS_rT12eg2Viyg#kj?@Hgf+U(TKN&j4b zA{65M#N2~(@H=IQM)jR=_pt=y?83V4sxSx=j~)+}vk($!eM~O|qN8;16@h#e*&13) z@<47#kz`5zpU_Bkaitw?nP@E<;yk)W&#nu$h~kkS@$l9COQZOOo;tS?p+1^Xj<*!6 zy*SK(AWusYVn!apNZ0ytC^EjL|;0n9#G;3m7^o2CEmrB z;4xI{7K4CIxKpz^C;~l9B(20_9-ijkjIo9zJFD_@5f>hh=jQ<>4@CdlA>QrUM^ExU zw9L>iGQ91+P5|BQ|3;4-Dfs^&BG{1w`G42Ry9EUf5c(8==!HSlXMII)T<@l~hPSo~f;wp` zfm5obs}rXY_h(B^iwubm<#k8bw@wE1QUc(er^JkdP>38MLZ%cEryJSiA+J#8^%a9O zqav6JUQVhJ2?+Y&m^?JwZ~&4Ll6xQKJG8rwW2tnlnr1Ipq1i~rENituM5T?kBX^&8 z3?3gGs~#nSD8+#dgr9;Rcp6BAc$kZZ(Fx?PSuvI6)WP3_J|(v4_B4A|#oGQXCeR6j zW`Ge#`ydJ!?nC~{;VB=3M1?_jxcPKckgmhhNy`&hZ~;!+oYA2~^?FeV8r0baYZUlav4%5baCLRAYUXln%r%Yavu7 zC184XuM%3&k0F|IcJkjElj`Dhzgz8XLJapwR0RK_9B(CS?CjZtyg5RaT&0 zKq$lg!(7OkE%z-ruDU1gM!?KH}Xu*T$ z9)V}q^!zNm+GeItg{)DH&Bc&0s?s+#J$J4)Ip3|&2r{D3y#)u^wPi#l#$8JN%*<&9 zKK77cBH~taD^s@FFg4Twfr2K={ zx}2;uWI_;YBpGAP8Ns4rKTZ zw#tvRT#ya@=}`+=F%h1S$J+Nh4gNxd>3{DviXP7O;-smVE@1;FUytt~IHq&%_}Ty@ixUEaVasrD@ybjZH6jGSV4U^45IJnnEsG z`i=&6%-B-)()JOiNP5~2FMPWrTi-$yXf8PaRz`swB3yJBL3*zU{;{~W+`I&oJt0$) zR&!5LFS@C^Ly4FAtAKjxhD;KgKlW4ES_d~FF323DcS(pF-KRM2!9oh2yZO$ZY-9{d zv(Ub}wwU(wd73oZ8kYzh;j*)q6u?bHCyLKk30U?N8i#qaP!H7Uie7Ijr3oU$B0Hab)( zw!`;!j1|HMYr{CsB!&eD3$Cd9>q98!3P%AHUC~Z~Iox|`q0{o14U*(A6{r$J zwE#`AiIkWvtyED26*Wa^AH^YKlYl49B@LoqRU;NeeD#|Qz7u_JrlG9my5}BaUtP)4 zrfmX(_&v??9=Wg*b)mEV45I9SMRC(`TDvg&Axc^>s&rakw&=UugnVUa3bABF< zB+L(nfT8Uu0+pMl-KMbvdX!%ZNSeI~9#rO@57LdaY_6Z;Tq$lN`?k+Z0tiE78ai=%h@Z>&W0$EENJA#`gqPM-66ppu;6^VkBo)lq zNf|rl86v-iuZETN0z{7|7a$N~W}EHHi5$~fFA-zfo2|r_O{YOl`H{$(ie3&a`LhOS z)uAz^s17?yYrS+sLG!KxBc;(npz?XlV~ z>WX2pqad8ibIiXK&v=RnOV$+X|K4PM-2_rYxMp$B4ZxX={`zd??T{AWo^Q&8L^Dca0>x#w6 zV5!2gR1!*s53rf4ZweqpQSF={QdpsSqv*1i@TyBo z?F!SFK=o2xu9(-D_{kd~@+v)hsu2abV~9(S6$nlYdjzk235=p6&V^{SczPDhM2bdq z$on=v=1syahL#5lEA!6wV=Mt1X?S8WDY9SZ0XPn>l%d+g zV24fioiLyd+&h;KK>G9?B%Qj#5M4QPD}qjzE+*cCqGSOG3%nyhu_pA+ZbYtD*{H%g zG;R*?n3f31vJ%J!na5zZKJ#F-9mNyxXTfzAgLtt494>$Ws0sR1KtkiIF-55A3KXQ{ zh0)I1GL~ngFA+Wzj@%A79Jx38A*{DWrI{i<<|aJNrEg`-v(O63GttvDP5BoL{p>&>-vH-1LLk^<+|)1wLbYA=D}^8o)y`a2fz8}Y(L(yg zH()=soK6E)k=5z|tiz^tAqr8ic(_-1mc|r;Hw##`TjP{Y4*W%LxlKS%_8;!PZ11KY z0&jQw556tre>ggJ5dZa-&vjXYJ-u$NiP>3I88EdTe~`bJ@{P-}UY-Z1K>NWuMz< zwKltz-tB$2y8nsa?LPlX26FspUfP!gzzIHf{9=)5Lm{wcUq{NVxV$ulIpFV%~{u<3_ z$8bcE#1YBEvGKdz!?QHIjS^JwRbSs%qQMXQ?`FU{!PF``QdK|!^Ro9kR8u?SF7K#? zb?5QC0#+27(6X=ck{tS9>TbobJBnb->;!;40dpW+;Ybsk%u}(9z`YFO&V~^6P<+$s zh|Y#wTT~qdmm*O(si~4e%}fmlxa@vFvEJIEt{TC|7uMiRS|t*pzXNJicjU%qS~-L$ zm%!E4{Pe-@BlJw96cpEq(mDdUEyFUuKl9OE&?rdIKgGU6D$QNN?GeCav2F)htg26GALP+gxVx?Hxjz>?t`w}yb^7@pzKf_+nnZ18c>1KrMTIh@^wPI^B7FRVBBb@;~^c0d1* z<@NuE#}4v;--g-$+~*F?{|D#)4(ES4#+NAozzTLd{oC=~pLD~KANC|$i<=N<^l&FW zrr7YR$gF}h;rM_(7e9+RzTnfr?C@ZAWH3AGUNgU*(*OilT+>)1sd8&$vmyx}-4PNk z`J;RMtcTBGcBwgxBTV4Y= zE$0Po_mXw=Ex!S48in;?*^E6n`co=*0Yx`(Ih<4Mu(+c7bOj;)SJK%299SNFWRGfJ zibZ6BQ**=Wfz5T2j>Jls7p#`ykJ3^C3kDrQmX@P^{pP|d-A=>YRP&fl>(D7}3>ElC z#hS0ceh(fEn^&bEr_RBiSE~T0?iH=;LM39zA)Mb$Qc96xy}1@3Y%{rAd8cM!6H!Ur zcT_XqGh{ol&9Rf5r*NEy^|_wP-18s+3fP70mk!u$71tv;o^3wafL9WdCNcy-lpK*~ zO4sIK#xtS&WtoiFe2Natfk4!K5=a~!r$NYQuM;8UCRqQ9j*bt?C&PE%WjX|gUM7UL zWjJaoUWD8AhJ5M3(2uhke7}*%Y2l)NqtTiq<%mvVF`A`vPl%Sq))CXX6!(g1sg;nu zJSj(Z0H6!D6^=clTQ`y3@U2}qgeRL$2PA8aU9e1kQ-WUMM!n-T;#&Ng$E5E>_;boV zq^ItflPrntOH|z)9UI>bUD6<`TtYTPL)S{{$qZSX3wV;E!gEc1p(_ume8$_GTUDm? zp@a~081g>D^97g~!2ZvKg_zsvBdG^gqQ8c>){w;M@oMEO58QwM$_fTS3q{|&lA`ua zB@F86j%kW$h!>b|2q5^cVuJ{!hwZC)NQ&?xog>qVaFaqoM-~yFFrP0kYwI*FI$kL1 zw!2~7;ryr-jKf{aoBR-WaHLSdrYSCT)KnzN*`K&OIQid}W7%!%r04Vh(n4dkwYD!6 zz;5UNqoYT2=l`+M1N*<*Ljniq|AX^?hx0#AHf6_u0a$^}$T^2dAda z)SMn74w41012ZG2m^K1$B{tU7+}zX*tq3$imJ0QVvBj3~v|oaPVQ?YFma|8|!~VX9 zr2?!GZ~<>s3d#$GeTg2;$dh6P>!;2@;ej6Xo<%OU7cW0v$8ikmh0Qt&O1-z#TG=!^bSb_>=A!JFKE^|q7usp`~Y0(QCu0o5C!^fE&a6%wVO8eHfdMv9c z#YRJ%_y6lo-TCSAsS=O*$YCr+zQ&7)USmGD*5F0{O;O|t4plVRMR7X^IUpbJ9{?Xd|T`I6?zaY;Gq z#;Y7We{-o%JKbD@qRAoJ!?-=i?+aM$#JrLzTtLOU;cc0;cH*IdqyR5g;o8 zCf4SX*5wkM8v93<8c&R@AySxnGx7qo&Yy7CM4%kB>EAQT&|O+exP=r82t%h!TkGRh z-@-5#=vu2f0Jm6R2oB`UHUBJ3g{v3Z^o4vy_+#mGm^{1wAk z3syrM2H25X9#&ef$z_DeTF~>e-I_Ct_!^7=jWh!Z%T6wPcQvk{$hOs@qJqX&3iDoE z7rN%P8K_|B3S3_lBgSa|l-CXmRtm8$Y3e9YxzX8dflAhyx>>N@<|ZXSLH6`pG($EM z6b2%Og@UV(b8&R90o6q7_$b#eFMAA)c_;?jU?BpH@mc*N9D8BKab2v-grQrN+<-l>CblGl?6el# zhMp{e|H?WMu!A}A=ouOt9vvPZX>Tl!tY3v?uM>eyfQ6VzVfwQ;{~taMpTETaV|;8B z?jqUf?KlGeh5LUToc|Bb|9d?DD<|mn{BH+%JF5q6fNc(XVpr9(JE_abim@x^MBRRl zws~?0%c%iihv2H{Zf+L~sTRN)v%8`iSqM89ZeejRPgD0*%%7z*W{9fag=FPSjzwPkfftZfKM7xjzCJlEGFzhN0byU2i({RA&w4E z?9z}Kvl2U(utVI_i-C=vWuhhEhUU?o0Fkny9F$HC^3{cvEoFvKV&>Rmt|w6sNDo4f zPA`)RRlCc7&e>>RqMvlO%l!{ur}E#}@#9Ddc_9C#Q}vnr+I{{_neMjHeUQN}^FMz4 zSU*x};`_P?LWeCHcIkpBb21z z5#=wFATVe_$_K%sDDQXYI+viip;*qPuFy@0gbSIoam5QfX=gd-HIWQVkckK3+<-1f zTaR_=EFPqyJ}1I>3oLB}m@{0aC>BT@S}CjSSME+kWf6K1t!8Pnj29X);w@t+R#|NFuA-s(X9U*Dp#5g7@Q&JUmL z+6OOR|KQcWN&lR|Z5No@Y75FKhBy{;7nPe(o54O1-D4Ahk8vfxgcLSwlIDsR^fgr3 z$c_$=jUu0}Pf1@rDBUHTj-pL-%LHBNw-jNM{5; z<}%bCI+;zLop`WTNAAY?i8E)A6tQ+{>fw7ww%QvbFcgA0i7#^h`KdD}>vRvfcUBF9 zzSegbZCwBGhd18&-uCM+=}^A5xRgmJ*%!W$5eWYmxgR$X(Ee*~4nO(?h%%J+-FWAB zH-7)D1K1EeblDZ&e*Ih5-h1uFAAWWHmA4g5r(ole?Y!}?*FN}rAXjj)^TT(yKX^Yo zg&Xktl`B`W?H9hY^Zj=O$eq`}d+q(-Ui;uTz-r$SKy78uF~Qqx1IZrSboi6$zy9K% zcV5E$a5An5kSMXRwAy&I{_w*OgVkg^Kl?{Cyz|Pl*WP<|`@64P|HV-4kztJCU#<1&5l{tW68%w0JB)pKPn zUwUGFAlrWK&FjB^cjw3NSW7!^JhSu0i`PE*)AlnT#w`I;D0KaU54XSj!i{(ScKhw0 zY(M{gd7noy4GNZGi&OdWPgz#xX6MD1x4-jB85WR}VZ97cJ|E~i7Mgwc*_%K6^|klk zzy9Js0jO*5J+t$x-+(lMmF@ragX_P2-(ls$pKO2c$2WiT+Rf*mNB29wd>tq$>sYWo z#Grlckum_{_Xja3cijxS?*Gd9KD8LQclqz=afAdukpFg_y-#1Dm-)wb%m671t99A) zVFj|dr;fYJe`90E5gRgR|8bP@KM&-;J)wDDh3{ytEv{@Kf6YChgc0fq8k@tH@9RsS zZ>^=zHX4^2SJFqYpS;*YR_mnhN)uP>?erP0uu(R7Rw0EkA1WM-kiyLOT>%3^Vk-s= zKqJYE0!i{UoVb~}V)ESi`uRCYAYGfCL~?C@g>y0^0K|pPOwG+h19=)o=#{^zUKkzv z;$wGJhxG5jp}{W?-1T|nQ+uKXyGkK^TG~_g+@dL39d6)@u$;nIBr8G)*+a0wY^`tB z@&CfcC8S}JGc*!tiTqQ)*jl=p9a>L*;{sU*2U}J~SggbFp`9J#zXN@mz8B)^C6UmX z;k>zdm71~5jm_3dHP_qp`7>t*vy-*Ssk0Mj2EOE!dsTtg>fQsth3uQ(OxnAzBen*j z48rJhZE^i-m7opXr>BLw9ta3@YTQ?Ov=aCFXvMjSGAJOzgn@Q8GNfvTDJ7gU^sHck z0Y?WhMUM?Gwl;8SyqP^(8N%seXees#(aNJt2|-tLq5GnmKwTBR z4@8f?)c17B{4(g_XE>#@cIwRF%s@|1VM?CX>B0l7f%WPa@MVzG66g~`ZtXOjCFX0> zCu=7MzLbrO*wJ9;t>0|F@DDq0{%ZTf*RQ?zZ-N9|FT-oUjo4G@W=%pVqNlMhx$Ecn zFB*b;cX`S}|VEVqCA`|anRz47*YICV**07_aylk4&YPOYUa zqV>{7x8Hbk`{U=d8$f#Py|*Z4zW4IZFQ32h%D1~Y-bP{ z@lV{s!ZP`>plcCahz)prA(s<;TSl`lXG3Gz3AY8sMY{Dul#)IInD@b7%2N$@Bz>zO z#4pQ`pKeASmy8=@Wu7V5*BgsXq)c1Fo?Nw|IA*?Vm;mhm$9&Rxt)b}g`g(Lw9<(&}S#Z>+ZinrkYR{`1|M;oQ=mEcLgKF94sQGb6HskgjO(~v=N>_h$#LFsMYmcuX$tc`Td0AxlLGen4%dB6?TC>Y( z4JWTY9LMxz4HN%`)%HobKWeNkUaj_Ycur2u>pXb7L$o_SQzOB>oE!|?V0i&?t#!y; zl)IoSSv-cXcw!C7=>_P!IE*EWWvP5A?qg3O}>2Ue|h$$|gCzY&ZSuEl}7x<+CV& zsfgLyY;91}t9yfZm~)eWLKQJoRYc>d4eBjCdg21Il0W6yB@1QXKx4%P1rFD3J%M}2 zD~LT}31Y(-)yLQyp@KFotsI(ViC$nbho>H>ojEs{nT&OI=H&S^HUD#d>MVYMqs+;_ z1+Da828AAiyG)!m9iZtAkXT2#06M5#UOlmSV&%lviHbr~wl^kPW&q_yg5exb6je)5 z^bE&)3f$C2qgOd58H#s{TTJdyYrVJ_MVTt_8pDD8K%mwtI@y~X{mbn6$S z#x3$z_qnx8pT+B%TQ+FiH!&T3C)z3FI&SKsUD|UWzEcf_{!2cDa^L|%GX)`ecVd~X z&7Ys0u20X**XL)?*AV;!t*Nv#^PsL8TRU>40L!VJU;N|t3-3ZR52GNc{$W`H6O-)^ zzqS3|d$h;-4oq@hyZ*sjT2d(9Mxxxi;qJGn7Y`R-pk2!QpV=;U(C~gS(3AWZZfEy) z@ozxwBL9sYL$dEg{zEFc@dN*_{Z8qg>IM0a0gzfb@h?m~Wiq)5*KIMdHWR?)@p82? z2gAF5NGM{%S+PA!Ps;b70SBrK4e?;(-neT0(D*5nh&lnEcE_MxFLDEaf>-R)&xM(D z^N7ng_ZV97yD%?`!;cwJ4zy1s4~H&OV?lTvTi(;(6LXgl_?UNZ(uDbyr#%#PB75oL zgenNDJd=J?zK>U-F0=1Ps!&nc_q(f53k{@w7d5MU>ieAPc|^Iba^1jS$qoJ{VxncE zY0vT9$6or>vM1)`w&tqSG(m94rcSm8c}JkvpNJqYS2YMqkB6(7q6yatNZiL#ncQ}Y z_9>7+!Iofeke%Q?60U)ot6QrS>0t;B+mDRs$$~9N22D39LWMh~3%CT;mR!)PjRP)k zp$Qi=2PZxpCH9p1_q29zMd4ogzdxMqV+YuU|HIceAOC0c=)AE$VEreDO&BZKY@;rdTuu2S-XkfLO9kNF_ z>^U2jEtnpzZy_V2p8fTdyuJZ*uPaUiAgW%ZCzWh|Y!PB}tnY;^t*_7!cn1%PdSauL zjjKv_kUY1o$$HVUxMPPUhe6duBcor+n#j3%~zWk zfsPl4)o@5BBjR0jbImD%`6OIcN-Oc$@|ELvfGQ}H*$zMH(3|O5H6DFbFfFtje82{- z+d@-JrI-iPzrE|O-5d9eo1>4wVn4p2d=E~ZxO0mp9#JimM2L&0^JXl37%C1G>LQ4u zqmQ^5jT6Jl{xxtP;Wp|XFMJ^#x*hR)99sYnTSeKx3WEwYFra*;;)vvOTb6=o;ler2 zbYEpkr$fmR&qOp7GM-#RZik$(npg^?oh zVflp^e^Zk%KANpX6X(`1mvLIJJ~#E(weFR*m%pyf&eZ1+cJLdGu%ZJ=hl4vDhs0#m zbUk!i*u*J3#zUf@8_XezBeg4^pll<_rER$8#YUhLM2m=+ORlIxn4CF}+pErFaP$KR zoG@cY{7h2gwp%MO;Dlu?On2HR5X47lEKxAPT&U7|yaI!vibf*^N4?pZ*;iCM1o}a>Pc{X$ER8mOp zKjyN^=bv)T@j_(o>1>E$x9-TUzwpQHx4ugg=b!!P=C?mm&%EnD{lm^{|9FpGAl(PI zKss~&>7DJ5X;V!7d@@o=ak%r1t|+GsY~)pO_XKQlGyMU2ls1zXks zQJch45)rtL5V&EHm%bfYU)e%*+YxdQWxohBdq=kY&WCUr_3-vBEFnVw)>>d=HGt|r1i~D%$}|xGU3$pgZStE*@@Yy+MI83ee*J` zTbHt-m8BJkrMcmWNA2~WzPJ6{{{~{QpRKG^wY@o~j`3KX#SViDFnA@*%}-5FP({O} zO#Z$0{Ydg28>W9hF+2Id)K}B`@%w=y1RmO~L*qJ0FZ>j~tZ+Uygt&oB&YY}e72D?O z7WKJ0Idk?b$U9qE%ZA#^M0JT*@6#g~*@#;n!NoUzJOFaOC0zjth+1?p%njlbW_)t& zNZ~m<1xf6@@ZIf~exY8(*FO2x&a?02KsgIbZTQJlAFW(qyynMXVlK1mqJJ;LnfEHh zldZL&-y6Ez&%_6t@b7)JA{`y5_-{Ff{%jlUW&e-8v-ShcF8lwHBS((q_y2MFe;(}r z`@#0!>fF&>M#!jT-eJ>_0U^1#k&xxhOxGU>-g4!yBUMp}`A7%#5%?6Z8GPI3vAwBl-@ySY z5{Pk~ipSMl1xO7yzWouZdC#2x`{A?tA9hT09L!&r=Gs8C;1)zA$=iY~Hc0%!tg?gI z@bK_J_NZ)IOD%YGeTt8^!Uja=lOF{&Dm}CwT5&BoU}+&iSB+lUNn2q{b|!R<4Q6-T zQLle>q7Hdtu3kqX4R>f1eYOIcw(_(--1EO_E8=(+<^t|ly?xSFQkvG1fM4jTjyqfQ7;;uBu+gHUtX&|KwUU_NX>+C*QxzN9rPhq++hXghJ+>|f z8UZ!Yxpi%ObGNpkaAK-^j2|LBZKS$3x6(s%1xrIVFV{4tP3+pk)^0sGt3?4?6X)_@ zOb~s4Z$A6U_H(b^{J|$TpLr3nZFb)HXZ8ApVbRWy{| z!5)9;P+xW^Q>Ld6Nd^c-gYYnTGg5AN@TFW$A23RL-G1-w?U(+^jykzUBEtf-atAc? z*lhuL2O;Rcv*A)t@_%dvyf^VuO79~7j~^}G|Kdz=Aph?J)qAcI$^TH*&7Pl}pG(F5 z=qaRqJuEPY>RkQo;nC6YFOJ@QoJWQkdV5oY248P~RyZ{?TYq2%K{g(lnEC6O>C=1# zy$t_E%{uA(*0$<_4V{$^EP5EZ;$~yxck}RZ{exdzd+!-#N{nvCgxx$YHE`o=7r1PH zhPgSx0rMZUFsI4WWO zZjoqREr|4vjSx6tqITB3951|t^_6>hr0^2vSnlQBg_mYOZ;&Ta3b(Zp#3^jvc;`<$ z|La?3hWqZb2xRC7(&bL}DVYZ1BWcyH@a#%fuZeg|nFulAF*S=ODCk2m*f^A2Rl=l~ zc&BnGzKm4(bDOa)Alr=H8cJk9`hB%+Mai9D0zThfh?b`X{)h|G4y#)+ldN_rQZN=T zDgSir!|){&ADkEw{_dF-|9`2_)BcYs4puyS2XmMGfAq+BKK|E%{ny^HyuTW5O9#)3 z^nbO7Ym?{axqEvhM8^U_2bd4n6(sxLGia(~X%zOqBHTN%V#v0I$>E|Z=6q6ES-Tts zK9z||7hyjd@n6&(Mlb8%ZaVizV5r6Qzqk1B$Hx!D{_`OI_r6iybHzQ(|7v5^go-^z z1l{xek0bvN@;@ESf6vpn?}GcA|GilR?Y95peeVA6$k^C{{m;JVbT1Y6F#mg-^azMu z=O4+wQu{9?fIG1Nk1+VzeJ$801*4LIqVZDL0$^0~*#ULX@@>Pw9@c-LjL&}mKQekC z|J@cE*x&pIQh@HK4Hx*~%Uz(|h5sKOJ)XD!!HvMd{@(?ox9*ic|HJ0pbe^-dx$|e{ z=#xir#RTS18L5bHaInaQ?~j*a`qogc{DBXjjrQ)g=06HARBJ?2Z@f0jNXYUHVU zVE>8@z)rR%X}se)=O(5k`gAtl=hd?Dr2s|+qv-is9oEil0J`A*(SB0|hnLgt-B{tB z>4;fzyq~&O=w#_^fq>|6yr3$>td?Y8(+C7ry)uAECs3$QCD+_QH-GZ|>%aS8=LgT-{K*?Tzhsc|o6o<_3_($O zBoBhQ0V9?C`^zvr7)M4Pn8PFej;{t?sY=r4{5C z5N{ycU#olr*CL?g#wXw3{^`#ZC5i?P!(n3kg&)Dfky*6-g&1Sm=2{~!F=yMRIdX}c z8lIsOfnpJ6jvKjZ286%zI5+~_x{dh%Q%n1k|MsQ|*hT&uJvyHE|35NzA!0!IV?@@VGHut~;9lYQnlT?CF5}ZwnPFT*9Ci1x&86LL( z(0sc!GSCNc@A1TStA|>ehffpNqlr!qPhrT3&(mHz0HoR#ym0=@sYUj;{<(i7`%~B8 zpxxI$LOmVM-~S#xjF3sJpM;XuBOuC-X@%>FD0>~?pa=h*K+y@7Ux(i z=`$F1tff(>N+XQiV~ZJCS0F4w{<0M(ek37^XnMx5Qgp-tF`V@OP_;6NIy@P{R%}6&VIcb`t4Z$G#f3FmkRex$!vSP=>qhN=cp8cP zC0w-25X2&YjHFBIvo1m|f$Apb4=^E>Wd}hZmC+<-h#LUHhD`~UGO zE|B5`q9Z{3K@LL0#4Ohp>zLpuMvO#v$qDRHkQX9ECM6wVY0Q_XY;;~|K87JyeGfQI z3>cPo4+axr_IGTX0a$uyT+HMCZTqbsBO?=tbmvFEy!pm6H=loX`?s$uAq%32EPk31 zu1ey{q=|5X&vY3szYcUqz$auf#QaoW&+cOYmF(+7Lopo_jHq6O(647uUN8!8hu3xp zV*~uqrwTxn1dia_l4z=7JmFh;SD0FbqPY}SOjI-;Y#jZjGkPO%jCMJ3Z@j5IX6KE6 z*?#xqXuiZ>%(oV!FM=XMUGy}DZ6Cuh=yL1=DWg- z?8%wwGmjY9oE5Z`n~O+vm_ZMsJ$Jb?7-2(9)(4>}MuS*SB3CfI{ydc2= zxQI$LVh0R}%SE>0MyktQiH4Dv^2sqG5N8>1!^<`-eCLt~HuxHWgY5O~20Dr!l|#sB z1Q-_-K-~?8_6y@8xTZyX<++_d{`=0WzlMD#c-enE^W#U?{^OZ{pj7bEOY*B9 zFA+O}{Q76VKoR9{0S@^c0bwa8j5dfQPOvH$ueR$F+$5G!I58^Lww6~GE;;fL=7JOE zWzn%xolv;K;gFshg-aj@%Z>mA9<|r(LV&d~w+PYITCVa3wEnrvCM?JGt%!`+LYG-O zjn}BaB!`*M5LdC-sx4yno|O?7}-Q{1D1` zd_h`F4sy&-DNM@EN_|v&cPeW<8@w=?IN?;s%iM(nJ6@ zP@yEbm$U0NMY^&$ff~yORMepy7qegN+-yC8;X95lU5H1PxKIzT?}TdNh+bWmw!bYb z1G#)90vef6Hy!bLbEy?(rM0}wUAp76B*fYQ`47!TD~L1I3H7*H%rjAk@eAWz%@q@Z z0R4qgJ_xg5x2QT>d>=TPp6gstzl}xIdvc?>Nj_-|8T_5OV`Yl8g&ekJT9G>>-OeNa z3_x&rHf+hl#-j^@&V^9|x$=)IIKBSx&^GGd4xx~j4WVJcfe zAR)QaE#M}WM53OAY#U~_nyJk+4@3rqtE-|ieL@A zLhU0P{UL6m!p9FGGmCx`-s_;^Odmr0jK>>miXf7f9X4(BP3hijU1h43HrdV4`n}|nS~UA66K}6){gUMT(MRtZbYJ?h zeN{Fbs!4p>Sn=F|Sl93otX69LBfC4Ze6qBwO;C&ibK8wg;WDt)u5!>CaM6t#K~>O< zq;atzB{D3wr$;%XIIW2@0}(TrjmJ|9$nAyj5K|2V{IfBB3_w>F$*g@}PV7~j8m}N= zS|l2pUp+a+px(>cwY9sNr-RXu)JtB`A%#|c0>lB@b!g0h&TO>1#%1*>hto(?wh^$0ni&FPusqUWe znZy*=$?p#lpFIr;ONxq|MX(`hLArr4QIWWGHgvMe)J>7vCW%Pr#!8kPZAxO_9KkusU$aC zQi}03AV+CL>6b~I8D#&?4&y96Y1<0GL~?iwD+?Q|_1OkvuvX7cY6`Ex40dAjk24k< z%q9@kdzPj{(Tee_D2H24G!_}rRL~ma26swG9;AR(Nf`L;3So*D0qZF7r{CC5nT+S> z2?84FX6C{C?5L4RnYy5ywLl$Spdd$)Q%H(wrZE!97r8O@M4PYVm~AIk3Vx|JEn|b@ zgNKngVQ_qWaQyJ#_z|ex^l)0C64729sce7r>+KI;_kv7E<)F|q)`ZglZeSG_nWbRO zc&X^b5Hktf7Q2q;iE7zn&l4R7X2RD`8Bak8eaPyA0_B|U`Gf5?I%AII$y+Zx7i=Jt zqp=j*TwSkIZ0=sv4~femjb0v&BjSdm#toB-KF~SF5;fvv6o%0E z)zSs!7O6Zc8^J`e?>|8#dROibdsOF1Q!erg*jC7ba{kE%q>3$}LL}-18&}k}-E{i|bdbo|(x+$1F^- zmt#KQ(PbKigp=;dMX|q{=^?jTS{&wXvP zAYJ`Jne7Sg%p!PIJzHjj(vvwb&yb(OB-n`5h&x^G9e@uKqhBU&F?6&nwr-TW!POAF zp%ZQpYsobGWw_%N4tl!{=2QRWyZE&Be_aj#3_H8o{~bA&_y0Y793f#2?Eh{9b$J`P zdv2|*pXkCauXLAdhVCwyY|Nn(z`Zp2|twCn`lu8%I< zxkRZ*!RT_Kl*IN+ggSo{23=Ep6{h3BzNzOn8DQBnmrqN71#9Os254E<# zr)48>o#sc9yMS}Dn-n7uHHiQiOQS4e0tjI$Lm6 z%+Usd(&HSky3l?+p(1PFfjOC z?sDCrlQ#|yrwRfIT(GG7jKfLiRu3tyx4{FCVqPLMbev2g(ImhcjBi( zadA!3rd%eeK|@EA3KBr)3Ec~}?eN5?&U2#a;GlZJI8T3OPZYFSM4LVQ>{0GI+7q^& z3Dj*zG|$uL=cmq0%|Eg`o+aIVZwm_|1@>}?7T|T!K@fb0_g;>r!((6#1uB12mHW=S zszdsB;LzZg2k!bj><-)k^!kTCyz$QWcHa68l1eaq=!dVvz*Es)7Yd*pop z>9XO$Y_@i~UVC^RQL$?$Mi;yRrPTL;B+4~D$9irAZ%F(@cb z<%5$dY<8~w`9%%3troUB;xoJb+_N{{rjx7;1VBh#G_o#V)R=;{RCKv5HUcZ3?T??+ zW`PZy_i@sP&BD$vpTF_Sx3_=#2hTIO<8HM96#8JqNJEsdGQ@S4{(w3A>#QqLTyi=P zukgXJmEjRF6#b+US2v2}82~H>hI)GY` zQ!wcHLfn|&jpB#Se$($i>2+H~BP!5DoO}0W*@e+xn}l&HCrf=&yB4htoWT@{Ns&xehv5hYmpDXnHm1+ z->!e~y4s+^ih^_!evI}bMbJO%Z&{7*UJP^tT8g(vv(haw+`BI&64DNIhLl9nSzZh+ znfquiqrvg1yWiyZsA9}F)U5`X02wnob|PuwfUx4IfI(Bs7I>t*kpSLG=&DnRVwvsrvd1gi`*w3Z;`}9AGf$b=fmM9;<|4v ztX*or*pOA6ILRIB!YD4eM<4stBZ?tWdk2#Wbu<(KzVn>es)X1Zv;+SBo7X;m8PPZ( zN6ZqTv5?b$ z^tknhPY*W5Nw|JB&_#z=oVY6F}{u+eYndPy3vL-G{-CKZ-J4s)cU! zfSGh+KlFeZ76PmG*pm>2-DL&@Mf01^QS`!L-)-iwHkspDQgG`JdH=G(VNJ~^^>5Z1Ukd5BE93G5?lKos398hB-n$3pw=w9R11DU#i0fRqkyP6>_ewyuAIidvNc^hTySMFMyub6tzutWA)$1R|e@U6~GodzQ~OK-AVCiT|3o)jL(kRwWpDnhB!sUD9Qi%^8! zwJ23N`WL>l{o9{_RU!KzykQg#U}HCb{BJvN{o3w3-~IOX`@cg0+Zr`@ttS{`8B^d=OTnI1cT+ z^L?EUIpFfZ@Z>^svrZxh7d<*U3NLl+66&FN{jC>qaKpo$cmC!2@BdB@D>(XZEyARX zDkC^nnRM=c+CoOVDpT`0BMEV%ruw#y+dn?5PM(>We6Wrzku&o%(^D{vGs)aF6cIqa zr7k)~v`&*!#C+54?BLd6&x_m%xFX%iRvqlCI-2nOkb%NX*BlZ?w6LN>tmfb!-h2Ya z2}qCPp05^1WX%+9q}?io{u+n&C*a?&_}*N=;UC6)zTZ8KPXLW41|wK~TxKH%Qs~1h zt^X?0&&Oh{Ly02-TFtnbQ=}20Bro@-Mbii(;t2z=R36=?O4hd+bMB#wucmhdi`LgkKBnkEyjc?Cn!TP(8)=0!b9T7+Y zWMqFt-ti2n&z2qg;CCS>SD&{9yd&eo+-O5R7%fMP-%ED3z2y93J9-@Jg4FJ`xhl^^ zxn5bSRJtH~I<`&r{-M2!~YqHST@m71K(O9o~P8yLA=jw@ zk_Z{Q^d6Q$sRfe3>196TPcjivq?JVSl$mjYis)6nz(h6?i9cnBPkB$V6@CgVim&l; zbC*#@>jnv=P~XWtuvOk%D9>H&z zOAtENWL9<66?F`Hst`%qU2}LLQWrKh#s~iZezcgmJ%fm~PsDb`$X5%dGHI z+y8Y7{N9uOAF_`l*n67)1NMIh@!xNQ_21_{R}%j{%J~+;cs_u5<1WtLJ?&k?ySuQ0 z4E+`hB-!C(P)<;4l@E6w+3`)eEV4F4s-R_l%MXydxFera~ zsc;l{BmzZQuW+eq>gESOzVY^NW1xIHXQ!rQgG|M5Zn_FdwEK9l{%zhO^Nj zL|hvh8$T}fy^U$^7{^9R$x9~&?RuIowiBZR4f%F#1^nc1Ne%JJm!;IP9hGCnFl6VI zXRm+omey6}Y88uVz!D@xM-rUeG;Bk24PwvUiML3d=?;`wAYw%DPd+G8+M^voy(8x9@@nx9?;e;mOX94nRm0geI8 z_(|=C4~(k;!HgXSBKYV_4Idx7FR}}z#8F2GaydjHRFbyjm;>y5xzZ=!-t}a|7$ipI zMSUBX{;q7D--%8-k^d;gGaZwzBtRBM~JjG|NNq?$HpHK=Rs3h{Kj=S7x+~I4qzQMXl`86E+cgquG!yYqe?Ku$vH<_)Gw`1E zTi=M!!sai(COPohFM^bKS5UDZGO<&Z??P?Sd9aF;i=VEQkgVfuwRpjK0k`!!z7A>T z4DvW6=EnO)y|9S*nI)J#Dh+(pu!w@6ymSFuL_Ci=Tywnc*m0i@?og*(tx#U_deBe5 zNkxthkT-iM^P3-kF56o3cmHu{WMC|fJ%3LJNFCjcF}WK)?U=mF4a+Wv?{vDkhXZrO zm%BH(%|G2}{`yiN8K_bakounI&!eeh*<hJ9i64%2CqP2M;d zU;p~Ut^avf)}=ENpY**K4*BzQvCiP5#NXl_>I7|8^~n+?k4S{+9d$Y5~}G1_|W4JJ$V`C6Gp z6sX)Qg{Kkf6!F^v3d;e$B56Bt;ze)rE9FXgWsO`We!5n!m5M3cZOtK?Rxma=GCGK? zgeE(Yt%>cfCx@^c6UQk?G9QT+nM$`sjlf zg7ji9QuV}=9}9SW-z^h=TT*Gg{x#!D+uEJI+&TXrKQMy)UtRv+8Gd(DYTNmTCI%7T z$+)xg-nQZY2Oo&#e@8~(F8|*-qHm+dQ2#y>qxWzV;P$+2K+W#ZS_Al;l0qDCgv9{t zeG!s6*T}3jma4Tw!8F_ga!aMkqQJ$MIS$FjGcueW9izuX`+D|4*_3rRWu|Rl_0i_-sCh5E6W*;lG!b=1yl%Sn=>_Bos#I zhr?5Ix!?zk1RWW zHQR4r{HFQx&n5X1e8ZXM*XFtPY06#o?xy`9qXoMyXq44tl!_ z5nY6{0FiZpTflAF%}=jOv+LKdZvNsoh~Fb0<82(len4lm64zL(yvBeq;B#0`0K>S~;@i2xatH_YNBMooE&kO%bG6mA zc~aTyQQ-S+t&PiArCk>|jBB~FPz|0#CH(l4 z1;0(5$A68y<|~BmJhUr7XxI1tm?0yBN(deLnwvF)JtOoVXC_ag^b7dEuZI(g3w7pK zs>LLHk|Jizj&KD>IkcvjEDJ6-!gxLdER zUmDa|!TF8TmkoyNcR87jywI-qalCZ0k;*)2xZTJUQ3tVDtAdNzIfRVXmq5u<3elh?oi)4=kN^PAb_4Y2y=x$6%6j$rYt5H`FGZ;-5Iv!T zHa(8xlj;}D;SR&{*#honEUZ=L`zUNa*@@*HQ$KEb5+xGI`*6wBd5yG8R$oI-yl|z^ zsU2dYlX;f|(?^&s(I^ecqK?WV94HC$)6AP`4cB*;_(w6J^&?AE+3h2n4l7$}ku zL}?T}d~N<@#dbiQ;zWQ5QznRSQJZqBkTr@WA3bgQ(b;yJ!+hHTOXON@8|HjuBh>f| zE`Fovx7C>ygtcj2cT=BJ-FMQ!auvb)`Ipl{NP-nSm5fN6*x1UWSnoOB>zfnyms_XNH2+>x-{f+WyU zLkET*1PYV-f{Rf8*wGvd-c)jQ+><6}vvVhBvv47r%tFEi_EHFB<+c>Q&Yhf?$O89( z8GG7!is7OOq(}~gHHv0h30r8V0mqSir2y}RkgABXydU06-B~2V7+OL|a7c3PY6|0! zu5`OE78(ViR0qI1-OHgj*rheghzF+bR5-mt<}0^RRlAh*`6OeX7`dzzoE{k*9vmI2 z*XD=B{x9owXb;T5+vxvr_!^J+e-6VPv}^yp4RwCURsMhMzhrm?mDLKh3n`&9lRG|Y zZkq4wp;LIArX%Ug(^?5eX;baQ?9^kKxon`$-9dW!C-rJ2jcV*(BvoN&zzKZAt9Bgm z6X&L~xd6#$er)W->BpA`($5amO3SrspkFH0XTpyZ)CdkEh$xhP9OC8*U3{*T>M$mE z+FislovuSkD;0zE!U^!m^a3}Y2EYyEJgnBzkk)9&PEY2KK#`p~ayp+oeiDHnA07?T z$MOGh$WQnUKL8lm*{U5tu0|`%%c;4{?9nVN1&ZP6sl&6G+0(&fHa9mlBe4~}_uQ#T z1fR>nA}|fJ!p!8<>iY{gMvs-a`P_t5kh2v4#0c!w(bu&~kaD+^GA-bi42$ z!o#c_X%6fc0V!j}uy;kC#lsUP2ULc+d9^kl1(B@KBD_VKrRN9=6@RKW>!>9gice&u z)O6IlPPWJ)`s25Z#U2gOJWr}Am2$gk6k&w$ANEEW1^n@>9FJ{9jq{FfCEHTyEUyOJ zqcKaZN}_BTh!to1Zq&A`HAI#UtBA#ofWhW}Y%0fR&2UW_wtz|rQ3L%#!(kzc7Meea z0MYqlQ`6H^IT)yCCdrJCGPb4Y`9OG=gYICc?N1E+ULE@;<(0cXNHTd--j%RRshXY1 z9G=diW!o$)=#S+mv(uT=`C|Z<0JZPt5NkF+lfi(D1~xi=kRp*eMj|`2VjyC}4=XET zt0`3q2qcg{TW+irR+EiIT!pMNwkgUuOWKpUl;^yHDMj9y7EY43WAaMf94a z6aLP>KN%U;pfu?dkL@PyLBJWEJnD9%DX}pSR~*!iOgQt6d}4&7-|lErQGg6VQGf|a zKZ)d`NZ|>isQ)C2S}Kx$#G0`Bya)}gdf|rQSV)V&jaJS!Oy2qOh{%6FNzEty-i$=) zh&B%>RW;PnxuS33V&JHvCK;zJM7m;&pz+i{J)qF-br2?hr`z6?+Zyo?2@@2*j=pO~ zO-fMa&b%hv?wyZ;&9x|YU1e=~*<3TG9+X%NL`o?PrR;;L%n@R-pmSoz`N^YjNFP9u zMghtYK9|pHQv~B; zl`{k`mkRZg(utubJB@|;c^tW?{JnJ`IKt`wFuh)+Vi^(crbE?^RB^g>f;@P&2_!QR zX~VL6b5bX+V4U3$gL2wzI~;l`5jvDk2m&2gC|Njz@6NF8IZ;I5GOpZa4RmrS^R-&0! zMl=Y3!c34{hC?nkwh@}C+#yo{)7*|Zv|s1;E?}5LbqimvDrY*3nPXQJTiD~9eGDF zLw(BI;#FOcsB%PV<(?fBXf{c=iwKu85U?dg6Q|70lDrl^$sEm0&G?TVKYnt0QY4c| zd^T!O!o5$iFGypHf|9v>nbZ& zRC&t{pp zaVObAKm%*rQajs6ypvEg9Dt`kIXHSI5M5W>j3h1_hSGLRScw=A^jd6MDmXZsdHZ@U z5jhf;OG94E+YHlHI$CzT0H0J#FOo#Vdch!zqx|H5w z7t!I_bsb}O4y8sbZATSGV*N?ATxq@j(W567ClQASp0kc(sEdMVD~4SEIZL&7as^>U zfHvD%MrfSG7^!4*>!x=uU+Bn6lE@5=M#O`@Q!;rJ>t+E11EH!OEfihHDyw20z;vMu zWtcEzWS4UB3HF+Nf=-{wED<@rQmLaeziVCj+K>3co2?XiUWPf(3pvO;hd9hvY^QwM z8TQ$kxAjiCb*}uh?YSdA@tt=}4%6K4ZM%ne&}S-TTknJQQxKjpS7bD}ejgk^lFbak z{dO_alG(PgH3W_83x-;>hOH9RCoZ?)#CGJ-zRghi!0ZrNc@bBtzt0TRxSw;sY9J)} z8CT%R<(NH>$?k8oM`Por6Fs#d%CUg7t2NkOt^ip}hTYjjub^s%+}+)5LAegxUSKSO zanC9`d9I?zcI|f?D@r(fnzEj-clRtq?F`QFlFu{}q0_Sr=MX{#C;u?R2#WmF5sH)p z80km&We8^ak)51qNNKUVJF{U8htYOpM;nIkF-3T)6cI!8z}_A&JGYyd6ugTOJoP@? z()YB)p{+7e8d`L1Ie-Qym+E2R8G|Aw==cOW)8U9!DGPCeKSB5|YBq2cHFs!eM65FI zB~~bZUYSR{GJHYOSN$Ah*an>&s9}4AE|^}VOZOhCrxPX;DNj7sHT(u{OAjMKL6?x4 zqQjHG*u?C7@i1T#YJ3-~hD|5oEJFOTcPWMakZHx68x-+&Ib!0{iachF+d{GAcH7$JihvuSw3B;n0aHc45;h+TdM-0`8L;mq8*sW1>SY`u&+5RjcRyy7i~ zk_3^mI#!E`k0IX3ZFnmr(2hB06xC47zV?JKWNH{lr>+b<7g*Ym&SD#lcXaCBmS&=N zR9&tgRok5tFv1D_C_@X`EBCGkHl(__vmrpyjh0504|h8_cEKb?xKHE+m^&0;V9Sag zp6}o^Sd7P)GUDY0U$TU)qg4f#-(g!C10n7h=xI zMI;ZM^1R8OT#ZdYWb#>6LL18TjuA z*m*b$1-L#gr`hy`est-E_7!>vAKwD*uO*iBP8$GlBD@W3qETq+Su(8d8|q#l&4%6c z1KeN_pHQ(o1iIcOkAbhxTMQxc)(!)A>F)&8lBSo)es4ml@aOMp9mFo7$L#eOGE9#n-`hu1hSJCtgtwkVgGUymgPyO zjHp{H^~IzT5vd{SCCh_uEzVIapY6R*6E?!X6M4=?zu7aHuCAs7XJW{Huolq2BL2bQ z)&$)MTxt<Iodm0Do}(N zBj)|X$=8asT?*Ki-`WqN7(G)fH%dGq4qfPquuqJi%s(_9d8E6<;%%xaZr3dH?@CLF z&rWBmS`Y&RZeFUF_+AA@FLB+NvE#X9K%@|AA>pUp>LVIp{)cC4WrQCh-7=s+6qL+Z zcslN8(xuHGiVuSFdEx~)7%QJQJ0V{Hpr=k*6B7{l!|kiES@Z#Jj{FRixffhFpnE4= zPZ=o*4fQp?;1J;UJKS(!XokN#_`Z2gx2E(7nY`9&B{U$B!gn1^!x3+2xW;xWqKxLa z#F!`+6TK&!Itw27tNaha#K|gChK~1a$>I?+DDBZ6m}Fi`l1EI9sF^0VnTh3(=1r|k z(gpNi-4}_16v5G$Dk5&F`3t~fBTxtf8XYd0Fc%}1ZVZ_8e6P*_B*H7h81(e^9gcut zb8JFDck(3$Q;g^qdmK**Z*$f}U|W*SFEkYbNoJprdKN)u&y|U5VcBBj`QF4-p1K1m ziS5_FuhMS5WtTTIX6{9W!*S3j(u=_6JnoP#grO0Cu*2m?DKl7I^1z?4G83OB;Q~&Z z0VQxDa~*;@f_E!Vhfpn%^)5!0P&EM=8d*q8IC{wEX*+ovNyrP_W1rd+87v|Du%pAf zuylBC7oR!LY-1+}VsK%3b*T_`iJN}y4?4o`?41LQ7kTcVDTC!BhHJL6h~RNcFt0UW z{7hso$-pQ{*be$|3=_Mh+*V$7LI zh4izT^uIl+e}~f#=AZ2UUa!y`JW@*Mk>g7Lnjlij{Xsuy0`H(Ht)}ja6ykKteuo1A zBHiY*r{=OVli5j~T$P{OB4mQDK=Zv2>Wad#b7vi*#KK9zZxZ0N=w?1or!M%g1L(eQ z8~A>Z+F$Qu>vsV`Y=ZLga%r)!%o9+U2Qoz4vgJMcURt61O=}iQ4G2Kw8cLU2CU6eS7$pu&QWq#4-s^Ngu~4h7 z;BrnW_)!j#_~Apog7_2MwjyG8VwWo80#z~-#P}*}D~njdaKddDkykmjf31(W3mx#5mm#N?8zQQn#dWJnaAr6?{v?Yg zZml>iwiU2wWm^M9$aX>|?Qv+GoVG1ez>j?O>`$QU2mMz51ru^!|Vur95N;M>(K+1|1y#^_;A?#wfoKz~G^P8+=HTYO*Jvd!; ztw-llZyeoj!*;$b&Nvhxns)Hnc@vf)wT8_WA^h@~{dLE)!n3AK#HRi*wXok;X^hQG{Rf9WkodpP>-Y5lm={kecne&H#fV{J&w-qNJZFip9Lp-1b>Tt z#ISgpY5eBLpwE(-GpCT>-O-bL!Ytbxh%8NN&YiN|pq=G0RT<8>UD|M@_a{N3rv;97 zYSj=p*mT}b;uZWv8dY~S?MA2U&XT0DCflLGt8f4C0Wl!Q#>kObW3#CI#F=Qb{t0*C zl!rk`wP7!cE^IfF?7`cn67jTCTIQWKBuvxK-c#Ck6q~>#Tov#(;<4TBDwaWL{MXs+ zG~%^q(=*5Cvbn*P;$FbhKK>uxM&tjD92g%N>&AcG3*2wl#=Z9j6!;f`%SHx+DKSO{ z&;;tp*78r325UPseP`cC*Z*X7zP^X49~p17{&$xDacpE{yqo{)ZV+AK{3FBZ#PpOT zN#1zvQuAkT>}n_2hX0R_jXe;J|2=;2fiC~Q8}ol}K<-7blh>-th$M-hVEx9W%^N@O znXu2v&{0so3W^RQdNjLUQ0)PjdvQM3fLcunck2Q2e3AVKF}EvO)?o7dmo`sRHU#S6(3t$iVZ%GM_@Z@lwMw80x3od8ry8`rOFe)J-;9#Jlq zTeswE^UX_}H~-LlSpaJ{L3{sn(tjHa__8^&4*kxr~dAcfV-fxE>rS6&m^T z=g$X}HvJdZ7|4ywSJtn8x_;vmV6_LSK?5y&g6uG}k}@4;Qzov>*S^|#{i5_D35~n zc^PE-S#ZV_vo(iG=U12Ex)in$d({WR`Q`eT|JnTDi{^(HF`o77KLb?-0@nJMuVYN> zH{aX5da-%)BB<}%kG}$SG3L%3ee6V1%Zuk$`vN2b+PwM&(gSHr8y{TU_~5nm8-HzH zyy>=a~!iJUGsjZKoSsG0??87Xw`kbTQDyKoSsG0??87Xw`kbTQDy UKo|HA!z&*|=T5`2v7Iv{57##a8-h)HKl=wf4S4pqwD7-ps{iu$#^(Laq2`9>P)p-y z4UM5t)Bev`%Rhpd>L+8ElFry?x~7?LTKBg3f6~XOvVW%%msLalpnmYK_HS%zX=w~$ zU1QTe{P|@6kMO+C{+-d*NLMWKL1>VH_cb@a#{QvD%l>NnH}7k1_>47t5bAGv{>AOj zc6#i^WovZRTG_B~ZI)KnthsC{dnUk&j~<1JlM{7yuUqnJea2nHXi=yk)L^}Q)@UzI zutC#If4+BbQutRet{L%UP}2v3adq#ZOj1%0$2voG0{7WvEBo@Nk;T$Rwvd1R)5xvj z&GXj1S!?3?-$u?9HnxhFm+56|#lHN~nwzpF#;m!E*4(w%G3D^|YJCDo?Zrpl98kYn zpCs%L&KB0D?ORLsqfLAka}VrWw_mN_~2FW+#FItDtGEj=cf; zt&~gS-xtq6FXVFe#8p}Y47r@O@@-+`64q1_ubLF}?Ya3v?w;Ta`0T|TR81~03R^5f zfu%E-UxTk|Qs9Gg2v^&g-KCkrTF$=uvNSSNS{}h-`@%!vh2(Ag9g-B*E`Yu@`ox-_ z@y0#w2)v@95gT`AsQ zh5^7v>BsZi;}<#j^FsbAa2E0}Ac8fq=1SifWcG!LLT*A>O-SkL#m>$Z*1oSc^_`dY z>`JXt1R6xAy}DL-IckqQ1UICM(dFX&Ip61O>Df#1zJ2T4;`AeH_Kx^0tjz+O{ms4Y z%Om9cK8{FDi2FH&)&1ASIw2Ra&du$`17LxvN@zN5zNvs%229H91?_TJ^J3$d7K{Kulh!(qZw! zD)In1=IWSz>%KjGAHIoCFF2}sfU0vp2uJph!2h^FLJ03PlNuq>~3{ z?Qa*!D1$VRXW7I2Mu>LMm5ZoDB7 zYRk0WH<0Z~EpzkWS#UyZa1s0r6kbl-*X|ed6ZYfp+@Nhj&|bS^T^Y0Qj8{hNPPuxP z%EIP?-;jCH zrPA}K=qH-Jc%{{MSMBQ;&>#RK`QX;*_EKKNqKIQxtY&l)Jb@u*5C&cr@)FgaXLq51 zO6~Z`h*G~pkSs0>){z=0u-6_4Jpya-9(5br`RjP0>N!0M?kR3vUEAkIt+5|b@gZAv zC78JC^g5Fx$X^&FZ|t*lTY zeteG>b8@^P%$Ck=pnyRwjH`WLkd?BJ(eu1M>hkv?VQKmaT2^arhE^{vqd0JmopI9L z;v;MH1@$`fGsVY~fBT~GzHe)X?ybnS|q`=Ab1A~pB z(iS$i%8_^b{`U1z0u)99v6UHYBE-(5um+pmcQ+)|)zMz7lzoA9r6ttvQL)YqBp!@^ z$rW#G5Jp}a1bJjQY6JK?Z8IKRwI3}}PF`BZPOXsNpe!^uMcHlxhRF%{ho_U9%+Fen zuf1AdaG3~|nSjL{0yCq$42%?W4~q*c7QzHj*bNV8X- zp(FEjN*f|HN0&FP(fg(BeX$bq`M79uxS-PSQ-U8-w0X#!0-kRtpKYI8wx2FqD(tdRSKm22Hj*zK4BzT$c2&*q5r5l9|@e;@<3_mQ2X{R|N89OzSsKNxICsu1Z~Q<#?Zl))r5 z0pe;xF%?ZUh!vWvz{_03DQTE#eWs+SOyV5Va{p2$nV;ABxBKt3HRt$yJEAcb>u&Eo z7Ve2KydUoAJ{oO{w6X3k7Vcud3CA${8+ITZi^c-%ShTmJ`$#Xt>Yi{{Z!{8P-R)R+ zoc(vSt1ZAHn2+?Z$p0ShiNrA1!=i@{cSa+aigvYj9%+kq9b^Z9psTx=bw&?GdqJtU z8=qbE(XRF$Oh*nyx_SfbP^71|10=!+qMgy+@kj?C_DE z!`-n+P{egimTMHr`Zb-UG}s~`Ly1X)*49<%%X=I+m`E`UWk_b8D>%n!@XVvKZop_P z#BuyA2)5hL*dBuaFGu0Qp`3awe31B`(nrcd~?H zn7Y!J;RBAFc`O=(QZmW<@RI5KlN>k6M=Mmfn>({3$sTq}F$Xz@el3~QPI0f^!JR*6 zZk}xjvNpM2QTez#NOp2LNXZi1pHM=S0bNRQRJxp$Ohj8ki)T`@Y8vd+pb{Ts1W^@= zeg+*jQfXbG4~W*Psh;$^jXgo;rjorsYydbFa@*pQOQNJ)nAPI&+?5!F=8VF;|bbPc9J13U2bW|9@?$~0wXKsrh0 zlPc9f3M9P|!Ek@Sl2l*^!;iH*>fq$~6q%4qCOLtrYsmmNmGhD0nhTJKG%C1;nNlF= zEGc`I1aAgk4wtnO2GF@&cc6Y6x#2L8P;_!%P*7C&{gjHZ8&uNd_24WiONPn%m42X2 z%Q}#uB(yaAc8?nsLmH=?)=tTK0NFgPn8Gd^W_4!B@r*7c<%Po&O>>1w4*Q3N1VQ)2 z=AbTtARx;H$&tuSMX_=WzPV2Ll_5(A3%_%LRT*wNK;#Ub&`ow zN&?WjHY_F0;rf1E#+Rz8_44U>GDCSRz!Y7VhqO3%w{o3ye3CG`=|}eG9${xM>&V=d z{5}>CwysE2fKW9v=$`I)x;hL$Q$0noe+EtM#<-j|dC?K0Ttc5C$7G(c_aKpQ>3kYR zPy*ecgffzn60Tmg{(is`LPJX8+mMrBPD!OT-SqtP6s0^9vKX{@2c?s^$n#J_64dx+)QM z5iZH3bQ-Jzo1_j``D`s80g*qkQltz&+kzWKh0`QtqXsm{qW#bHeTzUpil13J~a}XLgrx#6Iet9fr&U(gYLOjm&ST^glkGd zO|DAnm@~aOU&Q%%7kdU-tYfq@A(>qugW_Eu0wi-jLEmZn#9~oYybwgD6B!7y>GGd5 za2HOdKa=cNgjW+XXG(NeP`#6tSaZ-l*J|^3jy*Dc@FlKj?$*SY>nn7v2&1Lsgpx^# z5C!Y>SId|cvBa|zcRIir@cuaW5cxDpu4s2)$P`3}+;B9Khdhdm?uxt4Nf} z8>N~CQJ48v$}4FB$FF=!cRObuBUJlMtngIjmon%cd$pBUgXl3yCj$?2M4gmay`<5p zNPUQGbJ*1*c1-0RSh%Z=wRU&)_C&Euiee8I#lK!gQ{K+(y*FV&!q(NzB1f^sVzG{J zXD0{q2x2MzwRRsq-V;68(aSozJKKQiKm>cW@PW<g%3tzzRg`= zr`;W28tsX+VqZtxbh#g~!;#i#xKj}AibekC2pFRQ2QlQ1#%Rmf-qU@EDEF`y1UB>b9d%zO0>-!TvKAb+@VQEOrEdjsQXT?5$H$2Qt!tTsAuP+RniVZM%>4 zM8Op2@<4ZQ2W#z!c12>sA0k}`JEO6V0RP}}Ig>{tJ+Ua?r+bUMF=5?~@KHxK;X@He znn-&)l+S&EBMXy8+hHJp9*lIgMquyWW06Q#1+5)u!0tP7BPhf7j)tS1LIII3Pg&6X zp_uFDZBa5!jIZfH=YTX776rGwhkvh~HDTggWEaNMyXKbk>Qe$yMRdNi3h>H-4Fl4C zf(n{f^-Yo~ndB$Pg|a+0T) zGL)Bg0<2umQFdkB6uf$;OjHPXp=86dq`Nh(U(r#b)0w`cuW6|O6dem6;zwexv=Bt9 z+762i)gQekTv1r7F7qPF3nNF6U=`27PSi;AvU&2u{D%Wn>7jA(T3n$v^ttT zT*Eg~cKjs5>5pY@6c#p3$?5n!&AHtyjIL{GUE%zysDb=^!59W(rl8!-+ zZx@_eMB)YO{&kNNKbl#@%7LT>QO53A!xk${$E2w^Zrov z{lAu`eV^|CeT-)pdUZPMLb}ngd#tP5wTp32VD;#h`R@{)TE-yCrB1We-j`v?#Um5{GohG!eg=2>$z?+V~H*9g2LQz;~Xv#D7zBV{`TW@0L*Wr}+OE&o1V904wCDi}~3) zZ@|8uEv${$k2dWqquVzZtrtu7-4T0f%>LH-6}xzO*`9deT)tUcw&pKbbE~DTh0@ZD zAB28SKeu5`ExcM^@Q`nh+$?R;eNJm~maarxSS>!y7xTBpZ`Y-bY;oyPA^+UL^YS_U zD1GfYpo+KeiHZqnD{XFC6RXzvoV9vUAhFjkTXWZm zZP41dSjbIdoNfYvhuLv~)0)d}-?}5N@)Z|W_#KwBJSjjEwD!gdvB2fFRwhe#M?Hb+ zzW(~_KFJuY8_uNoX&ap#)ermhn;t$bB_%z@XsJM&o1x1x>m&3F{K%-lh>2ai`ER6q z$23BBT=TiY=B?6?S6@daPN%_aARe!e|Ni&J7+xh2NXIj)>N4{iw3$yjL6h0ruJ%SS(P(MS>Nj^FNoPn#HbF%^{b`q#aau* z6J@Q}aaH{8cS0qO!YVN}yO`{`Z}>fg337vx$ye)>Z$SB_SZ&SEepQXW-Zxi6%9_tw zQ_JG|nTMW3Qzn9iDk12O)Wn_b8~22|nO<+|h);p@IW@3AMmO_@p3&`!!+2Y2E z4^o$tnv|%46b40ZhUw`p*0+m3m;u>DBW7fhwdxXZ*2SsP;|GPUCHqXa3QnZXN-%EG zQuaYHKPGHi%09ENU!coCb7xB<=f0#&5}{1u!~_5D6<_U6XmKOxWf|I;?m?darEBpo z@l^MJaZT-420rY61AUMG|JTyk@ag~meT?Td_BSQtr17D=;OE=Fd0*3~|Nr?>o;TXx zziSU7@xCXGx1Illn)X%Me?Pg)r~ThYc)k>uk-n;vRr={P@x`!g)RFa1@UG^Iy%|H_ z+oz~|-OYqI;CBueLQ8*tW zUzXxQKwv;30s#aA=n{qR<7C%~lh|>xo7*PdWRuOsXLlX@>}Io$FUfy7N2KIW`3rAV z^<}zydIkfMqLVD~*dj2~U0q#WU0q#WU4>`lvuC}24J~|}=fd0uB~$pa)vWbr2i;Ga zR)3D4c>TY56zYEh{eLpF{+~NJ|2>w^A@u)dLiN9SK<)pNCe43~Pwf2fRsDL^1NJzS z`agXBPZZ4gKbfC6ng7S}Ni_d`oX3(GV0K5Z{bF(_S@0#qlSTd$`$WwDX4~(2wSEqN z9=eTT^FKdz-kAUA@)IZazhn8V^O4?2m4v!JnM$pP%$kFuTi%HgddO$k{MTKm4kiVL&i`|hrv3N)$@$Ome8ThJ znRWPhQ=}7ew?b!nug-!BNPjzLR6$_4;~lX$a4hlw`f?%>Vn0LXKXlH_PV7I&^0Cf;?xxPoJ#=M;9_(N=4h*&ckM30IVxe6MI5A>Cc=E2@>w3@@fV#!u zoJ`;2|3u7xe$VQY+y9*}ob3OO<1=LZ6%U|hyLrGihROeeY5$+d=ciBfzhn7?EPbhU z5VQV93P&C<6Mzh^dyOiH;Z(PZv7CS(@a{M~KbNOccxebj-po3MeEvi{|7m@O%zxDb z;J^Z4$owx9U~Mqk{C`3b08gZzNQpziCsNFnhaJclo=DZ3{;gRq4xUH> z<+i-wXR|{mfIY8TcklF?eUCs@TU#CYvAqM(`n_tiU8z_5)k=NPqZlKz6I0^(po>wp zls8YLWaSEh-t64%wmQ%h|5E8z(IVCNdb5Sx^b@H&0EG&+;T^^dX|@NQfzMv}&Ch$Y zlX+fhyNBnLt@`Xa!V&|Bwrlq)KH}#4bwC#TQ92aqF6_gyUSBCvs@zwluPJ;}7LjO?h&F*36r(mL zl`{N6SdiJ|Sjr;(KVcM5U|Zz1OC%fO%dCu07CR@kGcm*as4 z(1CqHhuVM6h3r35`4j)2WBHtNZh2nUnR2!}gPxD!Y%%b5!`ZCXZUGbcDaHoQR4R4K zsj&#IZSRhg@piiXdszp{P{32)sW$pv&!OTJ6tz`%&>1|%Dp7UJwc`1HP$?xI;p=7t zgUj<*PQ3##F{ZwEx9Rs?XR8ChL>SXH1%qvSGCp{yDg~&ZSvr zGB*M9HN|kN54sex7sEH6y3^1miMzK-Aqz%c!<5W4?&ZgT2G|2vjXVCrX{ zUjA&e!!5gE^Z!ouZpE+OhVOavEZ~V$=p^Bj?%K&G!6#z=Gbf#+x&AYJ|2J*k|Cv5F zb#njbSU&5Zxb!jfe6wEJsrt9rDSlzRP?)%Iau4Xw|6|U7zgIij;D7&PH2dEK%89~c zesb!>{&yT7+xRnm|LEEnCjUeJzi9t0oXG#<_?&u@oN{@vTPs%XD z?|4mT&4W!T6wA5JV!xhCrB>(5rR!^zg;II;)0tY`Nq;(1Z}up|N&3_IbT)mld#9dF zd3P!O;Psc*uB|LzpIf^&`|05O%%=k=sgUFF_>Lp55ILXz%abMONI$muf27X8|KmXC zKhqfh=|uiN{!iTaE6#t1;)T)2|J?bhllZU4@<}@Wb+6CtLEX_6gZTBo06L#}{#PhW zovi=I@J8GKEaq%6f#M#FexTHaKn*V&=Gw`dI%`J~0~6nmX( zcF4U~yk7D!4J;Rrd4+ddN#st`boI69nu~ zW0l+0*1)S^nF`0IQg7D!mCbviM4N80r)sUL?>h@zu<=BTb+6%6D$RDYU#Vn#uhnoJ zrPhqnBiE6gnF+e#I(iRgblkH+syqaqQ&Ga(ZVG~Wxqxu9LA$cdDH^^znE5y;$o#44 z$O$J~x7WmM$aqZDS4y$;>eQ|i*KW=aNQ~t&A^fvORbTBmpsf=ovYjgBWvp~I@y@B& z%UFP!Em4kYXkoe2=BYdX67x5Dz5YEps3z*lNc6n^pw|u$NWc-uUOM{3Sh0XKjB^Nq zs$>JGO&j5>|PjLo$>cQOGX zy_RoOgZE_kkFJ9sd<^(C8c-}FtBk9SW^?^czbftx`33;!K>*_tUMdP@lN2H-hbG8O zBb~V)8mir_Gv*8Ct~M3^Zu&FMeUryj9#G{!uPofL=? z*5K|~092Jt0BvM(m5E^rH_W+Nl|i(&tsJ6vXAIA#8_@5%kPB25n&}7*M-aI&oo4)1 z=1LV<$#v*Y)}^B1eGp>|K^YX@ZoC@zx{s%cRc#_ zk01T|pZCA|_j|9uiwynn{a^0A`mg)n`}yAQzo!nHiPw65W4*9pA2>EK#@4Hw?M`n8 z<{CS=+dz5Y-;xtR;gFnUpIz4(3o*a^G{(sB+X{l-B!a=*#jQYqo}l!3zE3xI50Fko+^EZ_Hg$nKdC$tN~kXEHksR&CJhAs+x7l_^vv4U1Qk@fSqf4r`mUy@`^> z`XMoN7uNb1Da5C!2~*X4dS183HJ|Xp22ChS(GaEzxML{ugsIyY!(~u%L~_otg3>g& zp?o9xqM|XHd>qk;p&w{J8j7T>Vx!4hR)epyg}SQ^a?(3~xkSXkOn4TC3LU^1WzL&LDwZxK$Nm zuG;N-QhY#ZU2)XPa`rGMn7ICN2b0^*I``9f2_p^s1N@jKA2Ir|8;lscjFGrJa)3Yt z8Nt#GeKh3?)am6S<_dDm7{`3fqO~b~TDQWo%-{`s$v|3c(1mnfBuX#WH50Fuv@@t> z0a-1bQRw}N^@UF%m9DOe*pcom>dop_yW;~SR4$0eHk%#bs&!Zs)4k7C>b)jx&%^_k z2NZ(_*KX0XL$KZ{z63t{{kQji_@liqzq0qM-~QplA7u8w|IXewf1&DEr4`Vv!jGBS zmHmG&fAQhJd~ffKe}*sB=EvWEb?@a5@!1FexcAO)*-xY0hd+D&!SB8fZH3fWhO{6c z>67B2s!GUbn|>v{O=pA1l}tJCp}~q7rqJFg&<*n&7Osup`+x@%=Xcq$PtLHUs}GMBEXCN!=d(u%s6;u35nUDm$J;c*r@sx)+s$`--0cAJ1t)^^FDQt0Q41U!}YJwa~7yyd!f0G0Qb3DA ziqs0UT`5{&N$zrZAXatAX$BgeD{bti)DdL<_6P)|QK#QZqk6E}PgCVu$ER@WvsL2gb@ra>IEE4(gr~)-53?BQq=-j>HDaFm8$8KGCyhluk~3)KL4FQiT`*!pOf?7lk?w*^IvfQ{8?Co z%)?-I25Ux#=C_6_On`{9*$fHvGHp~UOFV?AOAk>I;=3L80QJO<00T;3vmO+r6$sa| z6xz?M=gp;+Rn!6(5FY%%zp2#x%Hm>ieyy^)Hg^@qV%AwFkWwRJzep=$UxhZqeld-P zeMe1+e`QJy`z{n4@l$F{q8h0F{HxHqQXAX`SljkM!fIX4FzY>EkxBCQs;Fp%sUwP} zxH2UQ!|urIr#Fn<1`L+I8WZhiX@qmFPPOi50suVNE9_YU9q4Y&>-L=@{XtW8)psJy z+Ay2j@Hf*yYC41~a2AG%^&Jf0?7>#KQ>zVn94J5oRm0m&TMw1ABO{^U0j39Un*+Wt z`kSO-{%rjg+vm{s2CC;^#{+vBX6DrQmPdt=ijB%POlzt%hyih9037ls&q5=O`O%D1 zBz;lj0LiMiVURFCRz|rI$4t{BkXWtt$3|_kJ|rf$URv3}MAMd4VZ^PVI0u}Wfs+x3 z8~Sm_5S)HsMP-f1NT9rqVy)zLgGaj951y|xTe-B8!4TmFw;#P7CsKNn2$#W|c*u?0-y1oU@q;>$MRp;ZYuo&~iP!*X-{#2&? zFosSx+u3p7{oSLU{#3l~fuisAa(}hDvb;dwixe;+=@Q=Tu-3(L6~9*N1_Gdioo$Ii zyi;u0;K4??2%`*B=#0Z167jBcn;9E9IJ=$EN>Y=(M0>7tD{KnHj~_1R+9bs7ti|%o z24oKEH#Y};HUxHjjI;f8yW?Q9&iz{^-NlCoJoU^$VMG*jTi@$IYl79887<4FuP4Y#( zv<&?O^@)Cfj$g)Kg>c0f$0B)>Ptj>-oF;ktQe;={Ee}j?77959372)@F08!`;K6J9 zoBN;Sm1>}ojjE=i!PTYk3%zpNV;ZTblhgbu*1hnfPCW_IQHzdrhP?s&< z`=NeVM5rFYt%#j+9(?rshu?nd(J#NW_s)+Wz5L4l zTR++R@Qpuw_}bn_f6ttB_J8>6{qMfJ|GnS9!$)8Fm%XpOwfD`}_TT>b-mCwG&p-Iq z!+-hSgZJKk_=CSyoO;N&&vKQ-b;QqMS96rj6o ztWjN}L;XmK#G(iDKMPLC)94;+&&@1N1&ufP(FA4!Q}-Wgm9V8D_4U zm{tl+Mie@)6l#7Bdk00w0ci9J2?d@uiE;)jABk{3$ZBg`+6pdCqPvH=eC}?H`If7T z@&cDOLc7lRB?>|s-0ce9r41g5S?@=j+T4EUJnIy^@pHD-FWPWtvP*Z+=VMJf>{;e! zY8viX*(-xqD<0aG=AakXo=F@{L3|n&Xr;FyCT{>HnJn$bG9&Pk)9Go*#eI^(Wg0(V zk{GhB$3iEGkT%(@ZK)1P6AS6Xns?GQp^;uU#kk9EQz+~%=Qv(hq2x(S?A|v=Y^Qn` z-#{@z^FK8a?9uvvPfVSkHuL`#rq7+k|2me>$@$;O`QH%#Z;J25 zzhf?wYRh^w$bF`0fJAgKmK^}^v5LdL;Ia@aB5v5bS~P&6(ZcxR?CJ|i@&iU*KPr@% zu^X%a|7oXetQw1zj~hPUlO39;hu7(u=yO%xozZ>bWBtyU3!=q6+~N_hmJRu=rm_ix zv$5M^phDtXUg$-7=sIP3iq7=QS4vkaS4xXTywP^n>(`WgFA-_O(<0slP-`1#*}?-8 z;G_WnZuEkKHP%RuVNgL+IxN*aujVyxdz|%9!=##^cw)E{+y>@E_$F;%n7#rn``aLh z6l@%RHK8|Xql=TMh>$o&S*Z!GwmNr8D>5z%L$RU{x^mK6k?wvh!UTIJZo=VCRJ>HC zMJ8|$Wc5V_pLqgC8BH`SW;CV?onx68Yrl=Mtd%gYL;-9OW%X_2haHYu2*X|?orAp z@_htxhf5tFcM7DX@+2jM^4d68ohp2lGnZzbyM+155*IRf^MZi z#GEC6QN0r^I~Qy>TV53?L71{Vkmtx;Lo_)lDHqWq7oOlY^4EG$s8($qQ?n#03L(1T zjsFPR+3x&DQ7#C*W-!61v^XFjTkqHun2ll$>zv@0Ib$>gC$T(g=sk*{Df*s-qB#tc zgwPu3?{ukYl(AuFVdf_sb$4UPTyQ1u(CAVIIj z9{`pF&eea1qC-Yoc2jb+XgG#NzsYuH)?mnrDBF( zeW+Z+*Md%k#vEr5t$lPMpRJRjCMY6UM_ecoi=(v%#+`n-p0oKU@CdX?Ldgv}&O^bu zycO4x0WB;Z@{gxMeq#k2R2O$ztRouJX2zynjp~$3?u4Rn}>1s~*u;@M- zxI&Ug>1C!i#*DqR6{!z;RcumNfcefxJjO_`7S7b=Dj7?|AZaIMg{o+nimu7yA_Wl( zeCMqYtJ+@X{lM$kVt}J@w@FiA>fARUCOFrSrIbDZ)?4O={LQhM`i?0JUT9 zD0rc>@o0F)Y??JEV3LZ;?3-1-+w0obpg#1z)!d1gK{P0=MD&n^0;96UXh{625pb;D z-Wq}P8$D;O_D(KO^s>=(wk#CUXtAlsZu8En%gAtD6Rn7?thFlHE9XO^K2j5{3!z`NPzFC@_24P!s2Z2#pMfqt`FJPx*e zt?Hm%+ctN4cea}?kD_(NMWj#reXpC(IddgNP*A`}SYatnt-d#Hbvj*sqM|`P<*Wjr z!XG~T!J}{d*WMf7Ev@W-=iU9+f2EtLq9fk&O;&VxhR|pYHS-yk6x~1-Ly`v3q>yv2 z2G`cH@?MYyYqY@C4Q>&C;L``veR-g=uEdnTN64L=I|I z@yJ;>z_UM;PEjeH1n@ zRxyre1zn5n`VgvGhHp{~--GwR{@{b(Jb3SG`(OL{u*N2HoKp>H0jd7;{5 z@|70Pws+T*y@A+AHO-X1l-kX2d0uz8y8v_K*{L?$Vu1`4y#4=pYwydi?f>T0{V%?= z_p`r!_{}fxedQz2_NW+h-^sB*8UAavbhU(TY>H!0Hn#LyvAkpySCAlej9L!YwMXxK0NUusue|>7Bc_Y;PB%a~7K*3h?4%UI7mB}4#;llC z%Ch~YD1o{w7RzW@`t`v2;?tI=G5 zhs6Jxn4C2If8o)&6aU}i_?-Cvp7{Sp`TtrY{4@70>nOS94#@^Mn&eW(W7}K9x%d7U3EwG3u99Xcl%0q|LN-aCl zho`GHrf+;UDS%(0LkUk1x#nbSv7nOoTA>@yy^CzjgACB5)mhkt z3Z+BRDsmn_Yg<-;B{FvTCM8dVbTXE#r05OjT1=gtLHU)Hx1r4iZfLWbgcWIM&2p}V z{)f!j=j!XrNO!CwnM`<8IqF?$Z5Veo-Q&KMX1fYQpK$Y0a5HQze6Kd>H*b3~;d@?_ zwzw|Yg{{W(M!O9s^vvFz79K&V9xj<|A|;(r>Swd(Rc~o;><-rP#QM{iqQ^ErEjyHx zxjBSGb@XZJK^o?P07cZvxT02>u+GHVZo^m@~?Ju=%a% z5jLXSSJ-5*CRq=e8A5?K2#TTC+uWsR!?1GE%XvgV)<_jIK1fDlghs*UF{5JP2P+yj zF*r&9kI_l{N!wNbR6a@V|8zzE0I~n^GbI1xH2i1S|MOGR=TGea$MHF_|DV|Zhs6KT zqI{sfK})$OV_8&v7T{xO(2mhUbSR}G16Kq|9#f$RUCdV(k1sO%-L4rttQWbv@W#9P zG6%{Rs?wB^UWB&{sY3A~vl2iS;mw9XHiBNIw8c>7stc=4O5d#-lSTkb2_4r?zfzOy zZW!fJRHmeDj@I5G8)IsGl7=jLOw5>YPi4ww&kp}J$H2{dTDAd(tx{md!GE>OI5TR{ z7CwSnKuBJHhR+MUQmsH-4_#BiZyRcDEt4Ia*}|PMt5s(O_4=DtU)I%Ntq<|04?w6i z!>HZ@;(EkAhYfSkQ~>mj2fTdFyJuwEvJYCK+njV^6In{T$jMJ*+@Oegr&BATdus%D zJK33|7+wP(eo5YrfOEzmlmGtU_j3RA75}+iPXy zH$yDR=h|E)$EWaA6@F0*#mhIYvSDvU!a#@oXb`Z#t_*{~eZZoDz}mkItMo89+dXstcfN44 z|2vk?$^P$T{}=23W!dn_N3PRodax}K1wHf;R!Y%t_7+q<998~qg|bmsTj((+p@}3m zpz@*a%9bJ95Z+-2 z2hsHpR@*`$CBgIzU(>7g8A-UIfJMO!_j0EzsE4ResjClBpJ%*2AOtIc0^0Pqd!2r3 zMkU%%@LC3MPNU8Kp<#gDi|4_I1%t`u{ljB}tNfvXY&Jf3Xc*hI{@p_Zp}ok!J3O4v zRoIo9!;_}jIW!EEpSX=i=_Qh)!~+phH`ngm$z*f*mFk-_B?D6V^Im(F+;ik%walX- z=H)GJc7UB(@QB%vK!&kxtTY4!#VlL=;H75Iz=xl-4)ovolT`mVt^|#=3y9bM&*!Jk zpEK`&6sAw^e;mu_ME^h0|A(CabbRrR2>~8B+BhAGi6PEohmUYP6bi_0In{Av&plk- z$`~Dqod=6GOqv)tmI5flqPe(owE_fPn!^m!>C>62U!%K_zBBHe&JcLI_J_aMPiOFg zANxH6Z<}qe-O1KBs4Nd;n)TDE)ZF4?Wp!=tYO%6dT1Cj~v^b`jzRGWMrxt7 z`dnr0rR&AY(%f}<**-txq}#VqPx0#-cDwn}X>g4vta-9h~}h zW3aGQZdUzfja|KDWS@<&Bn$g!)^!-%*zgEDGw@mn~D?%q0yil(%csj`%HNqjkYL8R(u%(!{fn0k?BY< zMwwWwrCj@y!dlANp;%8Rz}iygNU`QI5$w1IanYcFy$!uc&_s&;%eJkN88m`h`?jB$;wy?Wznw9_}mliGgR`wTB|7C}K)HayRH!4BAPe znl*F`?n}742~mk@-^_Rs%C9Qu8cePXeGR2l4H66<1%WAVgu}8FVl3NON{_-&HX-pi zhDGhb{h-m^BO57T>8*a~P6k8g*;#mZRKu_ZOHrRp8P)dDj>T60@sELq{r#=}(GGwY zH30DHvCzNMb6}@XZJ~kaXvg9h=0jvp?r$CK#AroLj7zid?x+T)g?5yqpA}}3l*cg= z0qXZ#M>{22QB&d)ygGWyKeqYMI)?dR@5cSs5lja8kUP9FbturlX?vlVHu$!Hw8aWD zg)oKNx-nQ6){>2N%|5arqdZ4z`1nzqF?Ka7IQsZuz|x)o(e%^2>Y)S(8|jncsJuv2XkwulnQu4X?=-wZJyvfn&~yS=mR0UxrH z2*!-PYMU)w{Z7YeReSIQ!@+W$r5yzh=5 z-iXLz?RfZM4z|_Jt`c!2X+3UAHn&~jl0y~(U2?K~c!_p8qf6-nH#NetsAj3OQM=oM z@@Y41uR&qX#gK#&1X8IDmEw4vUC+K)2ol{DUJ<1M?ndmqV`1l=PP=svg9Olt*G}~=W?{lnF63R2hV{ON)2^I^ zJvTVuFl@Dn?7Fhb_A6#7c$U$gWlehPZ%Nt0NC5^Y<5qrur;8Dp z)Dp4C@-((4&?G|uQvyKL2f*qz*aK*pAnTs;eA5FC$F&ZSLp7+;C#vhiqmXzAyXu`Y zmJZ8Fh_-ylV~toMGK3bi>+`{-ov2o}W;j zg_dM{hPz~x752NR+Clh zk`zXZ!`4p&u%fss0vyltIVPC<*NW3_?6Q~`U7inR_%&xe)oF70vHJWFONDvS+Uo9* z8_`Cg13em}go)^VW2{C-dlZWVi?Nb?z-Qxcg(qaBuF-6wlB=L}fG`d04Dr0I0!)BNTQ4Ioc5l!#}bSvY>vb5i*t| zawSTtnUX&`>=vVz9Ojp#tu#R$tMpSr#@gH%=wV9=OU|8SC6NseSv*wy#CK`Kuu>){ zW|}P68?o;SpoRD7TFG_#rR~!7B;@vedhklws0R08wI~6GG-N22 zxdayc#CX{)2*bSx7`klzprl)-``>BbGC7m*UflX<-zxHoFowEMYMhZHVt7OGF9TMG zays~MWXa)5MnJn6k?Rb%^_gwx63*D%KAxklP#W_rc7#6&G*>(0)6TQ60z=KNJknmOBDN;^( znpphu_Kr^eCPwX`5}0Frq#{_bnUlEcaj;DT6b;=qzvi&>Ea3S*uBQ5vi4GDHw6qyUFNc)vDag|yDF-?Szd_4i>#)D}h4T9t`P>9uX_ z+$~^YO(FJnBX=FL*kde6ci3)kr7~y$QCm+Y4kx2G6A0VZ7lQH>@mS?SF&1MCgi@K08 zx?@NLZV8Oalli!#4oMona8$nDt-jd0=I=b_{0y4C)6jt)M3J+q|k)&--2ejQN7 z8ZWQ`zU=*dI{>TKjQ$A*kYa-`>b>Qmm5-Km63_{7t>to71ve z#(2gBz4zXOhl$|4_f-SV{@ee|s!L2Bz4LRkdNc<~1sxufRvzXVR?@YtNII;T zaXJV~TWk=AoEXuYW8?&J8B3byLrfaWkbG}u7#=3s#c=Ur&6Qo5)*IG!ZqT5!1p~xrI+r{15Fvx@kMjI;n&FsfQNWF=prKExW zCrV1~I{o8gNzuY%U`dJm9|21~Rzm7cJPwvT9#R@6!vtJ;X?8@>8a@uNX(A?^oZ2iq z#eCMW7ve2BG5Q96!ivk|Z__+&NJw&%ffLZ&kYfiVy4G+c1+=#Q6v#fRG8|Gr4yP6$ z09>-2w;5I@R+kc`Vsxnh&~Sk(%O0IBB|tuYT`C&+14~;8a>Rv8Mdhe;sTe{YwJsHd z{wGSxBhsaYvE@i#!v93o;r_q*m0Vj5W)e9YZ zQGa}Gj-X4$F>nH!A4->!eT&tlj8aEO_Q%lbQE5|B0XZ6NY6Ng2X;U)X-Ehkklp1E4 zB0veZuUPhoEmHvUm@HFPgdS+7+O72+Y~AwM`)W$1}>2{#l!Ggbtv}EqB)65(L$A%5K~zR);qc| zWtF4WA_erpg=h@AwupgB7%luHAwwu=@e$G?1p6om5k`02fBo{p%PCDz1U&2t#p%kp}OmWN6s>7>8)Q&*VaK#if!dU<4WYOXt zcj!jl=-?HdnVW68=t3P=TA&yMOh9r^UYYs{FK;lncFpILKcq`JlsQ5lb4S@sq zD9Ud=sl$H={lDz?I$f{Vzb89mMh6Y!YQJHoN6Cx;uKScFfFB>kCZd(7D{Lb)rQ8pT zXe26%*a(eS`7^$~WbxoaYvUCv$-R65FY*%+KE_hn|*;^DEC6;dz#$ z1?(2Q&(28z^a8-7d4M1TVRxCGu#v=TXIz#bJyoqft`@vIb3r5%>DF^k69cM@UCKX z6=ojFO_^zoF|f{k6a21&ATi1f0vO9C=tu*2;m{qqGFMtW;*KE5kvihvYIYCZk=5r) z*N?a(2vY3`XT%q2;>;+#URnS?EadYOah-)NM5o8QX~t@Thdd0C(gA!Lpsh{U48tKO zns=3{8=jRCek2jnf(QoO-2g5pgb2SB*U{={A7@9UP&(p{3b@HTD$U!EtFM~GJ=*T7 zplv;t91F7cMPAn@!#IusdU&irtcNSA_H58@LQ#AcJVXQkmeTqdwWpQq;w6a`TyH+l)URzmQa3+E0rluwppyd*%4wdHFQ8x@x4B}lw71YU5uvN^gAtfh$|09F0zV@LKE;6ou@Fq?{>9!hH`hK zL(Un8ClqEG&@m?clc=;P8PU02dc1Iv9+VyU~$DVt=Ub_PtRi6E9b=q49 zn*w?F`khvWntX~?abY)^y(pVxb)SUF&KJHQz~b+*oB6S!xSSDbl0k**XW>Tx7!d$4 z8N?>b$_)j_A3{HN8410+UJcKj@SFs=LZPE`hCdVZyHZ*#I%gW)ivf`~2aSf;TQ5vp zKyR!2`CX130EU*>j|Mupbvs?Jo#6moV9&Hdp(h(%6>(ta3^H7?TwVcPGfj0JG?;kx zn{83i+C|pjme=<)3^mum6T(h||IE68PF4&eYi6N%`NmafrZebw2Yobaax_`^RSa4M zt-6jxo`brW5){S$)mk0j%ShnZEqwE=laK8hRU4siM6MS~SBfv9?Gl^F#=%#5Qg7BN z&=4Aao9${J!;SWKs;$;NwRb}iVBH;m0TEbTn_IiFN=VpAsx*%dVP4(2)9i2KX<&bV zgstm@4FK`BiHA6&ZbOllSE=`!xbQgEoD2bt(l=(~Rq=tUsvLJl{_bB|kdGF?cY;A_=ba zQy;eUcGGWGc0Aa7R(+1NWyY91I0PuI65eOMT3-igbmMjhrT}VMbZAQZt_(hl<`)pw zw|T?xjb#XeFPZeU%S)A&<;Bu6?uoC@mCC_dnbw9`S)rgJED^S~E?#EXY5?TWB7HF= z1zwmdFPD~)Q?7b++~5!phm83mOdc_T%c6~iGKTUQ8fuGUyh~nLgt!&UX9IS{wa;4S zQo!CeNfz{B4X_LPAn?Pm+kx+8hY4`XLp!8THe{lEI{H9x+Jt+AdX&pT!4asLIE=EI z@YR_xO7|MVfl)eDJQqgQQV0-*&oIV}1btvWjlv|%wvlZ|EZjOfM%F$AUq-=9+`A(g ztzq07)xgZKZb*g(mZpVer`l|zZtK84spfLI3*T?AZ`jurLxN{*D}%aB^?L&^p10L? zLYZwaa|ShNx$0NgPRcn0-!YOUTI~A0Y7KhW#?4gC%c$xslrq4h%2ZG!NvGezJw=7B z(G~@(4_i9uhip58`?*WlCW)yuTRw3&R}s58RuG;RlIfrUD-T$j2E>Vd!)}H_=X3>C zcB0YqJgp1ptlDT*{cUu<28l+_ePW#W2nAHduIYn#n9k=9-uZgdPu6?rrsy}byW=-2iKy-bWEfzps{)-^PFfhT1NLUDDiObEvsb;5|kzIG&3l^re` z=MX?(xNBbI+7EF68}&ikZ*H}{I;+(3)(hvz;5X?`xCJ-wPPb=( z*8z#<#)i^M|4y@C+lCfYfVC=Uv6D0U?`h-rdE@sp`aPpF-KU=htjEU3E^5y=dtUWc z@Mu>hF3~j~Xf?Xg6*Zf%hEd^@GQ$;5reXTt=bSUXaWzzw@Vev-(5r)LN`IrvQ4`J? z`r|re3ps=7@VvWyRnG=KVqKCHCu6s;cnjW;nsK-rjwfv7KI+9gNigc za5AlPqq9DzamHv*Lh>M^bj4+&b+`hHy&@5is2`Et5ba8PHy2qFM}7Q&7u4~CK}oy8 z7{OZ{qv6N{P~0v!1eA_cl8*)u3!2^PZv$N=Yg+rpipjmHte6cLm6AjZ<5E~!)n3{- zk-_=G7ZmJ3#>2OytjPe3b|L`tv9{ACE^?h~rPUIdt6}jAErOKuPaBgu^rr4r$S}(; z%gERTPkFa>`;#ijK4}H(kNk5)`u}!Z8$c!} zaO#lCLC-(p2`2_MGHu_f$ZC^f;&(>B}7TKNHbcj1Z#l-~JLCg;>I0IKO z`IAoPJ0vs&5px`@r zg|*%p^mPENit99XK>AmETYfH=%R2Rrb06TL`GtYJG;8@)JUWs8$M89l+a92adgfk$t`q2w8nt%6HT;m+e!_guKkIjD zx4eG%`Jk;o1CAZsJ=^dt5c<`Af=elJmrs-xDf-q2KVLPSi`ZEkSFTk{2U%@|y@M~C z(tP9KsX=A4+4lpx&INkzpq~!Z2t5Z%Rm54T{ul^odjmLnJGK|?=FaB8w*nh%0!bsP z?R4q`*lXCUMZc_dde-L@p+S9KI7ctgizzULal!^?1`*fHaM0GK;Xyg|T9&MhpIE7!{_ z^TpNGm2!pIh<9CHrH&(E6kaG5&0>HZUVJtRuCC3^KNl(*DrFP^;#r~-*kJNPVA`_)x#N?s7JoHO{+R%xye%j)K!@A)`eK((&i=2%qF zpO{TAww&N3gBiLww|cEoTwW``q)!S?U(~F)lS1&MU_2>=$`nFnCPHNz!oo8X z^YhOX#-}EW=f#3n!hqPK6QR` zV*HuOX=r9{at>9~3+FFf7G=**PF{KDne)$#PZbxYpzH;}Vd31|cyVeWeL4BQAK!$w8(Ug52!6@WRqF6*2gkdKWvFgbSj&^{_zwYPMqu;~X< z+Pe!I8Q2(c3q1qGjq{4@DLrX4VHlvmIv!lW2)-x|F2toyQNVRX2^lmu;N>aFBG3#Rk|^$fV@&!dCvKQQ-(jx%SP=N-?*TFfUdC{5fMFd z67&eit5Mgqb8LwIF>kUSNvF@Ab?6rMZH^RU9lTFO7rTQtRx0I%mF2~k6ld2)g4)mMbUfR0GT~N-cFgXcB)x!>;obe#_GJ`*;kf%E3_JcfXBmr?e;b`+OS# zRc2I*T$)wyx!NIQ7}sP`tc&}r_PaGs%v{#jjF5yye5?~r)ry)MuA}N2Fl$v9)V8@N ze|lDD_aX4YjDJ)6%e-M{ zr+U{>udTr=9P?CMm(DuRU8ZwYmrh_|%CJ*b`CR3Cv+^aa&Jbn{I^g#^z3P_doc{a( z8NIgcC|3e&umS48w6EUwoVIs|yCyQ;HQHA%-YF^zMR=BjUlrET&&NVt8fsfSt4pG;K+wbM!_N2B?AQvig;sK_8hglJ!V8;DvDA|CFwI zc6!Y%Vqo9Jxxl_&OwO5)feDY9-^{g%olZ47wSJ3u2QV$I%vY`!*I-e&x(bV9Kr>QV z#tVqVr#@viPaw?dnve3QJ{2_nDaA8c4%esfi5wOri)_T3WiXHg|ck zSh>P))Z>k0+1ChAUvNTIB`!)8wT`yk(AidJ-B_Zo21J$ZB&X?W0-755eJVigAi4$D zDU`X$qVS8QAaWKmCqufjZtne1(kLs0BC3R1FO&;Gb2j(rcC?GOZBJVu2&2FX%UWjF z+iZ7KXssJE;UCXR(?#nHMp>64om4j=%(E0{9bKNN0#iwtV1`|}DW6#nST~^*hUn?k zGE_;C8RGq{vwUN5F-~?c0MtAwhzC+}cf6fi_nr)@^wjWWNULG#rt5GX;>t;&?qQGw zBBkuC!a2@UPuW+EU|{IxNtD7bsDd#Vc-4bZg>@2G7RG3$@#%dWP9ejI1ek)%sU+3Re)~+4L;=*yKNs8C! zHT>n&g>KMyl9hseOlA&BKRL@^#^B3ZJQ%`;FrHHKTFa zY^-Jmlt6!GQ9tyJEZcn!eB9H?om}*;Q4^eqP4=)G$%LCoBv*%Fo8%fAu7YSocDmd5 ze2h})5u`Uxv+p6x53VrMlgu1k>P(mPox{_L}}4!cHDT3!Q)FO(M6 zu7yfH9S3z{BN#Wln};mJbO-x!uVUI{4cQIqPidBO#Z1mB1&qnUJgJ8Xs->0NPSvy6 z4b8kNN(4uR8_Kj-ni;Y*G~2L9)74VJMhSW0vxE8;MOeZ(M@+8JMSme}fVD&vBspCe zeu_w%wOfK!LsHNc!=j|T;P|T35|lceKqcBF;JvHoDGt(N(fNXA)WVNe_VUWg8lI1q z=8KiNd8GcZ8s!_yYo#So+A4d)!GfZ!GNX1~)zp+>iwKTm;{dV<2uvJOiWN%As9gO{ zmxML+MzKDPJVp_G#du463!49eF_KKf0%feURt(!*NUCY#34?SILShvX+DfnuIsnCk zY3+uDw`RamSwOYj3=JD9buKm@>hgMUx?8V`+pM}~pICV#Ko~=DdvixO#`5Kl;mxB? z9tQC!!xC2Jc?6f~7Zu*Dh7d6hBgyn=3 zDi*~!Yhf39p-nc~Ss;!Ye;3`I^LJ0TT6by08JRDdQ%`IZToD)rYnszTQ-33Re1&F( zSZRRWGOY1zl75T5X{h_QU4%f(yjsr#zZFZ%&(G;mA)^D)VFit(3av#TQ^Vr`_^^&i z-YOXSWzg+=b$%)o9k&^MkJl%r&nb)>0Y!;i$EM9K?;hsuy2gvoi_$Y`!L}Isq(OX} zK8ucIXLr0Eoc?Ju?lYDgZode|kvLZanM51m-tH_aGrF#p>bPd;=$!cNAXES zq%-h@#r44~@(do(Bx{91)4>M?yMfhQ8`+%G({<+-ojo~EMGS(mI>rT_X&8a1IZYw} ztL!8^s*t6)FBw}{cnYry-XqR%8K?TJaMm9$F0ZU!pSJ{7K6K#t=uTj z7p-%GK$TX8!dby=e*kK3b!9mk(DI9=6%5A_K*q^96gZVCEr5_N&n;P|VjL>^48`ny zHLr4cX?~PFt1E|t4}~%YYPcSB?=c+#;1TUO25l$Uv$EA{r&ihVw?udvzzLlLSITRZ zD=X!txwWho#ofRF{f4N0R9vKQws;uYNy|DmQ?218Bvwi43{Ea`TFT5_soYpDz34it zEA!7`(1GIIk{;oa*BGKM1rnqNGY8ZX=r|u2-c%Af0wM<%lFU;~vCB&-5+=J?uJC8h zAGBc}?KE2`HhGWac(u_)DvL1{07nHoTwLQS0&hnvmc^Z~LI_~3Hn0yTi0hbGIcKs@ zX&R{w>k;V4nj1z)CW{zO!*`8moT~48JGjpPMs@tIS8Fz476HfNyQfGurJdj6sxCsJ(27QF6;u`6vXILe`de9KnA!C8 z)fxKrC1gNF5-JjULni2l!?|p-{Xh5_~R%SKuh5$^TswJ z&H|y!t8(5*XChpOw5Vm>Kxxzr8sgNWT5I{qR#A}R8D+(EzF;h3Y9pT;G<&es;9U_i zI3wEo3#y8>*rJ6F)HC!9mk0SQ>zoM(L8&}%zCtuwG;j6D2(vnDKe^mbfy4n$8k}TVDLI#XV9zR zBT7-^wL60?>&n1cpNe9Dzg{B-S0Ix*wI!Zqc}L3h|{9ux{M z<{aLlSjai%(i}~O%=B5jYJ1D;wY}C^E$2IJXPZFWysSKDDJHe=ha#Fx};p|t(i-Z&g+vm1rjOf`L7}roq z(`yFL%)rDUu@Z8cj$sYFbkl@cWMPrSY!b&R2K>d15c!UZYld(ResoGxVuOpJLJ^EY zLw5*9!4Jl|YqkeQqr0i_H;PTJpnG)lAe`Pqbzl$JnWx>tZ!`LeE(|SlRj)SsUXL5$ zGU~c3PB7S-$rabWYewld7ISNYD`m%(C{G5A8O14i&hxso4oG{t7(^PWV8KqKB%thK zIMS9ugz1aYerHhIcJ6rShJZm;s+*WBhcp)7+4g##I=xwBtYqBX^L9G7Bbl3zOT4i& zpid|cZt_Xij+$eMQr!J^nM`ueeDk9zQA042G=_!vse^n^0ZiBg2zT_QLDGXXXf`QvipVQ7g=+J+ks zp_>5Tz-UTtgxpHoNOZH{j8i5Bp0uD=6_lW{o9!{dIXGU5e~<{KkXP0so)m+yM|9FW z_l0a%VHNy;skpRK=2tl6MAd+R+>Qf+VqE|UJobe{Mzdw){Ika42$WYDHvRxsUDJSG zhKbj_4I_ao@mX5r8m(%d-BjVqCRzhMA0#&3u%;2?Do0kCCTC!qQ>lBm=?og(fN0;t zLVfvXy8shby`8jD0}Glnj`xK{w{yfbb>ACNrmK%>mW21{xxR(Q;NO8(%tnq6kYan%q;GCWKO$K-oMcBKKs z^Bji56 z)2(*25DS473Ob~$ymaT~fCs8X+f0V~D8h?EZ!rB1PjtiZTUuGTv4}nZAUU6UN}K#ky?3|(TtlB0Iocs> zn%Wgvobg8`m{)NFi($_L5S<>zk*%-?4m_xLJRcPjGSpZ5HTzvaZ8YhRG>9nE3j3Yh z1urB5!iH2fQPEAn*2)m)18w4DY|dVM-Dge*!Hm)IAcI&!JV^2Wg5lYg7!PJUio&B- zt!;Z1ejhyo4?<&Z{#p?^s)!9ciCSiLw5AnVrw_&#zF<&=JqXYnvTsCwBy=;U2U(OS zgZ8a<=S~}0Q&_vCTe?{*%cVOy;{iYz{JU7b`g>a1snJm}Ob?F7f1UlBsRF0FqZ_-RP_7!R}Fo zXBk-&q`{`b0HDLu6!w#ZoPVsJ1LuE>Px+vI;`2XDPEO}7`QOf;%o#@!kQ>T$>^Xzr6g+M$K6iLY^u4{uG&&f(DO!JPZpsjHoe87lo$!{5+**i z=g>`C(^90)o=G`poa=x)MI>lah}s4VwU?Ee?`Er0yOne1S!l`DJ(nl&>^8fY^X-n) z#w-H}iAwqoIW!`OoU__-(X*xPanIov`p)3FV3pa@FcOYaCBv^zhQsz20_t}<%qwR8 z`VFUcuhs&ZJ9AE>d6z0T`z(|~7nU&qjE^RBg?uiB#&MCi==#dyB4|_#b(L-}1(>$!Z%E``R|v!Rk#+S#|lPwPIzZyihDxDyRfj_>aZX+8S&`ipvY7 zx#dcQmLSm}d+0oO`NowiP?VQpHBzT!?6&ygmrGYgN+R=?Yu(!%Y&Cq$&c}~_lqaqT z8HTZ%hg^GAGaJJH>=rCD)rW7cb_O7m)2?}3(~WU>WfolZt||i(RxHD(F7=KAmmhGs z&Kahx2A1&vk+@5(x^0zP0p+tUyTsb>+~W1q&dq+>P+Xqu#sgtSZw>}8zea#Ti%TpZ zu9*&p!~&+rffCkdJhx$lOLI<1hlAD{w`IjgT+P#1jQ#-zQ?A}gIt1m`c?mp@8oQYv z8+tU$EpsFgG6i{|$iU3XWN5h*ZHM+4#pRXa@*3&j>Rdv`Wcenel3t2n-IQl%%eQ(D zT8>kBrciOhgXH7qkFtfVJpZX-cxHYSdNyr+u3G7t0bM;nWvGP@X=1jPO3$;>bVcB1 z$;GBWd%Di%Bzy@@D!C&AJ5|wRfLku)vm2&nLQj>_;vo?vI?9BSc$Fw$<4v2gP@M6y z+0V{aBk0U?yPg#o_-E5femT~$8ipLar2xi=axc7iLnU>MPU|sX7}5moPQ~JjrPZ|< zp?HH5rYcu((`w2?M_}Yo2yW{svDJ~E^I4d%dVS*>P4XP9e0c2nT!B(G7M!sl=}&oH z)ujS|$Jo``JSHee6@I9Ii!?UC+O?Y8P(-SfU_62-Ss4(Oqf31JLqn+|NP*&5vf3dn ziaCJEcz1<1wZ@v@33AWYn?8*@ZvRt9oP1XiOL(;lCrA>>WO#CD^M+0iPx~|GnJa4% z{o%+7?n(6>$5JLs$7i5y6&tJAP^X0E40Qbjs@*r)etyr76E@Cm44V78?;1#Dojmpbuk=Gwz< z{QBPK9AD~oSSZ&gFJB4tqWX@{1&%M9TI`}IKFVrk zsiU)WVOV?KX*7J&Y6TQbUQuCBEnt9VP?qR5eu=W&iJyYMw5v88&?T|nZ&r>sc=O> zqRo>k^Aypc3LbD&=N>B_4${>g>{MXbd>NHXWQn=w*E;P+rm{L;S(fnwR?)$CIRpro zMNaXq*Q+7zq9i=62ds>~f%BD^DQbwiA#z%8HXRj=?vfz}1LxXvip#pstu;ZcU3wZ5 zH|{iMDHQ!ts#qdS+XBZ44B1WX+32vBfmp2R8j0S2T`IES^sTo5i3s|V5qZ1Hs=UQD ztT(!A>`i@>eJHA^&tuIzIG$9urOc&Bjy zNVe&LCbG3>SSL`JedEQNTR7{``@z2z zpn0bXS~F~{I9%10To{H!X$M3ADN$DqA|;;55QDS?%jo|!9_#sy9FviTO2RC4@Nk5k zCKk7^?ZxGlrQ(vtQb=e;+B2Ml>2{G~EL?E8Nms>*7K7wCVR0QN+Kp1r65$D};T3Kq z$Dz;QB}wuh@8TK-D`;T4ryF;MC7&7aSwomBk4)7t|6BK*JI($!L4s!#Oh=!VxYFWV zb_0~bbyyZ>SHQco-K=dhPX$;g>Vq20jm>-Pk{=-3xr6&=p8Y@rmlF`S`~9vzbM|bj z)AMrtU8lEo76^EDB0n)v$iu&rg{g`2xv9dYH`$mt_m^9E7}9Fwki@Orc4x;syLR@} zK*C)0LVW>gmv{UQU{LJ~I^Jovp2M#=xOyf=#aF%;T=Mw zhBg**ws1&iCG%L;I&~Ic9Jn}RpQT@_Ld%vsasY^X?Pa?KV3hovoT)6R=t~5)RpyJ7b$X~4^S{RVPaeY za0#MEE9;n!WOxA;8~IMpTtY=y+2)qF)9v3Q8H&<4D^HA>`eVi{e!SZyimAORnImgaa z2$FUPM$13$d{7QS2V8BIx=2`^KUsws-VFMOF!jFe2Ga)C5}vT3!){_TYN)qEO`7(2 zKcC{m6p93^<#JQU0;7fty(8kThaR@k&|1vvhf>!K^1nsav|KKTW(7h%JbCdG6^kQ; znCxi|v_{0E-N7lsrmgDYDn8K_a3+sd119%q6==T4(8%-wWR&73NYywlc}c zNTe*{$e?j?gGyXmYs1C~Kc=8+G8u)6 zxXWx%gpO@5q9SgKizYr{5fd$0hD-(7)TQ{QWR_S^K2MV|5GR9kx>aXs+DL6>11YnT znXtqb2roKSq-~9?UW`7f<|RJz-W)Rp3K+B8|_9DhXd@L8r>C`{7{x|QJ-b?>_^e@hXyij&Bz&#b!!K+B5Y9>7jcY^F0t3W z-gdPMzqdgwHM}YeuNRJ>1UlNi&Y;K5ak-G1Yw4E4Eny!`L-CgbB014iiF7@QrUn1b z-W#UgBQowPQ8$ogY!nlBzQ(Qzsh%Oxxsiiu#^s-$vYe}p!_U!^#L-paBgif#GZ+U1 zH&Yg}MP8X7)cxS7ou5+jlS$ScHY16Ai3J(MU2RkP8bu96On`&?Xhi?ehNpQa5xHC^@XfP3@I_4+)|ie|(EAakqRx9^=4;9} zg7Zc{g6nh+a773;d45P1047+XxYL~{h zron0>Q>-+&7*~}vX5tm|DQarWSmP_V%};=Br#Y&6PHG)rTD9m=&?T`~5M^-YvDGco zxABv{!*`g^X!k$$duy9J zqjw>;88a42qGmJ@YCIq9lfOH0K`$7JBcC~l5H)x-g%ih_|omvia8Kt#~T6!)#kt|-%=TGM+?u!5O#z%h_{2Tm#f&LF_ zC;~AyNeoUkR-+JEK?$M8<|Zp22&Kr>reZQQXVM7JRuyV{$feIxwfiiO=EgH%Z|pri!5mT zm-rH7HDkZk>}A0d`GaL6<5fl0G4?ZC9pECyj(;vHrfGx)?&y8I)yJ-N1w3=F7O$~ z`52+V$z%%7v(Gxp6ZC%y02Hs3KoFNpbC+Q%p?df{PEibKAeM@Pz3kk^a)>2kO4!KRU% zz;WkIVVElYPQTic>DxqDiR04$N8A4`Rd0D1A$qhnhV1|HQ|G3Q{a@kSIj5 z^JoUD)l#W>VB@Q+=r{aAwbgRQJCmu@;(~nG3S!%&QmbXG_UX)>W~+{V7|xkoEt|Tw zP&S`#gRoe+{8#iCD$V3-v$+mDTbnCiEv~U(!#aZO^1W)!T8#8v>}( ze(w6UmF1Ua(2D9%**|H6kMW6{{}s9qSE=N>_YT#@u=$VQwfSE-cdjsTGXIa`6F2^N zGU2mb(xKYG0X{!HJ<9l>&!77r&h!!WG4^vZ{z=9^_<8_g5I_GXCO|nd*Z-;M$&>ki z9G_Dd^Q76{nsEmG#yDuM&U~kPuh-n#?mL-U*1^GXt_`Ypyr#3})wbK6R%ff}`A{t9 zI*a{!4ys5dLa&xqv*xvZa^A)}`gqd3G-y@Z&h^zr=k^4af2lK|G>QY?V@@%Cr_o0Z z+^N>^CN;l~K^L{W-i}X~;A$OEj4>Xn?{pfGV0WtboX%#y+H5;jsM5WMh1G&iwLb%8 z@q_o&@R}M!I9KcVHM*#ltF_Nk6{rDw5@+pNY1LU>xw7`cT)F7Lci1XFUs@0a-J(KFSt&zgvJr4_~P|4 z?q^oY&_rqJ`eF&T?oMfWe(}aaY5A&i89*$rtT~INCD_a?IBP5Lb9ub9e5DMpi%YQI zaGj-M8TZC(bC*ktrL~s;m~*AH=3IflbI$d-@>*&B#^M~daQ#O4`pRk%>9p#3A^Oo_ z-A<>6HwS@wEkBn^HFwZ82}f6a#i)cB{Ez?qtlw$NQskd6e)T-@4ZQ*SolZ-_==iA= zoq+;eE>Z@4XwVT=2klC22P5Ucmo7SOvAkC3tLXWDrQ58V^;wcD0iaC37C}{QQK~g~ z#?xS#_1K>ZUTkk}iR$y+!F478uUC839ert^#p3g5Ls?$EUYuX6%r7lqbG2%Jyw++G(4FdC5w03$ z4dtwF3w}r~6xZfTi>rj|(&|;bF-(SqS!ZH;8nx)E?<4UT&g{rFolcA6W`}ja@oT+i zx39fjn_GR3o%76+*E{cPnqRY`(aS^Sx!SD5Xdk8&o z&^k+P7F5a76|R0yFQ|9ij$cznuF}k5^w(}CYc{ABY5Mi%7W5$-ZcWfMSM6e?(oD16 z*U;4AdMyg1nxm8fnO^#T-K;-74*!2T?Wzr0+Ci-WqwTfp;j%Wzk|m`ncZ-M~&z%5+_MFq?6^9@*r|Ve;7ry#gy(g}Mco?nd;nGNp zb}S(6dXt5DMQ-Wgb?}Tr6m$&VFp2GZ;jWI_O=DzCk1v5_oCrWkF3MGKNvL4{B}y#n zcLu#0OENmECkw_>_&Ln51f&I=d=-sI zF=7sjE=4F100p8{V6l9DPStlBA!$JgDRbyjgMIVE6A(j{uv-qZvrfS}^L#dy{q=m9 zd)O5AQB0(7w$pZ=H(5+BC%JgB?PrHD0uFm5DIAlh5@yR z6s0?r{tkwM4vGs(guu?Ev+D)R@)ZVVK3Xr#j9Y37u0=UuX^(ky!j{FM$*_Lf-@rx6 zfE`|K?L#t}=rbF!s5f+vlnsUB-Suh%48wNIl}sauHf%7Wm<%0NR2EiB+RY%If(3&Eyqb`}EC!R=7`7_x1hZ>-p? z+Hl52z~BHzm?iWXD0Eo0M$CkkBZ4b61_QA(n8G8_xrk9k*)en6l65L&)PpLLA}B_6 zVR=eq-kn=sEjgZFt9Cu8^tpkDF{8DjD7_dYik;PL82fy*`FvqBZQ)J) zRNH>D!f}~(dSf^57B+9LXV&xM&ul!M&0OSP&Sci-#{arH{`rmUm=%tKEjl{q{J~~M zf$chDW3E!jy2W68pb{|)dO1E-Dmu?P5j%T(2aQpA8iwU!8o&l1M_(Uxn8A94voD&@ zXPx{o7PT>2{$Ol^Zl)R z*ZmV_ojLE>@80|KJSFg_+*FT@x&GGGu4i8c(L*!&0tc!*J%!NPgJgGwN|`n%>#Ob` za_umd4aG=C$(NLI9no~F{rHEU!(uWmN1E*Q*Zk^D`p1?AyvrT+4_iWyb2^xjul4I@ zBH`q|yVW_&jLme+yc--Pg|a<|_2+FmAd&c#xp zbjTV*ZL`BlbB;YjydUsjK)1x+JHDJbwrX4eMdKdQ*xmo;Tq&VFn2kjq*?oLepkg-U zf2tO&Hc;a0u9$ZZZtJLjs}9A*2DYJj@9~ekq9}y$WQqtJFTz}^XJ09Z8G^>+`Lc3b z^2>$3?($Y00?~*}=n}sVor!C%oTRayj-M$k*P7V>Vv;5o%%zZVhX)IZXp1rzSiWa2 zrcjqbsZg>9Q5|&uY)srN;(=KzJYBs7;PyDs3i#8Yv+=u^%`3F5>-EU8tHcE&ANL;*S+LStf^)U=Po zHfqhXW}6xYs`QT_(@1!vZic;xWMq8eKmN0}5b?*8eG zd&sWs!6Z`&GWN-o-z6hh8U9(u{cY5VQZV27zReKTOvVBu1m(GP2Z^!D28nd z$Lp{cQlY|6-Bc2b__;n_f$U?jzOo!hhLS)v;P53;$GPuAh0B5!I;7e&NF=>OjJ_vqvQ;gHsa|`kNjIjbBMQ4{ zlqZU0@{wkf+C2+Zjqu%aYUR`79`*DRdS%tXq+$khDI*j4!2Ahw^DZ@sl@|r&3(m8u z4PBZ9epBk`XUXhuMO>5x+JCVEPMB|Ocv1W@U<(>mOa$Yil_nK1IdM$H(g<;pZ@Fr^ ze`|GKDeneE6DMm{p~LzMkqo-2oz;E)a-ZB^wPub4W??AGzmJxpp8j?^PE=HCL4EPo z8A~UUOU?!K&5`Mefpy>Y5EzAz3d1mFitFV0)*RhmYft^oS06r&!yKB%j=olhvcs?o zFrgW|Ds-lk$s>L4Zp@y3Tu)2DP$q`^GLpZ59~E1kzN{n8ubq1oC_!+)`|C|$;MdB2whJsztTDpMG)+PJk*T%;tPr>F6V z*wGbQiz>W{4M@?0oENhc1)+!rAR6G@sG-ex$O^1^FO32pXm;@YCRR` z1!nTg1IY39GR4#HXzj0;IQ&KHB=wki&n(5820E)qDPh3sn5!t8WQWF9>5Z4pXxp3* zwi|d@%!CaXU+hG8H8(7}`@+{(*pcMsPZc@o#)%S4lu(W6De06`-LdYLY;i_VW#6G~ zw)iWqC(|I(kb6Q5y`~pJp|h9Rl^Lrf_ui;c+V1mix$2GTm8<{-7Hp?^wpkr_TLU81 zzkENVBw(3E+p|{fEd~^GJXp+gISB+u5a6R3RulG!QSk?jELJ@RC#R&Z@8c{hQ)Pd~ zTTB+>n#UygTP3Dw*40RA7TBXnHN{B!lCNgZppE~4YY6I_i%yxAj%EEi5M5s=8m+lr{v2XG4K3`_H7|nO$Jo-?%zGbs-wvqCG7${}H}0y8ML6jT*Axbti3e@v6RXs>KdPua>oXP=3D6Ha)C|dL)Qxpg zPfx4S-)CXJFx0O-1(tdIm7E`+hFSPF&~ji-vHRBrZ#}I7t0T%B;r?ULzN_5$>$Gqk zhwI|nL$*WgfzCZgl@$J}(UtUPr=0HaIChpUK9{jj$x1dqwKcYteXKxL=6#l*T!}Ao zgKA%m)0`jJ?YRmFy0tW_jSonNp1-n6c2tbXhHu#H)HyP|*JZKztbR4Wm(XDHxo^yeecF5pn{HJlA+I57 z1JA+QS7u=lt&`L+^UOZz;L4}tM@{{odWlO0G3y#>mUPBS&}5va+Uq+*Ww~AU%Dg;d zb|40`cmLr)CyM=ifNH-2EsgxY&jZx+ zBTFC(;!ct`KSF+t`ov9ba{SM=8NmT6YsttM%i|mY+<}HV z9!^49z*Q#>K3;LA@2@nMJkt@vDN`Gbn%IveJz)E}55r|O!tzwL1@>1>>^?v5y5x>Q z;^;xeaVP?Ajz}u{w}%fS3?o@RhD`ruZ*Lfxn&XL*h*0*1)nXY8{pV#Ntbq=!oGgx4 z?SRC9*TpmSXo)seMv+!E^tov=bQZ&FHJz5s0EkhaT^8S}u4zK4=M zq`nSE>Ed5)LctpO{hc_9%yh^O%PoGEdS>nH8$at5?!?F7=(cJ+fO~P06D;3(2i6y& z3^-eWyo(HEyo)m!3O~BUHA~PxI+JFNbu)DAxSn{ns7xIi5R9SYAE;>hP;;za>heWw9R%7>NpLnx zL^ZQ1Afi3ij2Sl~Hdbc)p!Bo}oOEJ@FSGWvY3du&v6!>&XO0hI_P(9wI$ia?&-&GZ z2mh~C1_}5sC`uTt_~j@w#B#%ZAR_(;KItS9%+>oj5@z7ra`$TB+bv_QC}1b8?G zXR`v|FVp>M{cq=MxBo(BI^hI^5x#UjSU~-BVFZZo!0FbzJ&FXN*$gMc#qiw#yZ`Y-`Z(q~0w;<3L-?mbz9UD<)Bqv)|-uWo9uyzJK6Od1Vk{ zmTV7^y{@-jcVFX=*$zJq&8w)I(Mq!by=rv47g0}{Hcmp*WHreJ0%QHsT#2)T@Sjum z+dWNMd@Qq!Fbu=bU{w4o7bczCSU#M)=9WGG(0g^!@Kg-VT^WppYk_lV zH~42`b-6rGg9gKfm8YsOQ`FQwy|*auKd`9qUWe{PQbs>?bT}B~drm)E+@^aaq zQVJXw3QF{faBsNDeWoFT`9G7FzP~PDm~zyi?#v!G5f@J3U?BmNV8k5-k03^|P4u~6 zLWvu2tG{npP7gF$126}xMEuY@*2?e-6<5tqJ%C3@{` zzhp(_#j9AR=Z+N6u~F$$Q(45&HH^3sE73(48=PUx?MG51_lje-;7u*6gpj01w4c|5 zBVHHjsL)OCj6-n+G)*5~^c2w^jFrBa_cmNLcMx(Xa|tF(DKv+=3^$vuCqYOjtA)P% zw29g-XCxX#-fl%xc5@bxgz;yLxw0UndBd*!uXLRVa#QLvW40mHIdp{T&FwJ;Q8c8&|Y8_|qT1Z#pv87`Uvt3^`zgHxY_t z334EulS$pj?2~3MeIn0(=%1~jh*(1ueZw>710U^g#ND}C_*hgoX!fW~VKwQ@)D6T+ zx<$<3JVIWM-pL{Uz-2FT@UGL-_VWVRYkSM@W$>N1x8rFq{c+n5*zZ>h40!q1?G$*h zhvd|l^oNIb)0|&Pz*T4APHCx2v}e}jU9<(eiTxWSJL7i-+6zG8_aOv|FNdNt;vDO~ zd{$1RPh-4~`TFS_P}c>_@9oO${40_pHf-FY(k@wU;#jsA}Y>waFf8XpMJ?b%jv;p=81{?J}WheALUHnfjHXHb|`|fo3 z)O&BIUA+4=XawF;@w{ZuYg!Y0x`FuDJr&goPVL9nqcj*9YhF*OxM=4CZvj6#(Cb_W^Rj4z z$fi-$F*w+EMM!ofoIYuxxVTK0m%UWsJK7E)piBIDE7{T_O`~}0_Tz2lOaLT!Q0#N* zt5xtLj9vWeg#a#36)wp0Xyi_4ggq1=fAA+`>d6U%+filZ!z&&82#I4vP&w>2?gw`? ze1}-el9YlVoRDl5W+NS0v(=EAu=6hCB_ETTE}<%=`SL*yE5cNXM(iTZDgm_@tC{hN zp(iYR4)hHI*`3P9kW9m^i+MzlN43b)Oq#`6zF6q^s5Kr~D(4$t&?&(t(CBbRv&G(n z&>!AbwEQh05OUQ`4vut~7XkOTl7sg{{woIFfn9gEnA@9>vL`5Jbh{|8D^=ob#FH{r z1huqQ+>FtN9BT1$P3h;f4s{sNHEg!}9t;#Fn_SsHU4dAu?bhAzj3c|fi)!&jLHvZs zCtsZmp8w$jPVZ`zT#lIBmotTss92=%4Y!P25}bPf?RpvQK+ajo-w(NS3?Yx$maFT4 z`y(jZbQJ7mxCNQd{*Pn|eV0X_Pl++NAs>R+Q(R7%A8H8_%5Kg!>$f#(LU6*xCcWMcM&8ROVI4ARegbt{CH_1byu9kKqUQm_!Up z)rJltaetnf4J1TG(u`RULUu*@peRXW8vwdPN(g*nCJ6<0F*1s1F|n$7CN(oERD=Y} zyH(B(pC7`59WLHvUFCIyw zb~JJAau#`yzyw78#Mz&sHg$uU$gjm8xpjsG0NziaKM86|2KhrRx1!)t@+-)w*Id^7 zx#(s0tpSh2Ub{sD(DV&OMT(Vx8%LmRio?)2&82F1!EWki3iBO$(7|gI`8+HF&{K#i zX7af;)NG7zYv~?Jn%D?YPugkl+i3*r9O4VH5Ph-8Yr9!{+98iW2z`;+&fH6WQ{$HW z@YQ%;w@9IUfAQCFk zoJEjrToO}yR*hS@LxRnP9S5IF9h4N1=lIR(_E7WTZ>C30tpOwc2_Azy?@Bc5K$3wj z<1{lACuETBpdq*=xn$q;M^E+-apyX7*ZQk}eOLH=EF7yuTJWi9^z~;tW`rkXReCNW zZaWMIZk3w}3Bo^NJE5#4@oxXcDts71d8751?75HtEEDA&grp1kq=<~}KltltiyhdQ z%+2cqTy6e6egHPM_-~$Zzq!`VE*4i@5Ol43dgMl<$bMJTo+diol^Ji*K1-GJ*$ZoM zMXwPzk)jf4g2QzY-=qn_{TeHINT2Q2*%;UnzkWby>}%?!)2*w|k|u6JgBo+n>85$4 zb(?uY@DCr$)u)3);CD$ZY?U6$$|k>pSH3zH09KrzW(5sHr1_s1m+fO{n6B+o>i0-E zTuJI^_nMCqTFc~JF<(y_ zRd{@FEZ6s3mJtqap6tVkoE_X`Enl)%2ghpjWqp)~xlf{vHOV5v#zmw}rHv^*2hw}x zmA~W52J!G`bjmK6xChIzjd5%%--ObWX+90(;%b!%0-m2`S%`?DHU^g8){v_&Ey5Qw zr6%OL>M0Ae2`F=E<0E5F2S>l~%pe!l5phY;gwG9>3^7}F^+f*SrR+YSswQD*L9rSh z98!&na;M~tNZ{{^lCYWj8Ro!YxPK0N8IW7Q2-fB+mDNpc9Zx3?phVN-l-V-B54AB+ z%sb2XJfe&WkO|9_Ry+*5o4DJTr#ho#K=ZIT@e%T4s-e{D|8A+16I2Boh-FHE zIPWJC3kZtHpFtx_W~g^lU_PeD&A^E}msBf>x)xTmULs~2_j6J)j#vMS1)_N54dE$~ zga#3U+NCC}(Ms2!78E>4GfmLgitb}wCw;W!wxJafXdcSly4haM7|!|I+VejTb^VhO z1bh?1t4#2I@IR&b7;S$7D5DW_+*`K;JL`Z!bU9{V5pJ7%i`A7<1;5j%bCR@e#-xOG zYA)h#MJM$5I8%@D0m6X0Ja5J8L{%&uq8f^1IXz$PIOR>WwuNKd#1~C|ZXjTWcwCrx zm-ngi54EiCm6lDkBNedj5;$Nph{^?S3D57Xc@Fh};hrjT6m}iTU z6iXo{&x$O80QRx)BpojoLGh^&rAqr^O7eY9c%c>)Hn1eC*DZi)($727^$0m@eXE7Y z2_vq(ZRov0wc7KYfJO)@SDEPR1mLp!ts2V8Hu@=$=x*T)vETf(ooW3rE1mg`1;c?2 z`#uRVZ(vO=F)Wro$G>y!swwLWmVw^LuJt$C)wJ4!z61)4Svg%F9`E=6s-z32M=d)#;bomO~;YBaY`() zT_)XrV7j7m4p}(x!NSx>XrPvz?r}rXL|voZeLtO^$;Z^|jLqZr`x8}H#eT8wzw>tc z_T~g8zJ?Kb)qrC;RH)wfGu}2s+hYrHvV`tyBlIWP2|k$!hUv{&_(`n=NG}&9&j;52 zb60!>FSDW=!92hjn;Tm_+;O3p|J{D;H7*NC2!y)nA zcjtkpfo&ja51bTO`bUlrFdu6pksELbBeLAkb#KR~ii zt@mG8Oj5@h^K+d5xIB(G&X8QdFfqrJ$QVXKRFOF2;YRy2q4jqEJxXtAbm{6u+G?2{ z`+W>FZ?BQ2nWnX(nI?a)%8CDTsoeNOF#$1^m{-R2MAiXuYb5tla zHk(CeB6M6?hRIymJawicX?^ZE4T(nzcR`g~l;h=B?3A$Cm5Jlre-^O_?i(A1|ynhB^ z>#^G({;^DtFtdI@j@5g}s{TEI&cO2-rR7g6QG43ZK$@dvTQ2BE{>=#j>i09+@jKA_ z;(dFa&+=6zqmlb&F#RcTJYXU87a;daHPb+0fc%+x_QJB5TJu+$YDSE2y~7sZ1bw}5 z?WKna#{zVNOu_?eKJ7T#ME0xi{4h(SL|$7DS+5qO>!pd`pYHE8MYsC#^d^TxOxrfq z9p>I+`NjY`SU5s*OloYYT}sG4phC5$zDr-X=T9~HMmycrL1U0b+YR)2?{72KUiUst z3_{*}xYQ6>f?(y>^nGx}uwhQ;Ss^}mS7(&-e{f8G9dTJC5X9;)MVd(wKa%O#`! zB^a8s9kfL|Q;X1iLXe^fqV)gmQcF&cr7S4Y5nlT1A^Itq!@5hg80fje^EDLs$;u)( z28Qmc6!@yIh{i1gzB9Djy&ZTX2^u# z(6pq_u<_V+KG14GKcTRj0#9Z9+DqWdy$)#zKAm?*RrqEuNs{wJZb|Rs0rln=eY1gw zIs}Di;#wu%Wc!bWdJ3nq90M>QAAWqY_xIU4;$1dKFS57QHY8^%?-@cm`rZ4FRt?Ke zc~#ASl#7G=JCXrfEL%8i^nch*0*KosT#iy!kWjS@hgcjfly>}a< zkY}|f(r%EKpXmEMm>28_eCHiYUblpVWPTtPU$O30GlA7mpPCsn@*gjod)=@>7ePHsQ+oe414FbRt{57!{}_4MuW*YLXBT5!p977|`e5Px)n4~tAj@5^6u^bS#PI_~@G7uTSUCYcL}n|o&V(cxrp(}~7ufwCpk43s{iPf5 z`aGR33ZA&|6ZOkwFV@s8(ZD*_@~3Q_;n5AmyAt!H6gW6dxYKDu=Gp^3duLF#C;{+T zDV$D?msg4L<9hfqXT#To3t4>u!A|psCzn3+n(9}Vi$5wyqUi1zP7#Pd5pd8jKJI{4 zyS)HqvG|WyvdD^8d8B(EH!k7C1w3?=-Ckw;$n@F7(ez>onbUuZu3 zK$L|~cSJ=t z2Mr9q$z{Q8^l>jz0%Vm-J+&D*!(`23o$A>F4?23eY5T6Z7>@gOOOh<{0c`Y67$fZ( zlbXQT%g~`dvZbd}<858`Xauu%@$6W|0m(>TM&CbtKFaL`2`L;fAKgiq{Ie=lZ@BC4 zGhEK$t*bb-5A5Sepz&0y{;*S5yA`|IOUSV(wD6!etO2zefp*e+0gdkfBQVkuO3x$r zEB6Oh6QtS8S!HUA-A}W`Oq4CFmg1<_FSyhJBVGIx1jR+$Un!0YCNUBVw2O(G2+#*n zvka(rw{@SXF0#bU#*8_$sJODk@~L7Yu+Ei@PH!b5ou*>GR9#RtxcwZVtFkwy@}E7o z?-#icdL02=b>D8iul6)NRg#NQo%Y^0?_BnRhLL(9<6!)qaj4`pDX578J1)%t!e|v zrGu-HsODuXU8Dd+)!jmu88X0kVaxKMq0`Yfea_r>`Q|cqI2b!8&)CmGvdXpwUEYtq z>gyolw3nn%?GA1-!cQDb(Q2R&LAK`3@aRw|vwfj+9O-Nv%c&~AM|5|Cu9~^W9UV`X z8QfZ+W$-b_(Q&0_GX?!>cRFoAq%jy0r6xF)#!6|)FpfP6$25U2KeVb_tihw7{>ruz z8b~x+pOwQ_!)bDQ8f@yr5k>I8i-lu>8eL(mCDq<4^FI^6e{qb_E*T2@`Wu_a z&G(n$WFk*XAYNxtc)5r8=lhHCb>+|j1W&~+?N7??k$L#8DCsudgfUMV#iI_dO6H=~ zf(~CTKDZ~$d}OewMYT_9XP;HQZ6g5AYlMh&%>1|763g;=A~ve9df(TWTa+QUQ({Ye zwuWr)=gu{yQmI+`w1}*dCLZaJ_(l5^avVXpmQ2*k!r>M9;o0Z)h4Tr@n*qLXtB#?6 z#W&6wmG;k7p~wTlf_q9GVO@gkdrAr&*lGzPue>e%ifTiokERUWPxC+69lMpyDdR$h zCZmlnE$jzWuhJpdRt#OsfASipd^I z-IL;G|IxEICQZF_DSg;QD6HZg7;TG!RZ#Rz=6v82S;MZl%umD-dTt$q9u02s!Zaxf zn>btZOmvua+GJna&cbW_7bhP^Jy<~n%{VxpO@wO(g1^OVNb&S)l>wHzI|Su;nB;++!2sQ6uFIHGjycN!uhbb*=m2z3Y}LF_4INvOmwC1P~|NOp5JP(zTE)_2a9gB`VjOIdU zjrhhPZX|hw@eq|U$jx(Jb`G>r31G^Dj{_u$Yl%pXLb?bcUbg?aOU|@K+LGQv6rrk+ zNgp0uUqS_tXnWZ8q1k(E21KXZII%5`;vP`wJkte{q~B9IhOqmmfMyBOA4WhMe|`hk z?{E@cV=R6bneW5BWDA3KN1?sUmjfrrV3F6h&+`wXrsHo^g6LT{`lBG2&}H%JV`%k{ z!u%`2v!4mwV%$nRZRp{#lDEdlh`zS}8va6|t?+Q8<%Fxt7BE+oUE)1GOzwWmHd84N zquV9|$3_o3#jj`h_k!EbT{V=~ibyd*(1&59z8$fr(YwOcqNYhktW+PNZY?MdaffLh zwyerx)2KvXk;SQn)Qd_p_@mT2<(HH4z0aa-rBm&OI~OMLXv5y_i67$s4sO4WZ;J-9 z+Acjq(`B|ZcW6E}FRm4B)P_o-hK8L?MoM9ZEJ((zp@yzT{P6<3JpUn+A3DWBcGbs4 zUSK;2#6QZQXvgmS-iB)swGs{e{NTIU?T zcJHLUF+GrQ*ZoH_?P(GuambfOw{t@3B`E<;?Lbdv!tVbpR8xmlvDvg0kn zBwUzoiPe~6JbbNV&b_AKiN2_p7{G&X>hgjS{sCj-0>FT&R$Y_xW54^C{6HDjZ+2|t zxfof0PKmy$hAdfwY)dTo{Nth%+m1B%WZj&YYKI@hZxTK;hBt=0ZD7uXwPXKu>rfIJ z3E8==-ZtVpo2hQ%gw8sI1Sd84q*4x+KkeDKiEW1;{Kxi?=d{^k&G8!A2gu$!az&Af z?oBU`Hd7b7=dy`~kk+Kc3@c=W7VxPT@XPJ_Xl_*dc~2edpCtYQ8OJbuCTP7DCj3Mo z{ujjH#$S*Y@bR6s9@aYJ-OF~M36gUP82zVmDY_aRL%`w~Sg%gsqn*=R-=eyrDE~U|f!ZU)GUYkLuIEC34SIaY zN!;fSq#FJ9LTDw==jE|1i`pv)HeMy%k+V`O%ZaE3TLQ0~8nFK)K#q&LdAEI_hX*MsECy#0d$!(dRs$$Xt51C%l|$rY&l znj#n&VIW+5d}V4F=#;64p)Foog>K(SnXc}K z6+`_heOCP16vIv7&ZwzA8*k6urx{f)z&{tBT;Z%zWt>4s^zs&v@A+_3{gj^v#lUiv zc9i#+1HA7D>4_27Pb=LWcu?^>#qR+m(>~ zh{2#y!Xh|pbZ9??s+=Zy9K1Kk+5lEjC z4GIv|r-jaLbF-V6mIY_t=UZ@JPf7!itxSK?+ER&s`i0OSCzJN6U^l|4H{^GofJTLo zs8-OjSF`GPhbutojgUBH3tlz-F&QhPln}655FW=2 z791gGlq7g_KL8=@e`3rj8XjCi18L6f!TX0p!{ZeyzBy>uN*i>Pk`0s&x@#AKsk&@Aqc6%OuGw_g)HB7M+VX{eW72MsGX*m1Zz1Xp`*U zC4$oKt-2~@%OOQYcdG2IYN9g4gEN5HJl=m0|GgnF6!x~Pji3fAG82N?1089ss@(@M zhuv{SF2)4SO6aOYJL9ooyUB39DQ#U0sNa!dVOax5eg#XFFSwOLppigezD|X z!$6c`*`Z@*QC~6aFw9}*EP3Rtko|dkGYAy%J<;=X@{jX4;%x4aj5F?1u+;_~tz^wc zvV%^a#ORb3mWsRrZ%@;Wa6ZZ(Ok1f&od8M}?0Mf(^^Kf`E6(PPF4tfp75ze3GV z1CY)ltw5Bz|8KncIc$mEs%No#)y~o3DOZ8{LgA3MG{-LFoV50=%BT6tvut_Y)>i~) zvVF$1_ct`47Nx9a&248$>=7#7sdNqdyAhgjHbH9cINki+cgO1SA;S$t1KB$P9G@!< zQNnFhS+>n3+m}2+QEmqQ^Yl+EMrqMeblu0Yy5y-7T=rA4YBX|a5x1xi*Q%@2ovV1X z%4Z6MLVm8N)*DIo^s5h6b=#Gk7`5#3N*I*Q;3jGe7^|TmV67FXRtul>Ik3Cy&ff+$ zfg?S)-GWxQ73Ig`%u)OD!k>#SW!=seSF`D$1!#DrS@r*?oK5QHDZ5zHa`$8MF{=hT zIk;4GJ9@Mk1|}EoK>KlsUFhEbFXCMCdD~83FDP&F04k`n@HWtAF!g-gk}Far(V5i>(d+F?5qAFJ7yV$GI?*~Yo_Dl7|RdY@*oHm^67P4)O<_w~W*7G&W=Q~|trz12~ zmm&xJ(ciN7@)cyWr;k0+?~o(po7;M=TCL4ZU0h{&am#-xF~N*2Lnua&pbd&9yky*t zwkR$INwEb^B?t>je#mzhh~v)sd2=ER&4gOkl8CWe^Yz0LSz=O{N{}OwsavhW34FsM zqls_s{H^)Bl-HZTkb&r{=P*1nlXrBE($cp(iFE33vilDs3dRvuVyhN$aWF~i(&iKo z`xxPA5t$h19@3h#A6j_C(p+kynEh-E?Y{d6!oZBbxHTTF1CgjM^&k5vKPQ$Zld#^W>biy;FTdd!`Rh z>ngg~ncTcmIN9yX?T@2ZyK_cdeM?*4tc?^K-7i^P6UHSZaX3j1{VJ}K`9HsRYZ(UI z^z#nr4@U3t@38hXL5g^x-GmMW@3yCNQDt1I>zn<=2b`ondm5yB_985Ubjt_LlW+KZ zG2y?2*d7aXq% z_AJv_MHw&CR%P0RE?iXZT()%SbTg)XUaQoJLy)7W;;-}&7V>}IlQXKhcED)iIMWzL z8mmY=U_~3}drL_E`6M^vC(CWn6~b4Spc`rD-4=hO^oDgX$RCORBY$5wo`P~CtD8^A zhj2)xJcc>4iB5t-LyNO!ssKZxgF1UJNW`Sv^FbG!ZyFScBt}xfJ3oZ7%a{E`Vkw$q zb?#b5tKKpD$~NELV5#*7`%C{?`G)N+bf_SY%#Q>&Fl%l>7>>^vx?QGYK&U?VA&&B= z1P(kOEUImvtHRC<0UMtqtc93-pT_6x!QM7ujoT04lZ4c3j%vJPF+^_`j1<-HVgs3G zTn!Wz0FB5@>{Y~XM*^ltRmRZ;_wq%qUJY@hSs=FtXJWDpdkTv zCCG(Qc-a?1!_xz3-*Z}F?>BIMQ?0u@P7506@hvow8A)QXyv4r{joH_j;2eu=O zHvE9XmO3v0RzTwWrr-VhJHY!yOd<8BB2$Zjhm5b6dbCVp7CVBa*<1}Wtk1+cB%$@O z2eh$`zq1K=E5i(gBzN%uZaBR zZ?1hn$iy9Lla44eO!Db^)$8Zr3qFIReOMmCjshb44Vxu<}P9-tt$4vUx6GKR2QtBpySvCHwR;g7NkO*b47R6(6LJ-$MFt-XY}`ul3R`z<9cirgkNJD^ny#nXk^E>wiio0wy&GV zc%6anC__9m#+ZuGEWzOa#WZLlPM*3QDQFiEdheW|t8(NcY%o5XiAatc&*kBX9|a@( zNk7Mre^~>4Gr!+VQE&HqV#%wBcJNy9{9qj-N{F1dt}2ngI=U9a-bj2|UmaJGGx#8C zxvF*fthr3kJYH;NRh3yR@I(=yhW!R`CiHp$8-=-ro&AEGytvW67{wo9yTv<^fV@EJ z5B=UUEA@rp|5#}cu@Bf(d|(l8iK@8hJw@XBTiF0KU)X5K)$yQ)J^VBF8jxAV@XQM3 zs*1MrFRrMUhgh69)Zt0qnuarBM?}3blsRWnMW4zj$T}Jw%(clvtSWF|7#M&OcPR%r6@hQGCyY=5YkUW=| ze)P`HQL{WFDsNg1r@QRTfEX)I?UYHOV6;B-sB-LMY}UY!gP(*jaAy3{wI*Ad0%U)R z=Lr7XIuo1KG-DB5XbMnWlvU@#jldL8DRpn&w=P?NR>Hl@mgnYDsa)F4=EamBM5xGeHg zH>Q%8BigFuuo~fE>nl@MD%!7PV?R~u-p1Phg0GGmvSh>=iBqjiBJ)xwWIW`o>d3K= z?-golyB4d+eTYFapm4r?_~>YW*g%09;4SMoxm=uRDCvS|OVAv=f`&x0WqGl@65>nT zUTXAYtJqX&KA0BDr}h+pTl;?ijzDq0-Q4{E=&sm;hOBUklB?iA>S#Y>M=$_^$04TG zB@$o^3DuHT>tb+_aidN4`k0>HW%??*re!#YpmdSb-vZwtk!PLTCn0hl}~c5&K*Po z$6fR&zN?|C+PDv;N@)0XOUJPBXQjg|K%5m44!P~BFJxXuQ6IALhhN-|&}gZ;k>@y5yvN8E2Qi7;d$=x^mfd(Ha)$b_6Axt5~sGp2Pq(p{k#17&FkyNN5Dg`eO zL8hR+!_bL36G?8q1#~oE`Wren5Q*a(IMuQIc` z2LR12kQVT&x)L;@FXORVMl?q-f(d0qaKZ?Tsgu*P__2mI*2AqHij_IyTv-4t1{@2E zeh}s?BF#ry!NeG8RV7eLg$_a_zgiT7kdb?sZXhc}Xoy2p z1GEH;tQcUWk<R-J{@mJLEvFjmYo{$t>KC z4Q#CIZ6)P$JEEMsMYD;7m}ay~IJRhY(#KYZ@~l&EV6D<0Uc#ZaJr?L5mJ1N8$HN5# z`A}S-QaF+o3?lf3J2(wc z^jXCk05w&6v+#$OwbmuDjgVBb2v!xeDW8VPSItN0$`B9?Ju4Be*5LxdHlyZ>UbRMso3sjW}^Io8(~ zBkt`^Zq(W_Q}_&vuSa;VP+gy}D({CM-4E9>t~&UlJHW8RJF`ABUf6hCJqrBhMJ0b7 zn!^Q}-TPC@;#~!0)NnHp=e4Yjp1ANyVc+EKL~2xYTNIgf#+auDFDHN7;~MBr@EV=O zN&1+*ta$@ASGp6;6yTlx%Tk#+F|+nbX*%c<=dJtlR`iS@4DkH~6cg8RXEQ zTMn&QXn#S{#?=d%62^E(>2ww~X}I%oC9 znZZeu?!%EA5wVN~vvH@8O^O~L+;y!rS1%YIVg@>Dl+9^m4mK;TjOUP+=^BVM0e1DRK`u>-cdzfTG*G92&2E zY$FvER|i8~-$s~B_B*Zo*7Ujoaf8kPXYibu50AJW6g(^7Gk&rzA5P%IbZE6zKaDi2 zA_JfSq5z-=VzX`$*se2~-Q{*k=-`Yz(GpFh#wb4p#r%od?I_O2vM8VAo+flUv%b2x@`6)Zkv#>d>tpp^6SNajmB-XB z(@SPggGaSnW6|Agq$N$L77&Kq{IQ1moEg^TE7l+6C;B3o!x$&ddlFr?-yc)=JP|b+ zwJjBe+8hu&{zCC+mx#m4-xjk7(O#gs*=ZwCSLh~~7)YE|(y1)Q)iDAO2qh`PVxbBy zJ}V;xosKS5qFSe%{U82%|GV$Pd*!P8NUP|GdThvOVZ+WHQSS~m8=8$lbde=qU>B^- zYyF2?-~ezcu&;+-9!(q{BU7nOebQF&|GuBZ=Rb$Y0C0f#{}U(izmMmWc>LAy$wwOh zJwG*B2<3mA$e-N*J(kbO`QOR;U&Q%es@AIdzB9)a3Klr?a#ffoQOg&LLdN%64H-0( zf@&7>mLQtvv#P$@YUGp}&a7JBT1@q>Gau;{3?n!?*pudpCeUH6@PjuzIx|Ykew)@9 z*8S>c1(1p8A_bG|cDk4hFY-Nuo8f@0?QykEZP67W(`WN$;|3i}1n@Qr7PIJ$254jJ zZw1b=M~yD-4q;qf8YHFTmh%j(9@bd>!A`~oIO|-RwewKWi2W?g-cH(0=RVtMwlh#F zbC=!KxJ$vtjIqRxFq=R!EO2-xWe+ixY{7x2qv8Qi=FLF0QyQ-fx^x3ZBB3-NZaW*n zW1>ZjI%hr`QU#H;Rm0~IRr&P-?YEtAv?(JZB8S+T4{(N=1}=#bAOa$KiHnjb9Nnud z?hZq#bTjsfJQ$#493^1!V`HAj#SKR?KAJ@1BNR=#lYQjKpTzpV67*%%E#MIUzlrlx z)29BPhb_U0{(lUg6aD{0|Bv_o!%*l^xuEXJm{3{{B1O2nntPe69m0$fBg&R_Nz1rY z_I$j_zr~Vo5PbEKXZRKYdP{sUjYrk-}dTyxI#WO zy+Iof!bH#!ASni==u^6H+O^n2539a};+<{|9o0Zs z_Jg;B$A=V0n89o1T1jx4#%_UxDH00CnK&+HM5s>vb`TxCF?LX+<sR+JLA%2q(6I=a$L0*bpcF2-T&l*6I!AmAS|{&UBr^;$J>#K(#q%iV z!~4{a85q(aUamO&o^kk1GgMijVGBd0cLgGNmx&PBz~l@0gSB_xScBFBs6gsn%Ux?V z4GIb4R;OC`GgMEIU_-lzN~Qj4b!BL!#(at}e)lQFMR9RU{d>1VCq<2gCdfIHUT?sfVtijK0`XYSpgCbw$Lt7Z1?qMo=HGbfilwtmkRHh&mPCnw5m*G4Wv>)_H(||LDe10NU4b~<$01dgDh(TstN*-AUol(8dRTz}0c@-Xp z5XBWVh7g@-E}J|ef~(CxewyF%dTp;2(475NW*5v?_>Pka_XGI1?p5m;(jUKX*ZOzy zZ_oSOz#DjxlT8KtM?KvXdVEqdPDSByGk~ReA@Es+!#As{nnMzLvB6T}|BHxR0b31A z=b@_Il^C5{vl}QWd4T}w5Hm%k%?YuSfx0;FGBW-K7H+k$a!)TVXYkC(n`H@2S^0UE zGRC%+4U0s;@AnQw#m1P3!oo6@Vf-_t&4cc0RJ@Gsjh?H+OvQ|`3$)tWAREQGbDt$f z31Mc$vQZWHq#r?51DufmgSh?tNhU642FKKCcPqSShsI)$e>TH?>5Pw+reAvcQl&#tH>OH^j?SS0% zyF9o~qX(ewbb6TRveUw*c|FnatcFG=%icEIwbr2SiRYvr&XS3P?H=*ez1e62LbK~q zNRM6bb+(|r9puWzYI|#d3h>N~bNya_8*qVdlS&SNB*k;*N3C#S)C$jxTA`31xkiEA z7T#d>*ojED1M8_jtAvGzdFyvtu(HyC`eQTBy0&PJ)w+WUuX;P3_6++W%t9;`bErvOlbVeP%VQbyGpsAp82?yA3kYKYK)L3@-BLr>B$oe@~yo_5U2^|Cv6b z_G3T)&Da0f@s|_naGWsG_@n>#&4hU))?!sMiR|092L z|KoT*C;I=1{vW6RU{srj|V%IeDeb484-zIF|d=FZ-(_RjVO z?X$A&vmCP=wwq2Z&ArINuvZp~%U7w=#PqpT3aE_{+Luz>JiF@1Vo?RJLTJN!PE zA9RHhDpgvZ>K{rK`4ZsJfi!cMtKF7vha5rusFpX#>dkThIrb;Rf6ZN~+*mHX=!&~HGm3+7=@ zm=9taz@(k09lR{7R?p@>-)Od4nMsU0sRFV1$m$(N?#U>Pw#8#=vFmPHSm3M#Um z`coe-1y{9@eJlemvuzB{@Zm+X7J0?*bbG5V8G5u_nj#GW(z$x+rJ?h-4R5v$! z-tAd_b044kgH47OUYp=KldAQ4C3v3M_FCQ9v~4G`Lmq_sP9mKUAv!ni&yE2Q78Eb5 z&x7WF=4XSItldEM zi*8#{4@2$q=MgZI<3O5XJPJ&i3}L_?+X0T)zEOM}={+_+j`x%*y=r?49d+s68i+Gj zBn+F4+R+M3R|D?EEB*{nG6d=j0cKC|@K?3QYh-i~i8FOpyClDVY=EhUBxcKSHgC z^mL(h-%s-k5oxh?!mARLJ7FfxFC@`MA+iM~s1h|nMFaR6OvyaS+5{Puu*pvMXj(M( zgfVRfd&r|`*}8ii(mE@DRIQuaOz8{@%XGimu6Y>^yqwNx!0A)K5&L`0!>QbDqdCOQ z2}+@+&-~<_ZpUx-o43)Sy69luZJH+-bE1D z0L3Z=(rzY#E@AazOo}MIj(Dus*iGr*;zyXTyNP&Q)u60v%<1(p7Nh%*?@&D9 z`R}#|io~W@?H|MeXjuH8LMZ+x{5x@S{(Bsslf&MV!``^VUODkuqGnjO^<&-UR=e5~ zzbFP9|K5o@#pNf#JZ=zfg1D_9xJOXw4cergu{W@q-dJ9SPFGgf=GJZii7_fbae@Ej zkhWT#&1#DU8<)UVflB3ks=s-P0ygWhEj zedXXz4iGQQ(;|rC%2^(bo=#d^{&eEnLkfu6t*VdOq-gS4ORNmOlOFU4duWEz{^9pr z*XuP{iYSaPDD^R|=p}1IPqU};fuu(G{F%(InKWsbr+p_xAJ;+H#&hR1D%sQq+ecWC zPy-#BESgcOYRsu!CZs~$W*s_dY{T@ZOShUWw0MMvmUqf^5D3UkE7tH%&k?6t<;ADt z=dhWKsN(XuoLenjT`QKCbP(K14wZ-|)0~_Xgo(tVcyL3P#0Lq)e z!DuVEN(z1O$mRH+ne0G z!QY~GIGc;k$0c3(i{aJNYi$p-sgjuK# z*tVVK2wLgi3|k+AgX8eH1w!4*Z-KwUfzJQq8ITUv4^H`*?h~^q%ZgkM)rAmEO;koi zk}%dGZq31`$wqYs_jFhV@7|It+vYuH<1_I&5Sxvtz3Lj3&<43}n@6r>zK2{ys`?hX zOdS!%VU^X;nvHSPR93{*YgadsM-)ZxAowIz$Zgv9tX^AVKb^iook4f2SFL-Fj2wbN zDbPoP)NPCIsx_gX$C+Ld#@g7Z^P9SgZpa=e_%!N9+(FP~yVd?Snil=OpV4=UmPs6p z4~@!Qc5-J_Gne-W*BBM|$K%)*IZ>fOPMe5DS;1LFGY6Xk(7y^6C3CaBXXX>6hu zQbNI=x6`@pg~|+}QEsz|i6zWR3Kfzs{y+eP7Vf^*pnWPSw1K|Vwn505Sh-HVb1t8^ zM4r*${b(0Jr-` zZ~vc|E=)~??f>Ub?ElB{IkEqr*#G0~|Kt;hx;&ZA8?e@B_WwvLKZ{n)oDjsbo+qsS zJbXS6X)SE^JdO%JOT`)WF{t2bWwi+XFEPJW+|;-f*A||yt(51k7Wrbl`qC;_(dJi{ zuh7FGrvR?QFY-|ELf2teaOelGS?hJ$VskEPU#Gh~2qX4jgBoRzQz_97VunWt{F2dF ziZ>~qvwK2gsT6eqdx=lcOT5z_V-I;EfiZH9;pG5fd_*VD>``%f?(!nA5+7#qt}AalUG-(TxVSR6AYXGA7gm7xrnJ0PEI&WDh=s&fh?lrlgqANC z=hn1B;hj^c_}u(jX=Pb2%sdKp^H=!K zVrhPjjYGQuB1F_;ZWkq@1rI=7*Ivyp0{Y}u85F6t0DTs-K9yPmVqv?L)f-o?lwQPA z$W3GPf<2QL6;pgjD_0ihuF{-<$C>m!k38Yp9ncGT|6)o67wX8*{5G&#<6cJm4)#>^ zcLwHkb%#o$($8MbFlA6LDT-9k;c`~)5-59`ZK{d}ou{2Nc31|d#}rXeQ-Oic@JmK* z!*D|3wQ@A1{+%Y24!0W4E~O;6M&O%O-xCHR(KM!v;r%!3A8iYHM;fgv-lnoNxvrunPieGzywQ-L7>X@6 z#V)Zh1D*;s0_m7Rq3EhdJ0i5gMGr?f@t(QX9kebigh{x-%IcAWh`remaHWbf8uv3- z^^25k;E+1j*5oI*K<7sf} zUaNZV=w`X8J|>xYq@fcY&eNL*vlvzFYAcI0N^H-Z9{nGNp@l_TL_85;Faw* ztBP6d)G+!z^1RSGVVx19PzlfegW|Ch<7gX>8yY-CFb6~BS9K&5J{2UFDz57c=T2$6zp4h z(~5Guz7Zu}sY}Z8=?m4;)t2znQA=nlBtujVRv2N{Q7kEuCwhbqv{&W1>#8Xb2~r)!hWt_gD$bD43WYoJyU*c9e_|f&D39rxA!UwC}vjj)|R1 znp9!hN9R#~*I1!~W{XQIlz>UN&2H$r!9lC*hJ=ztXdJXTyg}FQj71AWJmg?E&#U!& z6hDS8?HY4EXg5GYr=2ucCMBM*X;$jZ-Z0h?zzwVf2&k??(-D)#(xiw9i|R0>1YiRx zrMmd4X@yvb>w_e7$U9eY9ibx+dT7s2n)Xn3DRuc+os!xaEzdo26h7}5Hxh*rE*Ulj5Xveh=VE)i=A^BKh;5^F0RhKHqcEE@C@$C^=(Y>()E0K8;FL@%E6 zF#(kpM2#^e=Qk=8x%vxdHP<1drPxx4ikg%V$~+x|dF?v0DjW8C!v#E?SWH-iXC