From 05b09b0c5005066a876a28b5ff27d051db26ccc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=90=E8=89=BA=E4=B8=B9?= <53546877+Craven1701@users.noreply.github.com> Date: Fri, 11 Jul 2025 11:19:56 +0800 Subject: [PATCH 1/7] =?UTF-8?q?1.=E4=BD=BF=E7=94=A8=E5=8D=95=E5=BC=95?= =?UTF-8?q?=E5=8F=B7=E5=AF=B9=E9=85=8D=E7=BD=AE=E5=80=BC=E8=BF=9B=E8=A1=8C?= =?UTF-8?q?=E5=BC=BA=E5=BC=95=E7=94=A8=EF=BC=8C=E4=BF=AE=E5=A4=8D=E5=BD=93?= =?UTF-8?q?spark-sql=E7=9A=84=E9=85=8D=E7=BD=AE=E4=B8=AD=E5=87=BA=E7=8E=B0?= =?UTF-8?q?.*#@()=E7=AD=89=E7=89=B9=E6=AE=8A=E5=AD=97=E7=AC=A6=E6=97=B6?= =?UTF-8?q?=EF=BC=8C=E5=91=BD=E4=BB=A4=E5=9C=A8=E8=A7=A3=E6=9E=90=E5=90=8E?= =?UTF-8?q?=E9=87=8D=E6=96=B0=E6=8B=BC=E6=8E=A5=E6=97=B6=E5=8F=91=E7=94=9F?= =?UTF-8?q?=E9=94=99=E8=AF=AF=EF=BC=8C=E5=AF=BC=E8=87=B4=E6=89=A7=E8=A1=8C?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5=E7=9A=84=E9=97=AE=E9=A2=98=202.=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D--driver-java-options=E8=A7=A3=E6=9E=90=E9=94=99?= =?UTF-8?q?=E8=AF=AF=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../omniadvisor/service/spark_service/spark_cmd_parser.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/omniadvisor/src/omniadvisor/service/spark_service/spark_cmd_parser.py b/omniadvisor/src/omniadvisor/service/spark_service/spark_cmd_parser.py index 100b5cdd8..2f865a2b1 100644 --- a/omniadvisor/src/omniadvisor/service/spark_service/spark_cmd_parser.py +++ b/omniadvisor/src/omniadvisor/service/spark_service/spark_cmd_parser.py @@ -52,7 +52,6 @@ class SparkCMDParser: _parser.add_argument('--archives') _parser.add_argument('--conf', action='append', help="Arbitrary Spark configuration property") _parser.add_argument('--driver-memory', dest='driver-memory') - _parser.add_argument('--driver-java-options', dest='driver-java-options') _parser.add_argument('--driver-library-path', dest='driver-library-path') _parser.add_argument('--driver-class-path', dest='driver-class-path') _parser.add_argument('--executor-memory', dest='executor-memory') @@ -78,15 +77,12 @@ class SparkCMDParser: @staticmethod def _normalize_value(value): """ - 对spark-sql命令解析出来的value做标准化处理，将含有空格字符串类型的值用双引号包裹 + 对spark-sql命令解析出来的value做标准化处理，对所有value的值使用单引号包裹进行强引用，维持字符串字面量的原义 :param value: 原始命令解析出的value :return: 标准化的value """ - if isinstance(value, str) and ' ' in value: - return f'"{value}"' - else: - return value + return f"'{value}'" @staticmethod def _append_double_dash_args(cls, key, value): -- Gitee From ac59cd3b7bd78c5eaf5faa894fc9cea2842e45f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=90=E8=89=BA=E4=B8=B9?= <53546877+Craven1701@users.noreply.github.com> Date: Fri, 11 Jul 2025 11:21:00 +0800 Subject: [PATCH 2/7] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=89=B9=E6=80=A7?= =?UTF-8?q?=E8=BE=93=E5=87=BA=E7=9A=84spark-sql=E7=BB=93=E6=9E=9C=E6=9C=AB?= =?UTF-8?q?=E5=B0=BE=E6=9C=89=E5=A4=9A=E4=BD=99=E7=A9=BA=E8=A1=8C=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- omniadvisor/src/omniadvisor/interface/hijack_recommend.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/omniadvisor/src/omniadvisor/interface/hijack_recommend.py b/omniadvisor/src/omniadvisor/interface/hijack_recommend.py index ef6a67f4d..02eeb4eae 100644 --- a/omniadvisor/src/omniadvisor/interface/hijack_recommend.py +++ b/omniadvisor/src/omniadvisor/interface/hijack_recommend.py @@ -117,7 +117,7 @@ def hijack_recommend(argv: list): if exam_record.status == OA_CONF.ExecStatus.success: # 打印结果输出 global_logger.info("Spark execute success, going to print Spark output.") - print(output) + print(output, end="") # 若执行失败 则判断是否需要拉起安全机制 else: if exec_config != user_config: @@ -125,10 +125,10 @@ def hijack_recommend(argv: list): safe_exam_record, safe_output = spark_run(load, user_config) global_logger.info("Spark execute in security protection mechanism, going to print Spark output.") # 打印安全机制下任务的输出 - print(safe_output) + print(safe_output, end="") else: global_logger.warning("Spark execute failed in user config, going to print Spark output.") - print(output) + print(output, end="") if exec_config == load.test_config: _process_load_config(load=load) -- Gitee From aad478085fa9c577b297b6353a2fd4c44222fab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=90=E8=89=BA=E4=B8=B9?= <53546877+Craven1701@users.noreply.github.com> Date: Fri, 11 Jul 2025 11:22:24 +0800 Subject: [PATCH 3/7] =?UTF-8?q?=E6=97=A5=E5=BF=97=E7=AD=89=E7=BA=A7?= =?UTF-8?q?=E4=BF=AE=E6=94=B9=20=E5=AF=B9=E4=BA=8E=E6=97=A0=E6=B3=95?= =?UTF-8?q?=E6=89=BE=E5=88=B0optimal=20config=E7=9A=84=E6=83=85=E5=86=B5?= =?UTF-8?q?=20=E6=97=A5=E5=BF=97=E7=94=B1error=E7=BA=A7=E5=88=AB=E4=BF=AE?= =?UTF-8?q?=E6=94=B9=E5=88=B0warning=E7=BA=A7=E5=88=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- omniadvisor/src/omniadvisor/interface/config_tuning.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/omniadvisor/src/omniadvisor/interface/config_tuning.py b/omniadvisor/src/omniadvisor/interface/config_tuning.py index a735c323b..226a4eef4 100644 --- a/omniadvisor/src/omniadvisor/interface/config_tuning.py +++ b/omniadvisor/src/omniadvisor/interface/config_tuning.py @@ -184,7 +184,7 @@ def main(): try: unified_tuning(load, args.retest_way, args.tuning_method) except NoOptimalConfigError: - global_logger.error( + global_logger.warning( 'The tuning method: %s cannot find optimal config, please try other method.', args.tuning_method ) return @@ -197,5 +197,5 @@ def main(): try: unified_tuning(load, args.retest_way, tuning_method) except NoOptimalConfigError: - global_logger.info('The tuning method: %s cannot find optimal config, trying next.', tuning_method) + global_logger.warning('The tuning method: %s cannot find optimal config, trying next.', tuning_method) break -- Gitee From b9d5c59954494da34eb0e8f52b3a22da68acd613 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=90=E8=89=BA=E4=B8=B9?= <53546877+Craven1701@users.noreply.github.com> Date: Fri, 11 Jul 2025 11:24:52 +0800 Subject: [PATCH 4/7] =?UTF-8?q?=E5=B1=8F=E8=94=BD=E6=9D=A5=E8=87=AAsmac?= =?UTF-8?q?=E5=BA=93=E7=9A=84RuntimeWarning?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- omniadvisor/src/omniadvisor/utils/logger.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/omniadvisor/src/omniadvisor/utils/logger.py b/omniadvisor/src/omniadvisor/utils/logger.py index 937547623..3a2e0c8af 100755 --- a/omniadvisor/src/omniadvisor/utils/logger.py +++ b/omniadvisor/src/omniadvisor/utils/logger.py @@ -1,5 +1,6 @@ import os import logging +import warnings from logging.config import dictConfig from common.constant import OA_CONF @@ -65,6 +66,9 @@ global_logger = logging.getLogger('omniadvisor') # 禁止logger作用域上升到root global_logger.propagate = False +# 屏蔽warnings模块中的RuntimeWarning 避免smac库在运行时偶发的warning打印影响用户体验 +warnings.filterwarnings('ignore', category=RuntimeWarning) + # 屏蔽部分第三方库中的Logger modules_setting_error = ['smac'] for model in modules_setting_error: -- Gitee From 63a7665e8adb38593c8fb0401c848efd12d24dea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=90=E8=89=BA=E4=B8=B9?= <53546877+Craven1701@users.noreply.github.com> Date: Fri, 11 Jul 2025 11:35:41 +0800 Subject: [PATCH 5/7] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=AF=B9=E4=BA=8E?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E5=A4=B1=E8=B4=A5=E6=97=B6=E7=8A=B6=E6=80=81?= =?UTF-8?q?=E7=A0=81=E7=9A=84=E5=8C=BA=E5=88=86=EF=BC=8C=E5=9C=A8=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E4=B8=AD=E6=8F=90=E7=A4=BA=E7=94=A8=E6=88=B7=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E5=A4=B1=E8=B4=A5=E7=9A=84=E5=8E=9F=E5=9B=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/spark_service/spark_run.py | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/omniadvisor/src/omniadvisor/service/spark_service/spark_run.py b/omniadvisor/src/omniadvisor/service/spark_service/spark_run.py index 4c03a96ea..c1e93fed7 100644 --- a/omniadvisor/src/omniadvisor/service/spark_service/spark_run.py +++ b/omniadvisor/src/omniadvisor/service/spark_service/spark_run.py @@ -1,6 +1,6 @@ import multiprocessing import time - +from dataclasses import dataclass from requests.exceptions import HTTPError from common.constant import OA_CONF @@ -15,6 +15,24 @@ from omniadvisor.utils.logger import global_logger from omniadvisor.utils.utils import save_trace_data +@dataclass +class ExitCodeInfo: + code: int + description: str + + +class ExitCodes: + SUCCESS = ExitCodeInfo(0, "Success") + TIMEOUT = ExitCodeInfo(124, "timeout") + SPARK_SQL_FAIL = ExitCodeInfo(1, "Spark SQL failed") + + _CODE_MAP = { + 0: SUCCESS.description, + 124: TIMEOUT.description, + 1: SPARK_SQL_FAIL.description + } + + def spark_run(load: Load, conf: dict): """ 输入负载与配置，执行Spark任务 @@ -41,7 +59,11 @@ def spark_run(load: Load, conf: dict): # 不存在application_id的情况下不提取time_taken 直接返回 if exitcode != 0: - global_logger.info(f"Spark Load {load.id} execute failed, update the exam result.") + exitcode_describe = ExitCodes._CODE_MAP.get(exitcode, ExitCodeInfo(exitcode, "Abnormal exit code").description) + global_logger.warning(f"Spark Load {load.id} did not exit normally and return a non-zero exit code {exitcode}" + f", exitcode describe:{exitcode_describe}") + global_logger.info(f"Update exam result, set the exam exec status to %s because of %s", + OA_CONF.ExecStatus.fail, exitcode_describe) try: return ExamRecordRepository.update_exam_result( exam_record=exam_record, -- Gitee From f8f34e256ba0d3a8bfa4e4bc37a50d94bb8fb64a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=90=E8=89=BA=E4=B8=B9?= <53546877+Craven1701@users.noreply.github.com> Date: Fri, 11 Jul 2025 12:13:59 +0800 Subject: [PATCH 6/7] =?UTF-8?q?=E7=AE=80=E5=8C=96=E8=BF=94=E5=9B=9E?= =?UTF-8?q?=E7=8A=B6=E6=80=81=E7=A0=81=E7=9A=84=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/spark_service/spark_run.py | 24 +++++-------------- 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/omniadvisor/src/omniadvisor/service/spark_service/spark_run.py b/omniadvisor/src/omniadvisor/service/spark_service/spark_run.py index c1e93fed7..e63d522f7 100644 --- a/omniadvisor/src/omniadvisor/service/spark_service/spark_run.py +++ b/omniadvisor/src/omniadvisor/service/spark_service/spark_run.py @@ -1,6 +1,5 @@ import multiprocessing import time -from dataclasses import dataclass from requests.exceptions import HTTPError from common.constant import OA_CONF @@ -15,22 +14,11 @@ from omniadvisor.utils.logger import global_logger from omniadvisor.utils.utils import save_trace_data -@dataclass -class ExitCodeInfo: - code: int - description: str - - -class ExitCodes: - SUCCESS = ExitCodeInfo(0, "Success") - TIMEOUT = ExitCodeInfo(124, "timeout") - SPARK_SQL_FAIL = ExitCodeInfo(1, "Spark SQL failed") - - _CODE_MAP = { - 0: SUCCESS.description, - 124: TIMEOUT.description, - 1: SPARK_SQL_FAIL.description - } +CODE_MAP = { + 0: "Success", + 124: "timeout", + 1: "Spark SQL failed" +} def spark_run(load: Load, conf: dict): @@ -59,7 +47,7 @@ def spark_run(load: Load, conf: dict): # 不存在application_id的情况下不提取time_taken 直接返回 if exitcode != 0: - exitcode_describe = ExitCodes._CODE_MAP.get(exitcode, ExitCodeInfo(exitcode, "Abnormal exit code").description) + exitcode_describe = CODE_MAP.get(exitcode, "Abnormal exit code") global_logger.warning(f"Spark Load {load.id} did not exit normally and return a non-zero exit code {exitcode}" f", exitcode describe:{exitcode_describe}") global_logger.info(f"Update exam result, set the exam exec status to %s because of %s", -- Gitee From af70b946933891754637fb0049868c05c0d02c68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=90=E8=89=BA=E4=B8=B9?= <53546877+Craven1701@users.noreply.github.com> Date: Fri, 11 Jul 2025 14:49:41 +0800 Subject: [PATCH 7/7] =?UTF-8?q?=E6=97=A5=E5=BF=97=E8=BE=93=E5=87=BA?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/omniadvisor/service/spark_service/spark_run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/omniadvisor/src/omniadvisor/service/spark_service/spark_run.py b/omniadvisor/src/omniadvisor/service/spark_service/spark_run.py index e63d522f7..f73a4f692 100644 --- a/omniadvisor/src/omniadvisor/service/spark_service/spark_run.py +++ b/omniadvisor/src/omniadvisor/service/spark_service/spark_run.py @@ -48,8 +48,8 @@ def spark_run(load: Load, conf: dict): # 不存在application_id的情况下不提取time_taken 直接返回 if exitcode != 0: exitcode_describe = CODE_MAP.get(exitcode, "Abnormal exit code") - global_logger.warning(f"Spark Load {load.id} did not exit normally and return a non-zero exit code {exitcode}" - f", exitcode describe:{exitcode_describe}") + global_logger.warning(f"Spark Load %s did not exit normally and return a non-zero exit code {exitcode}" + f", exitcode describe: %s", str(load.id), exitcode_describe) global_logger.info(f"Update exam result, set the exam exec status to %s because of %s", OA_CONF.ExecStatus.fail, exitcode_describe) try: -- Gitee