From 42d538b2f2019cb631f4f58334ab664443484648 Mon Sep 17 00:00:00 2001 From: wtingkai <330445001@qq.com> Date: Mon, 14 Jul 2025 10:11:41 +0800 Subject: [PATCH 1/3] =?UTF-8?q?feat(OmniAdvisor):=20=E4=BF=AE=E5=A4=8DSpar?= =?UTF-8?q?k=E5=91=BD=E4=BB=A4=E8=A7=A3=E6=9E=90=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E5=B8=A6=E6=9D=A5=E7=9A=84=E5=8D=95=E5=85=83=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../spark_service/test_spark_cmd_parser.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/omniadvisor/tests/omniadvisor/service/spark_service/test_spark_cmd_parser.py b/omniadvisor/tests/omniadvisor/service/spark_service/test_spark_cmd_parser.py index 92cbc080a..2b12bb210 100644 --- a/omniadvisor/tests/omniadvisor/service/spark_service/test_spark_cmd_parser.py +++ b/omniadvisor/tests/omniadvisor/service/spark_service/test_spark_cmd_parser.py @@ -91,13 +91,13 @@ class TestSparkCMDParser: cmd = SparkCMDParser.reconstruct_cmd(exec_attr, conf_params) assert cmd.startswith(exec_attr['omniadvisor-cmd-prefix']) - assert '--master yarn' in cmd - assert '--name test_app' in cmd - assert '-e "SELECT * FROM table"' in cmd - assert '--unknown1 test' in cmd - assert '-uk test' in cmd - assert '--conf spark.executor.memory=4g' in cmd - assert '--conf spark.driver.extraJavaOptions="-XX:+UseG1GC -XX:+UseG1GC"' in cmd + assert "--master 'yarn'" in cmd + assert "--name 'test_app'" in cmd + assert "-e 'SELECT * FROM table'" in cmd + assert "'--unknown1' 'test'" in cmd + assert "'-uk' 'test'" in cmd + assert "--conf spark.executor.memory='4g'" in cmd + assert "--conf spark.driver.extraJavaOptions='-XX:+UseG1GC -XX:+UseG1GC'" in cmd # 场景 2: 缺少前缀字段抛异常 def test_reconstruct_cmd_missing_prefix(self): @@ -123,7 +123,7 @@ class TestSparkCMDParser: assert "--extra" in cmd assert "arg" in cmd - assert "--conf spark.executor.instances=4" in cmd + assert "--conf spark.executor.instances='4'" in cmd # 场景 3: 包含 boolean 参数 def test_reconstruct_cmd_with_boolean(self): @@ -143,7 +143,7 @@ class TestSparkCMDParser: assert "--extra" in cmd assert "arg" in cmd - assert "--conf spark.executor.instances=4" in cmd + assert "--conf spark.executor.instances='4'" in cmd assert "verbose" in cmd assert "supervise" in cmd assert "kill" in cmd -- Gitee From b0de706769bc8cdd6704736e0cd54b9dfba7e4ab Mon Sep 17 00:00:00 2001 From: wtingkai <330445001@qq.com> Date: Mon, 14 Jul 2025 10:12:15 +0800 Subject: [PATCH 2/3] =?UTF-8?q?feat(OmniAdvisor):=20=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E6=9C=AA=E6=89=BE=E5=88=B0=E4=BC=98=E5=8C=96=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E6=83=85=E5=86=B5=E4=B8=8B=E7=9A=84=E6=89=93=E5=8D=B0=E5=92=8C?= =?UTF-8?q?=E9=80=80=E5=87=BA=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- omniadvisor/src/tuning.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/omniadvisor/src/tuning.py b/omniadvisor/src/tuning.py index 37e8b3954..a75f59057 100644 --- a/omniadvisor/src/tuning.py +++ b/omniadvisor/src/tuning.py @@ -1,14 +1,22 @@ from common.constant import check_oa_conf -from omniadvisor.interface import config_tuning +from omniadvisor.interface.config_tuning import ( + main, + NoOptimalConfigError +) from omniadvisor.utils.logger import global_logger if __name__ == '__main__': try: check_oa_conf() - config_tuning.main() + main() # 无需进行逻辑处理的异常,直接抛至该层 # 若需进行逻辑处理(如环境清理等),则需在相应位置处理后重新抛至该层 + except NoOptimalConfigError as e: + # 若未找到优化配置,则通过warning打印即可 + global_logger.warning(e) + # 正常退出 + exit(0) except Exception as e: # 异常信息统一在此处打印,方便定位,抛出异常的地方无需打印log global_logger.error(e) -- Gitee From dd9efb502b78b66e88d1b1453cb5fdce886344b6 Mon Sep 17 00:00:00 2001 From: wtingkai <330445001@qq.com> Date: Mon, 14 Jul 2025 10:13:22 +0800 Subject: [PATCH 3/3] =?UTF-8?q?feat(OmniAdvisor):=20=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E6=96=87=E4=BB=B6=E4=B8=AD=E6=8A=93=E5=8F=96?= =?UTF-8?q?Trace=E7=9A=84=E9=BB=98=E8=AE=A4=E8=B6=85=E6=97=B6=E4=B8=8E?= =?UTF-8?q?=E9=97=B4=E9=9A=94=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- omniadvisor/config/common_config.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/omniadvisor/config/common_config.cfg b/omniadvisor/config/common_config.cfg index 53f7206a8..240e42c4e 100755 --- a/omniadvisor/config/common_config.cfg +++ b/omniadvisor/config/common_config.cfg @@ -10,8 +10,8 @@ tuning.strategy=[["transfer", 1],["expert", 2],["iterative", 10]] # Spark History Server的URL 仅用于Rest模式 spark.history.rest.url=http://localhost:18080 # Spark从History Sever抓取Trace的超时时间 -spark.fetch.trace.timeout=10 +spark.fetch.trace.timeout=30 # Spark从History Sever抓取Trace的间隔用时 -spark.fetch.trace.interval=2 +spark.fetch.trace.interval=5 # Spark任务执行的超时时间,对比基线的比例 spark.exec.timeout.ratio=10.0 \ No newline at end of file -- Gitee