From 0d27e865b666812dc44ddaa9c1508379ea1bdb39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=90=E8=89=BA=E4=B8=B9?= <53546877+Craven1701@users.noreply.github.com> Date: Wed, 3 Sep 2025 20:09:45 +0800 Subject: [PATCH 1/6] =?UTF-8?q?=E6=96=B0=E5=A2=9EDjangoView=E7=94=A8?= =?UTF-8?q?=E4=BA=8E=E5=B1=95=E7=A4=BA=E5=9F=BA=E7=BA=BF=E6=80=A7=E8=83=BD?= =?UTF-8?q?=E5=92=8C=E6=9C=80=E4=BC=98=E6=80=A7=E8=83=BD=E7=9B=B8=E5=85=B3?= =?UTF-8?q?=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- omniadvisor/logs/app.log | 78 ++++++ omniadvisor/logs/app.log.2025-08-09 | 230 ++++++++++++++++++ omniadvisor/logs/app.log.2025-08-11 | 78 ++++++ omniadvisor/logs/app.log.2025-08-19 | 113 +++++++++ omniadvisor/pyproject.toml | 1 + .../tuning_result/tuning_result_history.py | 4 + omniadvisor/src/server/app/admin.py | 16 ++ omniadvisor/src/server/app/models.py | 7 + .../app/templates/load_performance.html | 24 ++ omniadvisor/src/server/app/views.py | 6 + omniadvisor/src/server/engine/urls.py | 2 + omniadvisor/src/server/secret_key | 3 + .../tests/omniadvisor/interface/.coverage | Bin 0 -> 53248 bytes 13 files changed, 562 insertions(+) create mode 100644 omniadvisor/logs/app.log create mode 100644 omniadvisor/logs/app.log.2025-08-09 create mode 100644 omniadvisor/logs/app.log.2025-08-11 create mode 100644 omniadvisor/logs/app.log.2025-08-19 create mode 100644 omniadvisor/src/server/app/templates/load_performance.html create mode 100644 omniadvisor/src/server/secret_key create mode 100644 omniadvisor/tests/omniadvisor/interface/.coverage diff --git a/omniadvisor/logs/app.log b/omniadvisor/logs/app.log new file mode 100644 index 000000000..270a494da --- /dev/null +++ b/omniadvisor/logs/app.log @@ -0,0 +1,78 @@ +2025-08-22 16:11:35 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-22 16:11:35 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-22 16:11:35 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-22 16:11:35 [INFO] retest_service.py:36 in retest - Retest succeeded in round 1. Performance result: . +2025-08-22 16:11:35 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-22 16:11:35 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-22 16:11:35 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-22 16:11:35 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-22 16:11:35 [WARNING] retest_service.py:43 in retest - Retest failed in round 1. Exception source: Spark exception: . +2025-08-22 16:11:35 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-22 16:11:35 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-22 16:11:35 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-22 16:11:35 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-22 16:11:35 [ERROR] retest_service.py:29 in retest - Retest failed in round 1. Exception source: Non-Spark exception. Details: . +2025-08-22 16:11:35 [INFO] config_tuning.py:99 in _single_tuning - Retest process did not complete normally. Please check the detailed logs for more information. Remove tuning results. +2025-08-22 16:11:35 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-22 16:11:35 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-22 16:11:35 [INFO] config_tuning.py:116 in _single_tuning - The way of retest is hijacking, update the config from tuning to test config. +2025-08-22 16:11:35 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-22 16:11:35 [WARNING] hijack_recommend.py:286 in hijack_recommend - Spark execute failed in user config, going to print Spark output. +2025-08-22 16:11:35 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-22 16:11:35 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-22 16:11:35 [WARNING] hijack_recommend.py:286 in hijack_recommend - Spark execute failed in user config, going to print Spark output. +2025-08-22 16:11:35 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-22 16:11:35 [INFO] hijack_recommend.py:279 in hijack_recommend - Spark execute success, going to print Spark output. +2025-08-22 16:11:35 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-22 16:11:35 [INFO] hijack_recommend.py:75 in _process_load_config - Best config of load failed too much, roll back to next best config. +2025-08-22 16:11:35 [INFO] hijack_recommend.py:91 in _process_load_config - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-22 16:11:35 [INFO] hijack_recommend.py:226 in _create_or_update_load - Load not found in database, created new one and execute. +2025-08-22 16:11:35 [INFO] hijack_recommend.py:220 in _create_or_update_load - Load found in database, update and ready to get config to execute. +2025-08-22 16:11:35 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-22 16:11:35 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-22 16:11:35 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-22 16:11:35 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[primary_resource] +2025-08-22 16:11:35 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value', '--principal', 'principal1', '--version'] will be add to exec_attr[primary_resource] +2025-08-22 16:11:35 [ERROR] spark_fetcher.py:113 in get_spark_runtime_by_app - No job info returned for application_id=app-1 +2025-08-22 16:11:35 [INFO] spark_run.py:61 in spark_run - Spark Load 1 execute success +2025-08-22 16:11:35 [INFO] spark_run.py:93 in spark_run - Going to fetch Spark execute trace, the process is non-blocking. +2025-08-22 16:11:35 [WARNING] spark_run.py:66 in spark_run - Spark Load 1 execute failed, return code: 1, error code describe: Spark SQL failed +2025-08-22 16:17:21 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-22 16:17:21 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-22 16:17:21 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-22 16:17:21 [INFO] retest_service.py:36 in retest - Retest succeeded in round 1. Performance result: . +2025-08-22 16:17:21 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-22 16:17:21 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-22 16:17:21 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-22 16:17:21 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-22 16:17:21 [WARNING] retest_service.py:43 in retest - Retest failed in round 1. Exception source: Spark exception: . +2025-08-22 16:17:21 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-22 16:17:21 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-22 16:17:21 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-22 16:17:21 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-22 16:17:21 [ERROR] retest_service.py:29 in retest - Retest failed in round 1. Exception source: Non-Spark exception. Details: . +2025-08-22 16:17:21 [INFO] config_tuning.py:99 in _single_tuning - Retest process did not complete normally. Please check the detailed logs for more information. Remove tuning results. +2025-08-22 16:17:21 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-22 16:17:21 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-22 16:17:21 [INFO] config_tuning.py:116 in _single_tuning - The way of retest is hijacking, update the config from tuning to test config. +2025-08-22 16:17:21 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-22 16:17:21 [WARNING] hijack_recommend.py:286 in hijack_recommend - Spark execute failed in user config, going to print Spark output. +2025-08-22 16:17:21 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-22 16:17:21 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-22 16:17:21 [WARNING] hijack_recommend.py:286 in hijack_recommend - Spark execute failed in user config, going to print Spark output. +2025-08-22 16:17:21 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-22 16:17:21 [INFO] hijack_recommend.py:279 in hijack_recommend - Spark execute success, going to print Spark output. +2025-08-22 16:17:21 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-22 16:17:21 [INFO] hijack_recommend.py:75 in _process_load_config - Best config of load failed too much, roll back to next best config. +2025-08-22 16:17:21 [INFO] hijack_recommend.py:91 in _process_load_config - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-22 16:17:21 [INFO] hijack_recommend.py:226 in _create_or_update_load - Load not found in database, created new one and execute. +2025-08-22 16:17:21 [INFO] hijack_recommend.py:220 in _create_or_update_load - Load found in database, update and ready to get config to execute. +2025-08-22 16:17:21 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-22 16:17:21 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-22 16:17:21 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-22 16:17:21 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[primary_resource] +2025-08-22 16:17:21 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value', '--principal', 'principal1', '--version'] will be add to exec_attr[primary_resource] +2025-08-22 16:17:21 [ERROR] spark_fetcher.py:112 in get_spark_runtime_by_app - No job info returned for application_id=app-1 +2025-08-22 16:17:21 [INFO] spark_run.py:61 in spark_run - Spark Load 1 execute success +2025-08-22 16:17:21 [INFO] spark_run.py:93 in spark_run - Going to fetch Spark execute trace, the process is non-blocking. +2025-08-22 16:17:21 [WARNING] spark_run.py:66 in spark_run - Spark Load 1 execute failed, return code: 1, error code describe: Spark SQL failed diff --git a/omniadvisor/logs/app.log.2025-08-09 b/omniadvisor/logs/app.log.2025-08-09 new file mode 100644 index 000000000..d112c84d2 --- /dev/null +++ b/omniadvisor/logs/app.log.2025-08-09 @@ -0,0 +1,230 @@ +2025-08-09 17:33:01 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-09 17:33:01 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-09 17:33:01 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-09 17:33:01 [INFO] retest_service.py:29 in retest - Retest succeeded in round 1. Performance result: . +2025-08-09 17:33:01 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-09 17:33:01 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-09 17:33:01 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-09 17:33:01 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-09 17:33:01 [WARNING] retest_service.py:36 in retest - Retest failed in round 1. Exception source: Spark exception: . +2025-08-09 17:33:01 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-09 17:33:01 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-09 17:33:01 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-09 17:33:01 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-09 17:33:01 [ERROR] retest_service.py:22 in retest - Retest failed in round 1. Exception source: Non-Spark exception. Details: . +2025-08-09 17:33:01 [INFO] config_tuning.py:99 in _single_tuning - Retest process did not complete normally. Please check the detailed logs for more information. Remove tuning results. +2025-08-09 17:33:01 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-09 17:33:01 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-09 17:33:01 [INFO] config_tuning.py:116 in _single_tuning - The way of retest is hijacking, update the config from tuning to test config. +2025-08-09 17:33:01 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 17:33:01 [WARNING] hijack_recommend.py:211 in hijack_recommend - Spark execute failed in user config, going to print Spark output. +2025-08-09 17:33:01 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 17:33:01 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 17:33:01 [WARNING] hijack_recommend.py:211 in hijack_recommend - Spark execute failed in user config, going to print Spark output. +2025-08-09 17:33:01 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 17:33:01 [INFO] hijack_recommend.py:204 in hijack_recommend - Spark execute success, going to print Spark output. +2025-08-09 17:33:01 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 17:33:01 [INFO] hijack_recommend.py:69 in _process_load_config - Best config of load failed too much, roll back to next best config. +2025-08-09 17:33:01 [INFO] hijack_recommend.py:85 in _process_load_config - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-09 17:33:01 [INFO] hijack_recommend.py:174 in _create_or_update_load - Load not found in database, created new one and execute. +2025-08-09 17:33:01 [INFO] hijack_recommend.py:169 in _create_or_update_load - Load found in database, update and ready to get config to execute. +2025-08-09 17:33:01 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-09 17:33:01 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-09 17:33:01 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-09 17:33:01 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[unknown] +2025-08-09 17:33:01 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value'] will be add to exec_attr[unknown] +2025-08-09 17:33:01 [ERROR] spark_fetcher.py:106 in get_spark_runtime_by_app - No job info returned for application_id=app-1 +2025-08-09 17:33:01 [INFO] spark_run.py:51 in spark_run - Spark Load 1 execute success +2025-08-09 17:33:01 [INFO] spark_run.py:81 in spark_run - Going to fetch Spark execute trace, the process is non-blocking. +2025-08-09 17:33:01 [WARNING] spark_run.py:56 in spark_run - Spark Load 1 execute failed, return code: 1, error code describe: Spark SQL failed +2025-08-09 17:34:20 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-09 17:34:20 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-09 17:34:20 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-09 17:34:20 [INFO] retest_service.py:29 in retest - Retest succeeded in round 1. Performance result: . +2025-08-09 17:34:20 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-09 17:34:20 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-09 17:34:20 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-09 17:34:20 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-09 17:34:20 [WARNING] retest_service.py:36 in retest - Retest failed in round 1. Exception source: Spark exception: . +2025-08-09 17:34:20 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-09 17:34:20 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-09 17:34:20 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-09 17:34:20 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-09 17:34:20 [ERROR] retest_service.py:22 in retest - Retest failed in round 1. Exception source: Non-Spark exception. Details: . +2025-08-09 17:34:20 [INFO] config_tuning.py:99 in _single_tuning - Retest process did not complete normally. Please check the detailed logs for more information. Remove tuning results. +2025-08-09 17:34:20 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-09 17:34:20 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-09 17:34:20 [INFO] config_tuning.py:116 in _single_tuning - The way of retest is hijacking, update the config from tuning to test config. +2025-08-09 17:34:20 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 17:34:20 [WARNING] hijack_recommend.py:211 in hijack_recommend - Spark execute failed in user config, going to print Spark output. +2025-08-09 17:34:20 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 17:34:20 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 17:34:20 [WARNING] hijack_recommend.py:211 in hijack_recommend - Spark execute failed in user config, going to print Spark output. +2025-08-09 17:34:20 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 17:34:20 [INFO] hijack_recommend.py:204 in hijack_recommend - Spark execute success, going to print Spark output. +2025-08-09 17:34:20 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 17:34:20 [INFO] hijack_recommend.py:69 in _process_load_config - Best config of load failed too much, roll back to next best config. +2025-08-09 17:34:20 [INFO] hijack_recommend.py:85 in _process_load_config - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-09 17:34:20 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-09 17:34:20 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-09 17:34:20 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-09 17:34:20 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[unknown] +2025-08-09 17:34:20 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value'] will be add to exec_attr[unknown] +2025-08-09 17:34:20 [ERROR] spark_fetcher.py:106 in get_spark_runtime_by_app - No job info returned for application_id=app-1 +2025-08-09 17:34:21 [INFO] spark_run.py:51 in spark_run - Spark Load 1 execute success +2025-08-09 17:34:21 [INFO] spark_run.py:81 in spark_run - Going to fetch Spark execute trace, the process is non-blocking. +2025-08-09 17:34:21 [WARNING] spark_run.py:56 in spark_run - Spark Load 1 execute failed, return code: 1, error code describe: Spark SQL failed +2025-08-09 17:34:46 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-09 17:34:46 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-09 17:34:46 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-09 17:34:46 [INFO] retest_service.py:29 in retest - Retest succeeded in round 1. Performance result: . +2025-08-09 17:34:46 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-09 17:34:46 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-09 17:34:46 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-09 17:34:46 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-09 17:34:46 [WARNING] retest_service.py:36 in retest - Retest failed in round 1. Exception source: Spark exception: . +2025-08-09 17:34:46 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-09 17:34:46 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-09 17:34:46 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-09 17:34:46 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-09 17:34:46 [ERROR] retest_service.py:22 in retest - Retest failed in round 1. Exception source: Non-Spark exception. Details: . +2025-08-09 17:34:46 [INFO] config_tuning.py:99 in _single_tuning - Retest process did not complete normally. Please check the detailed logs for more information. Remove tuning results. +2025-08-09 17:34:46 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-09 17:34:46 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-09 17:34:46 [INFO] config_tuning.py:116 in _single_tuning - The way of retest is hijacking, update the config from tuning to test config. +2025-08-09 17:34:46 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 17:34:46 [WARNING] hijack_recommend.py:211 in hijack_recommend - Spark execute failed in user config, going to print Spark output. +2025-08-09 17:34:46 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 17:34:46 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 17:34:46 [WARNING] hijack_recommend.py:211 in hijack_recommend - Spark execute failed in user config, going to print Spark output. +2025-08-09 17:34:46 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 17:34:46 [INFO] hijack_recommend.py:204 in hijack_recommend - Spark execute success, going to print Spark output. +2025-08-09 17:34:46 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 17:34:46 [INFO] hijack_recommend.py:69 in _process_load_config - Best config of load failed too much, roll back to next best config. +2025-08-09 17:34:46 [INFO] hijack_recommend.py:85 in _process_load_config - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-09 17:34:46 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-09 17:34:46 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-09 17:34:46 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-09 17:34:46 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[unknown] +2025-08-09 17:34:46 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value'] will be add to exec_attr[unknown] +2025-08-09 17:34:46 [ERROR] spark_fetcher.py:106 in get_spark_runtime_by_app - No job info returned for application_id=app-1 +2025-08-09 17:34:46 [INFO] spark_run.py:51 in spark_run - Spark Load 1 execute success +2025-08-09 17:34:46 [INFO] spark_run.py:81 in spark_run - Going to fetch Spark execute trace, the process is non-blocking. +2025-08-09 17:34:46 [WARNING] spark_run.py:56 in spark_run - Spark Load 1 execute failed, return code: 1, error code describe: Spark SQL failed +2025-08-09 17:36:16 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-09 17:36:16 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-09 17:36:16 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-09 17:36:16 [INFO] retest_service.py:29 in retest - Retest succeeded in round 1. Performance result: . +2025-08-09 17:36:16 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-09 17:36:16 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-09 17:36:16 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-09 17:36:16 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-09 17:36:16 [WARNING] retest_service.py:36 in retest - Retest failed in round 1. Exception source: Spark exception: . +2025-08-09 17:36:16 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-09 17:36:16 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-09 17:36:16 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-09 17:36:16 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-09 17:36:16 [ERROR] retest_service.py:22 in retest - Retest failed in round 1. Exception source: Non-Spark exception. Details: . +2025-08-09 17:36:16 [INFO] config_tuning.py:99 in _single_tuning - Retest process did not complete normally. Please check the detailed logs for more information. Remove tuning results. +2025-08-09 17:36:16 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-09 17:36:16 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-09 17:36:16 [INFO] config_tuning.py:116 in _single_tuning - The way of retest is hijacking, update the config from tuning to test config. +2025-08-09 17:36:16 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 17:36:16 [WARNING] hijack_recommend.py:211 in hijack_recommend - Spark execute failed in user config, going to print Spark output. +2025-08-09 17:36:16 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 17:36:16 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 17:36:16 [WARNING] hijack_recommend.py:211 in hijack_recommend - Spark execute failed in user config, going to print Spark output. +2025-08-09 17:36:16 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 17:36:16 [INFO] hijack_recommend.py:204 in hijack_recommend - Spark execute success, going to print Spark output. +2025-08-09 17:36:16 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 17:36:16 [INFO] hijack_recommend.py:69 in _process_load_config - Best config of load failed too much, roll back to next best config. +2025-08-09 17:36:16 [INFO] hijack_recommend.py:85 in _process_load_config - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-09 17:36:16 [INFO] hijack_recommend.py:174 in _create_or_update_load - Load not found in database, created new one and execute. +2025-08-09 17:36:16 [INFO] hijack_recommend.py:169 in _create_or_update_load - Load found in database, update and ready to get config to execute. +2025-08-09 17:36:16 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-09 17:36:16 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-09 17:36:16 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-09 17:36:16 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[unknown] +2025-08-09 17:36:16 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value'] will be add to exec_attr[unknown] +2025-08-09 17:36:16 [ERROR] spark_fetcher.py:106 in get_spark_runtime_by_app - No job info returned for application_id=app-1 +2025-08-09 17:36:16 [INFO] spark_run.py:51 in spark_run - Spark Load 1 execute success +2025-08-09 17:36:16 [INFO] spark_run.py:81 in spark_run - Going to fetch Spark execute trace, the process is non-blocking. +2025-08-09 17:36:16 [WARNING] spark_run.py:56 in spark_run - Spark Load 1 execute failed, return code: 1, error code describe: Spark SQL failed +2025-08-09 17:55:12 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-09 17:55:12 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-09 17:55:12 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-09 17:55:12 [INFO] retest_service.py:29 in retest - Retest succeeded in round 1. Performance result: . +2025-08-09 17:55:12 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-09 17:55:12 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-09 17:55:12 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-09 17:55:12 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-09 17:55:12 [WARNING] retest_service.py:36 in retest - Retest failed in round 1. Exception source: Spark exception: . +2025-08-09 17:55:12 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-09 17:55:12 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-09 17:55:12 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-09 17:55:12 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-09 17:55:12 [ERROR] retest_service.py:22 in retest - Retest failed in round 1. Exception source: Non-Spark exception. Details: . +2025-08-09 17:55:12 [INFO] config_tuning.py:99 in _single_tuning - Retest process did not complete normally. Please check the detailed logs for more information. Remove tuning results. +2025-08-09 17:55:12 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-09 17:55:12 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-09 17:55:12 [INFO] config_tuning.py:116 in _single_tuning - The way of retest is hijacking, update the config from tuning to test config. +2025-08-09 17:55:12 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 17:55:12 [WARNING] hijack_recommend.py:211 in hijack_recommend - Spark execute failed in user config, going to print Spark output. +2025-08-09 17:55:12 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 17:55:12 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 17:55:12 [WARNING] hijack_recommend.py:211 in hijack_recommend - Spark execute failed in user config, going to print Spark output. +2025-08-09 17:55:12 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 17:55:12 [INFO] hijack_recommend.py:204 in hijack_recommend - Spark execute success, going to print Spark output. +2025-08-09 17:55:12 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 17:55:12 [INFO] hijack_recommend.py:69 in _process_load_config - Best config of load failed too much, roll back to next best config. +2025-08-09 17:55:12 [INFO] hijack_recommend.py:85 in _process_load_config - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-09 17:55:12 [INFO] hijack_recommend.py:174 in _create_or_update_load - Load not found in database, created new one and execute. +2025-08-09 17:55:12 [INFO] hijack_recommend.py:169 in _create_or_update_load - Load found in database, update and ready to get config to execute. +2025-08-09 17:55:12 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-09 17:55:12 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-09 17:55:12 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-09 17:55:12 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[unknown] +2025-08-09 17:55:12 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value'] will be add to exec_attr[unknown] +2025-08-09 17:55:12 [ERROR] spark_fetcher.py:112 in get_spark_runtime_by_app - No job info returned for application_id=app-1 +2025-08-09 17:55:12 [INFO] spark_run.py:51 in spark_run - Spark Load 1 execute success +2025-08-09 17:55:12 [INFO] spark_run.py:81 in spark_run - Going to fetch Spark execute trace, the process is non-blocking. +2025-08-09 17:55:12 [WARNING] spark_run.py:56 in spark_run - Spark Load 1 execute failed, return code: 1, error code describe: Spark SQL failed +2025-08-09 18:09:39 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-09 18:09:39 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-09 18:09:39 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-09 18:09:39 [INFO] retest_service.py:36 in retest - Retest succeeded in round 1. Performance result: . +2025-08-09 18:09:39 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-09 18:09:39 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-09 18:09:39 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-09 18:09:39 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-09 18:09:39 [WARNING] retest_service.py:43 in retest - Retest failed in round 1. Exception source: Spark exception: . +2025-08-09 18:09:39 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-09 18:09:39 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-09 18:09:39 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-09 18:09:39 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-09 18:09:39 [ERROR] retest_service.py:29 in retest - Retest failed in round 1. Exception source: Non-Spark exception. Details: . +2025-08-09 18:09:39 [INFO] config_tuning.py:99 in _single_tuning - Retest process did not complete normally. Please check the detailed logs for more information. Remove tuning results. +2025-08-09 18:09:39 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-09 18:09:39 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-09 18:09:39 [INFO] config_tuning.py:116 in _single_tuning - The way of retest is hijacking, update the config from tuning to test config. +2025-08-09 18:09:39 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 18:09:39 [WARNING] hijack_recommend.py:211 in hijack_recommend - Spark execute failed in user config, going to print Spark output. +2025-08-09 18:09:39 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 18:09:39 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 18:09:39 [WARNING] hijack_recommend.py:211 in hijack_recommend - Spark execute failed in user config, going to print Spark output. +2025-08-09 18:09:40 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 18:09:40 [INFO] hijack_recommend.py:204 in hijack_recommend - Spark execute success, going to print Spark output. +2025-08-09 18:09:40 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… +2025-08-09 18:09:40 [INFO] hijack_recommend.py:69 in _process_load_config - Best config of load failed too much, roll back to next best config. +2025-08-09 18:09:40 [INFO] hijack_recommend.py:85 in _process_load_config - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-09 18:09:40 [INFO] hijack_recommend.py:174 in _create_or_update_load - Load not found in database, created new one and execute. +2025-08-09 18:09:40 [INFO] hijack_recommend.py:169 in _create_or_update_load - Load found in database, update and ready to get config to execute. +2025-08-09 18:09:40 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-09 18:09:40 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-09 18:09:40 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-09 18:09:40 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[unknown] +2025-08-09 18:09:40 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value'] will be add to exec_attr[unknown] +2025-08-09 18:09:40 [ERROR] spark_fetcher.py:112 in get_spark_runtime_by_app - No job info returned for application_id=app-1 +2025-08-09 18:09:40 [INFO] spark_run.py:51 in spark_run - Spark Load 1 execute success +2025-08-09 18:09:40 [INFO] spark_run.py:81 in spark_run - Going to fetch Spark execute trace, the process is non-blocking. +2025-08-09 18:09:40 [WARNING] spark_run.py:56 in spark_run - Spark Load 1 execute failed, return code: 1, error code describe: Spark SQL failed diff --git a/omniadvisor/logs/app.log.2025-08-11 b/omniadvisor/logs/app.log.2025-08-11 new file mode 100644 index 000000000..63375a739 --- /dev/null +++ b/omniadvisor/logs/app.log.2025-08-11 @@ -0,0 +1,78 @@ +2025-08-11 16:54:25 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-11 16:54:25 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-11 16:54:25 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-11 16:54:25 [INFO] retest_service.py:36 in retest - Retest succeeded in round 1. Performance result: . +2025-08-11 16:54:25 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-11 16:54:25 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-11 16:54:25 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-11 16:54:25 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-11 16:54:25 [WARNING] retest_service.py:43 in retest - Retest failed in round 1. Exception source: Spark exception: . +2025-08-11 16:54:25 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-11 16:54:25 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-11 16:54:25 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-11 16:54:25 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-11 16:54:25 [ERROR] retest_service.py:29 in retest - Retest failed in round 1. Exception source: Non-Spark exception. Details: . +2025-08-11 16:54:25 [INFO] config_tuning.py:99 in _single_tuning - Retest process did not complete normally. Please check the detailed logs for more information. Remove tuning results. +2025-08-11 16:54:25 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-11 16:54:25 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-11 16:54:25 [INFO] config_tuning.py:116 in _single_tuning - The way of retest is hijacking, update the config from tuning to test config. +2025-08-11 16:54:25 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-11 16:54:25 [WARNING] hijack_recommend.py:279 in hijack_recommend - Spark execute failed in user config, going to print Spark output. +2025-08-11 16:54:25 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-11 16:54:25 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-11 16:54:25 [WARNING] hijack_recommend.py:279 in hijack_recommend - Spark execute failed in user config, going to print Spark output. +2025-08-11 16:54:25 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-11 16:54:25 [INFO] hijack_recommend.py:272 in hijack_recommend - Spark execute success, going to print Spark output. +2025-08-11 16:54:26 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-11 16:54:26 [INFO] hijack_recommend.py:75 in _process_load_config - Best config of load failed too much, roll back to next best config. +2025-08-11 16:54:26 [INFO] hijack_recommend.py:91 in _process_load_config - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-11 16:54:26 [INFO] hijack_recommend.py:226 in _create_or_update_load - Load not found in database, created new one and execute. +2025-08-11 16:54:26 [INFO] hijack_recommend.py:220 in _create_or_update_load - Load found in database, update and ready to get config to execute. +2025-08-11 16:54:26 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-11 16:54:26 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-11 16:54:26 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-11 16:54:26 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[unknown] +2025-08-11 16:54:26 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value'] will be add to exec_attr[unknown] +2025-08-11 16:54:26 [ERROR] spark_fetcher.py:112 in get_spark_runtime_by_app - No job info returned for application_id=app-1 +2025-08-11 16:54:26 [INFO] spark_run.py:59 in spark_run - Spark Load 1 execute success +2025-08-11 16:54:26 [INFO] spark_run.py:89 in spark_run - Going to fetch Spark execute trace, the process is non-blocking. +2025-08-11 16:54:26 [WARNING] spark_run.py:64 in spark_run - Spark Load 1 execute failed, return code: 1, error code describe: Spark SQL failed +2025-08-11 19:33:38 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-11 19:33:38 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-11 19:33:38 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-11 19:33:38 [INFO] retest_service.py:36 in retest - Retest succeeded in round 1. Performance result: . +2025-08-11 19:33:38 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-11 19:33:38 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-11 19:33:38 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-11 19:33:38 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-11 19:33:38 [WARNING] retest_service.py:43 in retest - Retest failed in round 1. Exception source: Spark exception: . +2025-08-11 19:33:38 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-11 19:33:38 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-11 19:33:38 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-11 19:33:38 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-11 19:33:38 [ERROR] retest_service.py:29 in retest - Retest failed in round 1. Exception source: Non-Spark exception. Details: . +2025-08-11 19:33:38 [INFO] config_tuning.py:99 in _single_tuning - Retest process did not complete normally. Please check the detailed logs for more information. Remove tuning results. +2025-08-11 19:33:38 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-11 19:33:38 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-11 19:33:38 [INFO] config_tuning.py:116 in _single_tuning - The way of retest is hijacking, update the config from tuning to test config. +2025-08-11 19:33:38 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-11 19:33:38 [WARNING] hijack_recommend.py:279 in hijack_recommend - Spark execute failed in user config, going to print Spark output. +2025-08-11 19:33:38 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-11 19:33:38 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-11 19:33:38 [WARNING] hijack_recommend.py:279 in hijack_recommend - Spark execute failed in user config, going to print Spark output. +2025-08-11 19:33:38 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-11 19:33:38 [INFO] hijack_recommend.py:272 in hijack_recommend - Spark execute success, going to print Spark output. +2025-08-11 19:33:38 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-11 19:33:38 [INFO] hijack_recommend.py:75 in _process_load_config - Best config of load failed too much, roll back to next best config. +2025-08-11 19:33:38 [INFO] hijack_recommend.py:91 in _process_load_config - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-11 19:33:38 [INFO] hijack_recommend.py:226 in _create_or_update_load - Load not found in database, created new one and execute. +2025-08-11 19:33:38 [INFO] hijack_recommend.py:220 in _create_or_update_load - Load found in database, update and ready to get config to execute. +2025-08-11 19:33:38 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-11 19:33:38 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-11 19:33:38 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-11 19:33:38 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[unknown] +2025-08-11 19:33:38 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value'] will be add to exec_attr[unknown] +2025-08-11 19:33:38 [ERROR] spark_fetcher.py:112 in get_spark_runtime_by_app - No job info returned for application_id=app-1 +2025-08-11 19:33:38 [INFO] spark_run.py:59 in spark_run - Spark Load 1 execute success +2025-08-11 19:33:38 [INFO] spark_run.py:89 in spark_run - Going to fetch Spark execute trace, the process is non-blocking. +2025-08-11 19:33:38 [WARNING] spark_run.py:64 in spark_run - Spark Load 1 execute failed, return code: 1, error code describe: Spark SQL failed diff --git a/omniadvisor/logs/app.log.2025-08-19 b/omniadvisor/logs/app.log.2025-08-19 new file mode 100644 index 000000000..370cdfe60 --- /dev/null +++ b/omniadvisor/logs/app.log.2025-08-19 @@ -0,0 +1,113 @@ +2025-08-19 15:27:58 [WARNING] spark_cmd_parser.py:189 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-19 15:27:58 [WARNING] spark_cmd_parser.py:189 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-19 15:27:58 [WARNING] spark_cmd_parser.py:189 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] +2025-08-19 15:27:58 [WARNING] spark_cmd_parser.py:189 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[unknown] +2025-08-19 15:27:58 [WARNING] spark_cmd_parser.py:189 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value', '--principal', 'principal1', '--version'] will be add to exec_attr[unknown] +2025-08-19 15:29:39 [WARNING] spark_cmd_parser.py:189 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-19 15:29:39 [WARNING] spark_cmd_parser.py:189 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-19 15:29:39 [WARNING] spark_cmd_parser.py:189 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-19 15:29:39 [WARNING] spark_cmd_parser.py:189 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[primary_resource] +2025-08-19 15:29:39 [WARNING] spark_cmd_parser.py:189 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value', '--principal', 'principal1', '--version'] will be add to exec_attr[primary_resource] +2025-08-19 15:52:03 [WARNING] spark_cmd_parser.py:187 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-19 15:52:03 [WARNING] spark_cmd_parser.py:187 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-19 15:52:03 [WARNING] spark_cmd_parser.py:187 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-19 15:52:03 [WARNING] spark_cmd_parser.py:187 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[primary_resource] +2025-08-19 15:52:03 [WARNING] spark_cmd_parser.py:187 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value', '--custom-arg', 'custom_value', '--principal', 'principal1', '--version'] will be add to exec_attr[primary_resource] +2025-08-19 15:53:50 [WARNING] spark_cmd_parser.py:187 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-19 15:53:50 [WARNING] spark_cmd_parser.py:187 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-19 15:53:50 [WARNING] spark_cmd_parser.py:187 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-19 15:53:50 [WARNING] spark_cmd_parser.py:187 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[primary_resource] +2025-08-19 15:53:50 [WARNING] spark_cmd_parser.py:187 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value', '--custom-arg', 'custom_value', '--principal', 'principal1', '--version'] will be add to exec_attr[primary_resource] +2025-08-19 15:55:42 [WARNING] spark_cmd_parser.py:183 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-19 15:55:42 [WARNING] spark_cmd_parser.py:183 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-19 15:55:42 [WARNING] spark_cmd_parser.py:183 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-19 15:55:42 [WARNING] spark_cmd_parser.py:183 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[primary_resource] +2025-08-19 15:55:42 [WARNING] spark_cmd_parser.py:183 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value'] will be add to exec_attr[primary_resource] +2025-08-19 15:59:26 [WARNING] spark_cmd_parser.py:185 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-19 15:59:26 [WARNING] spark_cmd_parser.py:185 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-19 15:59:26 [WARNING] spark_cmd_parser.py:185 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-19 15:59:26 [WARNING] spark_cmd_parser.py:185 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[primary_resource] +2025-08-19 15:59:26 [WARNING] spark_cmd_parser.py:185 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value', '--principal', 'principal1', '--version'] will be add to exec_attr[primary_resource] +2025-08-19 16:14:36 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-19 16:14:36 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-19 16:14:36 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-19 16:14:36 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[primary_resource] +2025-08-19 16:14:36 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value', '--principal', 'principal1', '--version'] will be add to exec_attr[primary_resource] +2025-08-19 19:21:49 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-19 19:21:49 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-19 19:21:49 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-19 19:21:49 [INFO] retest_service.py:36 in retest - Retest succeeded in round 1. Performance result: . +2025-08-19 19:21:49 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-19 19:21:49 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-19 19:21:49 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-19 19:21:49 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-19 19:21:49 [WARNING] retest_service.py:43 in retest - Retest failed in round 1. Exception source: Spark exception: . +2025-08-19 19:21:49 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-19 19:21:49 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-19 19:21:49 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-19 19:21:49 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-19 19:21:49 [ERROR] retest_service.py:29 in retest - Retest failed in round 1. Exception source: Non-Spark exception. Details: . +2025-08-19 19:21:49 [INFO] config_tuning.py:99 in _single_tuning - Retest process did not complete normally. Please check the detailed logs for more information. Remove tuning results. +2025-08-19 19:21:49 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-19 19:21:49 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-19 19:21:49 [INFO] config_tuning.py:116 in _single_tuning - The way of retest is hijacking, update the config from tuning to test config. +2025-08-19 19:21:49 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-19 19:21:49 [WARNING] hijack_recommend.py:279 in hijack_recommend - Spark execute failed in user config, going to print Spark output. +2025-08-19 19:21:49 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-19 19:21:49 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-19 19:21:49 [WARNING] hijack_recommend.py:279 in hijack_recommend - Spark execute failed in user config, going to print Spark output. +2025-08-19 19:21:49 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-19 19:21:49 [INFO] hijack_recommend.py:272 in hijack_recommend - Spark execute success, going to print Spark output. +2025-08-19 19:21:49 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-19 19:21:49 [INFO] hijack_recommend.py:75 in _process_load_config - Best config of load failed too much, roll back to next best config. +2025-08-19 19:21:49 [INFO] hijack_recommend.py:91 in _process_load_config - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-19 19:21:49 [INFO] hijack_recommend.py:226 in _create_or_update_load - Load not found in database, created new one and execute. +2025-08-19 19:21:49 [INFO] hijack_recommend.py:220 in _create_or_update_load - Load found in database, update and ready to get config to execute. +2025-08-19 19:21:49 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-19 19:21:49 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-19 19:21:49 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-19 19:21:49 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[primary_resource] +2025-08-19 19:21:49 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value', '--principal', 'principal1', '--version'] will be add to exec_attr[primary_resource] +2025-08-19 19:21:49 [ERROR] spark_fetcher.py:113 in get_spark_runtime_by_app - No job info returned for application_id=app-1 +2025-08-19 19:21:49 [INFO] spark_run.py:61 in spark_run - Spark Load 1 execute success +2025-08-19 19:21:49 [INFO] spark_run.py:93 in spark_run - Going to fetch Spark execute trace, the process is non-blocking. +2025-08-19 19:21:49 [WARNING] spark_run.py:66 in spark_run - Spark Load 1 execute failed, return code: 1, error code describe: Spark SQL failed +2025-08-19 20:44:36 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-19 20:44:36 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-19 20:44:36 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-19 20:44:36 [INFO] retest_service.py:36 in retest - Retest succeeded in round 1. Performance result: . +2025-08-19 20:44:36 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-19 20:44:36 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-19 20:44:36 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-19 20:44:36 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-19 20:44:36 [WARNING] retest_service.py:43 in retest - Retest failed in round 1. Exception source: Spark exception: . +2025-08-19 20:44:36 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-19 20:44:36 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-19 20:44:36 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-19 20:44:36 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… +2025-08-19 20:44:36 [ERROR] retest_service.py:29 in retest - Retest failed in round 1. Exception source: Non-Spark exception. Details: . +2025-08-19 20:44:36 [INFO] config_tuning.py:99 in _single_tuning - Retest process did not complete normally. Please check the detailed logs for more information. Remove tuning results. +2025-08-19 20:44:36 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. +2025-08-19 20:44:36 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. +2025-08-19 20:44:36 [INFO] config_tuning.py:116 in _single_tuning - The way of retest is hijacking, update the config from tuning to test config. +2025-08-19 20:44:36 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-19 20:44:36 [WARNING] hijack_recommend.py:279 in hijack_recommend - Spark execute failed in user config, going to print Spark output. +2025-08-19 20:44:36 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-19 20:44:36 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-19 20:44:36 [WARNING] hijack_recommend.py:279 in hijack_recommend - Spark execute failed in user config, going to print Spark output. +2025-08-19 20:44:36 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-19 20:44:36 [INFO] hijack_recommend.py:272 in hijack_recommend - Spark execute success, going to print Spark output. +2025-08-19 20:44:36 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… +2025-08-19 20:44:36 [INFO] hijack_recommend.py:75 in _process_load_config - Best config of load failed too much, roll back to next best config. +2025-08-19 20:44:36 [INFO] hijack_recommend.py:91 in _process_load_config - Current tuning result: , best tuning result ever: , best tuning speedup: % +2025-08-19 20:44:36 [INFO] hijack_recommend.py:226 in _create_or_update_load - Load not found in database, created new one and execute. +2025-08-19 20:44:36 [INFO] hijack_recommend.py:220 in _create_or_update_load - Load found in database, update and ready to get config to execute. +2025-08-19 20:44:37 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-19 20:44:37 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-19 20:44:37 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] +2025-08-19 20:44:37 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[primary_resource] +2025-08-19 20:44:37 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value', '--principal', 'principal1', '--version'] will be add to exec_attr[primary_resource] +2025-08-19 20:44:37 [ERROR] spark_fetcher.py:113 in get_spark_runtime_by_app - No job info returned for application_id=app-1 +2025-08-19 20:44:37 [INFO] spark_run.py:61 in spark_run - Spark Load 1 execute success +2025-08-19 20:44:37 [INFO] spark_run.py:93 in spark_run - Going to fetch Spark execute trace, the process is non-blocking. +2025-08-19 20:44:37 [WARNING] spark_run.py:66 in spark_run - Spark Load 1 execute failed, return code: 1, error code describe: Spark SQL failed diff --git a/omniadvisor/pyproject.toml b/omniadvisor/pyproject.toml index ea18fef83..d71aa5442 100755 --- a/omniadvisor/pyproject.toml +++ b/omniadvisor/pyproject.toml @@ -25,6 +25,7 @@ psycopg2-binary = "^2.9.10" django-sslserver = "0.22" keyring = "24.3.1" keyrings-alt = "5.0.2" +pandas = "2.3.2" [tool.poetry.group.test.dependencies] pytest = "^7.4.4" diff --git a/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result_history.py b/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result_history.py index e28be2747..e7826e95a 100644 --- a/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result_history.py +++ b/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result_history.py @@ -147,3 +147,7 @@ class TuningResultHistory: tuning_result.to_tuning_data() for tuning_result in self._tuning_results if tuning_result.status != OA_CONF.TuningResultStatus.empty ] + + +def get_all_loads_performance_data(): + pass \ No newline at end of file diff --git a/omniadvisor/src/server/app/admin.py b/omniadvisor/src/server/app/admin.py index 3d0f61252..b913b5cc6 100644 --- a/omniadvisor/src/server/app/admin.py +++ b/omniadvisor/src/server/app/admin.py @@ -1,4 +1,5 @@ from django.contrib import admin +from django.shortcuts import redirect from .models import DatabaseLoad, DatabaseTuningRecord, DatabaseExamRecord @@ -112,3 +113,18 @@ class ExamRecordAdmin(BaseAdmin): def runtime_f(self, model): return round(model.runtime, 3) runtime_f.short_description = 'Runtime' + + +@admin.register(PerformanceData) +class PerformanceAdmin(admin.ModelAdmin): + # 禁止 add/change/delete 相关的操作 + def has_add_permission(self, request): + return False + def has_change_permission(self, request, obj=None): + return False + def has_delete_permission(self, request, obj=None): + return False + + # 点击虚拟模型时直接跳转 + def changelist_view(self, request, extra_context=None): + return redirect('/admin/load_performance') \ No newline at end of file diff --git a/omniadvisor/src/server/app/models.py b/omniadvisor/src/server/app/models.py index 74021fca2..a4ef645a6 100644 --- a/omniadvisor/src/server/app/models.py +++ b/omniadvisor/src/server/app/models.py @@ -81,3 +81,10 @@ class DatabaseExamRecord(models.Model): class Meta: db_table = 'omniadvisor_exam_record' # 自定义表名 + + +class PerformanceData(models.Model): + """虚拟模型 不对应数据库表 显示PerformanceData""" + class Meta: + verbose_name = "Performance Data" + verbose_name_plural = "Performance Data" diff --git a/omniadvisor/src/server/app/templates/load_performance.html b/omniadvisor/src/server/app/templates/load_performance.html new file mode 100644 index 000000000..17ab2ac7c --- /dev/null +++ b/omniadvisor/src/server/app/templates/load_performance.html @@ -0,0 +1,24 @@ +{% extends "admin/base_site.html" %} + + +{% block content %} +

Performance Data

+ + + + {% for col in data.0.kyes %} + + {% endfor %} + + + + {% for row in data %} + + {% for value in row.values %} + + {{% endfor %}} + + {% endfor %} + +
{{ col }}
{{ value }}
+{% endblock %} diff --git a/omniadvisor/src/server/app/views.py b/omniadvisor/src/server/app/views.py index 91ea44a21..02a71ed03 100644 --- a/omniadvisor/src/server/app/views.py +++ b/omniadvisor/src/server/app/views.py @@ -1,3 +1,9 @@ from django.shortcuts import render +import pandas as pd +from omniadvisor.service.tuning_result.tuning_result_history import get_all_loads_performance_data # Create your views here. + +def load_performance_data_view(request): + datalist = get_all_loads_performance_data() + return render(request, 'admin/load_performance.html', {'data':datalist}) diff --git a/omniadvisor/src/server/engine/urls.py b/omniadvisor/src/server/engine/urls.py index 1e05d382b..f379e994e 100644 --- a/omniadvisor/src/server/engine/urls.py +++ b/omniadvisor/src/server/engine/urls.py @@ -16,7 +16,9 @@ Including another URLconf """ from django.contrib import admin from django.urls import path +from server.app.views import load_performance_data_view urlpatterns = [ + path('admin/load_performance/', load_performance_data_view, name='load_performance_data'), path('admin/', admin.site.urls), ] diff --git a/omniadvisor/src/server/secret_key b/omniadvisor/src/server/secret_key new file mode 100644 index 000000000..98b139987 --- /dev/null +++ b/omniadvisor/src/server/secret_key @@ -0,0 +1,3 @@ +[default] +django.secretkey = b2&3=0rb1@40eb(&2xr4v%*gk1pa$c79087)q8hijjv3^vynm! + diff --git a/omniadvisor/tests/omniadvisor/interface/.coverage b/omniadvisor/tests/omniadvisor/interface/.coverage new file mode 100644 index 0000000000000000000000000000000000000000..1a2870cd1bbde21de950c86fe669fc3460aaaa82 GIT binary patch literal 53248 zcmeI4TWlQF8OLXKW@m4+d)8jt>y1H~Qc))md+h+Cw3SF`QfOB4{5(Ta;y0!qJc zX2&}lU&O->5y>~U&zUoG&iSA3cW!fbeD;pLdvseUB-gn-#sR+Iq95*mIg6yKQL|oao>hOm!G>(oK(NCA|WvY^?@av`oFB zn#;<(w!F=a&{4Cs1GaYoGST!}6Jsb_I?1;xrdHKVty$JAuM}JK%GTu8V~3Nmd@jcy z*Ib)avkbqf9&6x2*OXhUkWtw%D_HtjQ*D;#G;5opu2@($pqvM6&sx)H6m3JP>rL0p zrls4u(NwepTDfg&l{F3AhS2;2;03LLyk^j#882l=D?q24c#V{u=$*t>C|uLK|G~d-L_Uv5eN=pi}f@izb4lhM*Uk2O|{yl zKagxRNct(dqom(QI(NXFiI|k%IkuiTop`0`&Pm*fiLw0fFz>r%i}o0SSGrx` zf{Eax7r&+2bIR(n<7FuLcvQ-Net5ltV-ZTz=t{@#j<`iWUU2C*V9>`up{K}rrCa3P z+VaGrT8Gi3rRpYhSr~)889>2^6%`_@vqsx?+b;&)=81Dpx4d2k-LTIdVr_M!L-+og zVOcs%t2k|0pYyLO<8!KoqmVVO%o;{rQ=8u004Z*@KvTCAw+yRirJK8XSj(74ZE@T! zlRvrOFuxBvo!`IUq|s2DmCk&H)3BE-jQ!X-QO6rZAfYl2ozj9XdO@Ckrk%SuV)^+` zwtxxd+E6QAcgIQzYAIz?gPSmyZXFjkY6e(a zrD=3=l^n-MIcGB;%U^pf@3cFe(Hmz<7EJ05)h~7@80l#w7u@W&m)l{$g{PfhJ3)2< z92oQzaB%Z#aMq*X$Lc)mB)8%MpJ(Sd_@#pg5CI}U1c(3;AOb{y2oM1xKm>>Y5xDdS z2z-Q(;rc(q{=u=gz@dW(5CI}U1c(3;AOb{y2oM1xKm>>Y5%^>hkR!sC%>NgUDaH$9 z`OZH9+&KB!$?Zw7D#G66*qiL#Po{xXY9c@chyW2F0z`la5CI}U1c(3;AOb|dCm;(~ z^8Tj)k)$w|=zIu(@Bhby7diF>+n#wovoHO(^gZ%B@?-L+QqQDrPW~}@PvT6X7Jn;l z#y*HS(mCmn6pbE_UM2oiydIL#K?I1vMJCW36FEmM?zdpSRIynuPfXu9H9euhHenO? zGFw{l7T8^O7i`ol!UiwPp4aV(S-pmvt%|TiS63^Gx@DL}%PjYXHFHrfYeiGDHOnqR z?ihqUR>mgKEB9Rw_avEB*z%2AOsjgWWWz4DW(@@IFM#0veHZLlRJ2;EIq%!o2~{;4 z_GpDlO z19rk2=5n!asFhL+x8_5Amby1U+!2OE?r17Bv%TgEnpV4R7kfe_*j0(egHkN#0Krn< z3(>W-s2xxnPzwA`fnEUiAV@dDlx`RmtsX1@*7-mvb?!Iy9?regL3}X_;tPE@d2R7h zNpI?Qsf4;L6x~2Ad#G2e$J$2NnNVN{bbAzg+gBqE;-$XB4v2S8!B$fSIXz4{cQEPE zY^1_A(W_Tf1``*OAXf|1L{Bcza_18ud3TtS0kKLP#BS@mm^atfjT(Ht3s9`aKxJ=8 zDgkQ_Nie_geb?~^o3$QW$_<$9R1>0ror_VhGrYmBS3h9)oCv!6L$f=eWD20v|1RHs z5}niUQ_J)2?Ad5&P3(jBML_e;zFXJT4A?irgVwITYXubW{r{No633oo*NHb|UXZVf z9hD9zUrzlZ_F-yAVlH`a;%sKW@KXA8`rhb6;!~0o|4-bKK9Jv)zZ?DZMPB%jJP{xQ z9A}7*8s0p!YW+Xf5}k7Y^|~YAh3o%K4~WkF{TJM5BJHmK3l-7X-GA}k_5bKW(K*CF1M2YbHn1c*3;j2FZSlZkRyG{nKrOGm{%7I94(P7%Uj6w} zrYSn5{=*K4cTb^i)0XJyp~|gZ|I4A9*t7mmnW9q*)x=fn|72Zs?haKl&`lDW=-k$S zu?yD!@tWxD4NE0pP3*9El<&WeKiKrHzr9e;`dGItt z(KXip!o29**?;T0ngRPFGorIA5$5`zzW>Y5g-B|l|Z^6@`)=7)g4?mtL!);k3w+M=<_1Hl;)Ip9zuipA4xMq zBXD-j$e+){4fi+#Zag*&SF^)sUmF>MlgXj?$MGu1eTS<~jgFq;viTfbpUOR3zW$9T z`6E0hbEDi8#5t87gd0~3R>ha`I6OK4!R-U@=Q!>NH?`@=i4!MyPB_I)<)kd!$z&rk z1Ah~%Myl27>nxXn(3Z^Nm)}0gV|MVvof=BR#cR@!FSJ^%sUy71&48Aam*MJ^{K}hO zxLM%f5=IrUAn6p`xFWTkzYdPKH%mzfW|9#pfqvp~76)I6|NYF9Z=d_qv9H|3OEJ6= zi;PI%Pf9zUKD6gIAD%ydehjjMLzbd=KN^ul^u@@CfboR)Zs)#z_1WtICnsK`{dPVl zMeu$k66M7N`1t<6!0zDKIra|wH+!9(VSj;70sg>#$DU)qX3wynu^+P&?ECBy_8>dR z7Ma23n9A;gByFO_n=WYYB#3D=9qT`v}MJxOxC zXw>yY(e(tu^&$}wJ^*mv|8v>Pxw^=9B0vO)01+SpM1Tko0U|&IhyW2F0+&7keE(1D z|4YA~$wDGP1c(3;AOb{y2oM1xKm>>Y5xBew;Qs$Hwv7M({~mk;z`xiV?4Rs3>;ZU< zy#n6_@FIJGJq!B)e#w5$o?<^?C)p2RFTkVl-2lhfAxK6C5g-CYfCvx)B0vO)01+Sp zM1Tko0V1#w0bvZjeP7swP63@!bn@topfill5IQ+@2GJQnCyNe4CxcEJ9T}YzI!SaA Y=)}>9p(CLaMMp$OKqrC@e*gdf0WFj6*8l(j literal 0 HcmV?d00001 -- Gitee From dea89b26b7a2712cdd5d71752205430c253e41c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=90=E8=89=BA=E4=B8=B9?= <53546877+Craven1701@users.noreply.github.com> Date: Wed, 3 Sep 2025 20:16:49 +0800 Subject: [PATCH 2/6] =?UTF-8?q?=E5=9B=BA=E5=8C=96=E4=B8=80=E4=B8=8B?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- omniadvisor/config/common_config.ini | 8 ++++---- omniadvisor/script/spark-submit | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/omniadvisor/config/common_config.ini b/omniadvisor/config/common_config.ini index cb390b5fe..e805a4b95 100755 --- a/omniadvisor/config/common_config.ini +++ b/omniadvisor/config/common_config.ini @@ -10,13 +10,13 @@ backend.retest.queue= [database] # 后端数据库postgresql的database名称 -postgresql.database.name= +postgresql.database.name=omniadvisor # 后端数据库postgresql的用户 -postgresql.database.user= +postgresql.database.user=postgres # 后端数据库postgresql的连接host -postgresql.database.host= +postgresql.database.host=localhost # 后端数据库postgresql的连接端口 -postgresql.database.port= +postgresql.database.port=5432 [spark] # Spark History Server的URL 仅用于Rest模式 diff --git a/omniadvisor/script/spark-submit b/omniadvisor/script/spark-submit index 58193f6ae..d59d0901e 100644 --- a/omniadvisor/script/spark-submit +++ b/omniadvisor/script/spark-submit @@ -28,7 +28,7 @@ export PYTHONHASHSEED=0 if [[ -v enable_omniadvisor && "$enable_omniadvisor" = "true" ]]; then echo "*** OmniAdvisor is enabled! We will find best config to SPEEDUP! ***" # 根据特性实际的部署路径进行修改 - hijack_path="" + hijack_path="/home/x30041342/OmniAdvisor2_source/src/hijack.py" # OmniAdvisor劫持优化 cmd_prefix="${SPARK_HOME}/bin/spark-class org.apache.spark.deploy.SparkSubmit" python3 ${hijack_path} --omniadvisor-cmd-prefix "${cmd_prefix}" "$@" -- Gitee From bc740615208a264c92f2f6bcf48f14d4a2521e40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=90=E8=89=BA=E4=B8=B9?= <53546877+Craven1701@users.noreply.github.com> Date: Wed, 3 Sep 2025 20:20:13 +0800 Subject: [PATCH 3/6] bugfix --- omniadvisor/src/server/app/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omniadvisor/src/server/app/admin.py b/omniadvisor/src/server/app/admin.py index b913b5cc6..02bf7d8ad 100644 --- a/omniadvisor/src/server/app/admin.py +++ b/omniadvisor/src/server/app/admin.py @@ -1,7 +1,7 @@ from django.contrib import admin from django.shortcuts import redirect -from .models import DatabaseLoad, DatabaseTuningRecord, DatabaseExamRecord +from .models import DatabaseLoad, DatabaseTuningRecord, DatabaseExamRecord, PerformanceData # Register your models here. _LOAD_FIELDS = ('id', 'name', 'exec_attr', 'default_config', 'best_config', 'test_config', 'backend_retest_enable') -- Gitee From 09f1a7b4d458256968c476668e8111792d1e6329 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=90=E8=89=BA=E4=B8=B9?= <53546877+Craven1701@users.noreply.github.com> Date: Wed, 3 Sep 2025 20:55:37 +0800 Subject: [PATCH 4/6] =?UTF-8?q?=E6=80=A7=E8=83=BD=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E8=8E=B7=E5=8F=96=E4=B8=8E=E7=AD=9B=E9=80=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../omniadvisor/repository/load_repository.py | 9 +++++ .../service/tuning_result/tuning_result.py | 4 ++ .../tuning_result/tuning_result_history.py | 40 ++++++++++++++++++- omniadvisor/src/server/app/views.py | 2 +- 4 files changed, 53 insertions(+), 2 deletions(-) diff --git a/omniadvisor/src/omniadvisor/repository/load_repository.py b/omniadvisor/src/omniadvisor/repository/load_repository.py index ac74c102c..7c29d30cc 100644 --- a/omniadvisor/src/omniadvisor/repository/load_repository.py +++ b/omniadvisor/src/omniadvisor/repository/load_repository.py @@ -1,4 +1,5 @@ from django.db import IntegrityError +from django.db.models.query import QuerySet from common.exceptions import DuplicateEntryError from server.app.models import DatabaseLoad @@ -159,3 +160,11 @@ class LoadRepository(Repository): model_attr=model_attr ) return Load(database_model=database_load) + + @classmethod + def get_all_loads(cls) -> QuerySet[Load]: + return Load.objects.only("id", "name") + + @classmethod + def get_load_id(cls, load: Load) -> int: + return load.id diff --git a/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result.py b/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result.py index f480883f3..7d667b2ce 100644 --- a/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result.py +++ b/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result.py @@ -90,6 +90,10 @@ class TuningResult: failed=sum([exam_record.status == OA_CONF.ExecStatus.fail for exam_record in self._exam_records]) ) + @property + def tuning_id(self): + return self._tuning_record.id + @property def config(self): return self._tuning_record.config diff --git a/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result_history.py b/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result_history.py index e7826e95a..d14a54bd6 100644 --- a/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result_history.py +++ b/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result_history.py @@ -9,6 +9,8 @@ from omniadvisor.repository.tuning_record_repository import TuningRecordReposito from omniadvisor.service.tuning_result.tuning_result import TuningResult from omniadvisor.service.tuning_result.tuning_result import get_tuning_result +import pandas as pd + def get_other_tuning_result_history(exclude_load: Load) -> List: """ @@ -150,4 +152,40 @@ class TuningResultHistory: def get_all_loads_performance_data(): - pass \ No newline at end of file + # 获取全量的load_id + loads_list = LoadRepository.get_all_loads() + records = [] + # 获取全量load的基线性能 + for load in loads_list: + load_id = LoadRepository.get_load_id(load) + load_tuning_history = get_tuning_result_history(load) + # 基线数据 + user_tuning = load_tuning_history.user_tuning_result + base_perf = user_tuning.runtime + + # 最优数据 + best_tuning = load_tuning_history.best_tuning_result + best_tuning_id = best_tuning.tuning_id + best_tuning_config = best_tuning.config + peak_perf = best_tuning.runtime + + # 提升比例 + boost_ratio = load_tuning_history.boost_percentage + + # 存到 dict 里 + records.append({ + "load_id": load_id, + "base_perf": base_perf, + "peak_perf": peak_perf, + "best_tuning_id": best_tuning_id, + "best_tuning_config": best_tuning_config, + "boost_ratio": boost_ratio + }) + + # 转换成 DataFrame + df = pd.DataFrame(records) + print(df.head()) + return df + +if __name__ == "__main__": + get_all_loads_performance_data() diff --git a/omniadvisor/src/server/app/views.py b/omniadvisor/src/server/app/views.py index 02a71ed03..f7f1898a4 100644 --- a/omniadvisor/src/server/app/views.py +++ b/omniadvisor/src/server/app/views.py @@ -6,4 +6,4 @@ from omniadvisor.service.tuning_result.tuning_result_history import get_all_load def load_performance_data_view(request): datalist = get_all_loads_performance_data() - return render(request, 'admin/load_performance.html', {'data':datalist}) + return render(request, 'admin/load_performance.html', {'data': datalist}) -- Gitee From 49435ed236f031bd4f9d1317fe1078961710f19e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=90=E8=89=BA=E4=B8=B9?= <53546877+Craven1701@users.noreply.github.com> Date: Wed, 3 Sep 2025 21:46:14 +0800 Subject: [PATCH 5/6] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E6=B8=B2=E6=9F=93?= =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../omniadvisor/repository/load_repository.py | 8 ++- .../src/omniadvisor/repository/model/load.py | 5 ++ .../tuning_result/tuning_result_history.py | 18 ++++--- .../app/templates/load_performance.html | 52 ++++++++++++------- omniadvisor/src/server/app/views.py | 1 + 5 files changed, 56 insertions(+), 28 deletions(-) diff --git a/omniadvisor/src/omniadvisor/repository/load_repository.py b/omniadvisor/src/omniadvisor/repository/load_repository.py index 7c29d30cc..67813322d 100644 --- a/omniadvisor/src/omniadvisor/repository/load_repository.py +++ b/omniadvisor/src/omniadvisor/repository/load_repository.py @@ -162,9 +162,13 @@ class LoadRepository(Repository): return Load(database_model=database_load) @classmethod - def get_all_loads(cls) -> QuerySet[Load]: - return Load.objects.only("id", "name") + def get_all_loads_in_db(cls) -> list[Load]: + return Load.get_all_loads_in_db() @classmethod def get_load_id(cls, load: Load) -> int: return load.id + + @classmethod + def get_load_name(cls, load: Load) -> int: + return load.name diff --git a/omniadvisor/src/omniadvisor/repository/model/load.py b/omniadvisor/src/omniadvisor/repository/model/load.py index 79b7ed2c9..73bd51776 100644 --- a/omniadvisor/src/omniadvisor/repository/model/load.py +++ b/omniadvisor/src/omniadvisor/repository/model/load.py @@ -53,6 +53,11 @@ class Load: def database_model(self): return self._database_model + @classmethod + def get_all_loads_in_db(cls): + qs = DatabaseLoad.objects.only("id", "name") + return [Load(database_model) for database_model in qs] + class FieldName: id = 'id' name = 'name' diff --git a/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result_history.py b/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result_history.py index d14a54bd6..5b20063b4 100644 --- a/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result_history.py +++ b/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result_history.py @@ -9,9 +9,6 @@ from omniadvisor.repository.tuning_record_repository import TuningRecordReposito from omniadvisor.service.tuning_result.tuning_result import TuningResult from omniadvisor.service.tuning_result.tuning_result import get_tuning_result -import pandas as pd - - def get_other_tuning_result_history(exclude_load: Load) -> List: """ 查询 exclude_load 以外的所有 tuning result history @@ -153,12 +150,14 @@ class TuningResultHistory: def get_all_loads_performance_data(): # 获取全量的load_id - loads_list = LoadRepository.get_all_loads() + loads_list = LoadRepository.get_all_loads_in_db() records = [] # 获取全量load的基线性能 for load in loads_list: load_id = LoadRepository.get_load_id(load) + load_name = LoadRepository.get_load_name(load) load_tuning_history = get_tuning_result_history(load) + # 基线数据 user_tuning = load_tuning_history.user_tuning_result base_perf = user_tuning.runtime @@ -167,6 +166,8 @@ def get_all_loads_performance_data(): best_tuning = load_tuning_history.best_tuning_result best_tuning_id = best_tuning.tuning_id best_tuning_config = best_tuning.config + best_tuning_method = best_tuning.method + best_tuning_rounds = best_tuning.rounds peak_perf = best_tuning.runtime # 提升比例 @@ -175,17 +176,18 @@ def get_all_loads_performance_data(): # 存到 dict 里 records.append({ "load_id": load_id, + "name": load_name, "base_perf": base_perf, "peak_perf": peak_perf, "best_tuning_id": best_tuning_id, "best_tuning_config": best_tuning_config, + "best_tuning_method": best_tuning_method, + "best_tuning_rounds": best_tuning_rounds, "boost_ratio": boost_ratio }) - # 转换成 DataFrame - df = pd.DataFrame(records) - print(df.head()) - return df + print(records) + return records if __name__ == "__main__": get_all_loads_performance_data() diff --git a/omniadvisor/src/server/app/templates/load_performance.html b/omniadvisor/src/server/app/templates/load_performance.html index 17ab2ac7c..f7f063739 100644 --- a/omniadvisor/src/server/app/templates/load_performance.html +++ b/omniadvisor/src/server/app/templates/load_performance.html @@ -3,22 +3,38 @@ {% block content %}

Performance Data

- - - - {% for col in data.0.kyes %} - - {% endfor %} - - - - {% for row in data %} - - {% for value in row.values %} - - {{% endfor %}} - - {% endfor %} - -
{{ col }}
{{ value }}
+ + + + + + + + + + + + + + + + {% for row in data %} + + + + + + + + + + + + {% empty %} + + + + {% endfor %} + +
load_idnamebase_perfpeak_perfbest_tuning_idbest_tuning_configbest_tuning_methodbest_tuning_roundsboost_ratio
{{ row.load_id }}{{ row.name }}{{ row.base_perf }}{{ row.peak_perf }}{{ row.best_tuning_id }}{{ row.best_tuning_config }}{{ row.best_tuning_method }}{{ row.best_tuning_rounds }}{{ row.boost_ratio }}
暂无数据
{% endblock %} diff --git a/omniadvisor/src/server/app/views.py b/omniadvisor/src/server/app/views.py index f7f1898a4..af1c664e9 100644 --- a/omniadvisor/src/server/app/views.py +++ b/omniadvisor/src/server/app/views.py @@ -4,6 +4,7 @@ from omniadvisor.service.tuning_result.tuning_result_history import get_all_load # Create your views here. + def load_performance_data_view(request): datalist = get_all_loads_performance_data() return render(request, 'admin/load_performance.html', {'data': datalist}) -- Gitee From c83173db9e9b447b98c255a585b56aeda1b1a761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=90=E8=89=BA=E4=B8=B9?= <53546877+Craven1701@users.noreply.github.com> Date: Wed, 3 Sep 2025 22:12:24 +0800 Subject: [PATCH 6/6] =?UTF-8?q?=E6=96=87=E4=BB=B6=E6=B8=85=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- omniadvisor/logs/app.log | 78 ---------- omniadvisor/logs/app.log.2025-08-09 | 230 ---------------------------- omniadvisor/logs/app.log.2025-08-11 | 78 ---------- omniadvisor/logs/app.log.2025-08-19 | 113 -------------- 4 files changed, 499 deletions(-) delete mode 100644 omniadvisor/logs/app.log delete mode 100644 omniadvisor/logs/app.log.2025-08-09 delete mode 100644 omniadvisor/logs/app.log.2025-08-11 delete mode 100644 omniadvisor/logs/app.log.2025-08-19 diff --git a/omniadvisor/logs/app.log b/omniadvisor/logs/app.log deleted file mode 100644 index 270a494da..000000000 --- a/omniadvisor/logs/app.log +++ /dev/null @@ -1,78 +0,0 @@ -2025-08-22 16:11:35 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-22 16:11:35 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-22 16:11:35 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-22 16:11:35 [INFO] retest_service.py:36 in retest - Retest succeeded in round 1. Performance result: . -2025-08-22 16:11:35 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-22 16:11:35 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-22 16:11:35 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-22 16:11:35 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-22 16:11:35 [WARNING] retest_service.py:43 in retest - Retest failed in round 1. Exception source: Spark exception: . -2025-08-22 16:11:35 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-22 16:11:35 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-22 16:11:35 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-22 16:11:35 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-22 16:11:35 [ERROR] retest_service.py:29 in retest - Retest failed in round 1. Exception source: Non-Spark exception. Details: . -2025-08-22 16:11:35 [INFO] config_tuning.py:99 in _single_tuning - Retest process did not complete normally. Please check the detailed logs for more information. Remove tuning results. -2025-08-22 16:11:35 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-22 16:11:35 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-22 16:11:35 [INFO] config_tuning.py:116 in _single_tuning - The way of retest is hijacking, update the config from tuning to test config. -2025-08-22 16:11:35 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-22 16:11:35 [WARNING] hijack_recommend.py:286 in hijack_recommend - Spark execute failed in user config, going to print Spark output. -2025-08-22 16:11:35 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-22 16:11:35 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-22 16:11:35 [WARNING] hijack_recommend.py:286 in hijack_recommend - Spark execute failed in user config, going to print Spark output. -2025-08-22 16:11:35 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-22 16:11:35 [INFO] hijack_recommend.py:279 in hijack_recommend - Spark execute success, going to print Spark output. -2025-08-22 16:11:35 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-22 16:11:35 [INFO] hijack_recommend.py:75 in _process_load_config - Best config of load failed too much, roll back to next best config. -2025-08-22 16:11:35 [INFO] hijack_recommend.py:91 in _process_load_config - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-22 16:11:35 [INFO] hijack_recommend.py:226 in _create_or_update_load - Load not found in database, created new one and execute. -2025-08-22 16:11:35 [INFO] hijack_recommend.py:220 in _create_or_update_load - Load found in database, update and ready to get config to execute. -2025-08-22 16:11:35 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-22 16:11:35 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-22 16:11:35 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-22 16:11:35 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[primary_resource] -2025-08-22 16:11:35 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value', '--principal', 'principal1', '--version'] will be add to exec_attr[primary_resource] -2025-08-22 16:11:35 [ERROR] spark_fetcher.py:113 in get_spark_runtime_by_app - No job info returned for application_id=app-1 -2025-08-22 16:11:35 [INFO] spark_run.py:61 in spark_run - Spark Load 1 execute success -2025-08-22 16:11:35 [INFO] spark_run.py:93 in spark_run - Going to fetch Spark execute trace, the process is non-blocking. -2025-08-22 16:11:35 [WARNING] spark_run.py:66 in spark_run - Spark Load 1 execute failed, return code: 1, error code describe: Spark SQL failed -2025-08-22 16:17:21 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-22 16:17:21 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-22 16:17:21 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-22 16:17:21 [INFO] retest_service.py:36 in retest - Retest succeeded in round 1. Performance result: . -2025-08-22 16:17:21 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-22 16:17:21 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-22 16:17:21 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-22 16:17:21 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-22 16:17:21 [WARNING] retest_service.py:43 in retest - Retest failed in round 1. Exception source: Spark exception: . -2025-08-22 16:17:21 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-22 16:17:21 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-22 16:17:21 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-22 16:17:21 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-22 16:17:21 [ERROR] retest_service.py:29 in retest - Retest failed in round 1. Exception source: Non-Spark exception. Details: . -2025-08-22 16:17:21 [INFO] config_tuning.py:99 in _single_tuning - Retest process did not complete normally. Please check the detailed logs for more information. Remove tuning results. -2025-08-22 16:17:21 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-22 16:17:21 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-22 16:17:21 [INFO] config_tuning.py:116 in _single_tuning - The way of retest is hijacking, update the config from tuning to test config. -2025-08-22 16:17:21 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-22 16:17:21 [WARNING] hijack_recommend.py:286 in hijack_recommend - Spark execute failed in user config, going to print Spark output. -2025-08-22 16:17:21 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-22 16:17:21 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-22 16:17:21 [WARNING] hijack_recommend.py:286 in hijack_recommend - Spark execute failed in user config, going to print Spark output. -2025-08-22 16:17:21 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-22 16:17:21 [INFO] hijack_recommend.py:279 in hijack_recommend - Spark execute success, going to print Spark output. -2025-08-22 16:17:21 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-22 16:17:21 [INFO] hijack_recommend.py:75 in _process_load_config - Best config of load failed too much, roll back to next best config. -2025-08-22 16:17:21 [INFO] hijack_recommend.py:91 in _process_load_config - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-22 16:17:21 [INFO] hijack_recommend.py:226 in _create_or_update_load - Load not found in database, created new one and execute. -2025-08-22 16:17:21 [INFO] hijack_recommend.py:220 in _create_or_update_load - Load found in database, update and ready to get config to execute. -2025-08-22 16:17:21 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-22 16:17:21 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-22 16:17:21 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-22 16:17:21 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[primary_resource] -2025-08-22 16:17:21 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value', '--principal', 'principal1', '--version'] will be add to exec_attr[primary_resource] -2025-08-22 16:17:21 [ERROR] spark_fetcher.py:112 in get_spark_runtime_by_app - No job info returned for application_id=app-1 -2025-08-22 16:17:21 [INFO] spark_run.py:61 in spark_run - Spark Load 1 execute success -2025-08-22 16:17:21 [INFO] spark_run.py:93 in spark_run - Going to fetch Spark execute trace, the process is non-blocking. -2025-08-22 16:17:21 [WARNING] spark_run.py:66 in spark_run - Spark Load 1 execute failed, return code: 1, error code describe: Spark SQL failed diff --git a/omniadvisor/logs/app.log.2025-08-09 b/omniadvisor/logs/app.log.2025-08-09 deleted file mode 100644 index d112c84d2..000000000 --- a/omniadvisor/logs/app.log.2025-08-09 +++ /dev/null @@ -1,230 +0,0 @@ -2025-08-09 17:33:01 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-09 17:33:01 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-09 17:33:01 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-09 17:33:01 [INFO] retest_service.py:29 in retest - Retest succeeded in round 1. Performance result: . -2025-08-09 17:33:01 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-09 17:33:01 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-09 17:33:01 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-09 17:33:01 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-09 17:33:01 [WARNING] retest_service.py:36 in retest - Retest failed in round 1. Exception source: Spark exception: . -2025-08-09 17:33:01 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-09 17:33:01 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-09 17:33:01 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-09 17:33:01 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-09 17:33:01 [ERROR] retest_service.py:22 in retest - Retest failed in round 1. Exception source: Non-Spark exception. Details: . -2025-08-09 17:33:01 [INFO] config_tuning.py:99 in _single_tuning - Retest process did not complete normally. Please check the detailed logs for more information. Remove tuning results. -2025-08-09 17:33:01 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-09 17:33:01 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-09 17:33:01 [INFO] config_tuning.py:116 in _single_tuning - The way of retest is hijacking, update the config from tuning to test config. -2025-08-09 17:33:01 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 17:33:01 [WARNING] hijack_recommend.py:211 in hijack_recommend - Spark execute failed in user config, going to print Spark output. -2025-08-09 17:33:01 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 17:33:01 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 17:33:01 [WARNING] hijack_recommend.py:211 in hijack_recommend - Spark execute failed in user config, going to print Spark output. -2025-08-09 17:33:01 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 17:33:01 [INFO] hijack_recommend.py:204 in hijack_recommend - Spark execute success, going to print Spark output. -2025-08-09 17:33:01 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 17:33:01 [INFO] hijack_recommend.py:69 in _process_load_config - Best config of load failed too much, roll back to next best config. -2025-08-09 17:33:01 [INFO] hijack_recommend.py:85 in _process_load_config - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-09 17:33:01 [INFO] hijack_recommend.py:174 in _create_or_update_load - Load not found in database, created new one and execute. -2025-08-09 17:33:01 [INFO] hijack_recommend.py:169 in _create_or_update_load - Load found in database, update and ready to get config to execute. -2025-08-09 17:33:01 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-09 17:33:01 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-09 17:33:01 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-09 17:33:01 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[unknown] -2025-08-09 17:33:01 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value'] will be add to exec_attr[unknown] -2025-08-09 17:33:01 [ERROR] spark_fetcher.py:106 in get_spark_runtime_by_app - No job info returned for application_id=app-1 -2025-08-09 17:33:01 [INFO] spark_run.py:51 in spark_run - Spark Load 1 execute success -2025-08-09 17:33:01 [INFO] spark_run.py:81 in spark_run - Going to fetch Spark execute trace, the process is non-blocking. -2025-08-09 17:33:01 [WARNING] spark_run.py:56 in spark_run - Spark Load 1 execute failed, return code: 1, error code describe: Spark SQL failed -2025-08-09 17:34:20 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-09 17:34:20 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-09 17:34:20 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-09 17:34:20 [INFO] retest_service.py:29 in retest - Retest succeeded in round 1. Performance result: . -2025-08-09 17:34:20 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-09 17:34:20 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-09 17:34:20 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-09 17:34:20 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-09 17:34:20 [WARNING] retest_service.py:36 in retest - Retest failed in round 1. Exception source: Spark exception: . -2025-08-09 17:34:20 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-09 17:34:20 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-09 17:34:20 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-09 17:34:20 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-09 17:34:20 [ERROR] retest_service.py:22 in retest - Retest failed in round 1. Exception source: Non-Spark exception. Details: . -2025-08-09 17:34:20 [INFO] config_tuning.py:99 in _single_tuning - Retest process did not complete normally. Please check the detailed logs for more information. Remove tuning results. -2025-08-09 17:34:20 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-09 17:34:20 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-09 17:34:20 [INFO] config_tuning.py:116 in _single_tuning - The way of retest is hijacking, update the config from tuning to test config. -2025-08-09 17:34:20 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 17:34:20 [WARNING] hijack_recommend.py:211 in hijack_recommend - Spark execute failed in user config, going to print Spark output. -2025-08-09 17:34:20 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 17:34:20 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 17:34:20 [WARNING] hijack_recommend.py:211 in hijack_recommend - Spark execute failed in user config, going to print Spark output. -2025-08-09 17:34:20 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 17:34:20 [INFO] hijack_recommend.py:204 in hijack_recommend - Spark execute success, going to print Spark output. -2025-08-09 17:34:20 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 17:34:20 [INFO] hijack_recommend.py:69 in _process_load_config - Best config of load failed too much, roll back to next best config. -2025-08-09 17:34:20 [INFO] hijack_recommend.py:85 in _process_load_config - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-09 17:34:20 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-09 17:34:20 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-09 17:34:20 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-09 17:34:20 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[unknown] -2025-08-09 17:34:20 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value'] will be add to exec_attr[unknown] -2025-08-09 17:34:20 [ERROR] spark_fetcher.py:106 in get_spark_runtime_by_app - No job info returned for application_id=app-1 -2025-08-09 17:34:21 [INFO] spark_run.py:51 in spark_run - Spark Load 1 execute success -2025-08-09 17:34:21 [INFO] spark_run.py:81 in spark_run - Going to fetch Spark execute trace, the process is non-blocking. -2025-08-09 17:34:21 [WARNING] spark_run.py:56 in spark_run - Spark Load 1 execute failed, return code: 1, error code describe: Spark SQL failed -2025-08-09 17:34:46 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-09 17:34:46 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-09 17:34:46 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-09 17:34:46 [INFO] retest_service.py:29 in retest - Retest succeeded in round 1. Performance result: . -2025-08-09 17:34:46 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-09 17:34:46 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-09 17:34:46 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-09 17:34:46 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-09 17:34:46 [WARNING] retest_service.py:36 in retest - Retest failed in round 1. Exception source: Spark exception: . -2025-08-09 17:34:46 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-09 17:34:46 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-09 17:34:46 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-09 17:34:46 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-09 17:34:46 [ERROR] retest_service.py:22 in retest - Retest failed in round 1. Exception source: Non-Spark exception. Details: . -2025-08-09 17:34:46 [INFO] config_tuning.py:99 in _single_tuning - Retest process did not complete normally. Please check the detailed logs for more information. Remove tuning results. -2025-08-09 17:34:46 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-09 17:34:46 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-09 17:34:46 [INFO] config_tuning.py:116 in _single_tuning - The way of retest is hijacking, update the config from tuning to test config. -2025-08-09 17:34:46 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 17:34:46 [WARNING] hijack_recommend.py:211 in hijack_recommend - Spark execute failed in user config, going to print Spark output. -2025-08-09 17:34:46 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 17:34:46 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 17:34:46 [WARNING] hijack_recommend.py:211 in hijack_recommend - Spark execute failed in user config, going to print Spark output. -2025-08-09 17:34:46 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 17:34:46 [INFO] hijack_recommend.py:204 in hijack_recommend - Spark execute success, going to print Spark output. -2025-08-09 17:34:46 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 17:34:46 [INFO] hijack_recommend.py:69 in _process_load_config - Best config of load failed too much, roll back to next best config. -2025-08-09 17:34:46 [INFO] hijack_recommend.py:85 in _process_load_config - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-09 17:34:46 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-09 17:34:46 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-09 17:34:46 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-09 17:34:46 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[unknown] -2025-08-09 17:34:46 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value'] will be add to exec_attr[unknown] -2025-08-09 17:34:46 [ERROR] spark_fetcher.py:106 in get_spark_runtime_by_app - No job info returned for application_id=app-1 -2025-08-09 17:34:46 [INFO] spark_run.py:51 in spark_run - Spark Load 1 execute success -2025-08-09 17:34:46 [INFO] spark_run.py:81 in spark_run - Going to fetch Spark execute trace, the process is non-blocking. -2025-08-09 17:34:46 [WARNING] spark_run.py:56 in spark_run - Spark Load 1 execute failed, return code: 1, error code describe: Spark SQL failed -2025-08-09 17:36:16 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-09 17:36:16 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-09 17:36:16 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-09 17:36:16 [INFO] retest_service.py:29 in retest - Retest succeeded in round 1. Performance result: . -2025-08-09 17:36:16 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-09 17:36:16 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-09 17:36:16 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-09 17:36:16 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-09 17:36:16 [WARNING] retest_service.py:36 in retest - Retest failed in round 1. Exception source: Spark exception: . -2025-08-09 17:36:16 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-09 17:36:16 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-09 17:36:16 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-09 17:36:16 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-09 17:36:16 [ERROR] retest_service.py:22 in retest - Retest failed in round 1. Exception source: Non-Spark exception. Details: . -2025-08-09 17:36:16 [INFO] config_tuning.py:99 in _single_tuning - Retest process did not complete normally. Please check the detailed logs for more information. Remove tuning results. -2025-08-09 17:36:16 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-09 17:36:16 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-09 17:36:16 [INFO] config_tuning.py:116 in _single_tuning - The way of retest is hijacking, update the config from tuning to test config. -2025-08-09 17:36:16 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 17:36:16 [WARNING] hijack_recommend.py:211 in hijack_recommend - Spark execute failed in user config, going to print Spark output. -2025-08-09 17:36:16 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 17:36:16 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 17:36:16 [WARNING] hijack_recommend.py:211 in hijack_recommend - Spark execute failed in user config, going to print Spark output. -2025-08-09 17:36:16 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 17:36:16 [INFO] hijack_recommend.py:204 in hijack_recommend - Spark execute success, going to print Spark output. -2025-08-09 17:36:16 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 17:36:16 [INFO] hijack_recommend.py:69 in _process_load_config - Best config of load failed too much, roll back to next best config. -2025-08-09 17:36:16 [INFO] hijack_recommend.py:85 in _process_load_config - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-09 17:36:16 [INFO] hijack_recommend.py:174 in _create_or_update_load - Load not found in database, created new one and execute. -2025-08-09 17:36:16 [INFO] hijack_recommend.py:169 in _create_or_update_load - Load found in database, update and ready to get config to execute. -2025-08-09 17:36:16 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-09 17:36:16 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-09 17:36:16 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-09 17:36:16 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[unknown] -2025-08-09 17:36:16 [WARNING] spark_cmd_parser.py:173 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value'] will be add to exec_attr[unknown] -2025-08-09 17:36:16 [ERROR] spark_fetcher.py:106 in get_spark_runtime_by_app - No job info returned for application_id=app-1 -2025-08-09 17:36:16 [INFO] spark_run.py:51 in spark_run - Spark Load 1 execute success -2025-08-09 17:36:16 [INFO] spark_run.py:81 in spark_run - Going to fetch Spark execute trace, the process is non-blocking. -2025-08-09 17:36:16 [WARNING] spark_run.py:56 in spark_run - Spark Load 1 execute failed, return code: 1, error code describe: Spark SQL failed -2025-08-09 17:55:12 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-09 17:55:12 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-09 17:55:12 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-09 17:55:12 [INFO] retest_service.py:29 in retest - Retest succeeded in round 1. Performance result: . -2025-08-09 17:55:12 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-09 17:55:12 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-09 17:55:12 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-09 17:55:12 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-09 17:55:12 [WARNING] retest_service.py:36 in retest - Retest failed in round 1. Exception source: Spark exception: . -2025-08-09 17:55:12 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-09 17:55:12 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-09 17:55:12 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-09 17:55:12 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-09 17:55:12 [ERROR] retest_service.py:22 in retest - Retest failed in round 1. Exception source: Non-Spark exception. Details: . -2025-08-09 17:55:12 [INFO] config_tuning.py:99 in _single_tuning - Retest process did not complete normally. Please check the detailed logs for more information. Remove tuning results. -2025-08-09 17:55:12 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-09 17:55:12 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-09 17:55:12 [INFO] config_tuning.py:116 in _single_tuning - The way of retest is hijacking, update the config from tuning to test config. -2025-08-09 17:55:12 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 17:55:12 [WARNING] hijack_recommend.py:211 in hijack_recommend - Spark execute failed in user config, going to print Spark output. -2025-08-09 17:55:12 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 17:55:12 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 17:55:12 [WARNING] hijack_recommend.py:211 in hijack_recommend - Spark execute failed in user config, going to print Spark output. -2025-08-09 17:55:12 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 17:55:12 [INFO] hijack_recommend.py:204 in hijack_recommend - Spark execute success, going to print Spark output. -2025-08-09 17:55:12 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 17:55:12 [INFO] hijack_recommend.py:69 in _process_load_config - Best config of load failed too much, roll back to next best config. -2025-08-09 17:55:12 [INFO] hijack_recommend.py:85 in _process_load_config - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-09 17:55:12 [INFO] hijack_recommend.py:174 in _create_or_update_load - Load not found in database, created new one and execute. -2025-08-09 17:55:12 [INFO] hijack_recommend.py:169 in _create_or_update_load - Load found in database, update and ready to get config to execute. -2025-08-09 17:55:12 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-09 17:55:12 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-09 17:55:12 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-09 17:55:12 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[unknown] -2025-08-09 17:55:12 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value'] will be add to exec_attr[unknown] -2025-08-09 17:55:12 [ERROR] spark_fetcher.py:112 in get_spark_runtime_by_app - No job info returned for application_id=app-1 -2025-08-09 17:55:12 [INFO] spark_run.py:51 in spark_run - Spark Load 1 execute success -2025-08-09 17:55:12 [INFO] spark_run.py:81 in spark_run - Going to fetch Spark execute trace, the process is non-blocking. -2025-08-09 17:55:12 [WARNING] spark_run.py:56 in spark_run - Spark Load 1 execute failed, return code: 1, error code describe: Spark SQL failed -2025-08-09 18:09:39 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-09 18:09:39 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-09 18:09:39 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-09 18:09:39 [INFO] retest_service.py:36 in retest - Retest succeeded in round 1. Performance result: . -2025-08-09 18:09:39 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-09 18:09:39 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-09 18:09:39 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-09 18:09:39 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-09 18:09:39 [WARNING] retest_service.py:43 in retest - Retest failed in round 1. Exception source: Spark exception: . -2025-08-09 18:09:39 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-09 18:09:39 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-09 18:09:39 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-09 18:09:39 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-09 18:09:39 [ERROR] retest_service.py:29 in retest - Retest failed in round 1. Exception source: Non-Spark exception. Details: . -2025-08-09 18:09:39 [INFO] config_tuning.py:99 in _single_tuning - Retest process did not complete normally. Please check the detailed logs for more information. Remove tuning results. -2025-08-09 18:09:39 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-09 18:09:39 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-09 18:09:39 [INFO] config_tuning.py:116 in _single_tuning - The way of retest is hijacking, update the config from tuning to test config. -2025-08-09 18:09:39 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 18:09:39 [WARNING] hijack_recommend.py:211 in hijack_recommend - Spark execute failed in user config, going to print Spark output. -2025-08-09 18:09:39 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 18:09:39 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 18:09:39 [WARNING] hijack_recommend.py:211 in hijack_recommend - Spark execute failed in user config, going to print Spark output. -2025-08-09 18:09:40 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 18:09:40 [INFO] hijack_recommend.py:204 in hijack_recommend - Spark execute success, going to print Spark output. -2025-08-09 18:09:40 [INFO] hijack_recommend.py:199 in hijack_recommend - Going to execute Spark load …… -2025-08-09 18:09:40 [INFO] hijack_recommend.py:69 in _process_load_config - Best config of load failed too much, roll back to next best config. -2025-08-09 18:09:40 [INFO] hijack_recommend.py:85 in _process_load_config - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-09 18:09:40 [INFO] hijack_recommend.py:174 in _create_or_update_load - Load not found in database, created new one and execute. -2025-08-09 18:09:40 [INFO] hijack_recommend.py:169 in _create_or_update_load - Load found in database, update and ready to get config to execute. -2025-08-09 18:09:40 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-09 18:09:40 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-09 18:09:40 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-09 18:09:40 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[unknown] -2025-08-09 18:09:40 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value'] will be add to exec_attr[unknown] -2025-08-09 18:09:40 [ERROR] spark_fetcher.py:112 in get_spark_runtime_by_app - No job info returned for application_id=app-1 -2025-08-09 18:09:40 [INFO] spark_run.py:51 in spark_run - Spark Load 1 execute success -2025-08-09 18:09:40 [INFO] spark_run.py:81 in spark_run - Going to fetch Spark execute trace, the process is non-blocking. -2025-08-09 18:09:40 [WARNING] spark_run.py:56 in spark_run - Spark Load 1 execute failed, return code: 1, error code describe: Spark SQL failed diff --git a/omniadvisor/logs/app.log.2025-08-11 b/omniadvisor/logs/app.log.2025-08-11 deleted file mode 100644 index 63375a739..000000000 --- a/omniadvisor/logs/app.log.2025-08-11 +++ /dev/null @@ -1,78 +0,0 @@ -2025-08-11 16:54:25 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-11 16:54:25 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-11 16:54:25 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-11 16:54:25 [INFO] retest_service.py:36 in retest - Retest succeeded in round 1. Performance result: . -2025-08-11 16:54:25 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-11 16:54:25 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-11 16:54:25 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-11 16:54:25 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-11 16:54:25 [WARNING] retest_service.py:43 in retest - Retest failed in round 1. Exception source: Spark exception: . -2025-08-11 16:54:25 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-11 16:54:25 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-11 16:54:25 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-11 16:54:25 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-11 16:54:25 [ERROR] retest_service.py:29 in retest - Retest failed in round 1. Exception source: Non-Spark exception. Details: . -2025-08-11 16:54:25 [INFO] config_tuning.py:99 in _single_tuning - Retest process did not complete normally. Please check the detailed logs for more information. Remove tuning results. -2025-08-11 16:54:25 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-11 16:54:25 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-11 16:54:25 [INFO] config_tuning.py:116 in _single_tuning - The way of retest is hijacking, update the config from tuning to test config. -2025-08-11 16:54:25 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-11 16:54:25 [WARNING] hijack_recommend.py:279 in hijack_recommend - Spark execute failed in user config, going to print Spark output. -2025-08-11 16:54:25 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-11 16:54:25 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-11 16:54:25 [WARNING] hijack_recommend.py:279 in hijack_recommend - Spark execute failed in user config, going to print Spark output. -2025-08-11 16:54:25 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-11 16:54:25 [INFO] hijack_recommend.py:272 in hijack_recommend - Spark execute success, going to print Spark output. -2025-08-11 16:54:26 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-11 16:54:26 [INFO] hijack_recommend.py:75 in _process_load_config - Best config of load failed too much, roll back to next best config. -2025-08-11 16:54:26 [INFO] hijack_recommend.py:91 in _process_load_config - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-11 16:54:26 [INFO] hijack_recommend.py:226 in _create_or_update_load - Load not found in database, created new one and execute. -2025-08-11 16:54:26 [INFO] hijack_recommend.py:220 in _create_or_update_load - Load found in database, update and ready to get config to execute. -2025-08-11 16:54:26 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-11 16:54:26 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-11 16:54:26 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-11 16:54:26 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[unknown] -2025-08-11 16:54:26 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value'] will be add to exec_attr[unknown] -2025-08-11 16:54:26 [ERROR] spark_fetcher.py:112 in get_spark_runtime_by_app - No job info returned for application_id=app-1 -2025-08-11 16:54:26 [INFO] spark_run.py:59 in spark_run - Spark Load 1 execute success -2025-08-11 16:54:26 [INFO] spark_run.py:89 in spark_run - Going to fetch Spark execute trace, the process is non-blocking. -2025-08-11 16:54:26 [WARNING] spark_run.py:64 in spark_run - Spark Load 1 execute failed, return code: 1, error code describe: Spark SQL failed -2025-08-11 19:33:38 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-11 19:33:38 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-11 19:33:38 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-11 19:33:38 [INFO] retest_service.py:36 in retest - Retest succeeded in round 1. Performance result: . -2025-08-11 19:33:38 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-11 19:33:38 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-11 19:33:38 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-11 19:33:38 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-11 19:33:38 [WARNING] retest_service.py:43 in retest - Retest failed in round 1. Exception source: Spark exception: . -2025-08-11 19:33:38 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-11 19:33:38 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-11 19:33:38 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-11 19:33:38 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-11 19:33:38 [ERROR] retest_service.py:29 in retest - Retest failed in round 1. Exception source: Non-Spark exception. Details: . -2025-08-11 19:33:38 [INFO] config_tuning.py:99 in _single_tuning - Retest process did not complete normally. Please check the detailed logs for more information. Remove tuning results. -2025-08-11 19:33:38 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-11 19:33:38 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-11 19:33:38 [INFO] config_tuning.py:116 in _single_tuning - The way of retest is hijacking, update the config from tuning to test config. -2025-08-11 19:33:38 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-11 19:33:38 [WARNING] hijack_recommend.py:279 in hijack_recommend - Spark execute failed in user config, going to print Spark output. -2025-08-11 19:33:38 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-11 19:33:38 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-11 19:33:38 [WARNING] hijack_recommend.py:279 in hijack_recommend - Spark execute failed in user config, going to print Spark output. -2025-08-11 19:33:38 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-11 19:33:38 [INFO] hijack_recommend.py:272 in hijack_recommend - Spark execute success, going to print Spark output. -2025-08-11 19:33:38 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-11 19:33:38 [INFO] hijack_recommend.py:75 in _process_load_config - Best config of load failed too much, roll back to next best config. -2025-08-11 19:33:38 [INFO] hijack_recommend.py:91 in _process_load_config - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-11 19:33:38 [INFO] hijack_recommend.py:226 in _create_or_update_load - Load not found in database, created new one and execute. -2025-08-11 19:33:38 [INFO] hijack_recommend.py:220 in _create_or_update_load - Load found in database, update and ready to get config to execute. -2025-08-11 19:33:38 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-11 19:33:38 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-11 19:33:38 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-11 19:33:38 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[unknown] -2025-08-11 19:33:38 [WARNING] spark_cmd_parser.py:175 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value'] will be add to exec_attr[unknown] -2025-08-11 19:33:38 [ERROR] spark_fetcher.py:112 in get_spark_runtime_by_app - No job info returned for application_id=app-1 -2025-08-11 19:33:38 [INFO] spark_run.py:59 in spark_run - Spark Load 1 execute success -2025-08-11 19:33:38 [INFO] spark_run.py:89 in spark_run - Going to fetch Spark execute trace, the process is non-blocking. -2025-08-11 19:33:38 [WARNING] spark_run.py:64 in spark_run - Spark Load 1 execute failed, return code: 1, error code describe: Spark SQL failed diff --git a/omniadvisor/logs/app.log.2025-08-19 b/omniadvisor/logs/app.log.2025-08-19 deleted file mode 100644 index 370cdfe60..000000000 --- a/omniadvisor/logs/app.log.2025-08-19 +++ /dev/null @@ -1,113 +0,0 @@ -2025-08-19 15:27:58 [WARNING] spark_cmd_parser.py:189 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-19 15:27:58 [WARNING] spark_cmd_parser.py:189 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-19 15:27:58 [WARNING] spark_cmd_parser.py:189 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[unknown] -2025-08-19 15:27:58 [WARNING] spark_cmd_parser.py:189 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[unknown] -2025-08-19 15:27:58 [WARNING] spark_cmd_parser.py:189 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value', '--principal', 'principal1', '--version'] will be add to exec_attr[unknown] -2025-08-19 15:29:39 [WARNING] spark_cmd_parser.py:189 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-19 15:29:39 [WARNING] spark_cmd_parser.py:189 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-19 15:29:39 [WARNING] spark_cmd_parser.py:189 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-19 15:29:39 [WARNING] spark_cmd_parser.py:189 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[primary_resource] -2025-08-19 15:29:39 [WARNING] spark_cmd_parser.py:189 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value', '--principal', 'principal1', '--version'] will be add to exec_attr[primary_resource] -2025-08-19 15:52:03 [WARNING] spark_cmd_parser.py:187 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-19 15:52:03 [WARNING] spark_cmd_parser.py:187 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-19 15:52:03 [WARNING] spark_cmd_parser.py:187 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-19 15:52:03 [WARNING] spark_cmd_parser.py:187 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[primary_resource] -2025-08-19 15:52:03 [WARNING] spark_cmd_parser.py:187 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value', '--custom-arg', 'custom_value', '--principal', 'principal1', '--version'] will be add to exec_attr[primary_resource] -2025-08-19 15:53:50 [WARNING] spark_cmd_parser.py:187 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-19 15:53:50 [WARNING] spark_cmd_parser.py:187 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-19 15:53:50 [WARNING] spark_cmd_parser.py:187 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-19 15:53:50 [WARNING] spark_cmd_parser.py:187 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[primary_resource] -2025-08-19 15:53:50 [WARNING] spark_cmd_parser.py:187 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value', '--custom-arg', 'custom_value', '--principal', 'principal1', '--version'] will be add to exec_attr[primary_resource] -2025-08-19 15:55:42 [WARNING] spark_cmd_parser.py:183 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-19 15:55:42 [WARNING] spark_cmd_parser.py:183 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-19 15:55:42 [WARNING] spark_cmd_parser.py:183 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-19 15:55:42 [WARNING] spark_cmd_parser.py:183 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[primary_resource] -2025-08-19 15:55:42 [WARNING] spark_cmd_parser.py:183 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value'] will be add to exec_attr[primary_resource] -2025-08-19 15:59:26 [WARNING] spark_cmd_parser.py:185 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-19 15:59:26 [WARNING] spark_cmd_parser.py:185 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-19 15:59:26 [WARNING] spark_cmd_parser.py:185 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-19 15:59:26 [WARNING] spark_cmd_parser.py:185 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[primary_resource] -2025-08-19 15:59:26 [WARNING] spark_cmd_parser.py:185 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value', '--principal', 'principal1', '--version'] will be add to exec_attr[primary_resource] -2025-08-19 16:14:36 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-19 16:14:36 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-19 16:14:36 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-19 16:14:36 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[primary_resource] -2025-08-19 16:14:36 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value', '--principal', 'principal1', '--version'] will be add to exec_attr[primary_resource] -2025-08-19 19:21:49 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-19 19:21:49 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-19 19:21:49 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-19 19:21:49 [INFO] retest_service.py:36 in retest - Retest succeeded in round 1. Performance result: . -2025-08-19 19:21:49 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-19 19:21:49 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-19 19:21:49 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-19 19:21:49 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-19 19:21:49 [WARNING] retest_service.py:43 in retest - Retest failed in round 1. Exception source: Spark exception: . -2025-08-19 19:21:49 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-19 19:21:49 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-19 19:21:49 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-19 19:21:49 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-19 19:21:49 [ERROR] retest_service.py:29 in retest - Retest failed in round 1. Exception source: Non-Spark exception. Details: . -2025-08-19 19:21:49 [INFO] config_tuning.py:99 in _single_tuning - Retest process did not complete normally. Please check the detailed logs for more information. Remove tuning results. -2025-08-19 19:21:49 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-19 19:21:49 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-19 19:21:49 [INFO] config_tuning.py:116 in _single_tuning - The way of retest is hijacking, update the config from tuning to test config. -2025-08-19 19:21:49 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-19 19:21:49 [WARNING] hijack_recommend.py:279 in hijack_recommend - Spark execute failed in user config, going to print Spark output. -2025-08-19 19:21:49 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-19 19:21:49 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-19 19:21:49 [WARNING] hijack_recommend.py:279 in hijack_recommend - Spark execute failed in user config, going to print Spark output. -2025-08-19 19:21:49 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-19 19:21:49 [INFO] hijack_recommend.py:272 in hijack_recommend - Spark execute success, going to print Spark output. -2025-08-19 19:21:49 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-19 19:21:49 [INFO] hijack_recommend.py:75 in _process_load_config - Best config of load failed too much, roll back to next best config. -2025-08-19 19:21:49 [INFO] hijack_recommend.py:91 in _process_load_config - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-19 19:21:49 [INFO] hijack_recommend.py:226 in _create_or_update_load - Load not found in database, created new one and execute. -2025-08-19 19:21:49 [INFO] hijack_recommend.py:220 in _create_or_update_load - Load found in database, update and ready to get config to execute. -2025-08-19 19:21:49 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-19 19:21:49 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-19 19:21:49 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-19 19:21:49 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[primary_resource] -2025-08-19 19:21:49 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value', '--principal', 'principal1', '--version'] will be add to exec_attr[primary_resource] -2025-08-19 19:21:49 [ERROR] spark_fetcher.py:113 in get_spark_runtime_by_app - No job info returned for application_id=app-1 -2025-08-19 19:21:49 [INFO] spark_run.py:61 in spark_run - Spark Load 1 execute success -2025-08-19 19:21:49 [INFO] spark_run.py:93 in spark_run - Going to fetch Spark execute trace, the process is non-blocking. -2025-08-19 19:21:49 [WARNING] spark_run.py:66 in spark_run - Spark Load 1 execute failed, return code: 1, error code describe: Spark SQL failed -2025-08-19 20:44:36 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-19 20:44:36 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-19 20:44:36 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-19 20:44:36 [INFO] retest_service.py:36 in retest - Retest succeeded in round 1. Performance result: . -2025-08-19 20:44:36 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-19 20:44:36 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-19 20:44:36 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-19 20:44:36 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-19 20:44:36 [WARNING] retest_service.py:43 in retest - Retest failed in round 1. Exception source: Spark exception: . -2025-08-19 20:44:36 [INFO] config_tuning.py:107 in _single_tuning - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-19 20:44:36 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-19 20:44:36 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-19 20:44:36 [INFO] config_tuning.py:94 in _single_tuning - The way of retest is backend, going to retest the config from tuning…… -2025-08-19 20:44:36 [ERROR] retest_service.py:29 in retest - Retest failed in round 1. Exception source: Non-Spark exception. Details: . -2025-08-19 20:44:36 [INFO] config_tuning.py:99 in _single_tuning - Retest process did not complete normally. Please check the detailed logs for more information. Remove tuning results. -2025-08-19 20:44:36 [INFO] config_tuning.py:137 in _get_next_config - Use AI iterative optimization method to tuning. -2025-08-19 20:44:36 [INFO] config_tuning.py:82 in _single_tuning - Load config tuning success, get new config to retest. -2025-08-19 20:44:36 [INFO] config_tuning.py:116 in _single_tuning - The way of retest is hijacking, update the config from tuning to test config. -2025-08-19 20:44:36 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-19 20:44:36 [WARNING] hijack_recommend.py:279 in hijack_recommend - Spark execute failed in user config, going to print Spark output. -2025-08-19 20:44:36 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-19 20:44:36 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-19 20:44:36 [WARNING] hijack_recommend.py:279 in hijack_recommend - Spark execute failed in user config, going to print Spark output. -2025-08-19 20:44:36 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-19 20:44:36 [INFO] hijack_recommend.py:272 in hijack_recommend - Spark execute success, going to print Spark output. -2025-08-19 20:44:36 [INFO] hijack_recommend.py:267 in hijack_recommend - Going to execute Spark load …… -2025-08-19 20:44:36 [INFO] hijack_recommend.py:75 in _process_load_config - Best config of load failed too much, roll back to next best config. -2025-08-19 20:44:36 [INFO] hijack_recommend.py:91 in _process_load_config - Current tuning result: , best tuning result ever: , best tuning speedup: % -2025-08-19 20:44:36 [INFO] hijack_recommend.py:226 in _create_or_update_load - Load not found in database, created new one and execute. -2025-08-19 20:44:36 [INFO] hijack_recommend.py:220 in _create_or_update_load - Load found in database, update and ready to get config to execute. -2025-08-19 20:44:37 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-19 20:44:37 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-19 20:44:37 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['example.jar'] will be add to exec_attr[primary_resource] -2025-08-19 20:44:37 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['app.jar'] will be add to exec_attr[primary_resource] -2025-08-19 20:44:37 [WARNING] spark_cmd_parser.py:165 in parse_cmd - The remainder unknown params ['my_app.jar', '--custom-arg', 'custom_value', '--principal', 'principal1', '--version'] will be add to exec_attr[primary_resource] -2025-08-19 20:44:37 [ERROR] spark_fetcher.py:113 in get_spark_runtime_by_app - No job info returned for application_id=app-1 -2025-08-19 20:44:37 [INFO] spark_run.py:61 in spark_run - Spark Load 1 execute success -2025-08-19 20:44:37 [INFO] spark_run.py:93 in spark_run - Going to fetch Spark execute trace, the process is non-blocking. -2025-08-19 20:44:37 [WARNING] spark_run.py:66 in spark_run - Spark Load 1 execute failed, return code: 1, error code describe: Spark SQL failed -- Gitee