From 829d2cd7395bb95dcc12842238d09b099652103a Mon Sep 17 00:00:00 2001 From: wtingkai <330445001@qq.com> Date: Tue, 12 Aug 2025 19:58:23 +0800 Subject: [PATCH] =?UTF-8?q?feat(OmniAdvisor):=20=E6=9E=84=E5=BB=BA?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E4=BE=9D=E8=B5=96=E8=A1=A5=E5=85=85=20&&=20L?= =?UTF-8?q?oad=E5=AD=97=E6=AE=B5=E7=BC=BA=E5=B0=91=E7=AD=89=E4=BB=A3?= =?UTF-8?q?=E7=A0=81BUG=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- omniadvisor/pyproject.toml | 3 ++- .../src/omniadvisor/interface/config_tuning.py | 4 ++-- .../src/omniadvisor/repository/model/load.py | 4 ++++ .../service/spark_service/spark_cmd_parser.py | 14 ++++++++++---- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/omniadvisor/pyproject.toml b/omniadvisor/pyproject.toml index 3dd09bc05..d2849e690 100755 --- a/omniadvisor/pyproject.toml +++ b/omniadvisor/pyproject.toml @@ -15,12 +15,13 @@ packages = [ [tool.poetry.dependencies] # 正式版本依赖 -python = "^3.8" +python = "~3.9.9" django = "~4.2.10" smac = "~2.2.0" colorlog = "~6.9.0" requests = "^2.32.3" python-dateutil = "2.9.0.post0" +psycopg2-binary = "^2.9.10" [tool.poetry.group.test.dependencies] pytest = "^7.4.4" diff --git a/omniadvisor/src/omniadvisor/interface/config_tuning.py b/omniadvisor/src/omniadvisor/interface/config_tuning.py index 2d17d4c31..654a21015 100644 --- a/omniadvisor/src/omniadvisor/interface/config_tuning.py +++ b/omniadvisor/src/omniadvisor/interface/config_tuning.py @@ -72,7 +72,7 @@ def _single_tuning(load: Load, retest_way: str, tuning_method: str) -> None: signal.signal(signal.SIGTERM, handler) signal.signal(signal.SIGINT, handler) - method_extend, next_config = _get_next_config(load, tuning_method) + next_config, method_extend = _get_next_config(load=load, tuning_method=tuning_method) if not next_config: raise NoOptimalConfigError(f'The recommending config of method {tuning_method} is empty,' @@ -158,7 +158,7 @@ def _get_next_config(load: Load, tuning_method: str) -> tuple[dict, str]: # 调优 next_config, method_extend = tuner.tune() global_logger.debug(f"The method_extend is: {method_extend}, config from tuning is: {next_config}") - return method_extend, next_config + return next_config, method_extend def _continuous_tuning_with_strategies(load: Load, tuning_strategies: list[tuple[str, int]]) -> None: diff --git a/omniadvisor/src/omniadvisor/repository/model/load.py b/omniadvisor/src/omniadvisor/repository/model/load.py index 18683a88f..624e47945 100644 --- a/omniadvisor/src/omniadvisor/repository/model/load.py +++ b/omniadvisor/src/omniadvisor/repository/model/load.py @@ -45,6 +45,10 @@ class Load: def tuning_needed(self): return self._database_model.tuning_needed + @property + def backend_retest_forbidden(self): + return self._database_model.backend_retest_forbidden + @property def database_model(self): return self._database_model diff --git a/omniadvisor/src/omniadvisor/service/spark_service/spark_cmd_parser.py b/omniadvisor/src/omniadvisor/service/spark_service/spark_cmd_parser.py index 0b69b3642..a344f6235 100644 --- a/omniadvisor/src/omniadvisor/service/spark_service/spark_cmd_parser.py +++ b/omniadvisor/src/omniadvisor/service/spark_service/spark_cmd_parser.py @@ -12,6 +12,8 @@ from omniadvisor.utils.logger import global_logger _CMD_PREFIX_KEY = 'omniadvisor-cmd-prefix' # 命令行开头的字段名称 _CMD_UNKNOWN_KEY = 'unknown' +# 命令行中class字段 +_CMD_CLASS_KEY = 'class' # 单横杠参数的字段 _SINGLE_HORIZONTAL_BAR_KEYS = ['e', 'f', 'i'] # 布尔类型的字段 @@ -118,7 +120,7 @@ class SparkCMDParser: """ return value - @staticmethod + @classmethod def _append_double_dash_args(cls, key, value) -> List: """ 用于处理spark命令中以 "--" 作为前缀的参数 @@ -136,7 +138,7 @@ class SparkCMDParser: return [f'--{key}', cls._normalize_value(value)] @classmethod - def parse_cmd(cls, argv: list) -> dict: + def parse_cmd(cls, argv: list) -> tuple[dict, dict]: """ 解析提交后的命令，得到解析的参数后，以字典的形式保存 返回一个包含所有提取出的基础参数的字典,每个键代表一个参数名,详见add_argument中的配置的参数。 @@ -194,9 +196,13 @@ class SparkCMDParser: # 把${SPARK_HOME}/bin/spark-class org.apache.spark.deploy.SparkSubmit通过空格拆分成含有两个元素的参数列表 cmd_fields += exec_attr.get(_CMD_PREFIX_KEY).split(" ") + # 将命令行中的class属性提前放置，符合Spark正常执行命令 + if _CMD_CLASS_KEY in exec_attr: + cmd_fields += cls._append_double_dash_args(_CMD_CLASS_KEY, exec_attr[_CMD_CLASS_KEY]) + # 处理执行属性 for key, value in exec_attr.items(): - if key == _CMD_PREFIX_KEY: + if key in [_CMD_PREFIX_KEY, _CMD_CLASS_KEY]: continue # 单独处理 unknown = ['xxxxxxx.jar', 'xxxxxx', 'xxxxx'] 这部分未被解析的残余参数追加到末尾 不在此处理 elif key == _CMD_UNKNOWN_KEY: @@ -206,7 +212,7 @@ class SparkCMDParser: cmd_fields += [f'-{key}', cls._normalize_value(value)] # 处理剩余 -- 作为前缀的参数 else: - cmd_fields += cls._append_double_dash_args(cls, key, value) + cmd_fields += cls._append_double_dash_args(key, value) # 处理参数配置 for key, value in conf_params.items(): -- Gitee