diff --git a/omniadvisor/pyproject.toml b/omniadvisor/pyproject.toml index 3dd09bc058e2dbf1971ca3bad77fb4d6636c89d6..d2849e690b254f6c70c9ca03f7502c8199fdd043 100755 --- a/omniadvisor/pyproject.toml +++ b/omniadvisor/pyproject.toml @@ -15,12 +15,13 @@ packages = [ [tool.poetry.dependencies] # 正式版本依赖 -python = "^3.8" +python = "~3.9.9" django = "~4.2.10" smac = "~2.2.0" colorlog = "~6.9.0" requests = "^2.32.3" python-dateutil = "2.9.0.post0" +psycopg2-binary = "^2.9.10" [tool.poetry.group.test.dependencies] pytest = "^7.4.4" diff --git a/omniadvisor/src/omniadvisor/interface/config_tuning.py b/omniadvisor/src/omniadvisor/interface/config_tuning.py index 2d17d4c31c4dee5a5a10c2ffc46e50d830e066d2..654a21015184eaf94cce53d2c27ce7cbe6e104b2 100644 --- a/omniadvisor/src/omniadvisor/interface/config_tuning.py +++ b/omniadvisor/src/omniadvisor/interface/config_tuning.py @@ -72,7 +72,7 @@ def _single_tuning(load: Load, retest_way: str, tuning_method: str) -> None: signal.signal(signal.SIGTERM, handler) signal.signal(signal.SIGINT, handler) - method_extend, next_config = _get_next_config(load, tuning_method) + next_config, method_extend = _get_next_config(load=load, tuning_method=tuning_method) if not next_config: raise NoOptimalConfigError(f'The recommending config of method {tuning_method} is empty,' @@ -158,7 +158,7 @@ def _get_next_config(load: Load, tuning_method: str) -> tuple[dict, str]: # 调优 next_config, method_extend = tuner.tune() global_logger.debug(f"The method_extend is: {method_extend}, config from tuning is: {next_config}") - return method_extend, next_config + return next_config, method_extend def _continuous_tuning_with_strategies(load: Load, tuning_strategies: list[tuple[str, int]]) -> None: diff --git a/omniadvisor/src/omniadvisor/repository/model/load.py b/omniadvisor/src/omniadvisor/repository/model/load.py index 18683a88f78bb17da9a5c405aebcccbb9a5ae142..624e479455df3151fc04254523490536befe3a57 100644 --- a/omniadvisor/src/omniadvisor/repository/model/load.py +++ b/omniadvisor/src/omniadvisor/repository/model/load.py @@ -45,6 +45,10 @@ class Load: def tuning_needed(self): return self._database_model.tuning_needed + @property + def backend_retest_forbidden(self): + return self._database_model.backend_retest_forbidden + @property def database_model(self): return self._database_model diff --git a/omniadvisor/src/omniadvisor/service/spark_service/spark_cmd_parser.py b/omniadvisor/src/omniadvisor/service/spark_service/spark_cmd_parser.py index 0b69b3642a128926640c549190ce3dd99a3f710c..a344f6235160ee3ff08ea2c4ab0ea85afe3dda53 100644 --- a/omniadvisor/src/omniadvisor/service/spark_service/spark_cmd_parser.py +++ b/omniadvisor/src/omniadvisor/service/spark_service/spark_cmd_parser.py @@ -12,6 +12,8 @@ from omniadvisor.utils.logger import global_logger _CMD_PREFIX_KEY = 'omniadvisor-cmd-prefix' # 命令行开头的字段名称 _CMD_UNKNOWN_KEY = 'unknown' +# 命令行中class字段 +_CMD_CLASS_KEY = 'class' # 单横杠参数的字段 _SINGLE_HORIZONTAL_BAR_KEYS = ['e', 'f', 'i'] # 布尔类型的字段 @@ -118,7 +120,7 @@ class SparkCMDParser: """ return value - @staticmethod + @classmethod def _append_double_dash_args(cls, key, value) -> List: """ 用于处理spark命令中以 "--" 作为前缀的参数 @@ -136,7 +138,7 @@ class SparkCMDParser: return [f'--{key}', cls._normalize_value(value)] @classmethod - def parse_cmd(cls, argv: list) -> dict: + def parse_cmd(cls, argv: list) -> tuple[dict, dict]: """ 解析提交后的命令,得到解析的参数后,以字典的形式保存 返回一个包含所有提取出的基础参数的字典,每个键代表一个参数名,详见add_argument中的配置的参数。 @@ -194,9 +196,13 @@ class SparkCMDParser: # 把${SPARK_HOME}/bin/spark-class org.apache.spark.deploy.SparkSubmit通过空格拆分成含有两个元素的参数列表 cmd_fields += exec_attr.get(_CMD_PREFIX_KEY).split(" ") + # 将命令行中的class属性提前放置,符合Spark正常执行命令 + if _CMD_CLASS_KEY in exec_attr: + cmd_fields += cls._append_double_dash_args(_CMD_CLASS_KEY, exec_attr[_CMD_CLASS_KEY]) + # 处理执行属性 for key, value in exec_attr.items(): - if key == _CMD_PREFIX_KEY: + if key in [_CMD_PREFIX_KEY, _CMD_CLASS_KEY]: continue # 单独处理 unknown = ['xxxxxxx.jar', 'xxxxxx', 'xxxxx'] 这部分未被解析的残余参数追加到末尾 不在此处理 elif key == _CMD_UNKNOWN_KEY: @@ -206,7 +212,7 @@ class SparkCMDParser: cmd_fields += [f'-{key}', cls._normalize_value(value)] # 处理剩余 -- 作为前缀的参数 else: - cmd_fields += cls._append_double_dash_args(cls, key, value) + cmd_fields += cls._append_double_dash_args(key, value) # 处理参数配置 for key, value in conf_params.items():