diff --git a/omniadvisor/config/common_config.ini b/omniadvisor/config/common_config.ini index cb390b5fec16e9904fca992ea33bde7f22a2dcb8..e805a4b957fc0ec8d83787d757450bfaca691ab4 100755 --- a/omniadvisor/config/common_config.ini +++ b/omniadvisor/config/common_config.ini @@ -10,13 +10,13 @@ backend.retest.queue= [database] # 后端数据库postgresql的database名称 -postgresql.database.name= +postgresql.database.name=omniadvisor # 后端数据库postgresql的用户 -postgresql.database.user= +postgresql.database.user=postgres # 后端数据库postgresql的连接host -postgresql.database.host= +postgresql.database.host=localhost # 后端数据库postgresql的连接端口 -postgresql.database.port= +postgresql.database.port=5432 [spark] # Spark History Server的URL 仅用于Rest模式 diff --git a/omniadvisor/pyproject.toml b/omniadvisor/pyproject.toml index ea18fef830254ccaf79fd6edd944b85fec664bf9..d71aa544259569976b02218d017c0f1f08b72389 100755 --- a/omniadvisor/pyproject.toml +++ b/omniadvisor/pyproject.toml @@ -25,6 +25,7 @@ psycopg2-binary = "^2.9.10" django-sslserver = "0.22" keyring = "24.3.1" keyrings-alt = "5.0.2" +pandas = "2.3.2" [tool.poetry.group.test.dependencies] pytest = "^7.4.4" diff --git a/omniadvisor/script/spark-submit b/omniadvisor/script/spark-submit index 58193f6ae504193e7914a358c7d0808c5bc43722..d59d0901e8e8fd016f862ce0aee20966a83ccb77 100644 --- a/omniadvisor/script/spark-submit +++ b/omniadvisor/script/spark-submit @@ -28,7 +28,7 @@ export PYTHONHASHSEED=0 if [[ -v enable_omniadvisor && "$enable_omniadvisor" = "true" ]]; then echo "*** OmniAdvisor is enabled! We will find best config to SPEEDUP! ***" # 根据特性实际的部署路径进行修改 - hijack_path="" + hijack_path="/home/x30041342/OmniAdvisor2_source/src/hijack.py" # OmniAdvisor劫持优化 cmd_prefix="${SPARK_HOME}/bin/spark-class org.apache.spark.deploy.SparkSubmit" python3 ${hijack_path} --omniadvisor-cmd-prefix "${cmd_prefix}" "$@" diff --git a/omniadvisor/src/omniadvisor/repository/load_repository.py b/omniadvisor/src/omniadvisor/repository/load_repository.py index ac74c102c075927108f08a5710a1c02f7b737bbb..67813322d82fc26e9e7c303fda23a93a75283bd1 100644 --- a/omniadvisor/src/omniadvisor/repository/load_repository.py +++ b/omniadvisor/src/omniadvisor/repository/load_repository.py @@ -1,4 +1,5 @@ from django.db import IntegrityError +from django.db.models.query import QuerySet from common.exceptions import DuplicateEntryError from server.app.models import DatabaseLoad @@ -159,3 +160,15 @@ class LoadRepository(Repository): model_attr=model_attr ) return Load(database_model=database_load) + + @classmethod + def get_all_loads_in_db(cls) -> list[Load]: + return Load.get_all_loads_in_db() + + @classmethod + def get_load_id(cls, load: Load) -> int: + return load.id + + @classmethod + def get_load_name(cls, load: Load) -> int: + return load.name diff --git a/omniadvisor/src/omniadvisor/repository/model/load.py b/omniadvisor/src/omniadvisor/repository/model/load.py index 79b7ed2c9a6ff640e92d01dc90baff0fa59b60d0..73bd51776714491524b3cc908731d1e210c4c2ac 100644 --- a/omniadvisor/src/omniadvisor/repository/model/load.py +++ b/omniadvisor/src/omniadvisor/repository/model/load.py @@ -53,6 +53,11 @@ class Load: def database_model(self): return self._database_model + @classmethod + def get_all_loads_in_db(cls): + qs = DatabaseLoad.objects.only("id", "name") + return [Load(database_model) for database_model in qs] + class FieldName: id = 'id' name = 'name' diff --git a/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result.py b/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result.py index f480883f3aba42a285ba813f881f1401095db715..7d667b2cecb79e555d07b0b03256a4afef63afa6 100644 --- a/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result.py +++ b/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result.py @@ -90,6 +90,10 @@ class TuningResult: failed=sum([exam_record.status == OA_CONF.ExecStatus.fail for exam_record in self._exam_records]) ) + @property + def tuning_id(self): + return self._tuning_record.id + @property def config(self): return self._tuning_record.config diff --git a/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result_history.py b/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result_history.py index e28be27479af71305f85d7a5690f001c97fe119f..5b20063b46ae22d4d117f49232f416a95b819a1f 100644 --- a/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result_history.py +++ b/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result_history.py @@ -9,7 +9,6 @@ from omniadvisor.repository.tuning_record_repository import TuningRecordReposito from omniadvisor.service.tuning_result.tuning_result import TuningResult from omniadvisor.service.tuning_result.tuning_result import get_tuning_result - def get_other_tuning_result_history(exclude_load: Load) -> List: """ 查询 exclude_load 以外的所有 tuning result history @@ -147,3 +146,48 @@ class TuningResultHistory: tuning_result.to_tuning_data() for tuning_result in self._tuning_results if tuning_result.status != OA_CONF.TuningResultStatus.empty ] + + +def get_all_loads_performance_data(): + # 获取全量的load_id + loads_list = LoadRepository.get_all_loads_in_db() + records = [] + # 获取全量load的基线性能 + for load in loads_list: + load_id = LoadRepository.get_load_id(load) + load_name = LoadRepository.get_load_name(load) + load_tuning_history = get_tuning_result_history(load) + + # 基线数据 + user_tuning = load_tuning_history.user_tuning_result + base_perf = user_tuning.runtime + + # 最优数据 + best_tuning = load_tuning_history.best_tuning_result + best_tuning_id = best_tuning.tuning_id + best_tuning_config = best_tuning.config + best_tuning_method = best_tuning.method + best_tuning_rounds = best_tuning.rounds + peak_perf = best_tuning.runtime + + # 提升比例 + boost_ratio = load_tuning_history.boost_percentage + + # 存到 dict 里 + records.append({ + "load_id": load_id, + "name": load_name, + "base_perf": base_perf, + "peak_perf": peak_perf, + "best_tuning_id": best_tuning_id, + "best_tuning_config": best_tuning_config, + "best_tuning_method": best_tuning_method, + "best_tuning_rounds": best_tuning_rounds, + "boost_ratio": boost_ratio + }) + + print(records) + return records + +if __name__ == "__main__": + get_all_loads_performance_data() diff --git a/omniadvisor/src/server/app/admin.py b/omniadvisor/src/server/app/admin.py index 3d0f61252fd6fd9033ce8a241a5c2ac4ef1e8a17..02bf7d8adea30898e75c38fbd94452cbe3ab6d8e 100644 --- a/omniadvisor/src/server/app/admin.py +++ b/omniadvisor/src/server/app/admin.py @@ -1,6 +1,7 @@ from django.contrib import admin +from django.shortcuts import redirect -from .models import DatabaseLoad, DatabaseTuningRecord, DatabaseExamRecord +from .models import DatabaseLoad, DatabaseTuningRecord, DatabaseExamRecord, PerformanceData # Register your models here. _LOAD_FIELDS = ('id', 'name', 'exec_attr', 'default_config', 'best_config', 'test_config', 'backend_retest_enable') @@ -112,3 +113,18 @@ class ExamRecordAdmin(BaseAdmin): def runtime_f(self, model): return round(model.runtime, 3) runtime_f.short_description = 'Runtime' + + +@admin.register(PerformanceData) +class PerformanceAdmin(admin.ModelAdmin): + # 禁止 add/change/delete 相关的操作 + def has_add_permission(self, request): + return False + def has_change_permission(self, request, obj=None): + return False + def has_delete_permission(self, request, obj=None): + return False + + # 点击虚拟模型时直接跳转 + def changelist_view(self, request, extra_context=None): + return redirect('/admin/load_performance') \ No newline at end of file diff --git a/omniadvisor/src/server/app/models.py b/omniadvisor/src/server/app/models.py index 74021fca20786bce9d873a48de2b5e075c41d26e..a4ef645a6435162ca6f1456ea338498ca5e33060 100644 --- a/omniadvisor/src/server/app/models.py +++ b/omniadvisor/src/server/app/models.py @@ -81,3 +81,10 @@ class DatabaseExamRecord(models.Model): class Meta: db_table = 'omniadvisor_exam_record' # 自定义表名 + + +class PerformanceData(models.Model): + """虚拟模型 不对应数据库表 显示PerformanceData""" + class Meta: + verbose_name = "Performance Data" + verbose_name_plural = "Performance Data" diff --git a/omniadvisor/src/server/app/templates/load_performance.html b/omniadvisor/src/server/app/templates/load_performance.html new file mode 100644 index 0000000000000000000000000000000000000000..f7f0637391d652df50e4a8cffa536c432ad61a22 --- /dev/null +++ b/omniadvisor/src/server/app/templates/load_performance.html @@ -0,0 +1,40 @@ +{% extends "admin/base_site.html" %} + + +{% block content %} +
load_id | +name | +base_perf | +peak_perf | +best_tuning_id | +best_tuning_config | +best_tuning_method | +best_tuning_rounds | +boost_ratio | +
---|---|---|---|---|---|---|---|---|
{{ row.load_id }} | +{{ row.name }} | +{{ row.base_perf }} | +{{ row.peak_perf }} | +{{ row.best_tuning_id }} | +{{ row.best_tuning_config }} | +{{ row.best_tuning_method }} | +{{ row.best_tuning_rounds }} | +{{ row.boost_ratio }} | +
暂无数据 | +