diff --git a/0001-Add-Bear-to-A-FOT.patch b/0001-Add-Bear-to-A-FOT.patch new file mode 100644 index 0000000000000000000000000000000000000000..24a4f88608e6bebb6ccd798d71c3919b3947997a --- /dev/null +++ b/0001-Add-Bear-to-A-FOT.patch @@ -0,0 +1,426 @@ +diff --git a/a-fot b/a-fot +index 1520a9e..c633098 100644 +--- a/a-fot ++++ b/a-fot +@@ -74,6 +74,10 @@ function parse_input_params() { + check_success=$2 + shift 2 + ;; ++ --build_mode) ++ build_mode=$2 ++ shift 2 ++ ;; + *) + suggest_info + exit 1 +@@ -124,6 +128,10 @@ function check_config_item() { + echo "[ERROR] The configuration item 'check_success' is missing, please check!" + exit 1 + fi ++ if [[ -z ${build_mode} ]]; then ++ echo "[ERROR] The configuration item 'build_mode' is missing, please check!" ++ exit 1 ++ fi + } + + function suggest_info() { +@@ -141,6 +149,7 @@ Usage: a-fot [OPTION1 ARG1] [OPTION2 ARG2] [...] + --run_script Script path for run application + --max_waiting_time Maximum binary startup time (unit: seconds) + --check_success Check optimization result ++--build_mode Execute build scrip mode (Wrapper/Bear) + """ + } + +@@ -181,11 +190,27 @@ function check_common_dependency() { + is_file_exist "${gcc_path}/bin/gcc" + } + ++# 拆分编译数据库 ++function split_option() { ++ if [ "$bear_prefix" ];then ++ python3 $current_path/split_json.py -i $PWD/compile_commands.json ++ mv $PWD/compile_commands.json $PWD/compile_commands_$1.json ++ mv $PWD/compile_commands.fail.json $PWD/compile_commands.fail_$1.json ++ fi ++} ++ + # 使用原始编译选项进行编译 + function first_compilation() { + echo "[INFO] Start raw compilation" + is_file_exist ${build_script} "build_script" +- /bin/bash ${build_script} >> ${log_file} 2>&1 ++ if [[ $build_mode =~ "Bear" ]]; then ++ bear_prefix="bear -- " ++ echo "[INFO] Build in Bear mode" ++ else ++ echo "[INFO] Build in Wrapper mode" ++ fi ++ $bear_prefix /bin/bash ${build_script} >> ${log_file} 2>&1 ++ split_option first + is_file_exist ${bin_file} + is_success $? + } +@@ -237,7 +262,7 @@ function detect_process() { + function second_compilation() { + echo "[INFO] Try compiling with the new compilation options" + if [[ ${check_success} -eq 1 ]]; then +- /bin/bash ${build_script} >> ${log_file} 2>&1 & build_id=$! ++ $bear_prefix /bin/bash ${build_script} >> ${log_file} 2>&1 & build_id=$! + echo "[INFO] Found build id: ${build_id}" + add_opt=$(cat ${gcc_wrapper}/gcc | awk -F " " '{print $2}') + build_status=`ps -p ${build_id} | grep -c ${build_id}` +@@ -254,9 +279,10 @@ function second_compilation() { + done + wait + else +- /bin/bash ${build_script} >> ${log_file} 2>&1 ++ $bear_prefix /bin/bash ${build_script} >> ${log_file} 2>&1 + fi + echo "[INFO] Finish compiling with new compilation options" ++ split_option second + is_success $? + } + +@@ -308,10 +334,12 @@ function is_opt_success() { + if [[ ${opt_success} -eq 0 ]]; then + echo "[WARNING] Optimization may fail or the build process is too short, please check!" + echo "[WARNING] Please try gcc/g++ at: ${gcc_wrapper} instead of the original compiler" ++ exit 1 + else + echo "[INFO] Optimization may success!" + fi + fi ++ exit 0 + } + + #执行入口,部分函数为加载不同优化脚本中得到 +@@ -324,15 +352,17 @@ function main() { + load_script + + check_dependency +- create_wrapper ++ prepare_env + first_compilation + execute_run_script + detect_process + perf_record +- create_new_wrapper + ++ prepare_new_env + second_compilation + is_opt_success ++ exit "$?" + } + + main "$@" ++exit "$?" +diff --git a/a-fot.ini b/a-fot.ini +index 03a4702..b395504 100644 +--- a/a-fot.ini ++++ b/a-fot.ini +@@ -18,4 +18,6 @@ perf_time=100 + gcc_path=/usr/ + # 检测是否优化成功(1=启用,0=禁用) + check_success=0 ++# 构建模式 (Bear、Wrapper) ++build_mode=Bear + # 结束行勿删 +\ No newline at end of file +diff --git a/auto_bolt.sh b/auto_bolt.sh +index 625d1d5..ebcb2e5 100644 +--- a/auto_bolt.sh ++++ b/auto_bolt.sh +@@ -9,11 +9,29 @@ function check_dependency() { + fi + } + ++# 根据模式选择Wrapper或者Bear模式构建 ++function prepare_env() { ++ case ${build_mode} in ++ "Wrapper") ++ create_wrapper ++ ;; ++ "Bear") ++ export COMPILATION_OPTIONS="-Wl,-q" ++ export LINK_OPTIONS="-q" ++ ;; ++ *) ++ echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear" ++ exit 1 ++ ;; ++ esac ++} ++ ++ + # 创建原始wrapper + function create_wrapper() { + echo "[INFO] Start generating the original wrapper" +- echo "${gcc_path}/bin/gcc -Wl,-q \$@" >${gcc_wrapper}/gcc +- echo "${gcc_path}/bin/g++ -Wl,-q \$@" >${gcc_wrapper}/g++ ++ echo "${gcc_path}/bin/gcc -Wl,-q \"\$@\"" >${gcc_wrapper}/gcc ++ echo "${gcc_path}/bin/g++ -Wl,-q \"\$@\"" >${gcc_wrapper}/g++ + post_create_wrapper + } + +@@ -28,9 +46,28 @@ function perf_record() { + pkill ${application_name} + } + ++ ++# 根据模式选择Wrapper或者Bear模式构建 ++function prepare_new_env() { ++ case ${build_mode} in ++ "Wrapper") ++ create_new_wrapper ++ ;; ++ "Bear") ++ export COMPILATION_OPTIONS="-fbolt-use=${profile_data_path}/${gcov_name} -fbolt-target=${bin_file} -Wl,-q" ++ export LINK_OPTIONS="-q" ++ ;; ++ *) ++ echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear" ++ exit 1 ++ ;; ++ esac ++} ++ ++ + #生成新的wrapper + function create_new_wrapper() { + echo "[INFO] Start to generate a new wrapper" +- echo "${gcc_path}/bin/gcc -fbolt-use=${profile_data_path}/${gcov_name} -fbolt-target=${bin_file} -Wl,-q \$@" >${gcc_wrapper}/gcc +- echo "${gcc_path}/bin/g++ -fbolt-use=${profile_data_path}/${gcov_name} -fbolt-target=${bin_file} -Wl,-q \$@" >${gcc_wrapper}/g++ ++ echo "${gcc_path}/bin/gcc -fbolt-use=${profile_data_path}/${gcov_name} -fbolt-target=${bin_file} -Wl,-q \"\$@\"" >${gcc_wrapper}/gcc ++ echo "${gcc_path}/bin/g++ -fbolt-use=${profile_data_path}/${gcov_name} -fbolt-target=${bin_file} -Wl,-q \"\$@\"" >${gcc_wrapper}/g++ + } +diff --git a/auto_fdo.sh b/auto_fdo.sh +index 3d7bbb8..8426b30 100644 +--- a/auto_fdo.sh ++++ b/auto_fdo.sh +@@ -9,11 +9,28 @@ function check_dependency() { + fi + } + ++# 根据模式选择Wrapper或者Bear模式构建 ++function prepare_env() { ++ case ${build_mode} in ++ "Wrapper") ++ create_wrapper ++ ;; ++ "Bear") ++ export COMPILATION_OPTIONS="-g" ++ export LINK_OPTIONS="-g" ++ ;; ++ *) ++ echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear" ++ exit 1 ++ ;; ++ esac ++} ++ + # 创建原始wrapper + function create_wrapper() { + echo "[INFO] Start generating the original wrapper" +- echo "${gcc_path}/bin/gcc -g \$@" >${gcc_wrapper}/gcc +- echo "${gcc_path}/bin/g++ -g \$@" >${gcc_wrapper}/g++ ++ echo "${gcc_path}/bin/gcc -g \"\$@\"" >${gcc_wrapper}/gcc ++ echo "${gcc_path}/bin/g++ -g \"\$@\"" >${gcc_wrapper}/g++ + post_create_wrapper + } + +@@ -39,9 +56,25 @@ function perf_record() { + pkill ${application_name} + } + ++# 根据模式选择Wrapper或者Bear模式构建 ++function prepare_new_env() { ++ case ${build_mode} in ++ "Wrapper") ++ create_new_wrapper ++ ;; ++ "Bear") ++ export COMPILATION_OPTIONS="-fauto-profile=${profile_data_path}/${gcov_name}" ++ ;; ++ *) ++ echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear" ++ exit 1 ++ ;; ++ esac ++} ++ + #生成新的wrapper + function create_new_wrapper() { + echo "[INFO] Start to generate a new wrapper" +- echo "${gcc_path}/bin/gcc -fauto-profile=${profile_data_path}/${gcov_name} \$@" >${gcc_wrapper}/gcc +- echo "${gcc_path}/bin/g++ -fauto-profile=${profile_data_path}/${gcov_name} \$@" >${gcc_wrapper}/g++ ++ echo "${gcc_path}/bin/gcc -fauto-profile=${profile_data_path}/${gcov_name} \"\$@\"" >${gcc_wrapper}/gcc ++ echo "${gcc_path}/bin/g++ -fauto-profile=${profile_data_path}/${gcov_name} \"\$@\"" >${gcc_wrapper}/g++ + } +diff --git a/auto_prefetch.sh b/auto_prefetch.sh +index 2562dd6..265828a 100644 +--- a/auto_prefetch.sh ++++ b/auto_prefetch.sh +@@ -9,11 +9,28 @@ function check_dependency() { + fi + } + ++# 根据模式选择Wrapper或者Bear模式构建 ++function prepare_env() { ++ case ${build_mode} in ++ "Wrapper") ++ create_wrapper ++ ;; ++ "Bear") ++ export COMPILATION_OPTIONS="-g" ++ export LINK_OPTIONS="-g" ++ ;; ++ *) ++ echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear" ++ exit 1 ++ ;; ++ esac ++} ++ + # 创建原始wrapper + function create_wrapper() { + echo "[INFO] Start generating the original wrapper" +- echo "${gcc_path}/bin/gcc -g \$@" >${gcc_wrapper}/gcc +- echo "${gcc_path}/bin/g++ -g \$@" >${gcc_wrapper}/g++ ++ echo "${gcc_path}/bin/gcc -g \"\$@\"" >${gcc_wrapper}/gcc ++ echo "${gcc_path}/bin/g++ -g \"\$@\"" >${gcc_wrapper}/g++ + post_create_wrapper + } + +@@ -40,9 +57,25 @@ function perf_record() { + pkill ${application_name} + } + ++# 根据模式选择Wrapper或者Bear模式构建 ++function prepare_new_env() { ++ case ${build_mode} in ++ "Wrapper") ++ create_new_wrapper ++ ;; ++ "Bear") ++ export COMPILATION_OPTIONS="-fauto-profile=${gcov_file_name} -fcache-misses-profile=${profile_data_path}/${gcov_name}.cache-misses\:u -fprefetch-loop-arrays=2" ++ ;; ++ *) ++ echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear" ++ exit 1 ++ ;; ++ esac ++} ++ + #生成新的wrapper + function create_new_wrapper() { + echo "[INFO] Start to generate a new wrapper" +- echo "${gcc_path}/bin/gcc -fauto-profile=${gcov_file_name} -fcache-misses-profile=${profile_data_path}/${gcov_name}.cache-misses\:u -fprefetch-loop-arrays=2 \$@" >${gcc_wrapper}/gcc +- echo "${gcc_path}/bin/g++ -fauto-profile=${gcov_file_name} -fcache-misses-profile=${profile_data_path}/${gcov_name}.cache-misses\:u -fprefetch-loop-arrays=2 \$@" >${gcc_wrapper}/g++ ++ echo "${gcc_path}/bin/gcc -fauto-profile=${gcov_file_name} -fcache-misses-profile=${profile_data_path}/${gcov_name}.cache-misses\:u -fprefetch-loop-arrays=2 \"\$@\"" >${gcc_wrapper}/gcc ++ echo "${gcc_path}/bin/g++ -fauto-profile=${gcov_file_name} -fcache-misses-profile=${profile_data_path}/${gcov_name}.cache-misses\:u -fprefetch-loop-arrays=2 \"\$@\"" >${gcc_wrapper}/g++ + } +diff --git a/split_json.py b/split_json.py +new file mode 100644 +index 0000000..bb6547f +--- /dev/null ++++ b/split_json.py +@@ -0,0 +1,98 @@ ++#!/usr/bin/env python3 ++# _*_ coding:utf-8 _*_ ++ ++ ++""" ++ Copyright@2022-2022, All rights reserved, Powered by: Huawei Tech.Co.,Ltd. ++ ++split_json is a interface to split compile commands according to its execution result. ++ ++It will generate a new JSON file for the failed compile commands and remove these commands from source file. ++ ++Input: ++ compile commands json ++Output: ++ compile commands json including commands which has been successfully executed. ++ compile commands json including commands which executed unsuccessfully. ++ ++""" ++__author__ = 'z00500762' ++ ++import argparse ++import os ++import json ++import sys ++import logging ++ ++ ++class SplitJson: ++ compile_commands_success = list() ++ compile_commands_fail = list() ++ ++ def __init__(self, input_json): ++ self.input = input_json ++ ++ @staticmethod ++ def validate(execution): ++ if "arguments" not in execution: ++ return False ++ if "directory" not in execution: ++ return False ++ if "exec_result" not in execution: ++ return False ++ return True ++ ++ def get_compile_commands(self): ++ compile_commands = list() ++ try: ++ with open(self.input, "r", encoding='utf-8', errors='ignore') as json_file: ++ compile_commands = json.load(json_file) ++ if len(compile_commands) == 0: ++ logging.info("compile commands json file is empty: %s", self.input) ++ except IOError as exception: ++ logging.error("open compile commands json file failed: %s", exception) ++ except json.decoder.JSONDecodeError as exception: ++ logging.error("json decode file failed: %s", exception) ++ ++ return compile_commands ++ ++ def split_commands(self): ++ compile_commands = self.get_compile_commands() ++ for item in compile_commands: ++ if not self.validate(item): ++ logging.info("discard invalid commands: %s", str(item)) ++ if not item.get("rebuild"): ++ self.compile_commands_success.append(item) ++ else: ++ self.compile_commands_fail.append(item) ++ self.write_json() ++ ++ def write_json(self): ++ compile_commands_success_file = os.path.splitext(self.input)[0] + ".json" ++ compile_commands_fail_file = os.path.splitext(self.input)[0] + ".fail.json" ++ with open(compile_commands_success_file, 'w+') as fw: ++ json.dump(self.compile_commands_success, fw, sort_keys=False, indent=4) ++ ++ with open(compile_commands_fail_file, 'w+') as fw: ++ json.dump(self.compile_commands_fail, fw, sort_keys=False, indent=4) ++ ++ ++def main(input_json): ++ if not os.path.isabs(input_json): ++ input_json = os.path.join(os.getcwd(), input_json) ++ if not os.path.exists(input_json): ++ logging.error("compile_command_file not exists : %s", input_json) ++ return -1 ++ ++ sj = SplitJson(input_json) ++ sj.split_commands() ++ ++ ++if __name__ == "__main__": ++ cmd_parser = argparse.ArgumentParser(description="split compile commands json") ++ cmd_parser.add_argument( ++ '-i', '--input', dest='input_json', metavar='store', action='store', ++ help='json to split' ++ ) ++ args = cmd_parser.parse_args() ++ sys.exit(main(args.input_json)) diff --git a/A-FOT.spec b/A-FOT.spec index 59694869da6fa9199aa4d9a87db682cee966f061..beed81a51c9499a0333e3e343f846ffefff1551a 100644 --- a/A-FOT.spec +++ b/A-FOT.spec @@ -1,13 +1,15 @@ Name: A-FOT Version: v1.0 -Release: 0 +Release: 1 Summary: automatic feedback-directed optimization tool for openEuler License: MulanPSL-2.0 URL: https://gitee.com/openeuler/A-FOT Source0: %{name}-%{version}.tar.gz -Requires: gcc gcc-c++ autofdo llvm-bolt +Requires: gcc gcc-c++ autofdo llvm-bolt Bear python3 + +Patch1: 0001-Add-Bear-to-A-FOT.patch %description A-FOT is an automatic feedback-directed optimization tool for openEuler @@ -25,6 +27,7 @@ cp a-fot.ini %{buildroot}/%{_bindir} cp auto_fdo.sh %{buildroot}/%{_bindir} cp auto_prefetch.sh %{buildroot}/%{_bindir} cp auto_bolt.sh %{buildroot}/%{_bindir} +cp split_json.py %{buildroot}/%{_bindir} %files %attr(755, root, root) %{_bindir}/a-fot @@ -32,10 +35,17 @@ cp auto_bolt.sh %{buildroot}/%{_bindir} %attr(644, root, root) %{_bindir}/auto_fdo.sh %attr(644, root, root) %{_bindir}/auto_prefetch.sh %attr(644, root, root) %{_bindir}/auto_bolt.sh +%attr(644, root, root) %{_bindir}/split_json.py %license LICENSE %doc README.md %changelog +* Thu Dec 01 2022 huitailangzju <804544223@qq.com> - v1.0-1 +- Type:Sync +- ID:NA +- SUG:NA +- DESC:Add Bear to A-FOT + * Wed Mar 23 2022 liyancheng <412998149@qq.com> - 0 - Type:Init - ID:NA