diff --git a/.gitignore b/.gitignore index 0f17484a38d6c5728d2e0a43a0340ea87e335b9b..c795b054e5ade51b7031abab1581a5b7e2d2f5ba 100755 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1 @@ -build -tools \ No newline at end of file +build \ No newline at end of file diff --git a/tools/.gitkeep b/tools/.gitkeep deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/tools/bazel.rc b/tools/bazel.rc new file mode 100644 index 0000000000000000000000000000000000000000..3734fab715bc604274276396a7fbe225a5dbfcdb --- /dev/null +++ b/tools/bazel.rc @@ -0,0 +1,86 @@ +# Android configs. Bazel needs to have --cpu and --fat_apk_cpu both set to the +# target CPU to build transient dependencies correctly. See +# https://docs.bazel.build/versions/master/user-manual.html#flag--fat_apk_cpu +build:android --crosstool_top=//external:android/crosstool +build:android --host_crosstool_top=@bazel_tools//tools/cpp:toolchain +build:android_arm --config=android +build:android_arm --cpu=armeabi-v7a +build:android_arm --fat_apk_cpu=armeabi-v7a +build:android_arm64 --config=android +build:android_arm64 --cpu=arm64-v8a +build:android_arm64 --fat_apk_cpu=arm64-v8a + +# Config to use a mostly-static build and disable modular op registration +# support (this will revert to loading TensorFlow with RTLD_GLOBAL in Python). +# By default, TensorFlow will build with a dependence on +# //tensorflow:libtensorflow_framework.so. +build:monolithic --define framework_shared_object=false + +# For projects which use TensorFlow as part of a Bazel build process, putting +# nothing in a bazelrc will default to a monolithic build. The following line +# opts in to modular op registration support by default. +build --define framework_shared_object=true + +# Please note that MKL on MacOS or windows is still not supported. +# If you would like to use a local MKL instead of downloading, please set the +# environment variable "TF_MKL_ROOT" every time before build. +build:mkl --define=build_with_mkl=true --define=enable_mkl=true +build:mkl -c opt + +# This config option is used to enable MKL-DNN open source library only, +# without depending on MKL binary version. +build:mkl_open_source_only --define=build_with_mkl_dnn_only=true +build:mkl_open_source_only --define=build_with_mkl=true --define=enable_mkl=true + +build:download_clang --crosstool_top=@local_config_download_clang//:toolchain +build:download_clang --define=using_clang=true +# Instruct clang to use LLD for linking. +# This only works with GPU builds currently, since Bazel sets -B/usr/bin in +# auto-generated CPU crosstool, forcing /usr/bin/ld.lld to be preferred over +# the downloaded one. +build:download_clang_use_lld --linkopt='-fuse-ld=lld' + +build:cuda --crosstool_top=@local_config_cuda//crosstool:toolchain +build:cuda --define=using_cuda=true --define=using_cuda_nvcc=true + +build:rocm --crosstool_top=@local_config_rocm//crosstool:toolchain +build:rocm --define=using_rocm=true --define=using_rocm_hipcc=true + +build:cuda_clang --crosstool_top=@local_config_cuda//crosstool:toolchain +build:cuda_clang --define=using_cuda=true --define=using_cuda_clang=true --define=using_clang=true + +build:sycl --crosstool_top=@local_config_sycl//crosstool:toolchain +build:sycl --define=using_sycl=true --define=using_trisycl=false + +build:sycl_nodouble --crosstool_top=@local_config_sycl//crosstool:toolchain +build:sycl_nodouble --define=using_sycl=true --cxxopt -DTENSORFLOW_SYCL_NO_DOUBLE + +build:sycl_asan --crosstool_top=@local_config_sycl//crosstool:toolchain +build:sycl_asan --define=using_sycl=true --define=using_trisycl=false --copt -fno-omit-frame-pointer --copt -fsanitize-coverage=3 --copt -DGPR_NO_DIRECT_SYSCALLS --linkopt -fPIC --linkopt -fsanitize=address + +build:sycl_trisycl --crosstool_top=@local_config_sycl//crosstool:toolchain +build:sycl_trisycl --define=using_sycl=true --define=using_trisycl=true + +# Options extracted from configure script +build:gdr --define=with_gdr_support=true +build:ngraph --define=with_ngraph_support=true +build:verbs --define=with_verbs_support=true + +build --define=use_fast_cpp_protos=true +build --define=allow_oversize_protos=true +build --define=grpc_no_ares=true + +build --spawn_strategy=standalone +build --genrule_strategy=standalone +build -c opt + +# Other build flags. +build --define=grpc_no_ares=true + +# Modular TF build options +build:dynamic_kernels --define=dynamic_loaded_kernels=true + +# Default paths for TF_SYSTEM_LIBS +build --define=PREFIX=/usr +build --define=LIBDIR=$(PREFIX)/lib +build --define=INCLUDEDIR=$(PREFIX)/include diff --git a/tools/build_wheel.sh b/tools/build_wheel.sh new file mode 100644 index 0000000000000000000000000000000000000000..aef0f8770f7e3fdd5de80d25f70651a77c5496b0 --- /dev/null +++ b/tools/build_wheel.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. +# Description: Build npu_bridge pip-packages +# ============================================================================== + +set -e +set -o pipefail + +dst=$1 +PRODUCT=$2 + +if [[ -z "${dst}" ]]; then + echo -e "\033[7mNo destination dir provided\033[0m" + exit 1 +fi +if [[ "${PRODUCT}" = "cloud" ]]; then + echo "Build npu_bridge in cloud." + CUSTOM_COMMANDS="--define product=cloud" +elif [[ "${PRODUCT}" = "mini" ]]; then + echo "Build npu_bridge in mini." + CUSTOM_COMMANDS="--define product=mini" +elif [[ "${PRODUCT}" = "onetrack" ]]; then + echo "Build npu_bridge in onetrack." + CUSTOM_COMMANDS="--define product=onetrack" +else + echo "TF_Adapter not support this product." + exit 1 +fi + +#PYTHON_BIN_PATH=$(which python3.7) +PYTHON_BIN_PATH=${HI_PYTHON_PATH} +if [[ -z "${PYTHON_BIN_PATH}" ]]; then + echo -e "\033[7mNo python3 installed\033[0m" + exit 1 +fi +PYTHON_INCLUDE_PATH=${HI_PYTHON_INC} +if [[ -z "${PYTHON_INCLUDE_PATH}" ]]; then + echo -e "\033[7mNot found python3 include path\033[0m" + exit 1 +fi + +cd "$(dirname "${BASH_SOURCE[0]}")/../" +TOP_DIR="$(cd ../../../../../ && pwd)" +INSTALL_DIR="${TOP_DIR}/out/${PRODUCT}/host" +OUTPUT_DIR="${INSTALL_DIR}/obj/tf_adapter" +SOFT_DP_DIR="${INSTALL_DIR}/obj/lib" +mkdir -p "${OUTPUT_DIR}/genfiles" +mkdir -p "${OUTPUT_DIR}/wheel" + +TF_INSTALL_LIB_PATH=$("${PYTHON_BIN_PATH}" -c 'import tensorflow as tf; print(tf.sysconfig.get_lib())') +echo "Found installed tensorflow ${TF_INSTALL_LIB_PATH}" +TF_PACKAGE_PATH="$(cd ../../../compile_deps/tf-1.15.0 && pwd)" + +cp workspace.bzl.tpl workspace.bzl +sed -i "s|{TF_INSTALL_LIB_PATH}|"${TF_INSTALL_LIB_PATH}"|g" workspace.bzl +sed -i "s|{PYTHON_INCLUDE_PATH}|"${PYTHON_INCLUDE_PATH}"|g" workspace.bzl +cp WORKSPACE.tpl WORKSPACE +sed -i "s|TF_PACKAGE_PATH|"${TF_PACKAGE_PATH}"|g" WORKSPACE + +## use gcc-7.3.0 +#bazel --output_base=${OUTPUT_DIR}/genfiles build ${CUSTOM_COMMANDS} --config=opt //tf_adapter:_tf_adapter.so +/opt/buildtools/bazel-0.24.1/bin/bazel --output_base=${OUTPUT_DIR}/genfiles build ${CUSTOM_COMMANDS} --config=opt //tf_adapter:_tf_adapter.so + +cp -r tf_adapter/python/. "${OUTPUT_DIR}/wheel" +cp -f bazel-bin/tf_adapter/_tf_adapter.so "${OUTPUT_DIR}/wheel/npu_bridge" +cp ${SOFT_DP_DIR}/libSoftDp.so "${OUTPUT_DIR}/wheel/npu_bridge" + +cd "${OUTPUT_DIR}/wheel" +"${PYTHON_BIN_PATH}" setup.py bdist_wheel >/dev/null + +cp -f dist/*.whl "$1" diff --git a/tools/gen_sc_makefile_bazel.py b/tools/gen_sc_makefile_bazel.py new file mode 100644 index 0000000000000000000000000000000000000000..3db18b66dae4b4aa39fc72b81b6484f22a76f110 --- /dev/null +++ b/tools/gen_sc_makefile_bazel.py @@ -0,0 +1,205 @@ +#!/usr/bin/python +#-*- coding: UTF-8 -*- + +import os +import sys +import string +import datetime +import time +import re + +source_suffix = ["c","cpp","cc","cce"] + +def get_gcc_cmd(input_file, cpp_file_list, code_top_dir, custom_code_top_dir=""): + gcc_cmd_set = {} + lint_cmd_set = {} + file = open(input_file, "r") + lines = file.readlines() + file.close() + compile_cmd_regex = re.compile('(gcc|g\+\+|\"aarch64-linux-gnu-gcc\"|ccec)\s+[^ ].*\s-o\s+[^ ]+\.(o|s|cpp|obj)(\s|$)') + if(custom_code_top_dir != "") : + custom_code_top_dir = custom_code_top_dir.replace(code_top_dir + "/", "") + for line in lines: + line = line.strip() + line = line.strip(')') + if not compile_cmd_regex.search(line): + continue + items = line.split() + item_list = [] + compiler = "" + compiler_path = "" + for i in range(len(items)): + items[i] = items[i].strip("\"") + if items[i].endswith("gcc") or items[i].endswith("g++") or items[i].endswith("clang") or items[i].endswith("clang++"): + compiler = items[i].split('/')[-1] + if compiler.startswith("aarch64"): + items[i] = "external/hcc/bin/" + compiler + if custom_code_top_dir != "" and not items[i].startswith("/") : + items[i] = code_top_dir + "/" + custom_code_top_dir + items[i] + item_list = items[i:] + break + if len(item_list) == 0: + continue + if "-MD" in item_list and "-MF" in item_list : + index = item_list.index("-MF") + if custom_code_top_dir != "" : + item_list[index + 1] = custom_code_top_dir + item_list[index + 1] + + cpp_file = "" + if item_list[-1].strip().split(".")[-1] in source_suffix: + if custom_code_top_dir != "" : + item_list[-1] = custom_code_top_dir + item_list[-1].strip() + cpp_file = item_list[-1].strip() + else: + if "-c" not in item_list: + continue + index = item_list.index("-c") + if custom_code_top_dir != "" : + item_list[index+1] = custom_code_top_dir + item_list[index+1].strip() + cpp_file = item_list[index+1] + if cpp_file[-1].strip().split(".")[-1] not in source_suffix: + continue + + if "-o" not in item_list: + continue + try: + index = item_list.index("-o") + except: + continue + obj_file = item_list[index+1] + if custom_code_top_dir != "" : + obj_file = custom_code_top_dir + obj_file + item_list[index+1] = obj_file + + cpp_file_rel = cpp_file + if cpp_file_rel.startswith("/"): + code_top_prefix_len = len(code_top_dir) + cpp_file_rel = cpp_file[code_top_prefix_len+1:] + + if cpp_file_rel in cpp_file_list: + # gcc_cmd_set[obj_file] = " ".join(item_list) + c_flags_list = [] + rest_list = [] + is_add_I = False + is_add_D = False + for item in item_list: + item = item.strip() + if is_add_I: + is_add_I = False + item = "-I" + custom_code_top_dir + item + c_flags_list.append(item) + continue + if is_add_D: + is_add_D = False + item = "-D" + item + c_flags_list.append(item) + continue + is_add_I = False + is_add_D = False + if item == "-I": + is_add_I = True + elif item == "-D": + is_add_D = True + elif item.startswith("-I") or item.startswith("-D"): + c_flags_list.append(item) + else: + rest_list.append(item) + continue + gcc_options = " ".join(c_flags_list) + c_flags_list.append("-D_lint") + c_flags_list.append("-DLINUX_PC_LINT") + c_flags = " ".join(c_flags_list) + if compiler.startswith("aarch64-linux-gnu"): + wine_str = "export LINT_PATH=%s/vendor/hisi/llt/ci/tools/pc-lint;wine $(LINT_PATH)/LINT-NT.EXE %s $(LINT_PATH)/davinci_arm64.lnt %s"%(code_top_dir,c_flags, cpp_file) + else: + wine_str = "export LINT_PATH=%s/vendor/hisi/llt/ci/tools/pc-lint;wine $(LINT_PATH)/LINT-NT.EXE %s $(LINT_PATH)/davinci_x86.lnt %s"%(code_top_dir,c_flags, cpp_file) + + rest_option = " ".join(rest_list) + gcc_cmd_str = rest_option + " " + gcc_options + gcc_cmd_set[obj_file] = gcc_cmd_str + lint_out_path = code_top_dir + "/out/tools/lint/" + obj_file + lint_cmd_set[lint_out_path] = wine_str + + return gcc_cmd_set,lint_cmd_set + +def walkDir(top_dir,directory): + fileArray = [] + for root, dirs, files in os.walk(directory): + for name in files: + if name.endswith(".c") or name.endswith(".cc") or name.endswith(".cpp"): + fileArray.append(os.path.abspath(os.path.join(root, name))[len(top_dir)+1:]) + return fileArray + +def get_cpp_file_list(top_dir,input_file, custom_code_top_dir=""): + cpp_file_list = [] + file = open(input_file, "r") + lines = file.readlines() + file.close() + for line in lines: + line = line.strip() + if custom_code_top_dir != "" : + line_path = os.path.join(custom_code_top_dir,line) + else : + line_path = os.path.join(top_dir,line) + if os.path.isfile(line_path): + if line.endswith(".c") or line.endswith(".cc") or line.endswith(".cpp"): + cpp_file_list.append(line_path[len(top_dir)+1:]) + if os.path.isdir(line_path): + filelist = walkDir(top_dir,line_path) + if len(filelist) > 0: + cpp_file_list.extend(filelist) + return cpp_file_list + +def main(): + analysis_file = sys.argv[1] + cpp_file = sys.argv[2] + output_file = sys.argv[3] + code_top_dir = os.getcwd() + custom_code_top_dir = "" + if len(sys.argv) == 5 : + custom_code_top_dir = sys.argv[4] + cpp_file_list = get_cpp_file_list(code_top_dir,cpp_file, custom_code_top_dir) + else : + cpp_file_list = get_cpp_file_list(code_top_dir,cpp_file) + + gcc_cmd_set,lint_cmd_set = get_gcc_cmd(analysis_file, cpp_file_list, code_top_dir, custom_code_top_dir) + if len(gcc_cmd_set) == 0: + print "Error: can not get sc gcc cmd " + sys.exit(-1) + if len(lint_cmd_set) == 0: + print "Error: can not get lint gcc cmd " + sys.exit(-1) + + fd = open(output_file, "w") + # content = "GCC_PATH_1 = " + code_top_dir + "/prebuilts/gcc/linux-x86/aarch64/aarch64-linux-gnu-4.8/bin" + "\n" + # content += "GCC_PATH = $(GCC_PATH_1)" + "\n" + # content += "export GCC_PATH\n\n" + content = "" + content += ".PHONY: all\n" + content += "\n" + content += "ifeq ($(PCLINT_ENABLE),true)\n" + for (obj_file,wine_cmd) in lint_cmd_set.items(): + obj_file = obj_file + ".lint" + content += "all:" + obj_file + "\n" + content += obj_file + ": FORCE\n" + content += "\tmkdir -p $(dir $@)\n" + content += "\t"+ wine_cmd + " > $@ \n" + content += "\n" + content += "else\n" + + for (obj_file,gcc_cmd) in gcc_cmd_set.items(): + content += "all:" + obj_file + "\n" + content += obj_file + ": FORCE\n" + content += "\trm -rf $@\n" + content += "\t $(SOURCEANALYZER) " + gcc_cmd + "\n" + content += "\n" + + content += "endif\n" + content += "\n" + content += ".PHONY: FORCE\n" + content += "FORCE: \n" + fd.write(content) + fd.close() + +if __name__ == '__main__': + main() diff --git a/tools/sc_list.txt b/tools/sc_list.txt new file mode 100644 index 0000000000000000000000000000000000000000..96f08ce69893e2c653757749979e2a961378b039 --- /dev/null +++ b/tools/sc_list.txt @@ -0,0 +1 @@ +tf_adapter \ No newline at end of file