diff --git a/.gitee/ISSUE_TEMPLATE.en.md b/.gitee/ISSUE_TEMPLATE.en.md new file mode 100644 index 0000000000000000000000000000000000000000..e1522ec4010ae7bfeebde068aa48e11358d87ec2 --- /dev/null +++ b/.gitee/ISSUE_TEMPLATE.en.md @@ -0,0 +1,21 @@ +## Bug Description +A clear and concise description of what the bug is + +## Reproduction +Steps to reproduce the behavior +1. ... +2. ... +3. ... + +## Expected behavior +A clear and concise description of what is expected to happen + +## Environment +- Architecture +- OS, version, etc +- Version of the toolchain +- ... + +## Additional context +- Any other context which may help to reproduce and/or fix the issue. +- Supporting files (minimal test case) diff --git a/.gitee/PULL_REQUEST_TEMPLATE.en.md b/.gitee/PULL_REQUEST_TEMPLATE.en.md new file mode 100644 index 0000000000000000000000000000000000000000..aa60947a3ec94a92aa3320fa9a1ae3508d654637 --- /dev/null +++ b/.gitee/PULL_REQUEST_TEMPLATE.en.md @@ -0,0 +1,2 @@ +## PR Description +A clear and concise description of changes and rationale. Provide a link to external references / discussion if appropriate diff --git a/README.md b/README.md index 1273ba17c2fa0c73d93a66462f7f25111144b540..7a3f4e6fd07661d933cfced19cb147def9e476eb 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ the [libc++ C++ standard library](https://libcxx.llvm.org), the [LLD linker](https://lld.llvm.org), and more. ### Getting the Source Code and Building LLVM +We recommend that you base on the guide [llvm-build](https://gitee.com/openharmony-sig/third_party_llvm-project/blob/master/llvm-build/README.md) to get the source code and build it for OpenHarmony platform. The LLVM Getting Started documentation may be out of date. The [Clang Getting Started](http://clang.llvm.org/get_started.html) page might have more @@ -120,3 +121,5 @@ Join [LLVM Discourse forums](https://discourse.llvm.org/), [discord chat](https: The LLVM project has adopted a [code of conduct](https://llvm.org/docs/CodeOfConduct.html) for participants to all modes of communication within the project. + +For the llvm construction of openharmony, please refer to [llvm-build](https://gitee.com/openharmony-sig/third_party_llvm-project/blob/master/llvm-build/README.md). diff --git a/lldb/include/lldb/Target/ModuleCache.h b/lldb/include/lldb/Target/ModuleCache.h index 9dc0e0903196a6a3b64ac06a87994ec5c895c3d5..010b367aa966e3e9d20cefaf58d06a1bee93d6b3 100644 --- a/lldb/include/lldb/Target/ModuleCache.h +++ b/lldb/include/lldb/Target/ModuleCache.h @@ -67,6 +67,7 @@ private: bool *did_create_ptr); std::unordered_map m_loaded_modules; + std::recursive_mutex m_cache_mutex; }; } // namespace lldb_private diff --git a/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp b/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp index 8a708c1f9898e74f2b255e4a130ed88a4d93b1f4..540067eedc86ccc0bf99c3af2206920686c740bc 100644 --- a/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp +++ b/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp @@ -24,6 +24,7 @@ #include "lldb/Utility/LLDBLog.h" #include "lldb/Utility/Log.h" #include "lldb/Utility/ProcessInfo.h" +#include "llvm/Support/ThreadPool.h" #include @@ -638,22 +639,26 @@ void DynamicLoaderPOSIXDYLD::LoadAllCurrentModules() { m_process->PrefetchModuleSpecs( module_names, m_process->GetTarget().GetArchitecture().GetTriple()); + llvm::ThreadPool pool(llvm::hardware_concurrency(DynamicLoaderPOSIXDYLD::DYLD_CONCURRENCY_THREADING)); for (I = m_rendezvous.begin(), E = m_rendezvous.end(); I != E; ++I) { - ModuleSP module_sp = - LoadModuleAtAddress(I->file_spec, I->link_addr, I->base_addr, true); - if (module_sp.get()) { - LLDB_LOG(log, "LoadAllCurrentModules loading module: {0}", - I->file_spec.GetFilename()); - module_list.Append(module_sp); - } else { - Log *log = GetLog(LLDBLog::DynamicLoader); - LLDB_LOGF( - log, - "DynamicLoaderPOSIXDYLD::%s failed loading module %s at 0x%" PRIx64, - __FUNCTION__, I->file_spec.GetCString(), I->base_addr); - } + constexpr const auto &func_name = __FUNCTION__; + pool.async([&](const FileSpec &file_spec, addr_t link_addr, addr_t base_addr) { + ModuleSP module_sp = + LoadModuleAtAddress(file_spec, link_addr, base_addr, true); + if (module_sp.get()) { + LLDB_LOG(log, "LoadAllCurrentModules loading module: {0}", file_spec.GetFilename()); + module_list.Append(module_sp); + } else { + LLDB_LOGF( + log, + "DynamicLoaderPOSIXDYLD::%s failed loading module %s at 0x%" PRIx64, + func_name, file_spec.GetCString(), base_addr); + } + }, I->file_spec, I->link_addr, I->base_addr); } + pool.wait(); + m_process->GetTarget().ModulesDidLoad(module_list); m_initial_modules_added = true; } diff --git a/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.h b/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.h index 8d3e4cde54c7237911924b854954b3e4d02c0417..142b059233bce9db1dc233dfd97698bec719fae6 100644 --- a/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.h +++ b/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.h @@ -91,6 +91,9 @@ protected: std::map> m_loaded_modules; + /// number of woker in ThreadPool + static constexpr int DYLD_CONCURRENCY_THREADING {3}; + /// If possible sets a breakpoint on a function called by the runtime /// linker each time a module is loaded or unloaded. bool SetRendezvousBreakpoint(); diff --git a/lldb/source/Target/ModuleCache.cpp b/lldb/source/Target/ModuleCache.cpp index 5d8b30946456489f8e2d3ef24b24f985d6be8ecc..69faa0e6ef6a619082b7b05bdb645583344ea288 100644 --- a/lldb/source/Target/ModuleCache.cpp +++ b/lldb/source/Target/ModuleCache.cpp @@ -133,21 +133,6 @@ Status CreateHostSysRootModuleLink(const FileSpec &root_dir_spec, const auto sysroot_module_path_spec = JoinPath(JoinPath(root_dir_spec, hostname), platform_module_spec.GetPath().c_str()); - UUID module_uuid; - { - auto module_sp = - std::make_shared(ModuleSpec(sysroot_module_path_spec)); - module_uuid = module_sp->GetUUID(); - } - - if (!module_uuid.IsValid()) { - LLDB_LOGF(log, "Try CreateHostSysRootModuleLink but uuid is invalid %s", - module_uuid.GetAsString().c_str()); - return Status(); - } - - LLDB_LOGF(log, "CreateHostSysRootModuleLink with uuid %s", - module_uuid.GetAsString().c_str()); if (FileSystem::Instance().Exists(sysroot_module_path_spec)) { if (!delete_existing) @@ -156,6 +141,23 @@ Status CreateHostSysRootModuleLink(const FileSpec &root_dir_spec, DecrementRefExistingModule(root_dir_spec, sysroot_module_path_spec); } + // sysroot_module_path_spec might still exist. + // It means that module UUID is not valid. + if (FileSystem::Instance().Exists(sysroot_module_path_spec)) { + auto module_sp = + std::make_shared(ModuleSpec(sysroot_module_path_spec)); + UUID module_uuid = module_sp->GetUUID(); + + if (!module_uuid.IsValid()) { + LLDB_LOGF(log, "Try CreateHostSysRootModuleLink but uuid is invalid %s", + module_uuid.GetAsString().c_str()); + return Status(); + } + + LLDB_LOGF(log, "CreateHostSysRootModuleLink with uuid %s", + module_uuid.GetAsString().c_str()); + } + const auto error = MakeDirectory( FileSpec(sysroot_module_path_spec.GetDirectory().AsCString())); if (error.Fail()) @@ -232,13 +234,16 @@ Status ModuleCache::Put(const FileSpec &root_dir_spec, const char *hostname, Status ModuleCache::Get(const FileSpec &root_dir_spec, const char *hostname, const ModuleSpec &module_spec, ModuleSP &cached_module_sp, bool *did_create_ptr) { - const auto find_it = - m_loaded_modules.find(module_spec.GetUUID().GetAsString()); - if (find_it != m_loaded_modules.end()) { - cached_module_sp = (*find_it).second.lock(); - if (cached_module_sp) - return Status(); - m_loaded_modules.erase(find_it); + { + std::lock_guard lock(m_cache_mutex); + const auto find_it = + m_loaded_modules.find(module_spec.GetUUID().GetAsString()); + if (find_it != m_loaded_modules.end()) { + cached_module_sp = (*find_it).second.lock(); + if (cached_module_sp) + return Status(); + m_loaded_modules.erase(find_it); + } } const auto module_spec_dir = @@ -277,6 +282,7 @@ Status ModuleCache::Get(const FileSpec &root_dir_spec, const char *hostname, if (FileSystem::Instance().Exists(symfile_spec)) cached_module_sp->SetSymbolFileFileSpec(symfile_spec); + std::lock_guard lock(m_cache_mutex); m_loaded_modules.insert( std::make_pair(module_spec.GetUUID().GetAsString(), cached_module_sp)); diff --git a/llvm-build/Makefile b/llvm-build/Makefile index 2e3562419e15fb56ea8718c797611a4bfde944b0..011686dd41bca796ae74ba08d821e505cc93c083 100644 --- a/llvm-build/Makefile +++ b/llvm-build/Makefile @@ -123,7 +123,7 @@ all: $(TARGETS:%=musl_install_for_%) $(TARGETS:%=musl_copy_for_%): $(HIDE) mkdir -p $@ - $(HIDE) cp -rfu $(MUSLDIR)/[^p]* $@ + $(HIDE) cp -rfu $(MUSLDIR)/[!p]* $@ optimized_routines_install_for_liteos_a_user: musl_copy_for_liteos_a_user ifneq ($(ARCH),) diff --git a/llvm-build/README.md b/llvm-build/README.md index e12de6c793fa98dd4ffffc1737c0acb374bc0913..fd76e9fcf7559f59ce802ea6c7e313fb0e4655e7 100644 --- a/llvm-build/README.md +++ b/llvm-build/README.md @@ -18,10 +18,9 @@ MacOS X >= 10.15.4 ### Get Code ``` -repo init -u https://gitee.com/OpenHarmony/manifest.git -b add_llvm_toolchain-dev (not ready yet) +repo init -u https://gitee.com/OpenHarmony/manifest.git -b master -m llvm-toolchain.xml repo sync -c repo forall -c 'git lfs pull' -cp -r toolchain/llvm-project/llvm-build toolchain ```
@@ -32,7 +31,7 @@ Here is an example of starting build process on Linux or MacOS: # update prebuilts, no need to run each time ./toolchain/llvm-project/llvm-build/env_prepare.sh # build -python3 ./toolchain/llvm-build/build.py +python3 ./toolchain/llvm-project/llvm-build/build.py ``` 1. env_prepare (one time only) diff --git a/llvm-build/build.py b/llvm-build/build.py index e5a74de8ed96eb81c709cb62f17c36b8fa050082..f2c08612a5b6cd4953cd49608ee3cd3e109722ea 100755 --- a/llvm-build/build.py +++ b/llvm-build/build.py @@ -51,10 +51,8 @@ class BuildConfig(): self.no_build_mipsel = args.skip_build or args.no_build_mipsel self.no_build_x86_64 = args.skip_build or args.no_build_x86_64 - self.CURRENT_DIR = os.path.abspath(os.path.dirname(__file__)) - self.REPOROOT_DIR = os.path.abspath(os.path.join(self.CURRENT_DIR, '../../')) - self.LLVM_PROJECT_DIR = os.path.abspath(os.path.join(self.REPOROOT_DIR, 'toolchain', 'llvm-project')) - self.OUT_PATH = os.path.join(self.REPOROOT_DIR, 'out') + self.discover_paths() + self.TARGETS = 'AArch64;ARM;BPF;Mips;RISCV;X86' self.ORIG_ENV = dict(os.environ) self.VERSION = None # autodetected @@ -66,6 +64,20 @@ class BuildConfig(): self.MINGW_TRIPLE = 'x86_64-windows-gnu' logging.basicConfig(level=logging.INFO) + def discover_paths(self): + # Location of llvm-build directory + self.LLVM_BUILD_DIR = os.path.abspath(os.path.dirname(__file__)) + + parent_of_llvm_build = os.path.basename(os.path.dirname(self.LLVM_BUILD_DIR)) + if parent_of_llvm_build == 'toolchain': + self.REPOROOT_DIR = os.path.abspath(os.path.join(self.LLVM_BUILD_DIR, '../..')) + else: + assert parent_of_llvm_build == 'llvm-project' + self.REPOROOT_DIR = os.path.abspath(os.path.join(self.LLVM_BUILD_DIR, '../../..')) + + self.LLVM_PROJECT_DIR = os.path.join(self.REPOROOT_DIR, 'toolchain', 'llvm-project') + self.OUT_PATH = os.path.join(self.REPOROOT_DIR, 'out') + @staticmethod def parse_add_argument(parser): @@ -706,7 +718,7 @@ class SysrootComposer(BuildUtils): ohos_cmake = 'OHOS.cmake' dst_dir = self.merge_out_path( '../prebuilts/cmake/%s/share/cmake-3.16/Modules/Platform' % self.platform_prefix()) - src_file = '%s/%s' % (self.build_config.CURRENT_DIR, ohos_cmake) + src_file = '%s/%s' % (self.build_config.LLVM_BUILD_DIR, ohos_cmake) if os.path.exists(os.path.join(dst_dir, ohos_cmake)): os.remove(os.path.join(dst_dir, ohos_cmake)) shutil.copy2(src_file, dst_dir) @@ -714,10 +726,12 @@ class SysrootComposer(BuildUtils): def build_musl(self, llvm_install, target, *extra_args): cur_dir = os.getcwd() - os.chdir(self.build_config.CURRENT_DIR) + os.chdir(self.build_config.LLVM_BUILD_DIR) self.logger().info('build musl %s', self.merge_out_path('install')) - args = ['./build_musl.sh', '-t', target, '-c', - self.merge_out_path(llvm_install, 'bin'), '-o', self.merge_out_path()] + list(extra_args) + args = ['./build_musl.sh', '-t', target, + '-c', self.merge_out_path(llvm_install, 'bin'), + '-o', self.merge_out_path(), + '-T', self.build_config.REPOROOT_DIR] + list(extra_args) self.check_call(args) os.chdir(cur_dir) @@ -1676,8 +1690,8 @@ class LlvmPackage(BuildUtils): self.check_rm_tree(install_host_dir) self.check_copy_tree(build_dir, install_dir) # copy readme file to install_dir - shutil.copyfile(os.path.join(self.build_config.CURRENT_DIR, "toolchain_readme.md"), - os.path.join(install_dir, "README.md")) + shutil.copyfile(os.path.join(self.build_config.LLVM_BUILD_DIR, "toolchain_readme.md"), + os.path.join(install_dir, "README.md")) # Remove unnecessary binaries. necessary_bin_files = [] diff --git a/llvm-build/build_musl.sh b/llvm-build/build_musl.sh index 9fb8737c032a015587d0a42f46add2c31c7ff372..d6c3a0add8d5c7630cc59951bf3a9559937402c1 100755 --- a/llvm-build/build_musl.sh +++ b/llvm-build/build_musl.sh @@ -18,11 +18,12 @@ set -e #default variables CLANG_BIN_ROOT="${PWD}/../../out/install/linux-x86_64/clang-dev/bin/" TARGET_TRIPLE="" +TOPDIR="${PWD}/../.." OUT="${PWD}/../../out" make_libs=0 #argument parser -while getopts "c:t:o:lh" arg +while getopts "c:t:T:o:lh" arg do case "${arg}" in "c") @@ -31,6 +32,9 @@ do "t") TARGET_TRIPLE=${OPTARG} ;; + "T") + TOPDIR=${OPTARG} + ;; "o") OUT=${OPTARG} ;; @@ -43,6 +47,7 @@ do echo " Options are:" echo " -c Specify clang bin path" echo " -t Specify target tripple" + echo " -T Specify top of repo tree" echo " -o Specify the build output directory" echo " -l Install libs" exit 0 @@ -90,25 +95,25 @@ fi echo "CLANG_BIN_ROOT=${CLANG_BIN_ROOT}" echo "ARCH=${TARGETS_PREFIX}" echo "TARGET=${TARGET_TRIPLE}" -echo "TOPDIR=${PWD}/../.." +echo "TOPDIR=${TOPDIR}" echo "TARGETS=${TARGET_USER}" echo "OUT=${OUT}" echo "SYSROOTDIR=${OUT}/sysroot" # build musl_headers -make musl_header_install_for_${TARGET_USER} CLANG="${CLANG_BIN_ROOT}/clang" TOPDIR=${PWD}/../../\ +make musl_header_install_for_${TARGET_USER} CLANG="${CLANG_BIN_ROOT}/clang" TOPDIR=${TOPDIR} \ SYSROOTDIR=${OUT}/sysroot TARGETS=${TARGET_USER} TARGET=${TARGET_TRIPLE} ARCH=${TARGETS_PREFIX} -f Makefile # build musl_libs if ((make_libs == 1)); then if [ $TARGET_TRIPLE == "aarch64-linux-ohos" ] || [ $TARGET_TRIPLE == "riscv64-linux-ohos" ] || \ [ $TARGET_TRIPLE == "mipsel-linux-ohos" ] || [ $TARGET_TRIPLE == "x86_64-linux-ohos" ]; then - make CLANG="${CLANG_BIN_ROOT}/clang" TOPDIR=${PWD}/../../ SYSROOTDIR=${OUT}/sysroot TARGETS=${TARGET_USER}\ + make CLANG="${CLANG_BIN_ROOT}/clang" TOPDIR=${TOPDIR} SYSROOTDIR=${OUT}/sysroot TARGETS=${TARGET_USER}\ TARGET=${TARGET_TRIPLE} ARCH=${TARGETS_PREFIX} -f Makefile else for ARCH_CFLAG in "${CFLAGS_FOR_TARGET[@]}" do - make CLANG="${CLANG_BIN_ROOT}/clang" TOPDIR=${PWD}/../../ SYSROOTDIR=${OUT}/sysroot TARGETS=${TARGET_USER}\ + make CLANG="${CLANG_BIN_ROOT}/clang" TOPDIR=${TOPDIR} SYSROOTDIR=${OUT}/sysroot TARGETS=${TARGET_USER}\ TARGET=${TARGET_TRIPLE} ARCH=${TARGETS_PREFIX} ARCH_CFLAGS="${ARCH_CFLAG}" -f Makefile done fi diff --git a/llvm-build/mingw.py b/llvm-build/mingw.py index 78e1f5e5b217de4b0171e6920741d536b44b3f1e..5396ce8803ed2b80a9e7f0385756d03809899263 100755 --- a/llvm-build/mingw.py +++ b/llvm-build/mingw.py @@ -25,22 +25,30 @@ import sys class BuildConfig(): def __init__(self, clang_version): - self.THIS_DIR = os.path.realpath(os.path.dirname(__file__)) + self.LLVM_BUILD_DIR = os.path.realpath(os.path.dirname(__file__)) + + parent_of_llvm_build = os.path.basename(os.path.dirname(self.LLVM_BUILD_DIR)) + if parent_of_llvm_build == 'toolchain': + self.REPOROOT_DIR = os.path.realpath(os.path.join(self.LLVM_BUILD_DIR, '../..')) + else: + assert parent_of_llvm_build == 'llvm-project' + self.REPOROOT_DIR = os.path.realpath(os.path.join(self.LLVM_BUILD_DIR, '../../..')) + self.OUT_DIR = os.environ.get('OUT_DIR', self.repo_root('out')) self.CLANG_VERSION = clang_version self.LLVM_TRIPLE = 'x86_64-windows-gnu' def repo_root(self, *args): - return os.path.realpath(os.path.join(self.THIS_DIR, '../../', *args)) + return os.path.join(self.REPOROOT_DIR, *args) def out_root(self, *args): - return os.path.realpath(os.path.join(self.OUT_DIR, *args)) + return os.path.join(self.OUT_DIR, *args) def mingw64_dir(self): return self.out_root('mingw', self.LLVM_TRIPLE) def llvm_path(self, *args): - return os.path.realpath(os.path.join(self.THIS_DIR, '../llvm-project', *args)) + return os.path.join(self.REPOROOT_DIR, 'toolchain/llvm-project', *args) class LlvmMingw():