diff --git a/es2panda/BUILD.gn b/es2panda/BUILD.gn index 9e33605992d90cb4183e8ea18ec12a20bba6a74a..9c1aaa5fabab23078779d4d3ef104fa867b506c6 100644 --- a/es2panda/BUILD.gn +++ b/es2panda/BUILD.gn @@ -253,6 +253,7 @@ es2panda_src = [ "util/moduleHelpers.cpp", "util/symbolTable.cpp", "util/ustring.cpp", + "util/workerQueue.cpp", ] config("es2abc_config_src") { @@ -381,7 +382,10 @@ ohos_static_library("es2panda_lib") { use_rtti = true } - configs = [ ":es2abc_config_common" ] + configs = [ + ":es2abc_config_common", + "../merge_abc:panda_assembly_proto_public_config", + ] include_dirs = [ "${target_out_dir}", @@ -398,6 +402,7 @@ ohos_static_library("es2panda_lib") { "$ark_root/libpandafile:libarkfile_frontend_static", "$ark_root/libziparchive:libarkziparchive_frontend_static", "$ark_third_party_root/icu/icu4c:static_icuuc", + "../merge_abc:panda_assembly_proto_static", ] if (enable_bytecode_optimizer) { @@ -420,6 +425,7 @@ ohos_executable("es2panda") { } sources = [ + "aot/emitFiles.cpp", "aot/main.cpp", "aot/options.cpp", ] diff --git a/es2panda/aot/emitFiles.cpp b/es2panda/aot/emitFiles.cpp new file mode 100644 index 0000000000000000000000000000000000000000..897b30539f6fd8dc7f51c0ba5899ad9d663c28c4 --- /dev/null +++ b/es2panda/aot/emitFiles.cpp @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "emitFiles.h" + +#include +#include +#include +#include + +namespace panda::es2panda::aot { +void EmitFileQueue::Schedule() +{ + ASSERT(jobsCount_ == 0); + std::unique_lock lock(m_); + + // generate abcs + if (mergeAbc_) { + auto emitMergedAbcJob = new EmitMergedAbcJob(options_->CompilerOutput(), progsInfo_, isDebug_); + jobs_.push_back(emitMergedAbcJob); + jobsCount_++; + } else { + for (const auto &info: progsInfo_) { + try { + auto outputFileName = options_->OutputFiles().empty() ? options_->CompilerOutput() : + options_->OutputFiles().at(info.first); + auto emitSingleAbcJob = new EmitSingleAbcJob(outputFileName, &(info.second->program), statp_, isDebug_); + jobs_.push_back(emitSingleAbcJob); + jobsCount_++; + } catch (std::exception &error) { + throw Error(ErrorType::GENERIC, error.what()); + } + } + } + + // generate cache protoBins + for (const auto &info: progsInfo_) { + if (!info.second->needUpdateCache) { + continue; + } + auto outputCacheIter = options_->CompilerOptions().cacheFiles.find(info.first); + if (outputCacheIter != options_->CompilerOptions().cacheFiles.end()) { + auto emitProtoJob = new EmitCacheJob(outputCacheIter->second, info.second); + jobs_.push_back(emitProtoJob); + jobsCount_++; + } + } + + lock.unlock(); + jobsAvailable_.notify_all(); +} + +void EmitSingleAbcJob::Run() +{ + if (!panda::pandasm::AsmEmitter::Emit(panda::os::file::File::GetExtendedFilePath(outputFileName_), *prog_, statp_, + nullptr, isDebug_)) { + throw Error(ErrorType::GENERIC, "Failed to emit " + outputFileName_ + ", error: " + + panda::pandasm::AsmEmitter::GetLastError()); + } +} + +void EmitMergedAbcJob::Run() +{ + std::vector progs; + progs.reserve(progsInfo_.size()); + for (const auto &info: progsInfo_) { + progs.push_back(&(info.second->program)); + } + if (!panda::pandasm::AsmEmitter::EmitPrograms(panda::os::file::File::GetExtendedFilePath(outputFileName_), progs, + isDebug_)) { + throw Error(ErrorType::GENERIC, "Failed to emit " + outputFileName_ + ", error: " + + panda::pandasm::AsmEmitter::GetLastError()); + } +} + +void EmitCacheJob::Run() +{ + panda::proto::ProtobufSnapshotGenerator::UpdateCacheFile(progCache_, outputProtoName_); +} + +} // namespace panda::es2panda::util diff --git a/es2panda/aot/emitFiles.h b/es2panda/aot/emitFiles.h new file mode 100644 index 0000000000000000000000000000000000000000..27495790e2c5bdd09f6daded5e1c80b20e65223a --- /dev/null +++ b/es2panda/aot/emitFiles.h @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ES2PANDA_AOT_EMITFILES_H +#define ES2PANDA_AOT_EMITFILES_H + +#include +#include + +#include + +namespace panda::es2panda::aot { +class EmitSingleAbcJob : public util::WorkerJob { +public: + explicit EmitSingleAbcJob(const std::string &outputFileName, panda::pandasm::Program *prog, + std::map *statp, bool isDebug) + : outputFileName_(outputFileName), prog_(prog), statp_(statp), isDebug_(isDebug) {}; + NO_COPY_SEMANTIC(EmitSingleAbcJob); + NO_MOVE_SEMANTIC(EmitSingleAbcJob); + ~EmitSingleAbcJob() override = default; + + void Run() override; +private: + std::string outputFileName_; + panda::pandasm::Program *prog_; + std::map *statp_; + bool isDebug_; +}; + +class EmitMergedAbcJob : public util::WorkerJob { +public: + explicit EmitMergedAbcJob(const std::string &outputFileName, + const std::map &progsInfo, + bool isDebug) + : outputFileName_(outputFileName), progsInfo_(progsInfo), isDebug_(isDebug) {}; + NO_COPY_SEMANTIC(EmitMergedAbcJob); + NO_MOVE_SEMANTIC(EmitMergedAbcJob); + ~EmitMergedAbcJob() override = default; + + void Run() override; +private: + std::string outputFileName_; + const std::map &progsInfo_; + bool isDebug_; +}; + +class EmitCacheJob : public util::WorkerJob { +public: + explicit EmitCacheJob(const std::string &outputProtoName, panda::es2panda::util::ProgramCache *progCache) + : outputProtoName_(outputProtoName), progCache_(progCache) {}; + NO_COPY_SEMANTIC(EmitCacheJob); + NO_MOVE_SEMANTIC(EmitCacheJob); + ~EmitCacheJob() override = default; + + void Run() override; +private: + std::string outputProtoName_; + panda::es2panda::util::ProgramCache *progCache_; +}; + +class EmitFileQueue : public util::WorkerQueue { +public: + explicit EmitFileQueue(const std::unique_ptr &options, + std::map *statp, + const std::map &progsInfo) + : WorkerQueue(options->CompilerOptions().fileThreadCount), options_(options), statp_(statp), + progsInfo_(progsInfo) { + mergeAbc_ = options_->CompilerOptions().mergeAbc; + isDebug_ = options_->CompilerOptions().isDebug; + } + + NO_COPY_SEMANTIC(EmitFileQueue); + NO_MOVE_SEMANTIC(EmitFileQueue); + ~EmitFileQueue() override = default; + + void Schedule() override; + +private: + const std::unique_ptr &options_; + std::map *statp_; + const std::map &progsInfo_; + bool mergeAbc_ { false }; + bool isDebug_ { true }; +}; +} // namespace panda::es2panda::aot + +#endif diff --git a/es2panda/aot/main.cpp b/es2panda/aot/main.cpp index 1f1ae0f651c422b841d1b2c3ce325c60c9c8b00f..9c57e484d73405ec031aa9dd2eaea7efc3ddccef 100644 --- a/es2panda/aot/main.cpp +++ b/es2panda/aot/main.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -23,6 +24,7 @@ #include #include #include +#include #include @@ -48,10 +50,9 @@ public: } }; -static void GenerateBase64Output(panda::pandasm::Program *prog, - panda::pandasm::AsmEmitter::PandaFileToPandaAsmMaps *mapsp) +static void GenerateBase64Output(panda::pandasm::Program *prog) { - auto pandaFile = panda::pandasm::AsmEmitter::Emit(*prog, mapsp); + auto pandaFile = panda::pandasm::AsmEmitter::Emit(*prog); const uint8_t *buffer = pandaFile->GetBase(); size_t size = pandaFile->GetPtr().GetSize(); std::string content(reinterpret_cast(buffer), size); @@ -80,34 +81,25 @@ static void DumpPandaFileSizeStatistic(std::map &stat) std::cout << "total: " << totalSize << std::endl; } -static bool GenerateMultiProgram(const std::map &programsInfo, - const std::unique_ptr &options) +static bool GenerateProgramsByWorkers(const std::map &programsInfo, + const std::unique_ptr &options, std::map *statp) { - if (options->CompilerOptions().mergeAbc) { - std::vector progs; - for (const auto &progInfo: programsInfo) { - progs.push_back(progInfo.second->program); - } + auto queue = new panda::es2panda::aot::EmitFileQueue(options, statp, programsInfo); - const auto &output = options->CompilerOutput(); - if (!panda::pandasm::AsmEmitter::EmitPrograms(panda::os::file::File::GetExtendedFilePath(output), progs, - true)) { - std::cerr << "Failed to emit merged program, error: " << - panda::pandasm::AsmEmitter::GetLastError() << std::endl; - return false; - } - } else { - for (const auto &progInfo: programsInfo) { - const auto &output = options->OutputFiles().at(progInfo.first); - if (!panda::pandasm::AsmEmitter::Emit(panda::os::file::File::GetExtendedFilePath(output), - *(progInfo.second->program), nullptr, nullptr, true)) { - std::cout << "Failed to emit single program, error: " << - panda::pandasm::AsmEmitter::GetLastError() << std::endl; - return false; - } - } + bool emitResult = true; + try { + queue->Schedule(); + queue->Consume(); + queue->Wait(); + } catch (const class Error &e) { + emitResult = false; + std::cerr << e.Message() << std::endl; } - return true; + + delete queue; + queue = nullptr; + + return emitResult; } static void DumpProgramInfos(const std::map &programsInfo, @@ -117,11 +109,11 @@ static void DumpProgramInfos(const std::mapprogram); + es2panda::Compiler::DumpAsm(&(progInfo.second->program)); } if (compilerOptions.dumpLiteralBuffer) { - panda::es2panda::util::Dumper::DumpLiterals(progInfo.second->program->literalarray_table); + panda::es2panda::util::Dumper::DumpLiterals(progInfo.second->program.literalarray_table); } } } @@ -130,23 +122,10 @@ static void DumpProgramInfos(const std::map &programsInfo, const std::unique_ptr &options) { - DumpProgramInfos(programsInfo, options); - if (programsInfo.size() > 1) { - return GenerateMultiProgram(programsInfo, options); - } else { - auto *prog = programsInfo.begin()->second->program; - - int optLevel = options->OptLevel(); - bool dumpSize = options->SizeStat(); - std::map stat; - std::map *statp = optLevel != 0 ? &stat : nullptr; - panda::pandasm::AsmEmitter::PandaFileToPandaAsmMaps maps {}; - panda::pandasm::AsmEmitter::PandaFileToPandaAsmMaps *mapsp = optLevel != 0 ? &maps : nullptr; - - const auto &output = options->OutputFiles().empty() ? options->CompilerOutput() : - options->OutputFiles().at(programsInfo.begin()->first); - if (output.empty()) { - GenerateBase64Output(prog, mapsp); + if (programsInfo.size() == 1) { + auto *prog = &(programsInfo.begin()->second->program); + if (options->OutputFiles().empty() && options->CompilerOutput().empty()) { + GenerateBase64Output(prog); return true; } @@ -154,17 +133,22 @@ static bool GenerateProgram(const std::mapcompilerProtoOutput()); return true; } + } - if (!panda::pandasm::AsmEmitter::Emit(panda::os::file::File::GetExtendedFilePath(output), *prog, statp, mapsp, - true)) { - return false; - } + bool dumpSize = options->SizeStat(); + std::map stat; + std::map *statp = dumpSize ? &stat : nullptr; - if (dumpSize && optLevel != 0) { - DumpPandaFileSizeStatistic(stat); - } + if (!GenerateProgramsByWorkers(programsInfo, options, statp)) { + return false; } + if (dumpSize) { + DumpPandaFileSizeStatistic(stat); + } + + DumpProgramInfos(programsInfo, options); + return true; } @@ -204,29 +188,18 @@ int Run(int argc, const char **argv) std::map programsInfo; size_t expectedProgsCount = options->CompilerOptions().sourceFiles.size(); panda::ArenaAllocator allocator(panda::SpaceType::SPACE_TYPE_COMPILER, nullptr, true); - std::map *cachePrograms = nullptr; - - if (!options->CacheFile().empty()) { - cachePrograms = proto::ProtobufSnapshotGenerator::GetCacheContext(options->CacheFile(), - options->CompilerOptions().isDebug, &allocator); - } - int ret = Compiler::CompileFiles(options->CompilerOptions(), cachePrograms, programsInfo, &allocator); + int ret = Compiler::CompileFiles(options->CompilerOptions(), programsInfo, &allocator); if (options->ParseOnly()) { return ret; } if (!options->NpmModuleEntryList().empty()) { - es2panda::util::ModuleHelpers::CompileNpmModuleEntryList(options->NpmModuleEntryList(), cachePrograms, - programsInfo, &allocator); + es2panda::util::ModuleHelpers::CompileNpmModuleEntryList(options->NpmModuleEntryList(), + options->CompilerOptions(), programsInfo, &allocator); expectedProgsCount++; } - if (!options->CacheFile().empty()) { - proto::ProtobufSnapshotGenerator::UpdateCacheFile(programsInfo, options->CompilerOptions().isDebug, - options->CacheFile()); - } - if (!GenerateAbcFiles(programsInfo, options, expectedProgsCount)) { return 1; } diff --git a/es2panda/aot/options.cpp b/es2panda/aot/options.cpp index 5f7e8219641dd1ae657b8d1a0fe346db72ea1824..50e38c8907b9d84bdbcda393eef02fdcf6b04cd3 100644 --- a/es2panda/aot/options.cpp +++ b/es2panda/aot/options.cpp @@ -30,6 +30,8 @@ #include namespace panda::es2panda::aot { +constexpr char PROCESS_AS_LIST_MARK = '@'; +const std::string LIST_ITEM_SEPERATOR = ";"; template T RemoveExtension(T const &filename) @@ -56,6 +58,7 @@ static std::vector GetStringItems(std::string &input, const std::st return items; } +// Options bool Options::CollectInputFilesFromFileList(const std::string &input) { std::ifstream ifs; @@ -69,8 +72,7 @@ bool Options::CollectInputFilesFromFileList(const std::string &input) constexpr size_t ITEM_COUNT_MERGE = 5; // item list: [filePath; recordName; moduleKind; sourceFile, pkgName] constexpr size_t ITEM_COUNT_NOT_MERGE = 5; // item list: [filePath; recordName; moduleKind; sourceFile; outputfile] while (std::getline(ifs, line)) { - const std::string seperator = ";"; - std::vector itemList = GetStringItems(line, seperator); + std::vector itemList = GetStringItems(line, LIST_ITEM_SEPERATOR); if ((compilerOptions_.mergeAbc && itemList.size() != ITEM_COUNT_MERGE) || (!compilerOptions_.mergeAbc && itemList.size() != ITEM_COUNT_NOT_MERGE)) { std::cerr << "Failed to parse input file" << std::endl; @@ -116,7 +118,31 @@ bool Options::CollectInputFilesFromFileDirectory(const std::string &input, const return true; } -// Options +void Options::ParseCacheFileOption(const std::string &cacheInput) +{ + if (cacheInput[0] != PROCESS_AS_LIST_MARK) { + compilerOptions_.cacheFiles.insert({sourceFile_, cacheInput}); + return; + } + + std::ifstream ifs; + std::string line; + ifs.open(panda::os::file::File::GetExtendedFilePath(cacheInput.substr(1))); + if (!ifs.is_open()) { + std::cerr << "Failed to open cache file list: " << cacheInput << std::endl; + return; + } + + constexpr int cacheListItemCount = 2; + while (std::getline(ifs, line)) { + std::vector itemList = GetStringItems(line, LIST_ITEM_SEPERATOR); + if (itemList.size() != cacheListItemCount) { + continue; + } + compilerOptions_.cacheFiles.insert({itemList[0], itemList[1]}); + } +} + Options::Options() : argparser_(new panda::PandArgParser()) {} Options::~Options() @@ -299,7 +325,7 @@ bool Options::Parse(int argc, const char **argv) bool isInputFileList = false; if (!inputIsEmpty) { std::string rawInput = inputFile.GetValue(); - isInputFileList = rawInput[0] == '@'; + isInputFileList = rawInput[0] == PROCESS_AS_LIST_MARK; std::string input = isInputFileList ? rawInput.substr(1) : rawInput; sourceFile_ = input; } @@ -359,7 +385,10 @@ bool Options::Parse(int argc, const char **argv) functionThreadCount_ = opFunctionThreadCount.GetValue(); fileThreadCount_ = opFileThreadCount.GetValue(); npmModuleEntryList_ = opNpmModuleEntryList.GetValue(); - cacheFile_ = opCacheFile.GetValue(); + + if (!opCacheFile.GetValue().empty()) { + ParseCacheFileOption(opCacheFile.GetValue()); + } if (opParseOnly.GetValue()) { options_ |= OptionFlags::PARSE_ONLY; diff --git a/es2panda/aot/options.h b/es2panda/aot/options.h index f2bae2a776c546fc16832cb5f24b8d4f501bb46a..04bd5a8c7c71dce3c92c4f0850413cf4f7e7a4f0 100644 --- a/es2panda/aot/options.h +++ b/es2panda/aot/options.h @@ -122,11 +122,6 @@ public: return compilerProtoOutput_; } - const std::string &CacheFile() const - { - return cacheFile_; - } - const std::string &NpmModuleEntryList() const { return npmModuleEntryList_; @@ -139,6 +134,7 @@ public: bool CollectInputFilesFromFileList(const std::string &input); bool CollectInputFilesFromFileDirectory(const std::string &input, const std::string &extension); + void ParseCacheFileOption(const std::string &cacheInput); private: es2panda::ScriptExtension extension_ {es2panda::ScriptExtension::JS}; @@ -156,7 +152,6 @@ private: int optLevel_ {0}; int functionThreadCount_ {0}; int fileThreadCount_ {0}; - std::string cacheFile_; std::string npmModuleEntryList_; std::vector sourceFiles_; std::unordered_map outputFiles_; diff --git a/es2panda/compiler/core/compileQueue.cpp b/es2panda/compiler/core/compileQueue.cpp index ea11d2bd33e33b34c914a75c9296c6559dc50e12..cdc08adfbec1203a8567fe2329b06f584eb67cbe 100644 --- a/es2panda/compiler/core/compileQueue.cpp +++ b/es2panda/compiler/core/compileQueue.cpp @@ -24,47 +24,14 @@ #include #include #include +#include #include #include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include - -#include -#include -#include - -#include - namespace panda::es2panda::compiler { std::mutex CompileFileJob::global_m_; -void CompileJob::DependsOn(CompileJob *job) -{ - job->dependant_ = this; - dependencies_++; -} - -void CompileJob::Signal() -{ - { - std::lock_guard lock(m_); - dependencies_--; - } - - cond_.notify_one(); -} - void CompileFunctionJob::Run() { std::unique_lock lock(m_); @@ -102,14 +69,30 @@ void CompileModuleRecordJob::Run() void CompileFileJob::Run() { - es2panda::Compiler compiler(options_->extension, options_->functionThreadCount); + std::stringstream ss; + if (!util::Helpers::ReadFileToBuffer(src_->fileName, ss)) { + return; + } + std::string buffer = ss.str(); + src_->source = buffer; - auto *prog = compiler.CompileFile(*options_, src_, symbolTable_); + auto cacheFileIter = options_->cacheFiles.find(src_->fileName); + if (cacheFileIter != options_->cacheFiles.end()) { + src_->hash = GetHash32String(reinterpret_cast(buffer.c_str())); - if (prog == nullptr) { - return; + ArenaAllocator allocator(SpaceType::SPACE_TYPE_COMPILER, nullptr, true); + auto *cacheProgramInfo = proto::ProtobufSnapshotGenerator::GetCacheContext(cacheFileIter->second, &allocator); + + if (cacheProgramInfo != nullptr && cacheProgramInfo->hashCode == src_->hash) { + std::unique_lock lock(global_m_); + auto *cache = allocator_->New(src_->hash, std::move(cacheProgramInfo->program)); + progsInfo_.insert({src_->fileName, cache}); + return; + } } + es2panda::Compiler compiler(options_->extension, options_->functionThreadCount); + auto *prog = compiler.CompileFile(*options_, src_, symbolTable_); if (prog == nullptr) { return; } @@ -120,94 +103,11 @@ void CompileFileJob::Run() { std::unique_lock lock(global_m_); - auto *cache = allocator_->New(src_->hash, prog); + auto *cache = allocator_->New(src_->hash, std::move(*prog), true); progsInfo_.insert({src_->fileName, cache}); } } -CompileQueue::CompileQueue(size_t threadCount) -{ - threads_.reserve(threadCount); - - for (size_t i = 0; i < threadCount; i++) { - threads_.push_back(os::thread::ThreadStart(Worker, this)); - } -} - -CompileQueue::~CompileQueue() -{ - void *retval = nullptr; - - std::unique_lock lock(m_); - terminate_ = true; - lock.unlock(); - jobsAvailable_.notify_all(); - - for (const auto handle_id : threads_) { - os::thread::ThreadJoin(handle_id, &retval); - } -} - -void CompileQueue::Worker(CompileQueue *queue) -{ - while (true) { - std::unique_lock lock(queue->m_); - queue->jobsAvailable_.wait(lock, [queue]() { return queue->terminate_ || queue->jobsCount_ != 0; }); - - if (queue->terminate_) { - return; - } - - lock.unlock(); - - queue->Consume(); - queue->jobsFinished_.notify_one(); - } -} - -void CompileQueue::Consume() -{ - std::unique_lock lock(m_); - activeWorkers_++; - - while (jobsCount_ > 0) { - --jobsCount_; - auto &job = *(jobs_[jobsCount_]); - - lock.unlock(); - - try { - job.Run(); - } catch (const Error &e) { - lock.lock(); - errors_.push_back(e); - lock.unlock(); - } - - lock.lock(); - } - - activeWorkers_--; -} - -void CompileQueue::Wait() -{ - std::unique_lock lock(m_); - jobsFinished_.wait(lock, [this]() { return activeWorkers_ == 0 && jobsCount_ == 0; }); - for (auto it = jobs_.begin(); it != jobs_.end(); it++) { - if (*it != nullptr) { - delete *it; - *it =nullptr; - } - } - jobs_.clear(); - - if (!errors_.empty()) { - // NOLINTNEXTLINE - throw errors_.front(); - } -} - void CompileFuncQueue::Schedule() { ASSERT(jobsCount_ == 0); diff --git a/es2panda/compiler/core/compileQueue.h b/es2panda/compiler/core/compileQueue.h index fe1f0da29090b54ee92606849af43b87edb76d36..cf041fcaf07f4b658b53401bc40930ca368015a9 100644 --- a/es2panda/compiler/core/compileQueue.h +++ b/es2panda/compiler/core/compileQueue.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -32,25 +33,7 @@ namespace panda::es2panda::compiler { class CompilerContext; -class CompileJob { -public: - explicit CompileJob() {}; - NO_COPY_SEMANTIC(CompileJob); - NO_MOVE_SEMANTIC(CompileJob); - virtual ~CompileJob() = default; - - virtual void Run() = 0; - void DependsOn(CompileJob *job); - void Signal(); - -protected: - std::mutex m_; - std::condition_variable cond_; - CompileJob *dependant_ {}; - size_t dependencies_ {0}; -}; - -class CompileFunctionJob : public CompileJob { +class CompileFunctionJob : public util::WorkerJob { public: explicit CompileFunctionJob(CompilerContext *context) : context_(context) {}; NO_COPY_SEMANTIC(CompileFunctionJob); @@ -74,7 +57,7 @@ private: binder::FunctionScope *scope_ {}; }; -class CompileModuleRecordJob : public CompileJob { +class CompileModuleRecordJob : public util::WorkerJob { public: explicit CompileModuleRecordJob(CompilerContext *context) : context_(context) {}; NO_COPY_SEMANTIC(CompileModuleRecordJob); @@ -87,7 +70,7 @@ private: CompilerContext *context_ {}; }; -class CompileFileJob : public CompileJob { +class CompileFileJob : public util::WorkerJob { public: explicit CompileFileJob(es2panda::SourceFile *src, es2panda::CompilerOptions *options, std::map &progsInfo, @@ -108,35 +91,10 @@ private: panda::ArenaAllocator *allocator_; }; -class CompileQueue { -public: - explicit CompileQueue(size_t threadCount); - NO_COPY_SEMANTIC(CompileQueue); - NO_MOVE_SEMANTIC(CompileQueue); - virtual ~CompileQueue(); - - virtual void Schedule() = 0; - void Consume(); - void Wait(); - -protected: - static void Worker(CompileQueue *queue); - - std::vector threads_; - std::vector errors_; - std::mutex m_; - std::condition_variable jobsAvailable_; - std::condition_variable jobsFinished_; - std::vector jobs_ {}; - size_t jobsCount_ {0}; - size_t activeWorkers_ {0}; - bool terminate_ {false}; -}; - -class CompileFuncQueue : public CompileQueue { +class CompileFuncQueue : public util::WorkerQueue { public: explicit CompileFuncQueue(size_t threadCount, CompilerContext *context) - : CompileQueue(threadCount), context_(context) {} + : util::WorkerQueue(threadCount), context_(context) {} NO_COPY_SEMANTIC(CompileFuncQueue); NO_MOVE_SEMANTIC(CompileFuncQueue); @@ -148,12 +106,12 @@ private: CompilerContext *context_; }; -class CompileFileQueue : public CompileQueue { +class CompileFileQueue : public util::WorkerQueue { public: explicit CompileFileQueue(size_t threadCount, es2panda::CompilerOptions *options, std::map &progsInfo, util::SymbolTable *symbolTable, panda::ArenaAllocator *allocator) - : CompileQueue(threadCount), options_(options), progsInfo_(progsInfo), + : util::WorkerQueue(threadCount), options_(options), progsInfo_(progsInfo), symbolTable_(symbolTable), allocator_(allocator) {} NO_COPY_SEMANTIC(CompileFileQueue); diff --git a/es2panda/es2panda.cpp b/es2panda/es2panda.cpp index 867615913a86a54974444b39cdceb8994c9ef099..ac2441e9bff4d4f4b8d081f75d4d940e6d1f9f83 100644 --- a/es2panda/es2panda.cpp +++ b/es2panda/es2panda.cpp @@ -25,10 +25,7 @@ #include #include -#include - #include -#include namespace panda::es2panda { // Compiler @@ -141,59 +138,8 @@ void Compiler::DumpAsm(const panda::pandasm::Program *prog) compiler::CompilerImpl::DumpAsm(prog); } -static bool ReadFileToBuffer(const std::string &file, std::stringstream &ss) -{ - std::ifstream inputStream(panda::os::file::File::GetExtendedFilePath(file), std::ios::binary); - if (inputStream.fail()) { - std::cerr << "Failed to read file to buffer: " << file << std::endl; - return false; - } - ss << inputStream.rdbuf(); - return true; -} - -void Compiler::SelectCompileFile(CompilerOptions &options, - std::map *cacheProgs, - std::map &progsInfo, - panda::ArenaAllocator *allocator) -{ - if (cacheProgs == nullptr) { - return; - } - - auto fullList = options.sourceFiles; - std::vector inputList; - - for (auto &input: fullList) { - if (input.fileName.empty()) { - // base64 source - inputList.push_back(input); - continue; - } - - std::stringstream ss; - if (!ReadFileToBuffer(input.fileName, ss)) { - continue; - } - - uint32_t hash = GetHash32String(reinterpret_cast(ss.str().c_str())); - - auto it = cacheProgs->find(input.fileName); - if (it != cacheProgs->end() && hash == it->second->hashCode) { - auto *cache = allocator->New(it->second->hashCode, it->second->program); - progsInfo.insert({input.fileName, cache}); - } else { - input.hash = hash; - inputList.push_back(input); - } - } - options.sourceFiles = inputList; -} - int Compiler::CompileFiles(CompilerOptions &options, - std::map *cacheProgs, - std::map &progsInfo, - panda::ArenaAllocator *allocator) + std::map &progsInfo, panda::ArenaAllocator *allocator) { util::SymbolTable *symbolTable = nullptr; if (!options.hotfixOptions.symbolTable.empty() || !options.hotfixOptions.dumpSymbolTable.empty()) { @@ -204,8 +150,6 @@ int Compiler::CompileFiles(CompilerOptions &options, } } - SelectCompileFile(options, cacheProgs, progsInfo, allocator); - bool failed = false; auto queue = new compiler::CompileFileQueue(options.fileThreadCount, &options, progsInfo, symbolTable, allocator); @@ -231,18 +175,8 @@ int Compiler::CompileFiles(CompilerOptions &options, panda::pandasm::Program *Compiler::CompileFile(const CompilerOptions &options, SourceFile *src, util::SymbolTable *symbolTable) { - std::string buffer; if (src->source.empty()) { - std::stringstream ss; - if (!ReadFileToBuffer(src->fileName, ss)) { - return nullptr; - } - buffer = ss.str(); - src->source = buffer; - - if (src->hash == 0) { - src->hash = GetHash32String(reinterpret_cast(buffer.c_str())); - } + return nullptr; } auto *program = Compile(*src, options, symbolTable); diff --git a/es2panda/es2panda.h b/es2panda/es2panda.h index d4d201855e8d2135a07a6160efabbb4cc4308471..54ef49659359e6cfd5b623903234c9e9cdd494df 100644 --- a/es2panda/es2panda.h +++ b/es2panda/es2panda.h @@ -90,6 +90,7 @@ struct CompilerOptions { HotfixOptions hotfixOptions; bool bcVersion {false}; bool bcMinVersion {false}; + std::unordered_map cacheFiles; }; enum class ErrorType { @@ -175,14 +176,7 @@ public: panda::pandasm::Program *CompileFile(const CompilerOptions &options, SourceFile *src, util::SymbolTable *symbolTable); static int CompileFiles(CompilerOptions &options, - std::map *cacheProgs, - std::map &progsInfo, - panda::ArenaAllocator *allocator); - - static void SelectCompileFile(CompilerOptions &options, - std::map *cacheProgs, - std::map &progsInfo, - panda::ArenaAllocator *allocator); + std::map &progsInfo, panda::ArenaAllocator *allocator); inline panda::pandasm::Program *Compile(const SourceFile &input) { diff --git a/es2panda/util/helpers.cpp b/es2panda/util/helpers.cpp index 72be34a1e87be0a64d244c5ec20031de23530c5b..10a24baefd003e09540c0074a08d2cf962a2d67b 100644 --- a/es2panda/util/helpers.cpp +++ b/es2panda/util/helpers.cpp @@ -43,6 +43,8 @@ #include #endif +#include + namespace panda::es2panda::util { // Helpers @@ -460,4 +462,15 @@ bool Helpers::OptimizeProgram(panda::pandasm::Program * prog, es2panda::Compiler return true; } +bool Helpers::ReadFileToBuffer(const std::string &file, std::stringstream &ss) +{ + std::ifstream inputStream(panda::os::file::File::GetExtendedFilePath(file), std::ios::binary); + if (inputStream.fail()) { + std::cerr << "Failed to read file to buffer: " << file << std::endl; + return false; + } + ss << inputStream.rdbuf(); + return true; +} + } // namespace panda::es2panda::util diff --git a/es2panda/util/helpers.h b/es2panda/util/helpers.h index eab6632661202788d133e34491b8d421dc2661d2..6e4b8d0481ab444a1676bcf4dbdc362091170233 100644 --- a/es2panda/util/helpers.h +++ b/es2panda/util/helpers.h @@ -81,6 +81,7 @@ public: static bool OptimizeProgram(panda::pandasm::Program *prog, es2panda::CompilerOptions *options); template static T BaseName(T const &path, T const &delims = std::string(panda::os::file::File::GetPathDelim())); + static bool ReadFileToBuffer(const std::string &file, std::stringstream &ss); static const uint32_t INVALID_INDEX = 4294967295L; static const uint32_t MAX_INT32 = 2147483647; diff --git a/es2panda/util/moduleHelpers.cpp b/es2panda/util/moduleHelpers.cpp index 847cbea45c466249f47f2631b63d036dc40a89ff..f2a6bda869847859489ed0087632043ba9f26e36 100644 --- a/es2panda/util/moduleHelpers.cpp +++ b/es2panda/util/moduleHelpers.cpp @@ -15,28 +15,29 @@ #include "moduleHelpers.h" +#include #include +#include namespace panda::es2panda::util { void ModuleHelpers::CompileNpmModuleEntryList(const std::string &entriesInfo, - std::map *cacheProgs, - std::map &progsInfo, + panda::es2panda::CompilerOptions &options, std::map &progsInfo, panda::ArenaAllocator *allocator) { std::stringstream ss; - std::ifstream inputStream(panda::os::file::File::GetExtendedFilePath(entriesInfo)); - if (inputStream.fail()) { - std::cerr << "Failed to read file to buffer: " << entriesInfo << std::endl; + if (!util::Helpers::ReadFileToBuffer(entriesInfo, ss)) { return; } - ss << inputStream.rdbuf(); - uint32_t hash = GetHash32String(reinterpret_cast(ss.str().c_str())); + uint32_t hash = 0; + auto cacheFileIter = options.cacheFiles.find(entriesInfo); + if (cacheFileIter != options.cacheFiles.end()) { + hash = GetHash32String(reinterpret_cast(ss.str().c_str())); - if (cacheProgs != nullptr) { - auto it = cacheProgs->find(entriesInfo); - if (it != cacheProgs->end() && hash == it->second->hashCode) { - auto *cache = allocator->New(it->second->hashCode, it->second->program); + auto cacheProgramInfo = panda::proto::ProtobufSnapshotGenerator::GetCacheContext(cacheFileIter->second, + allocator); + if (cacheProgramInfo != nullptr && cacheProgramInfo->hashCode == hash) { + auto *cache = allocator->New(hash, std::move(cacheProgramInfo->program)); progsInfo.insert({entriesInfo, cache}); return; } @@ -61,7 +62,7 @@ void ModuleHelpers::CompileNpmModuleEntryList(const std::string &entriesInfo, prog->record_table.emplace(recordName, std::move(*entryRecord)); } - auto *cache = allocator->New(hash, prog); + auto *cache = allocator->New(hash, std::move(*prog), true); progsInfo.insert({entriesInfo, cache}); } } // namespace panda::es2panda::util diff --git a/es2panda/util/moduleHelpers.h b/es2panda/util/moduleHelpers.h index f7952e705dfb353f63b1b667e89fac03d432e879..3aff9c0a58d19b55bb19d8ed797003b6d3be5bb1 100644 --- a/es2panda/util/moduleHelpers.h +++ b/es2panda/util/moduleHelpers.h @@ -17,16 +17,15 @@ #define ES2PANDA_UTIL_MODULE_HELPERS_H #include +#include #include #include namespace panda::es2panda::util { class ModuleHelpers { public: - static void CompileNpmModuleEntryList(const std::string &entriesInfo, - std::map *cacheProgs, - std::map &progsInfo, - panda::ArenaAllocator *allocator); + static void CompileNpmModuleEntryList(const std::string &entriesInfo, panda::es2panda::CompilerOptions &options, + std::map &progsInfo, panda::ArenaAllocator *allocator); }; } // panda::es2panda::util diff --git a/es2panda/util/programCache.h b/es2panda/util/programCache.h index 24f71ae7d616df3f0d1fe099ddb9d4e0b8448e4c..946dd3c4fdeb1f54094032c121f942689e6f2afd 100644 --- a/es2panda/util/programCache.h +++ b/es2panda/util/programCache.h @@ -17,13 +17,20 @@ #define ES2PANDA_UTIL_PROGRAM_CACHE_H #include +#include namespace panda::es2panda::util { struct ProgramCache { uint32_t hashCode; - panda::pandasm::Program* program; + panda::pandasm::Program program; + bool needUpdateCache { false }; - ProgramCache(uint32_t hashCode, panda::pandasm::Program* program) : hashCode(hashCode), program(program) + ProgramCache(uint32_t hashCode, panda::pandasm::Program program) : hashCode(hashCode), program(std::move(program)) + { + } + + ProgramCache(uint32_t hashCode, panda::pandasm::Program program, bool needUpdateCache) + : hashCode(hashCode), program(std::move(program)), needUpdateCache(needUpdateCache) { } }; diff --git a/es2panda/util/workerQueue.cpp b/es2panda/util/workerQueue.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ed4d8dbc95fd584532f87b81b280382fed307192 --- /dev/null +++ b/es2panda/util/workerQueue.cpp @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2021-2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "workerQueue.h" + +namespace panda::es2panda::util { + +void WorkerJob::DependsOn(WorkerJob *job) +{ + job->dependant_ = this; + dependencies_++; +} + +void WorkerJob::Signal() +{ + { + std::lock_guard lock(m_); + dependencies_--; + } + + cond_.notify_one(); +} + +WorkerQueue::WorkerQueue(size_t threadCount) +{ + threads_.reserve(threadCount); + + for (size_t i = 0; i < threadCount; i++) { + threads_.push_back(os::thread::ThreadStart(Worker, this)); + } +} + +WorkerQueue::~WorkerQueue() +{ + void *retval = nullptr; + + std::unique_lock lock(m_); + terminate_ = true; + lock.unlock(); + jobsAvailable_.notify_all(); + + for (const auto handle_id : threads_) { + os::thread::ThreadJoin(handle_id, &retval); + } +} + +void WorkerQueue::Worker(WorkerQueue *queue) +{ + while (true) { + std::unique_lock lock(queue->m_); + queue->jobsAvailable_.wait(lock, [queue]() { return queue->terminate_ || queue->jobsCount_ != 0; }); + + if (queue->terminate_) { + return; + } + + lock.unlock(); + + queue->Consume(); + queue->jobsFinished_.notify_one(); + } +} + +void WorkerQueue::Consume() +{ + std::unique_lock lock(m_); + activeWorkers_++; + + while (jobsCount_ > 0) { + --jobsCount_; + auto &job = *(jobs_[jobsCount_]); + + lock.unlock(); + + try { + job.Run(); + } catch (const Error &e) { + lock.lock(); + errors_.push_back(e); + lock.unlock(); + } + + lock.lock(); + } + + activeWorkers_--; +} + +void WorkerQueue::Wait() +{ + std::unique_lock lock(m_); + jobsFinished_.wait(lock, [this]() { return activeWorkers_ == 0 && jobsCount_ == 0; }); + for (auto it = jobs_.begin(); it != jobs_.end(); it++) { + if (*it != nullptr) { + delete *it; + *it = nullptr; + } + } + jobs_.clear(); + + if (!errors_.empty()) { + // NOLINTNEXTLINE + throw errors_.front(); + } +} +} // namespace panda::es2panda::util diff --git a/merge_abc/src/compositeProgramProto.h b/es2panda/util/workerQueue.h similarity index 30% rename from merge_abc/src/compositeProgramProto.h rename to es2panda/util/workerQueue.h index 3291a666a1acb16e4810f9672712b24ce8dcc877..7b38f6419f7c09d7a1a2c123652f39cb0ff070cf 100644 --- a/merge_abc/src/compositeProgramProto.h +++ b/es2panda/util/workerQueue.h @@ -1,5 +1,5 @@ -/** - * Copyright (c) 2022 Huawei Device Co., Ltd. +/* + * Copyright (c) 2021-2023 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -13,21 +13,60 @@ * limitations under the License. */ -#ifndef MERGE_ABC_BUILD_COMPOSITE_PROGRAM_H -#define MERGE_ABC_BUILD_COMPOSITE_PROGRAM_H +#ifndef ES2PANDA_UTIL_WORKERQUEUE_H +#define ES2PANDA_UTIL_WORKERQUEUE_H -#include "assemblyProgramProto.h" -#include "compositeProgram.pb.h" -#include "programCache.h" +#include +#include +#include -namespace panda::proto { -class CompositeProgram { +#include +#include + +namespace panda::es2panda::util { + +class WorkerJob { public: - static void Serialize(const std::map &compositeProgramMap, - bool isDebug, protoPanda::CompositeProgram &protoCompositeProgram); - static void Deserialize(const protoPanda::CompositeProgram &protoCompositeProgram, - std::map &compositeProgramMap, - panda::ArenaAllocator *allocator); + explicit WorkerJob() {}; + NO_COPY_SEMANTIC(WorkerJob); + NO_MOVE_SEMANTIC(WorkerJob); + virtual ~WorkerJob() = default; + + virtual void Run() = 0; + void DependsOn(WorkerJob *job); + void Signal(); + +protected: + std::mutex m_; + std::condition_variable cond_; + WorkerJob *dependant_ {}; + size_t dependencies_ {0}; }; -} // panda::proto + +class WorkerQueue { +public: + explicit WorkerQueue(size_t threadCount); + NO_COPY_SEMANTIC(WorkerQueue); + NO_MOVE_SEMANTIC(WorkerQueue); + virtual ~WorkerQueue(); + + virtual void Schedule() = 0; + void Consume(); + void Wait(); + +protected: + static void Worker(WorkerQueue *queue); + + std::vector threads_; + std::vector errors_; + std::mutex m_; + std::condition_variable jobsAvailable_; + std::condition_variable jobsFinished_; + std::vector jobs_ {}; + size_t jobsCount_ {0}; + size_t activeWorkers_ {0}; + bool terminate_ {false}; +}; +} // namespace panda::es2panda::util + #endif diff --git a/merge_abc/BUILD.gn b/merge_abc/BUILD.gn index 05c1b0f16064fe03f9edd427f02a6e4411ae30b8..fc9706109689c747175554eec3838663f0f9047a 100644 --- a/merge_abc/BUILD.gn +++ b/merge_abc/BUILD.gn @@ -30,7 +30,6 @@ protobuf_snapshot_generator_sources = [ "src/assemblyProgramProto.cpp", "src/assemblyRecordProto.cpp", "src/assemblyTypeProto.cpp", - "src/compositeProgramProto.cpp", "src/ideHelpersProto.cpp", "src/mergeProgram.cpp", "src/metaProto.cpp", @@ -48,7 +47,8 @@ config("panda_assembly_proto_public_config") { "//third_party/protobuf/src", "//third_party/protobuf/src/google", "//third_party/protobuf/src/google/protobuf", - "//arkcompiler/ets_frontend/es2panda/util", + "../es2panda", + "../es2panda/util", ] } diff --git a/merge_abc/protos/compositeProgram.proto b/merge_abc/protos/compositeProgram.proto index 0177aba504bd216c91aac086f0f8c5a915483146..1eccc686a27eceec6cebc9eaa01c3488f4784ecd 100644 --- a/merge_abc/protos/compositeProgram.proto +++ b/merge_abc/protos/compositeProgram.proto @@ -19,12 +19,6 @@ package protoPanda; import "assemblyProgram.proto"; message ProgramCache { - bytes fileName = 1; - uint32 hashCode = 2; - Program program = 3; -} - -message CompositeProgram { - repeated ProgramCache ProgramCache = 1; - bool isDebug = 2; + uint32 hashCode = 1; + Program program = 2; } diff --git a/merge_abc/src/compositeProgramProto.cpp b/merge_abc/src/compositeProgramProto.cpp deleted file mode 100644 index 7cfb1c2590ac825a1a6109068deb1991aee9d690..0000000000000000000000000000000000000000 --- a/merge_abc/src/compositeProgramProto.cpp +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright (c) 2022 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "compositeProgramProto.h" - -namespace panda::proto { -void CompositeProgram::Serialize(const std::map &compositeProgramMap, - bool isDebug, protoPanda::CompositeProgram &protoCompositeProgram) -{ - for (const auto &[fileName, programCache] : compositeProgramMap) { - auto *protoProgramcache = protoCompositeProgram.add_programcache(); - protoProgramcache->set_filename(fileName); - protoProgramcache->set_hashcode(programCache->hashCode); - auto *protoProgram = protoProgramcache->mutable_program(); - Program::Serialize(*(programCache->program), *protoProgram); - } - protoCompositeProgram.set_isdebug(isDebug); -} - -void CompositeProgram::Deserialize(const protoPanda::CompositeProgram &protoCompositeProgram, - std::map &compositeProgramMap, - panda::ArenaAllocator *allocator) -{ - for (const auto &protoProgramcache : protoCompositeProgram.programcache()) { - auto &fileName = protoProgramcache.filename(); - auto hashCode = protoProgramcache.hashcode(); - auto &protoProgram = protoProgramcache.program(); - auto *program = allocator->New(); - Program::Deserialize(protoProgram, *program, allocator); - auto *programCache = allocator->New(hashCode, program); - compositeProgramMap.insert({fileName, programCache}); - } -} -} // namespace panda::proto diff --git a/merge_abc/src/protobufSnapshotGenerator.cpp b/merge_abc/src/protobufSnapshotGenerator.cpp index 4c7fb86e046657d096b897171ee4d1a025f73511..ba772e5557d0d52191e183529b63689d8a1c54de 100644 --- a/merge_abc/src/protobufSnapshotGenerator.cpp +++ b/merge_abc/src/protobufSnapshotGenerator.cpp @@ -50,24 +50,8 @@ void ProtobufSnapshotGenerator::GenerateProgram(const std::string &inputName, pa Program::Deserialize(proto_program, prog, allocator); } -void ProtobufSnapshotGenerator::UpdateCacheFile( - const std::map &compositeProgramMap, - const bool &isDebug, const std::string &cacheFilePath) -{ - protoPanda::CompositeProgram protoCompositeProgram; - CompositeProgram::Serialize(compositeProgramMap, isDebug, protoCompositeProgram); - std::fstream output(panda::os::file::File::GetExtendedFilePath(cacheFilePath), - std::ios::out | std::ios::trunc | std::ios::binary); - if (!output) { - std::cerr << "Failed to create cache file: " << cacheFilePath << std::endl; - return; - } - protoCompositeProgram.SerializeToOstream(&output); - output.close(); -} - -std::map *ProtobufSnapshotGenerator::GetCacheContext( - const std::string &cacheFilePath, bool isDebug, panda::ArenaAllocator *allocator) +panda::es2panda::util::ProgramCache *ProtobufSnapshotGenerator::GetCacheContext(const std::string &cacheFilePath, + panda::ArenaAllocator *allocator) { std::fstream input(panda::os::file::File::GetExtendedFilePath(cacheFilePath), std::ios::in | std::ios::binary); @@ -75,19 +59,37 @@ std::map *ProtobufSnapshotGen std::cout << "Cache file: " << cacheFilePath << " doesn't exist" << std::endl; return nullptr; } - protoPanda::CompositeProgram protoCompositeProgram; - if (!protoCompositeProgram.ParseFromIstream(&input)) { + + protoPanda::ProgramCache protoCache; + if (!protoCache.ParseFromIstream(&input)) { std::cerr << "Failed to parse cache file: " << cacheFilePath << std::endl; return nullptr; } - if (protoCompositeProgram.isdebug() != isDebug) { - return nullptr; - } + auto *program = allocator->New(); + Program::Deserialize(protoCache.program(), *program, allocator); + uint32_t hashCode = protoCache.hashcode(); + auto *programCache = allocator->New(hashCode, std::move(*program)); + + return programCache; +} - auto compositeProgramMap = allocator->New>(); - CompositeProgram::Deserialize(protoCompositeProgram, *compositeProgramMap, allocator); +void ProtobufSnapshotGenerator::UpdateCacheFile(const panda::es2panda::util::ProgramCache *programCache, + const std::string &cacheFilePath) +{ + protoPanda::ProgramCache protoCache; + protoCache.set_hashcode(programCache->hashCode); + auto *protoProgram = protoCache.mutable_program(); + Program::Serialize(programCache->program, *protoProgram); - return compositeProgramMap; + std::fstream output(panda::os::file::File::GetExtendedFilePath(cacheFilePath), + std::ios::out | std::ios::trunc | std::ios::binary); + if (!output) { + std::cerr << "Failed to create cache file: " << cacheFilePath << std::endl; + return; + } + protoCache.SerializeToOstream(&output); + output.close(); } + } // panda::proto diff --git a/merge_abc/src/protobufSnapshotGenerator.h b/merge_abc/src/protobufSnapshotGenerator.h index 598143b42dc97f59cd233555929b3369b70376ca..3cc0d4cdec0ca483fabff7ced766bd1f89e82520 100644 --- a/merge_abc/src/protobufSnapshotGenerator.h +++ b/merge_abc/src/protobufSnapshotGenerator.h @@ -17,7 +17,8 @@ #define MERGE_ABC_POROBUFSNAPSHOTGENERATOR_H #include "assemblyProgramProto.h" -#include "compositeProgramProto.h" +#include "compositeProgram.pb.h" +#include "es2panda.h" namespace panda::proto { class ProtobufSnapshotGenerator { @@ -25,10 +26,10 @@ public: static void GenerateSnapshot(const panda::pandasm::Program &program, const std::string &outputName); static void GenerateProgram(const std::string &inputName, panda::pandasm::Program &prog, panda::ArenaAllocator *allocator); - static std::map *GetCacheContext( - const std::string &cacheFilePath, bool isDebug, panda::ArenaAllocator *allocator); - static void UpdateCacheFile(const std::map &compositeProgramMap, - const bool &isDebug, const std::string &cacheFilePath); + static panda::es2panda::util::ProgramCache *GetCacheContext(const std::string &cacheFilePath, + panda::ArenaAllocator *allocator); + static void UpdateCacheFile(const panda::es2panda::util::ProgramCache *programCache, + const std::string &cacheFilePath); }; } // panda::proto #endif