From e1b5c5cd44523e7eb5e6ba747ab52539d953dfba Mon Sep 17 00:00:00 2001 From: nieben Date: Thu, 25 Jan 2024 14:26:21 +0800 Subject: [PATCH] incre restore 2 Signed-off-by: nieben --- .../native/backup_ext/include/ext_extension.h | 14 + .../native/backup_ext/src/ext_extension.cpp | 487 +++++++++++++++++- 2 files changed, 499 insertions(+), 2 deletions(-) diff --git a/frameworks/native/backup_ext/include/ext_extension.h b/frameworks/native/backup_ext/include/ext_extension.h index b822b23f4..09a916cde 100644 --- a/frameworks/native/backup_ext/include/ext_extension.h +++ b/frameworks/native/backup_ext/include/ext_extension.h @@ -43,6 +43,7 @@ public: void AsyncTaskRestoreForUpgrade(void); void ExtClear(void); + void AsyncTaskIncrementalRestoreForUpgrade(void); public: explicit BackupExtExtension(const std::shared_ptr &extension) : extension_(extension) @@ -75,6 +76,13 @@ private: */ int DoRestore(const string &fileName); + /** + * @brief incremental restore + * + * @param fileName name of the file that to be untar + */ + int DoIncrementalRestore(const string &fileName); + /** @brief clear backup restore data */ void DoClear(); @@ -100,6 +108,12 @@ private: */ void AsyncTaskRestore(); + /** + * @brief Executing Incremental Restoration Tasks Asynchronously + * + */ + void AsyncTaskIncrementalRestore(); + void AsyncTaskOnBackup(); private: diff --git a/frameworks/native/backup_ext/src/ext_extension.cpp b/frameworks/native/backup_ext/src/ext_extension.cpp index 0c0c71961..541a3d92f 100644 --- a/frameworks/native/backup_ext/src/ext_extension.cpp +++ b/frameworks/native/backup_ext/src/ext_extension.cpp @@ -38,6 +38,7 @@ #include "b_filesystem/b_file.h" #include "b_json/b_json_cached_entity.h" #include "b_json/b_json_entity_ext_manage.h" +#include "b_json/b_report_entity.h" #include "b_resources/b_constants.h" #include "b_tarball/b_tarball_factory.h" #include "filemgmt_libhilog.h" @@ -141,9 +142,60 @@ UniqueFd BackupExtExtension::GetFileHandle(const string &fileName) } } +static string GetReportFileName(const string &fileName) +{ + string reportName = fileName + "." + string(BConstants::REPORT_FILE_EXT); + return reportName; +} + ErrCode BackupExtExtension::GetIncrementalFileHandle(const string &fileName) { - return ERR_OK; + try { + if (extension_->GetExtensionAction() != BConstants::ExtensionAction::RESTORE) { + HILOGI("Failed to get file handle, because action is %{public}d invalid", extension_->GetExtensionAction()); + throw BError(BError::Codes::EXT_INVAL_ARG, "Action is invalid"); + } + + VerifyCaller(); + + string path = string(BConstants::PATH_BUNDLE_BACKUP_HOME).append(BConstants::SA_BUNDLE_BACKUP_RESTORE); + if (mkdir(path.data(), S_IRWXU) && errno != EEXIST) { + string str = string("Failed to create restore folder. ").append(std::generic_category().message(errno)); + throw BError(BError::Codes::EXT_INVAL_ARG, str); + } + + string tarName = path + fileName; + if (fileName.find('/') != string::npos) { + HILOGD("GetFileHandle: fileName include path symbol, need to make hash."); + tarName = path + GenerateHashForFileName(fileName); + if (CheckIfTarSuffix(fileName)) { + tarName += ".tar"; + } + } + if (access(tarName.c_str(), F_OK) == 0) { + throw BError(BError::Codes::EXT_INVAL_ARG, string("The file already exists")); + } + UniqueFd fd = UniqueFd(open(tarName.data(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR)); + + // 对应的简报文件 + string reportName = GetReportFileName(tarName); + if (access(reportName.c_str(), F_OK) == 0) { + throw BError(BError::Codes::EXT_INVAL_ARG, string("The report file already exists")); + } + UniqueFd reportFd = UniqueFd(open(reportName.data(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR)); + + auto proxy = ServiceProxy::GetInstance(); + auto ret = proxy->AppIncrementalFileReady(fileName, move(fd), move(reportFd)); + if (ret != ERR_OK) { + HILOGI("Failed to AppIncrementalFileReady %{public}d", ret); + } + + return ERR_OK; + } catch (...) { + HILOGE("Failed to get incremental file handle"); + DoClear(); + return BError(BError::Codes::EXT_INVAL_ARG).GetCode(); + } } ErrCode BackupExtExtension::HandleClear() @@ -290,7 +342,53 @@ ErrCode BackupExtExtension::PublishFile(const string &fileName) ErrCode BackupExtExtension::PublishIncrementalFile(const string &fileName) { - return ERR_OK; + HILOGE("begin publish incremental file. fileName is %{public}s", fileName.data()); + try { + if (extension_->GetExtensionAction() != BConstants::ExtensionAction::RESTORE) { + throw BError(BError::Codes::EXT_INVAL_ARG, "Action is invalid"); + } + VerifyCaller(); + + string path = string(BConstants::PATH_BUNDLE_BACKUP_HOME).append(BConstants::SA_BUNDLE_BACKUP_RESTORE); + string tarName = path + fileName; + // 带路径的文件名在这里转换为hash值 + if (fileName.find('/') != string::npos) { + HILOGD("PublishFile: fileName include path symbol, need to make hash."); + tarName = path + GenerateHashForFileName(fileName); + if (CheckIfTarSuffix(fileName)) { + tarName += ".tar"; + } + } + { + BExcepUltils::VerifyPath(tarName, true); + unique_lock lock(lock_); + if (find(tars_.begin(), tars_.end(), fileName) != tars_.end() || access(tarName.data(), F_OK) != 0) { + throw BError(BError::Codes::EXT_INVAL_ARG, "The file does not exist"); + } + tars_.push_back(fileName); + if (!IsAllFileReceived(tars_)) { + return ERR_OK; + } + } + + // 异步执行解压操作 + if (extension_->AllowToBackupRestore()) { + AsyncTaskIncrementalRestore(); + } + + return ERR_OK; + } catch (const BError &e) { + DoClear(); + return e.GetCode(); + } catch (const exception &e) { + HILOGE("Catched an unexpected low-level exception %{public}s", e.what()); + DoClear(); + return BError(BError::Codes::EXT_BROKEN_FRAMEWORK).GetCode(); + } catch (...) { + HILOGE("Unexpected exception"); + DoClear(); + return BError(BError::Codes::EXT_BROKEN_FRAMEWORK).GetCode(); + } } ErrCode BackupExtExtension::HandleBackup() @@ -425,6 +523,65 @@ int BackupExtExtension::DoRestore(const string &fileName) return ERR_OK; } +static unordered_map GetTarIncludes(const string &tarName) +{ + // 根据简报文件获取待解压的文件列表，如果简报文件内容为空，则进行全量解压，返回空 + unordered_map includes; + + // 获取简报文件内容 + string reportName = GetReportFileName(tarName); + + // 获取简报内容 + BReportEntity rp(UniqueFd(open(reportName.data(), O_RDONLY))); + auto infos = rp.GetReportInfos(); + for (auto iter : infos) { + includes.emplace(iter.first, true); + } + + return includes; +} + +int BackupExtExtension::DoIncrementalRestore(const string &fileName) +{ + HILOGI("Do incremental restore"); + if (extension_->GetExtensionAction() != BConstants::ExtensionAction::RESTORE) { + return EPERM; + } + // REM: 给定version + // REM: 解压启动Extension时即挂载好的备份目录中的数据 + string path = string(BConstants::PATH_BUNDLE_BACKUP_HOME).append(BConstants::SA_BUNDLE_BACKUP_RESTORE); + string tarName = path + fileName; + + // 带路径的恢复需要找到hash后的待解压文件 + if (fileName.find('/') != string::npos) { + HILOGD("DoIncrementalRestore: fileName include path symbol, need to make hash."); + string filePath = path + GenerateHashForFileName(fileName); + size_t pos = filePath.rfind('/'); + if (pos == string::npos) { + return EPERM; + } + string folderPath = filePath.substr(0, pos); + if (!ForceCreateDirectory(folderPath.data())) { + HILOGE("Failed to create directory"); + return EPERM; + } + tarName = filePath; + if (CheckIfTarSuffix(fileName)) { + tarName += ".tar"; + } + } + + // 当用户指定fullBackupOnly字段或指定版本的恢复，解压目录当前在/backup/restore + if (extension_->SpeicalVersionForCloneAndCloud() || extension_->UseFullBackupOnly()) { + UntarFile::GetInstance().IncrementalUnPacket(tarName, path, GetTarIncludes(tarName)); + } else { + UntarFile::GetInstance().IncrementalUnPacket(tarName, "/", GetTarIncludes(tarName)); + } + HILOGI("Application recovered successfully, package path is %{public}s", tarName.c_str()); + + return ERR_OK; +} + void BackupExtExtension::AsyncTaskBackup(const string config) { auto task = [obj {wptr(this)}, config]() { @@ -638,6 +795,36 @@ static void DeleteBackupTars() } } +static void DeleteBackupIncrementalTars() +{ + // The directory include tars and manage.json which would be deleted + BJsonCachedEntity cachedEntity(UniqueFd(open(INDEX_FILE_RESTORE.data(), O_RDONLY))); + auto cache = cachedEntity.Structuralize(); + auto info = cache.GetExtManage(); + auto path = string(BConstants::PATH_BUNDLE_BACKUP_HOME).append(BConstants::SA_BUNDLE_BACKUP_RESTORE); + for (auto &item : info) { + if (ExtractFileExt(item) != "tar" || IsUserTar(item, INDEX_FILE_RESTORE)) { + continue; + } + string tarPath = path + item; + if (!RemoveFile(tarPath)) { + HILOGE("Failed to delete the backup tar %{public}s", tarPath.c_str()); + } + // 删除简报文件 + string reportPath = GetReportFileName(tarPath); + if (!RemoveFile(reportPath)) { + HILOGE("Failed to delete the backup report"); + } + } + if (!RemoveFile(INDEX_FILE_RESTORE)) { + HILOGE("Failed to delete the backup index %{public}s", INDEX_FILE_RESTORE.c_str()); + } + string reportManagePath = GetReportFileName(INDEX_FILE_RESTORE); // GetIncrementalFileHandle创建的空fd + if (!RemoveFile(reportManagePath)) { + HILOGE("Failed to delete the backup report index %{public}s", reportManagePath.c_str()); + } +} + void BackupExtExtension::AsyncTaskRestore() { auto task = [obj {wptr(this)}, tars {tars_}]() { @@ -700,6 +887,59 @@ void BackupExtExtension::AsyncTaskRestore() }); } +void BackupExtExtension::AsyncTaskIncrementalRestore() +{ + auto task = [obj {wptr(this)}, tars {tars_}]() { + auto ptr = obj.promote(); + BExcepUltils::BAssert(ptr, BError::Codes::EXT_BROKEN_FRAMEWORK, + "Ext extension handle have been already released"); + try { + // 解压 + int ret = ERR_OK; + for (auto item : tars) { // 处理要解压的tar文件 + if (ExtractFileExt(item) == "tar" && !IsUserTar(item, INDEX_FILE_RESTORE)) { + ret = ptr->DoIncrementalRestore(item); + } + } + // 恢复用户tar包以及大文件 + // 目的地址是否需要拼接path(临时目录)，FullBackupOnly为true并且非特殊场景 + bool appendTargetPath = ptr->extension_->UseFullBackupOnly() && + !ptr->extension_->SpeicalVersionForCloneAndCloud(); + RestoreBigFiles(appendTargetPath); + + // delete 1.tar/manage.json + DeleteBackupIncrementalTars(); + + if (ret == ERR_OK) { + HILOGI("after extra, do incremental restore."); + ptr->AsyncTaskIncrementalRestoreForUpgrade(); + } else { + ptr->AppIncrementalDone(ret); + ptr->DoClear(); + } + } catch (const BError &e) { + ptr->AppIncrementalDone(e.GetCode()); + } catch (const exception &e) { + HILOGE("Catched an unexpected low-level exception %{public}s", e.what()); + ptr->AppIncrementalDone(BError(BError::Codes::EXT_INVAL_ARG).GetCode()); + } catch (...) { + HILOGE("Failed to restore the ext bundle"); + ptr->AppIncrementalDone(BError(BError::Codes::EXT_INVAL_ARG).GetCode()); + } + }; + + // REM: 这里异步化了，需要做并发控制 + // 在往线程池中投入任务之前将需要的数据拷贝副本到参数中，保证不发生读写竞争， + // 由于拷贝参数时尚运行在主线程中，故在参数拷贝过程中是线程安全的。 + threadPool_.AddTask([task]() { + try { + task(); + } catch (...) { + HILOGE("Failed to add task to thread pool"); + } + }); +} + void BackupExtExtension::AsyncTaskRestoreForUpgrade() { auto task = [obj {wptr(this)}]() { @@ -749,6 +989,49 @@ void BackupExtExtension::ExtClear() DoClear(); } +void BackupExtExtension::AsyncTaskIncrementalRestoreForUpgrade() +{ + auto task = [obj {wptr(this)}]() { + auto ptr = obj.promote(); + try { + BExcepUltils::BAssert(ptr, BError::Codes::EXT_BROKEN_FRAMEWORK, + "Ext extension handle have been already released"); + BExcepUltils::BAssert(ptr->extension_, BError::Codes::EXT_INVAL_ARG, + "extension handle have been already released"); + + auto callBackup = [obj]() { + HILOGI("begin call restore"); + auto extensionPtr = obj.promote(); + BExcepUltils::BAssert(extensionPtr, BError::Codes::EXT_BROKEN_FRAMEWORK, + "Ext extension handle have been already released"); + extensionPtr->AppIncrementalDone(BError(BError::Codes::OK)); + // 清空恢复目录 + extensionPtr->DoClear(); + }; + ptr->extension_->OnRestore(callBackup); + } catch (const BError &e) { + ptr->AppIncrementalDone(e.GetCode()); + } catch (const exception &e) { + HILOGE("Catched an unexpected low-level exception %{public}s", e.what()); + ptr->AppIncrementalDone(BError(BError::Codes::EXT_INVAL_ARG).GetCode()); + } catch (...) { + HILOGE("Failed to restore the ext bundle"); + ptr->AppIncrementalDone(BError(BError::Codes::EXT_INVAL_ARG).GetCode()); + } + }; + + // REM: 这里异步化了，需要做并发控制 + // 在往线程池中投入任务之前将需要的数据拷贝副本到参数中，保证不发生读写竞争， + // 由于拷贝参数时尚运行在主线程中，故在参数拷贝过程中是线程安全的。 + threadPool_.AddTask([task]() { + try { + task(); + } catch (...) { + HILOGE("Failed to add task to thread pool"); + } + }); +} + void BackupExtExtension::DoClear() { try { @@ -848,6 +1131,206 @@ ErrCode BackupExtExtension::HandleRestore() return 0; } +static map GetIncrementalFile(const UniqueFd &fd) +{ + vector> infos = BFile::ReadIncrementalFile(fd); + map files; + if (infos.begin() != infos.end()) { + for (auto info : infos) { + string filePath = info.find("path") != info.end() ? info.find("path")->second : ""; + string mode = info.find("mode") != info.end() ? info.find("mode")->second : ""; + string dir = info.find("dir") != info.end() ? info.find("dir")->second : "0"; + int64_t size = 0; + try { + size = info.find("size") != info.end() ? stoll(info.find("size")->second) : 0; + } catch (const std::out_of_range &e) { + size = 0; + } + string mtime = info.find("mtime") != info.end() ? info.find("mtime")->second : "0"; + string isIncr = info.find("isIncr") != info.end() ? info.find("isIncr")->second : "0"; + string hash = info.find("hash") != info.end() ? info.find("hash")->second : ""; + struct FileInfo file; + file.filePath = filePath; + file.mode = mode; + file.isDir = dir; + file.lmtime = mtime; + file.size = size; + file.fileHash = hash; + file.isIncr = isIncr; + files.try_emplace(filePath, file); + } + } + return files; +} + +static tuple, map, map, + map> CompareFiles(const UniqueFd &cloudFd, const UniqueFd &storageFd) +{ + map cloudFiles = GetIncrementalFile(cloudFd); + map storageFiles = GetIncrementalFile(storageFd); + map allFiles = {}; + map smallFiles = {}; + map bigFiles = {}; + map bigInfos = {}; + for (auto &item : storageFiles) { + // 进行文件对比 + string path = item.first; + if (cloudFiles.find(path) == cloudFiles.end() || (item.second.isIncr == "1" + && (cloudFiles[path].size != item.second.size || cloudFiles[path].lmtime != item.second.lmtime))) { + auto [res, fileHash] = BFileHash::HashWithSHA256(path); + if (fileHash.empty()) { + continue; + } + item.second.fileHash = fileHash; + item.second.isIncr = "1"; + } else { + item.second.fileHash = cloudFiles[path].fileHash; + } + if (cloudFiles.find(path) == cloudFiles.end() || + (item.second.isIncr == "1" && cloudFiles.find(path)->second.fileHash != item.second.fileHash)) { + // 在云空间简报里不存在或者hash不一致 + // 区分大小文件 + struct stat sta = {}; + if (stat(path.c_str(), &sta) == -1) { + continue; + } + if (sta.st_size < BConstants::BIG_FILE_BOUNDARY) { + HILOGI("compareFiles Find small file"); + item.second.size = sta.st_size; + smallFiles.try_emplace(path, item.second); + } else { + HILOGI("compareFiles Find big file"); + bigFiles.try_emplace(path, sta); + bigInfos.try_emplace(path, item.second); + } + } + allFiles.try_emplace(path, item.second); + } + HILOGI("compareFiles Find small files total: %{public}d", smallFiles.size()); + HILOGI("compareFiles Find big files total: %{public}d", bigFiles.size()); + return {allFiles, smallFiles, bigFiles, bigInfos}; +} + +static void WriteFile(const string &filename, const map &srcFiles) +{ + fstream f; + f.open(filename.data(), ios::out); + // 前面2行先填充进去 + f << "version=1.0&attrNum=6" << endl; + f << "path;mode;dir;size;mtime;hash" << endl; + for (auto item : srcFiles) { + struct FileInfo info = item.second; + string str = item.first + ";" + info.mode + ";" + info.isDir + ";" + to_string(info.size); + str += ";" + info.lmtime + ";" + info.fileHash + ";"; + f << str << endl; + } + f.close(); + HILOGI("WriteFile path: %{public}s", filename.c_str()); +} + +/** + * 获取增量的大文件的信息 +*/ +static TarMap GetIncrmentBigInfos(const map &files) +{ + auto getStringHash = [](const TarMap &m, const string &str) -> string { + ostringstream strHex; + strHex << hex; + + hash strHash; + size_t szHash = strHash(str); + strHex << setfill('0') << setw(BConstants::BIG_FILE_NAME_SIZE) << szHash; + string name = strHex.str(); + for (int i = 0; m.find(name) != m.end(); ++i, strHex.str("")) { + szHash = strHash(str + to_string(i)); + strHex << setfill('0') << setw(BConstants::BIG_FILE_NAME_SIZE) << szHash; + name = strHex.str(); + } + + return name; + }; + + TarMap bigFiles; + for (const auto &item : files) { + string md5Name = getStringHash(bigFiles, item.first); + if (!md5Name.empty()) { + bigFiles.emplace(md5Name, make_tuple(item.first, item.second, true)); + } + } + + return bigFiles; +} + +/** + * 增量tar包和简报信息回传 +*/ +static ErrCode IncrementalTarFileReady(const TarMap &bigFileInfo, const map &srcFiles, sptr proxy) +{ + string tarFile = bigFileInfo.begin()->first; + string manageFile = ""; + if (tarFile.rfind(".tar") != string::npos) { + manageFile = tarFile.substr(0, tarFile.length() - 4).append(".txt"); + } else { + manageFile = tarFile + ".txt"; + } + + string file = string(INDEX_FILE_INCREMENTAL_BACKUP).append(manageFile); + WriteFile(file, srcFiles); + + string tarName = string(INDEX_FILE_INCREMENTAL_BACKUP).append(tarFile); + ErrCode ret = + proxy->AppIncrementalFileReady(tarName, UniqueFd(open(tarName.data(), O_RDONLY)), + UniqueFd(open(file.data(), O_RDONLY))); + if (SUCCEEDED(ret)) { + HILOGI("IncrementalTarFileReady: The application is packaged successfully"); + // 删除文件 + RemoveFile(file); + RemoveFile(tarName); + } else { + HILOGI("IncrementalTarFileReady interface fails to be invoked: %{public}d", ret); + } + return ret; +} + +/** + * 增量大文件和简报信息回传 +*/ +static ErrCode IncrementalBigFileReady(const TarMap &pkgInfo, const map &bigInfos, sptr proxy) +{ + ErrCode ret {ERR_OK}; + for (auto &item : pkgInfo) { + if (item.first.empty()) { + continue; + } + auto [path, sta, isBeforeTar] = item.second; + + UniqueFd fd(open(path.data(), O_RDONLY)); + if (fd < 0) { + HILOGE("IncrementalBigFileReady open file failed, file name is %{public}s, err = %{public}d", + path.c_str(), errno); + continue; + } + + struct FileInfo info = bigInfos.find(path)->second; + string file = string(INDEX_FILE_INCREMENTAL_BACKUP).append(item.first).append(".txt"); + HILOGE("IncrementalBigFileReady write name is %{public}s", path.c_str()); + map bigInfo; + bigInfo.try_emplace(path, info); + WriteFile(file, bigInfo); + + ret = proxy->AppIncrementalFileReady(item.first, std::move(fd), UniqueFd(open(file.data(), O_RDONLY))); + if (SUCCEEDED(ret)) { + HILOGI("IncrementalBigFileReady ： The application is packaged successfully, package name is %{public}s", + item.first.c_str()); + RemoveFile(file); + } else { + HILOGI( + "IncrementalBigFileReady interface fails to be invoked: %{public}d", ret); + } + } + return ret; +} + ErrCode BackupExtExtension::HandleIncrementalBackup(UniqueFd incrementalFd, UniqueFd manifestFd) { return 0; -- Gitee