diff --git a/backup.gni b/backup.gni index 972c4a00babaeb099d4891d0e9b2f0abede5f5c5..d6149f38fc0d9cf5085fafd1ef12571c46b97db7 100644 --- a/backup.gni +++ b/backup.gni @@ -41,3 +41,16 @@ backup_mock_proxy_src = [ "$path_backup_mock/utils_mock/src/utils_mock_global_variable.cpp", "$path_backup/frameworks/native/backup_kit_inner/src/b_file_info.cpp", ] + +declare_args() { + brotli_enabled = true + if (defined(global_parts_info) && + (!defined(global_parts_info.thirdparty_brotli) || !global_parts_info.thirdparty_brotli)) { + brotli_enabled = false + } + if (current_cpu == "arm" || current_cpu == "x86") { + system_bit = 32 + } else { + system_bit = 64 + } +} \ No newline at end of file diff --git a/bundle.json b/bundle.json index 48ba56c11d81fa4746c2bac36edbf8cd8a2a9a53..2fb93834cd953f8807f3e34038b538276fd487f0 100644 --- a/bundle.json +++ b/bundle.json @@ -23,6 +23,7 @@ "ability_base", "ability_runtime", "access_token", + "brotli", "bundle_framework", "common_event_service", "cJSON", diff --git a/frameworks/native/backup_ext/BUILD.gn b/frameworks/native/backup_ext/BUILD.gn index 85d6d2369add6f537dcc589edcde191f2e40c03a..27a36aae524a477fe7aeeb4f22758f3f01e17338 100644 --- a/frameworks/native/backup_ext/BUILD.gn +++ b/frameworks/native/backup_ext/BUILD.gn @@ -79,6 +79,16 @@ ohos_shared_library("backup_extension_ability_native") { "samgr:samgr_proxy", ] + if (brotli_enabled) { + external_deps += ["brotli:brotli_shared"] + defines += ["BROTLI_ENABLED"] + } + if (system_bit == 32) { + defines += ["SYSTEM_BIT_32"] + } else { + defines += ["SYSTEM_BIT_64"] + } + cflags_cc = [ "-fdata-sections", "-ffunction-sections", diff --git a/frameworks/native/backup_ext/include/tar_file.h b/frameworks/native/backup_ext/include/tar_file.h index 0a875704c644cf5b60b5d73ebd36963ae92d8abf..bed12aacd16b76acd035d2fff867d3cdb5a3c7f6 100644 --- a/frameworks/native/backup_ext/include/tar_file.h +++ b/frameworks/native/backup_ext/include/tar_file.h @@ -59,6 +59,12 @@ const char GNUTYPE_LONGNAME = 'L'; const char EXTENSION_HEADER = 'x'; const uint32_t OTHER_HEADER = 78; const int ERR_NO_PERMISSION = 13; +#ifdef SYSTEM_BIT_32 +constexpr int SIZE_T_BYTE_LEN = 4; +#else +constexpr int SIZE_T_BYTE_LEN = 8; +#endif +constexpr size_t MAX_BUFFER_SIZE = 4096; } // namespace // 512 bytes @@ -81,7 +87,29 @@ using TarHeader = struct { char prefix[PREFIX_LEN]; char pad[PADDING_LEN]; }; + using TarMap = std::map>; + +class UniqueFile { +public: + UniqueFile(const char* filePath, const char* mode); + UniqueFile(const UniqueFile&) = delete; + UniqueFile& operator=(const UniqueFile&) = delete; + ~UniqueFile(); + FILE* file_ = nullptr; +}; + +template +class Buffer { +public: + Buffer(size_t size); + Buffer(const Buffer&) = delete; + Buffer& operator=(const Buffer&) = delete; + ~Buffer(); + T* data_ = nullptr; + size_t size_ = 0; +}; + class TarFile { public: static TarFile &GetInstance(); @@ -100,7 +128,16 @@ public: void SetPacketMode(bool isReset); uint64_t GetTarFileSize() { return static_cast(currentTarFileSize_); } + + bool Compress(const uint8_t* inputBuffer, size_t inputSize, uint8_t* outputBuffer, size_t* outputSize); + bool Decompress(const uint8_t* inputBuffer, size_t inputSize, uint8_t* outputBuffer, size_t* outputSize); + bool CompressFile(const std::string &srcFile, const std::string &compFile); + bool DecompressFile(const std::string &compFile, const std::string &srcFile); + std::string DecompressTar(const std::string &tarPath); private: + bool WriteCompressData(Buffer& compressBuffer, const Buffer& ori, UniqueFile& fout); + bool WriteDecompressData(const Buffer& compressBuffer, Buffer& decompressBuffer, + UniqueFile& fout, std::chrono::duration& decompSpan); TarFile() {} ~TarFile() = default; TarFile(const TarFile &instance) = delete; diff --git a/frameworks/native/backup_ext/src/ext_extension.cpp b/frameworks/native/backup_ext/src/ext_extension.cpp index ee46dd698d34f3adeb6bcb96fa9c76a7addace8d..3f19642797412d74164a9ef0bfad504cf22d5bf6 100644 --- a/frameworks/native/backup_ext/src/ext_extension.cpp +++ b/frameworks/native/backup_ext/src/ext_extension.cpp @@ -1006,6 +1006,7 @@ int BackupExtExtension::DoIncrementalRestore() HILOGE("Check incre tarfile path : %{public}s err, path is forbidden", GetAnonyPath(tarName).c_str()); return BError(BError::Codes::EXT_FORBID_BACKUP_RESTORE).GetCode(); } + tarName = TarFile::GetInstance().DecompressTar(tarName); unordered_map result; GetTarIncludes(tarName, result); if ((!extension_->SpecialVersionForCloneAndCloud()) && (!extension_->UseFullBackupOnly())) { diff --git a/frameworks/native/backup_ext/src/tar_file.cpp b/frameworks/native/backup_ext/src/tar_file.cpp index bf77dc21246b1686250eccd7a055dc4525b10a2a..5b5d718b0171a786b0f5fa74a3532af2f3e797a0 100644 --- a/frameworks/native/backup_ext/src/tar_file.cpp +++ b/frameworks/native/backup_ext/src/tar_file.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023-2024 Huawei Device Co., Ltd. + * Copyright (c) 2023-2025 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -15,14 +15,16 @@ #include "tar_file.h" +#include +#include #include #include +#include #include #include #include #include #include - #include "b_anony/b_anony.h" #include "b_error/b_error.h" #include "b_hiaudit/hi_audit.h" @@ -31,6 +33,11 @@ #include "filemgmt_libhilog.h" #include "securec.h" +#ifdef BROTLI_ENABLED +#include "brotli/encode.h" +#include "brotli/decode.h" +#endif + namespace OHOS::FileManagement::Backup { using namespace std; namespace { @@ -44,8 +51,57 @@ const uint32_t WAIT_INDEX = 100000; const uint32_t WAIT_TIME = 5; const string VERSION = "1.0"; const string LONG_LINK_SYMBOL = "longLinkSymbol"; +const string COMPRESS_FILE_SUFFIX = "__A"; +const string TAR_EXTENSION = ".tar"; +#ifdef BROTLI_ENABLED +constexpr int MEGA_BYTE = 1024 * 1024; +constexpr int BROTLI_QUALITY = 3; +constexpr bool USE_COMPRESS = false; // 默认关闭 +#else +constexpr bool USE_COMPRESS = false; +#endif } // namespace +UniqueFile::UniqueFile(const char* filePath, const char* mode) +{ + file_ = fopen(filePath, mode); + if (file_ == nullptr) { + HILOGE("open file fail err: %{public}s", strerror(errno)); + } +} + +UniqueFile::~UniqueFile() +{ + if (file_ != nullptr) { + if (fclose(file_) == EOF) { + HILOGE("close file fail err: %{public}s", strerror(errno)); + } + } + file_ = nullptr; +} + +template +Buffer::Buffer(size_t size) +{ + if (size <= 0 || size > MAX_BUFFER_SIZE) { + return; + } + data_ = new (std::nothrow) T[size]; + if (data_ == nullptr) { + HILOGE("new fail, size=%{public}zu", size); + return; + } + size_ = size; +} + +template +Buffer::~Buffer() +{ + if (data_ != nullptr) { + delete[] data_; + } +} + TarFile &TarFile::GetInstance() { static TarFile instance; @@ -449,13 +505,27 @@ bool TarFile::FillSplitTailBlocks() if (isReset_) { tarMap_.clear(); } - - tarMap_.emplace(tarFileName_, make_tuple(currentTarName_, staTar, false)); + if (USE_COMPRESS) { + std::filesystem::path fullTarPath = currentTarName_; // 全量路径 + std::filesystem::path parentPath = fullTarPath.parent_path(); + std::filesystem::path fileNameWithoutExtension = fullTarPath.stem(); // 文件名前缀 + std::string tarNewName = fileNameWithoutExtension.string() + COMPRESS_FILE_SUFFIX + TAR_EXTENSION; + string newTarPath = parentPath.string() + "/" + tarNewName; + HILOGI("tarFileName:%{public}s, currentTarName:%{public}s, newTarPath:%{public}s", + GetAnonyPath(tarFileName_).c_str(), GetAnonyPath(currentTarName_).c_str(), + GetAnonyPath(newTarPath).c_str()); + if (!CompressFile(fullTarPath, newTarPath)) { + tarMap_.emplace(tarFileName_, make_tuple(currentTarName_, staTar, false)); + } else { + tarMap_.emplace(tarNewName, make_tuple(newTarPath, staTar, false)); + } + } else { + tarMap_.emplace(tarFileName_, make_tuple(currentTarName_, staTar, false)); + } fclose(currentTarFile_); currentTarFile_ = nullptr; tarFileCount_++; - return true; } @@ -648,4 +718,187 @@ void TarFile::SetPacketMode(bool isReset) { isReset_ = isReset; } + +bool TarFile::Compress(const uint8_t* inputBuffer, size_t inputSize, uint8_t* outputBuffer, size_t* outputSize) +{ +#ifdef BROTLI_ENABLED + int ret = BrotliEncoderCompress(BROTLI_QUALITY, BROTLI_MAX_WINDOW_BITS, BROTLI_DEFAULT_MODE, + inputSize, inputBuffer, outputSize, outputBuffer); + if (ret != BROTLI_TRUE) { + HILOGE("compress fail, error: %{public}d", ret); + return false; + } +#endif + return true; +} + +bool TarFile::Decompress(const uint8_t* inputBuffer, size_t inputSize, uint8_t* outputBuffer, size_t* outputSize) +{ +#ifdef BROTLI_ENABLED + int ret = BrotliDecoderDecompress(inputSize, inputBuffer, outputSize, outputBuffer); + if (ret != BROTLI_TRUE) { + HILOGE("decompress fail, error: %{public}d", ret); + return false; + } +#endif + return true; +} + +bool TarFile::WriteCompressData(Buffer& compressBuffer, const Buffer& ori, UniqueFile& fout) +{ + size_t sizeCount = 1; + char* writeData = (char *)compressBuffer.data_; + size_t writeDataSize = compressBuffer.size_; + if (compressBuffer.size_ >= ori.size_) { // 压缩后大小大于原始大小则直接存储原始内容 + compressBuffer.size_ = ori.size_; + writeData = ori.data_; + writeDataSize = ori.size_; + } + if (fwrite(&compressBuffer.size_, SIZE_T_BYTE_LEN, sizeCount, fout.file_) != sizeCount) { + HILOGE("write compress size fail error: %{public}s", strerror(errno)); + return false; + } + if (fwrite(&ori.size_, SIZE_T_BYTE_LEN, sizeCount, fout.file_) != sizeCount) { + HILOGI("write ori size fail error: %{public}s", strerror(errno)); + return false; + } + if (fwrite(writeData, 1, writeDataSize, fout.file_) != writeDataSize) { + HILOGI("write compress data fail error: %{public}s", strerror(errno)); + return false; + } + return true; +} + +bool TarFile::CompressFile(const std::string &srcFile, const std::string &compFile) +{ +#ifdef BROTLI_ENABLED + HILOGI("BEGIN strF: %{public}s, compF: %{public}s", GetAnonyPath(srcFile).c_str(), GetAnonyPath(compFile).c_str()); + UniqueFile fin(srcFile.c_str(), "rb"); + UniqueFile fout(compFile.c_str(), "wb"); + if (fin.file_ == nullptr || fout.file_ == nullptr) { + HILOGE("open file fail!"); + return false; + } + Buffer ori(BLOCK_SIZE); + if (ori.data_ == nullptr) { + HILOGE("new ori buffer fail!"); + return false; + } + size_t maxSize = BrotliEncoderMaxCompressedSize(BLOCK_SIZE); + Buffer compressBuffer(maxSize); + if (compressBuffer.data_ == nullptr) { + HILOGE("new compress buffer fail!"); + return false; + } + size_t inTotal = 0; + size_t outTotal = 0; + auto compSpan = std::chrono::duration(std::chrono::seconds(0)); + while ((ori.size_ = fread(ori.data_, 1, BLOCK_SIZE, fin.file_)) > 0) { + compressBuffer.size_ = maxSize; + auto startTime = std::chrono::high_resolution_clock::now(); + if (!Compress((const uint8_t*)(ori.data_), ori.size_, compressBuffer.data_, &compressBuffer.size_)) { + return false; + } + compSpan += (std::chrono::high_resolution_clock::now() - startTime); + if (!WriteCompressData(compressBuffer, ori, fout)) { + return false; + } + inTotal += ori.size_; + outTotal += compressBuffer.size_; + } + HILOGI("END srcSize:%{public}zu, destSize:%{public}zu, rate:%{public}f, time:%{public}f ms, speed:%{public}f MB/s", + inTotal, outTotal, (outTotal == 0) ? 0 : (inTotal * 1.0f / outTotal), compSpan.count(), + (compSpan.count() == 0) ? 0 : inTotal * 1000.0f / MEGA_BYTE / compSpan.count()); +#endif + return true; +} + +bool TarFile::WriteDecompressData(const Buffer& compressBuffer, Buffer& decompressBuffer, + UniqueFile& fout, std::chrono::duration& decompSpan) +{ + size_t written = 0; + if (compressBuffer.size_ == decompressBuffer.size_) { + written += fwrite(compressBuffer.data_, 1, compressBuffer.size_, fout.file_); + } else { + auto startTime = std::chrono::high_resolution_clock::now(); + if (!Decompress((const uint8_t*)compressBuffer.data_, compressBuffer.size_, decompressBuffer.data_, + &decompressBuffer.size_)) { + return false; + } + decompSpan += (std::chrono::high_resolution_clock::now() - startTime); + written += fwrite(decompressBuffer.data_, 1, decompressBuffer.size_, fout.file_); + } + if (written != decompressBuffer.size_) { + HILOGI("write data fail error: %{public}s", strerror(errno)); + return false; + } + return true; +} + +bool TarFile::DecompressFile(const std::string &compFile, const std::string &srcFile) +{ +#ifdef BROTLI_ENABLED + HILOGI("BEGIN, compF:%{public}s, srcF:%{public}s", GetAnonyPath(compFile).c_str(), GetAnonyPath(srcFile).c_str()); + UniqueFile fin(compFile.c_str(), "rb"); + UniqueFile fout(srcFile.c_str(), "wb"); + if (fin.file_ == nullptr || fout.file_ == nullptr) { + HILOGE("open file fail!"); + return false; + } + Buffer decompressBuffer(BLOCK_SIZE); + Buffer compressBuffer(BLOCK_SIZE); + if (decompressBuffer.data_ == nullptr || compressBuffer.data_ == nullptr) { + HILOGE("new buffer fail!"); + return false; + } + size_t inTotal = 0; + size_t outTotal = 0; + size_t size; + auto decompSpan = std::chrono::duration(std::chrono::seconds(0)); + while ((size = fread(&compressBuffer.size_, SIZE_T_BYTE_LEN, 1, fin.file_)) > 0) { + if (size != 1) { + HILOGE("read comp size fail"); + return false; + } + if (compressBuffer.size_ > BLOCK_SIZE) { + HILOGE("compress size is too big."); + return false; + } + if (fread(&decompressBuffer.size_, SIZE_T_BYTE_LEN, 1, fin.file_) != 1 || + fread(compressBuffer.data_, 1, compressBuffer.size_, fin.file_) != compressBuffer.size_) { + HILOGE("read comp buffer fail"); + return false; + } + if (!WriteDecompressData(compressBuffer, decompressBuffer, fout, decompSpan)) { + return false; + } + inTotal += compressBuffer.size_; + outTotal += decompressBuffer.size_; + } + HILOGI("srcSize:%{public}zu, destSize:%{public}zu, time:%{public}f ms, speed:%{public}f MB/s", inTotal, outTotal, + decompSpan.count(), (decompSpan.count() == 0) ? 0 : outTotal * 1000.0f / MEGA_BYTE / decompSpan.count()); +#endif + return true; +} + +std::string TarFile::DecompressTar(const std::string &tarPath) +{ + if (!USE_COMPRESS) { + return tarPath; + } + std::filesystem::path filePath = tarPath; + std::filesystem::path parentPath = filePath.parent_path(); + std::filesystem::path fileNameWithoutExtension = filePath.stem(); + size_t pos = fileNameWithoutExtension.string().rfind(COMPRESS_FILE_SUFFIX); + if (pos == std::string::npos) { + return tarPath; + } + std::string oriTarName = fileNameWithoutExtension.string().substr(0, pos); + std::string decompressFileName = parentPath.string() + std::filesystem::path::preferred_separator + + oriTarName + TAR_EXTENSION; + if (!DecompressFile(tarPath, decompressFileName)) { + return tarPath; + } + return decompressFileName; +} } // namespace OHOS::FileManagement::Backup \ No newline at end of file diff --git a/test/fuzztest/backupext_fuzzer/BUILD.gn b/test/fuzztest/backupext_fuzzer/BUILD.gn index 94ac5b15e505a98c6227b216cac9366ffbcb2048..5e569b6b32278e827bffa09d7199fb12d6e416e1 100644 --- a/test/fuzztest/backupext_fuzzer/BUILD.gn +++ b/test/fuzztest/backupext_fuzzer/BUILD.gn @@ -82,6 +82,12 @@ ohos_fuzztest("BackupExtFuzzTest") { "private = public", ] + if (system_bit == 32) { + defines += ["SYSTEM_BIT_32"] + } else { + defines += ["SYSTEM_BIT_64"] + } + use_exceptions = true } ############################################################################### diff --git a/tests/unittests/backup_ext/BUILD.gn b/tests/unittests/backup_ext/BUILD.gn index 1f2fccc2362e8c9e4fbcbc666acaf243afa056ef..b350117a43ea02544294d1b770afe943a31d35fe 100644 --- a/tests/unittests/backup_ext/BUILD.gn +++ b/tests/unittests/backup_ext/BUILD.gn @@ -188,6 +188,11 @@ ohos_unittest("tar_file_test") { ] defines = [ "private=public" ] + if (system_bit == 32) { + defines += ["SYSTEM_BIT_32"] + } else { + defines += ["SYSTEM_BIT_64"] + } use_exceptions = true } @@ -269,7 +274,11 @@ ohos_unittest("untar_file_sup_test") { ] defines = [ "private=public" ] - + if (system_bit == 32) { + defines += ["SYSTEM_BIT_32"] + } else { + defines += ["SYSTEM_BIT_64"] + } use_exceptions = true } @@ -344,7 +353,11 @@ ohos_unittest("untar_file_test") { ] defines = [ "private=public" ] - + if (system_bit == 32) { + defines += ["SYSTEM_BIT_32"] + } else { + defines += ["SYSTEM_BIT_64"] + } use_exceptions = true } @@ -450,7 +463,11 @@ ohos_unittest("tar_file_sub_test") { ] defines = [ "private=public" ] - + if (system_bit == 32) { + defines += ["SYSTEM_BIT_32"] + } else { + defines += ["SYSTEM_BIT_64"] + } use_exceptions = true } @@ -485,7 +502,11 @@ ohos_unittest("installd_un_tar_file_test") { ] defines = [ "private=public" ] - + if (system_bit == 32) { + defines += ["SYSTEM_BIT_32"] + } else { + defines += ["SYSTEM_BIT_64"] + } use_exceptions = true }