From 81ccca485b9bd68643d25b8b2654506a6cacccf0 Mon Sep 17 00:00:00 2001 From: yaoruozi <2319695901@qq.com> Date: Thu, 6 Mar 2025 23:58:01 +0800 Subject: [PATCH] =?UTF-8?q?=E8=BE=B9=E6=89=AB=E6=8F=8F=E8=BE=B9=E4=BC=A0?= =?UTF-8?q?=E8=BE=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: yaoruozi <2319695901@qq.com> --- .../native/backup_ext/include/ext_extension.h | 3 + .../native/backup_ext/src/TestScannerFile.cpp | 236 ++++++++++++++++++ 2 files changed, 239 insertions(+) create mode 100644 frameworks/native/backup_ext/src/TestScannerFile.cpp diff --git a/frameworks/native/backup_ext/include/ext_extension.h b/frameworks/native/backup_ext/include/ext_extension.h index db933dad2..9c6a00277 100644 --- a/frameworks/native/backup_ext/include/ext_extension.h +++ b/frameworks/native/backup_ext/include/ext_extension.h @@ -337,6 +337,9 @@ private: std::function ReportOnProcessResultCallback(wptr obj, BackupRestoreScenario scenario); + // new function20250306 + void ScannerAnsSendFiles(); + private: std::shared_mutex lock_; std::shared_ptr extension_; diff --git a/frameworks/native/backup_ext/src/TestScannerFile.cpp b/frameworks/native/backup_ext/src/TestScannerFile.cpp new file mode 100644 index 000000000..6b348be14 --- /dev/null +++ b/frameworks/native/backup_ext/src/TestScannerFile.cpp @@ -0,0 +1,236 @@ +// +// Created by zxyrz on 2025/3/6. +// +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std::chrono_literals; + +// 配置参数 +constexpr size_t SMALL_FILE_LIMIT = 2 * 1024 * 1024; // 2MB +constexpr size_t PACK_SIZE_LIMIT = 100 * 1024 * 1024; // 100MB +constexpr size_t PACK_COUNT_LIMIT = 6000; // 6000文件 +constexpr int SEND_RATE_LIMIT = 60; // 60次/秒 + +// 全局统计量 +std::atomic total_size{0}; // 累计文件总大小 + +template +class ThreadSafeQueue { +private: + std::queue queue; + mutable std::mutex mtx; + std::condition_variable cv; + bool producer_done = false; + +public: + void push(const T& item) { + std::lock_guard lock(mtx); + queue.push(item); + cv.notify_one(); + } + + bool pop(T& item) { + std::unique_lock lock(mtx); + cv.wait(lock, [&](){ return !queue.empty() || producer_done; }); + if(queue.empty() && producer_done) return false; + item = queue.front(); + queue.pop(); + return true; + } + + void set_producer_done() { + std::lock_guard lock(mtx); + producer_done = true; + cv.notify_all(); + } +}; + +ThreadSafeQueue small_files_queue; +ThreadSafeQueue large_files_queue; +ThreadSafeQueue send_queue; + +void scan_directory(const std::string& path) { + DIR* dir = opendir(path.c_str()); + if(!dir) return; + + dirent* entry; + while((entry = readdir(dir))) { + // 跳过特殊目录 + if(strcmp(entry->d_name, ".") == 0 || + strcmp(entry->d_name, "..") == 0) continue; + + std::string full_path = path + "/" + entry->d_name; + + if(entry->d_type == DT_DIR) { + // 递归扫描子目录 + scan_directory(full_path); + } else if(entry->d_type == DT_REG) { + // 获取文件大小 + struct stat st; + if(stat(full_path.c_str(), &st) == 0) { + // 累计总大小 + total_size.fetch_add(st.st_size); + + // 分类文件 + if(st.st_size <= SMALL_FILE_LIMIT) { + small_files_queue.push(full_path); + } else { + large_files_queue.push(full_path); + } + } + } + } + closedir(dir); +} + +void producer(const std::string& root_dir) { + scan_directory(root_dir); + small_files_queue.set_producer_done(); + large_files_queue.set_producer_done(); +} + +void process_small_files() { + std::vector batch; + size_t batch_size = 0; + std::string path; + + while(small_files_queue.pop(path)) { + struct stat st; + if(stat(path.c_str(), &st) != 0) continue; + + batch.push_back(path); + batch_size += st.st_size; + + // 达到打包条件 + if(batch.size() >= PACK_COUNT_LIMIT || + batch_size >= PACK_SIZE_LIMIT) { + // 生成临时文件列表 + char tmpfile[] = "/tmp/tarlistXXXXXX"; + int fd = mkstemp(tmpfile); + if(fd == -1) continue; + + // 写入文件列表 + std::ofstream ofs(tmpfile); + for(const auto& p : batch) ofs << p << "\n"; + close(fd); + + // 创建tar包 + static std::atomic_int pack_counter{0}; + std::string tar_name = "/tmp/pack_" + + std::to_string(time(nullptr)) + "_" + + std::to_string(pack_counter++) + ".tar"; + + // 执行打包命令 + std::string cmd = "tar -cf " + tar_name + " -T " + tmpfile; + if(system(cmd.c_str()) == 0) { + send_queue.push(tar_name); + } + + // 清理临时文件 + remove(tmpfile); + batch.clear(); + batch_size = 0; + } + } + + // 处理剩余文件 + if(!batch.empty()) { + char tmpfile[] = "/tmp/tarlistXXXXXX"; + int fd = mkstemp(tmpfile); + if(fd != -1) { + std::ofstream ofs(tmpfile); + for(const auto& p : batch) ofs << p << "\n"; + close(fd); + + std::string tar_name = "/tmp/pack_final.tar"; + std::string cmd = "tar -cf " + tar_name + " -T " + tmpfile; + if(system(cmd.c_str()) == 0) { + send_queue.push(tar_name); + } + remove(tmpfile); + } + } +} + +void process_large_files() { + std::string path; + while(large_files_queue.pop(path)) { + send_queue.push(path); + } +} + +void sender() { + auto time_window = std::chrono::steady_clock::now(); + int send_count = 0; + + std::string path; + while(send_queue.pop(path)) { + // 打开文件获取fd（示例操作） + int fd = open(path.c_str(), O_RDONLY); + if(fd != -1) { + // TODO: 实际发送操作 + close(fd); + + // 删除临时tar包 + if(path.find("/tmp/pack_") == 0) { + remove(path.c_str()); + } + } + + // 精确限速控制 + if(++send_count >= SEND_RATE_LIMIT) { + auto now = std::chrono::steady_clock::now(); + auto elapsed = now - time_window; + auto remaining = 1s - elapsed; + + if(remaining > 0s) { + std::this_thread::sleep_for(remaining); + } + + time_window = now + remaining; + send_count = 0; + } + } +} + +int main() { + auto start = std::chrono::steady_clock::now(); + + // 启动工作线程 + std::thread producer_thread(producer, "/storage/media/local"); + std::thread small_worker(process_small_files); + std::thread large_worker(process_large_files); + std::thread sender_thread(sender); + + // 等待线程完成 + producer_thread.join(); + small_worker.join(); + large_worker.join(); + send_queue.set_producer_done(); + sender_thread.join(); + + // 输出统计结果 + auto duration = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start + ); + + std::cout << "=== 执行结果 ===" << "\n" + << "总耗时: " << duration.count() << " ms\n" + << "处理文件总数: " << total_size.load() << " bytes\n" + << "约合: " << total_size.load()/1024/1024 << " MB\n"; + + return 0; +} \ No newline at end of file -- Gitee