diff --git a/frameworks/native/include/hilogtool_msg.h b/frameworks/native/include/hilogtool_msg.h index 6ff89ec28fbe53af38e3ccc948285d3d1a3b906a..bf3b9f9184f8c9ace1e270a259de2f05c1426fee 100644 --- a/frameworks/native/include/hilogtool_msg.h +++ b/frameworks/native/include/hilogtool_msg.h @@ -23,7 +23,7 @@ #include "hilog_common.h" #define FILE_PATH_MAX_LEN 100 - +#define JOB_ID_ALL 0xffffffff typedef enum { LOG_QUERY_REQUEST = 0x01, LOG_QUERY_RESPONSE, @@ -59,11 +59,6 @@ typedef enum { OT_FLOW_SWITCH, } OperateType; -typedef enum { - NONE = 0, - STREAM = 1, - DIVIDED = 2, -} CompressStrategy; typedef enum { CREATE_SUCCESS = 1, @@ -76,7 +71,7 @@ typedef enum { } PersisterResponse; typedef enum { - COMPRESS_TYPE_OFF = 0, + COMPRESS_TYPE_NONE = 0, COMPRESS_TYPE_ZSTD, COMPRESS_TYPE_ZLIB, } CompressAlg; @@ -245,7 +240,6 @@ typedef struct { typedef struct { std::string logTypeStr; - std::string compressTypeStr; std::string compressAlgStr; std::string fileSizeStr; std::string fileNumStr; @@ -254,7 +248,6 @@ typedef struct { } LogPersistParam; typedef struct { uint16_t logType; // union logType - uint16_t compressType; uint16_t compressAlg; char filePath[FILE_PATH_MAX_LEN]; uint32_t fileSize; @@ -305,7 +298,6 @@ typedef struct { int32_t result; uint32_t jobId; uint16_t logType; - uint16_t compressType; uint16_t compressAlg; char filePath[FILE_PATH_MAX_LEN]; uint32_t fileSize; diff --git a/services/hilogd/BUILD.gn b/services/hilogd/BUILD.gn index 6b703428c98ed33c0a4b7b7142565cc095ecbc8d..59814b3bf92d7688fc7e4894216e1465b6f4f524 100644 --- a/services/hilogd/BUILD.gn +++ b/services/hilogd/BUILD.gn @@ -22,7 +22,6 @@ config("hilogd_config") { ohos_executable("hilogd") { sources = [ "cmd_executor.cpp", - "compressing_rotator.cpp", "flow_control_init.cpp", "log_buffer.cpp", "log_collector.cpp", @@ -33,9 +32,7 @@ ohos_executable("hilogd") { "log_reader.cpp", "main.cpp", ] - configs = [ ":hilogd_config" ] - deps = [ "//base/hiviewdfx/hilog/adapter:libhilog_os_adapter", "//base/hiviewdfx/hilog/frameworks/native:libhilogutil", diff --git a/services/hilogd/compressing_rotator.cpp b/services/hilogd/compressing_rotator.cpp deleted file mode 100644 index 3ec2fd05fd541c02d03ff1162172b0bc1da1f170..0000000000000000000000000000000000000000 --- a/services/hilogd/compressing_rotator.cpp +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "compressing_rotator.h" - -#include -#include - -#include -#include -namespace OHOS { -namespace HiviewDFX { -using namespace std; - -void CompressingRotator::InternalRotate() -{ - cout << "compressing internal rotate" << endl; - if ((uint32_t)fileIndex == resFileNum) { - stringstream ss; - ss << resPath << "."; - int pos = ss.tellp(); - ss << 1; - remove(ss.str().c_str()); - - for (uint32_t i = 2; i <= fileNum; ++i) { - ss.seekp(pos); - ss << (i - 1); - string newName = ss.str(); - ss.seekp(pos); - ss << i; - string oldName = ss.str(); -#ifdef DEBUG - cout << "OLD NAME " << oldName << " NEW NAME " << newName << endl; -#endif - rename(oldName.c_str(), newName.c_str()); - } - fileIndex -= 1; - } - - fileIndex += 1; - gzFile fd; - stringstream resName; - resName << resPath << "." << fileIndex << ".gz"; - fd = gzopen(resName.str().c_str(), "wb"); - if (!fd) { -#ifdef DEBUG - cout << "gzopen failed" << endl; -#endif - } - - uint32_t i = 1; - for (; i <= fileNum; i++) { - stringstream ss; - ss << fileName << "." << i; -#ifdef DEBUG - cout << "ss name " << ss.str() << endl; -#endif - ifstream file(ss.str(), ios::in); - file.seekg(0, file.end); - int size = file.tellg(); -#ifdef DEBUG - cout << "size is " << size << endl; -#endif - unique_ptr memblock = make_unique(size); - file.seekg(0, ios::beg); - file.read(memblock.get(), size); - gzwrite(fd, memblock.get(), size); - - file.close(); - remove(ss.str().c_str()); - } - gzclose(fd); - - leftSize = 0; - index = 0; -} -} // namespace HiviewDFX -} // namespace OHOS diff --git a/services/hilogd/etc/hilogd.rc b/services/hilogd/etc/hilogd.rc index ceace436b228ce1921fd8ced054574088188afb5..627c786c94c6fc8d0b20517f0cb69a51709e6683 100644 --- a/services/hilogd/etc/hilogd.rc +++ b/services/hilogd/etc/hilogd.rc @@ -15,6 +15,7 @@ on early-init write /proc/sys/net/unix/max_dgram_qlen 600 on post-fs-data + mkdir /data/misc/logd/ 0770 logd logd start hilogd service hilogd /system/bin/hilogd diff --git a/services/hilogd/include/compressing_rotator.h b/services/hilogd/include/compressing_rotator.h deleted file mode 100644 index eb2462ed1c4f1bb653d3019890c66246ca975959..0000000000000000000000000000000000000000 --- a/services/hilogd/include/compressing_rotator.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef _COMPRESSING_ROTATOR_H -#define _COMPRESSING_ROTATOR_H - -#include - -#include "log_persister_rotator.h" -namespace OHOS { -namespace HiviewDFX { -class CompressingRotator : public LogPersisterRotator { -public: - CompressingRotator(std::string resPath, uint32_t resFileSize, - uint32_t resFileNum, uint32_t tmpFileSize, - uint32_t tmpFileNum, std::string tmpPath = "") - : LogPersisterRotator(tmpPath, tmpFileSize, tmpFileNum) - { - this->resPath = resPath; - this->resFileSize = resFileSize; - this->resFileNum = resFileNum; - this->fileIndex = 0; - } - ~CompressingRotator() = default; -protected: - void InternalRotate() override; - -private: - std::string resPath; - uint32_t resFileSize; - uint32_t resFileNum; - int fileIndex; -}; -} // namespace HiviewDFX -} // namespace OHOS -#endif diff --git a/services/hilogd/include/log_compress.h b/services/hilogd/include/log_compress.h index e57aa23086cc5457ff54147e780fd44d471ac52f..eafcce6c34b4e5cb2bd3caaff10e80451ec285ac 100644 --- a/services/hilogd/include/log_compress.h +++ b/services/hilogd/include/log_compress.h @@ -15,33 +15,53 @@ #ifndef HILOG_COMPRESS_H #define HILOG_COMPRESS_H +#include #ifdef USING_ZSTD_COMPRESS #define ZSTD_STATIC_LINKING_ONLY #include "include/common.h" #include "zstd.h" #endif #include - namespace OHOS { namespace HiviewDFX { +const uint32_t MAX_PERSISTER_BUFFER_SIZE = 64 * 1024; +typedef struct { + uint32_t offset; + char content[MAX_PERSISTER_BUFFER_SIZE]; +} LogPersisterBuffer; + +const uint16_t CHUNK = 16384; class LogCompress { public: LogCompress(); - virtual ~LogCompress(); - virtual int Compress(Bytef *data, uLong dlen) = 0; - uLong zdlen = 0; + virtual ~LogCompress() = default; + virtual int Compress(LogPersisterBuffer* &buffer, LogPersisterBuffer* &compressBuffer) = 0; unsigned char *zdata = nullptr; + char buffIn[CHUNK]; + char buffOut[CHUNK]; +}; + +class NoneCompress : public LogCompress { +public: + int Compress(LogPersisterBuffer* &buffer, LogPersisterBuffer* &compressBuffer); }; class ZlibCompress : public LogCompress { public: - int Compress(Bytef *data, uLong dlen); + int Compress(LogPersisterBuffer* &buffer, LogPersisterBuffer* &compressBuffer); +private: + z_stream cStream; }; class ZstdCompress : public LogCompress { public: - int Compress(Bytef *data, uLong dlen); + int Compress(LogPersisterBuffer* &buffer, LogPersisterBuffer* &compressBuffer); +private: +#ifdef USING_ZSTD_COMPRESS + ZSTD_CCtx* cctx; +#endif }; } } -#endif /* HILOG_COMPRESS_H */ +#endif + /* HILOG_COMPRESS_H */ diff --git a/services/hilogd/include/log_persister.h b/services/hilogd/include/log_persister.h index f2a15ba827e8b1680cab8a20f358409157bb37af..bc80893249d3e7431a6112d40052100abd56380b 100644 --- a/services/hilogd/include/log_persister.h +++ b/services/hilogd/include/log_persister.h @@ -26,21 +26,22 @@ #include "log_persister_rotator.h" #include "log_reader.h" #include "log_compress.h" + namespace OHOS { namespace HiviewDFX { using namespace std; -typedef struct { - uint16_t offset; - char content[]; -} LogPersisterBuffer; -const uint16_t MAX_PERSISTER_BUFFER_SIZE = 4096; +typedef struct { + uint8_t index; + uint16_t types; + uint8_t levels; + LogPersistStartMsg msg; +} PersistRecoveryInfo; class LogPersister : public LogReader { public: - LogPersister(uint32_t id, std::string path, uint16_t compressType, - uint16_t compressAlg, int sleepTime, LogPersisterRotator *rotator, - HilogBuffer *buffer); + LogPersister(uint32_t id, std::string path, uint32_t fileSize, uint16_t compressAlg, int sleepTime, + LogPersisterRotator& rotator, HilogBuffer &buffer); ~LogPersister(); void SetBufferOffset(int off); void NotifyForNewData(); @@ -48,9 +49,9 @@ public: int ThreadFunc(); static int Kill(uint32_t id); void Exit(); - static int Query(uint16_t logType, - std::list &results); + static int Query(uint16_t logType, std::list &results); int Init(); + int InitCompress(); void Start(); bool Identify(uint32_t id); void FillInfo(LogPersistQueryResult *response); @@ -58,13 +59,14 @@ public: bool writeUnCompressedBuffer(HilogData *data); uint8_t GetType() const; std::string getPath(); + int SaveInfo(LogPersistStartMsg& pMsg); LogPersisterBuffer *buffer; - + LogPersisterBuffer *compressBuffer; private: uint32_t id; std::string path; + uint32_t fileSize; std::string mmapPath; - uint16_t compressType; uint16_t compressAlg; int sleepTime; std::mutex cvMutex; @@ -76,10 +78,13 @@ private: bool hasExited; inline void WriteFile(); bool isExited(); - FILE *fdinfo; + FILE *fdinfo = nullptr; int fd = -1; - LogCompress *LogCompress; + LogCompress *compressor; list persistList; + uint32_t plainLogSize; + PersistRecoveryInfo info; + bool restore = false; }; int GenPersistLogHeader(HilogData *data, list& persistList); diff --git a/services/hilogd/include/log_persister_rotator.h b/services/hilogd/include/log_persister_rotator.h index 4d203f26b4ef78874d14a65def895e66483ca611..fd2a147015861ab859acfa303fc769abe4aeb8ca 100644 --- a/services/hilogd/include/log_persister_rotator.h +++ b/services/hilogd/include/log_persister_rotator.h @@ -18,27 +18,33 @@ #include namespace OHOS { namespace HiviewDFX { +const std::string ANXILLARY_FILE_NAME = "persisterInfo_"; class LogPersisterRotator { public: LogPersisterRotator(std::string path, uint32_t fileSize, uint32_t fileNum, std::string suffix = ""); - virtual ~LogPersisterRotator(){}; - int Input(const char *buf, int length); + ~LogPersisterRotator() + { + fclose(fdinfo); + } + void Init(); + int Input(const char *buf, uint32_t length); void FillInfo(uint32_t *size, uint32_t *num); void FinishInput(); - + void SetIndex(int pIndex); + void SetId(uint32_t pId); protected: - virtual void InternalRotate(); + void InternalRotate(); uint32_t fileNum; uint32_t fileSize; std::string fileName; std::string fileSuffix; int index; - int leftSize; std::fstream output; - private: void Rotate(); bool needRotate = false; + FILE* fdinfo; + uint32_t id = 0; }; } // namespace HiviewDFX } // namespace OHOS diff --git a/services/hilogd/include/log_querier.h b/services/hilogd/include/log_querier.h index 5f89d8f86bfd55d540d1d6c5fc9cdcb4b457143b..64c1508ffcf2d926f8815f0fceef4e2a7d693f3d 100644 --- a/services/hilogd/include/log_querier.h +++ b/services/hilogd/include/log_querier.h @@ -28,6 +28,7 @@ public: int WriteData(HilogData* data); void NotifyForNewData(); uint8_t GetType() const; + int RestorePersistJobs(HilogBuffer& _buffer); ~LogQuerier() = default; }; } // namespace HiviewDFX diff --git a/services/hilogd/log_buffer.cpp b/services/hilogd/log_buffer.cpp index a3a12fd387c5e6d1c81cdd98e1eedf6aeccf5b87..6d5b0a957acaf28bf21f152ea0b2bb2c94ef6d25 100644 --- a/services/hilogd/log_buffer.cpp +++ b/services/hilogd/log_buffer.cpp @@ -335,11 +335,9 @@ bool HilogBuffer::ConditionMatch(std::shared_ptr reader) * strict mode: 0xdxxxxxx (full) * fuzzy mode: 0xdxxxx (using last 2 digits of full domain as mask) */ - if (((static_cast((0b01 << (reader->readPos->type)) & (reader->queryCondition.types)) == 0) || (static_cast((0b01 << (reader->readPos->level)) & (reader->queryCondition.levels)) == 0))) return false; - int ret = 0; if (reader->queryCondition.nPid > 0) { for (int i = 0; i < reader->queryCondition.nPid; i++) { diff --git a/services/hilogd/log_compress.cpp b/services/hilogd/log_compress.cpp index c06eb5959b99295a4645fdaf5f97bde44289ef55..7b35f78d424e717bef3bf78fda50b2b425c978e1 100644 --- a/services/hilogd/log_compress.cpp +++ b/services/hilogd/log_compress.cpp @@ -26,102 +26,147 @@ LogCompress::LogCompress() { } -LogCompress::~LogCompress() +int NoneCompress::Compress(LogPersisterBuffer* &buffer, LogPersisterBuffer* &compressBuffer) { - delete zdata; + if (memcpy_s(compressBuffer->content, compressBuffer->offset, buffer->content, buffer->offset) != 0) { + return -1; + } + return 0; } -int ZlibCompress::Compress(Bytef *data, uLong dlen) +int ZlibCompress::Compress(LogPersisterBuffer* &buffer, LogPersisterBuffer* &compressBuffer) { - zdlen = compressBound(dlen); + uint32_t zdlen = compressBound(buffer->offset); zdata = new unsigned char [zdlen]; - int err = 0; - z_stream c_stream; - - if (data && dlen > 0) { - c_stream.zalloc = NULL; - c_stream.zfree = NULL; - c_stream.opaque = NULL; - if (deflateInit2(&c_stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, MAX_WBITS + 16, 8, Z_DEFAULT_STRATEGY) != Z_OK) { - return -1; - } - c_stream.next_in = data; - c_stream.avail_in = dlen; - c_stream.next_out = zdata; - c_stream.avail_out = zdlen; - while (c_stream.avail_in != 0 && c_stream.total_out < zdlen) { - if (deflate(&c_stream, Z_NO_FLUSH) != Z_OK) { + if (zdata == nullptr) { + cout << "no enough memory!" << endl; + return -1; + } + size_t const toRead = CHUNK; + auto src_pos = 0; + auto dst_pos = 0; + size_t read = buffer->offset; + int flush = 0; + cStream.zalloc = Z_NULL; + cStream.zfree = Z_NULL; + cStream.opaque = Z_NULL; + if (deflateInit2(&cStream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, MAX_WBITS + 16, 8, Z_DEFAULT_STRATEGY) != Z_OK) { + delete zdata; + return -1; + } + do { + bool flag = read - src_pos < toRead; + if (flag) { + memset_s(buffIn, CHUNK, 0, CHUNK); + if (memmove_s(buffIn, CHUNK, buffer->content + src_pos, read - src_pos) != 0) { + delete zdata; return -1; } + cStream.avail_in = read - src_pos; + src_pos += read - src_pos; + } else { + memset_s(buffIn, CHUNK, 0, CHUNK); + if (memmove_s(buffIn, CHUNK, buffer->content + src_pos, toRead) != 0) { + delete zdata; + return -1; + }; + src_pos += toRead; + cStream.avail_in = toRead; } - if (c_stream.avail_in != 0) { - return c_stream.avail_in; - } - for (;;) { - if ((err = deflate(&c_stream, Z_FINISH)) == Z_STREAM_END) - break; - if (err != Z_OK) { + flush = flag ? Z_FINISH : Z_NO_FLUSH; + cStream.next_in = (Bytef *)buffIn; + /* run deflate() on input until output buffer not full, finish + compression if all of source has been read in */ + do { + cStream.avail_out = CHUNK; + cStream.next_out = (Bytef *)buffOut; + if (deflate(&cStream, flush) == Z_STREAM_ERROR) { + delete zdata; return -1; } - } - if (deflateEnd(&c_stream) != Z_OK) { - return -1; - } - zdlen = c_stream.total_out; - return 0; + unsigned have = CHUNK - cStream.avail_out; + if (memmove_s(zdata + dst_pos, CHUNK, buffOut, have) != 0) { + delete zdata; + return -1; + } + dst_pos += have; + } while (cStream.avail_out == 0); + } while (flush != Z_FINISH); + /* clean up and return */ + (void)deflateEnd(&cStream); + if (memcpy_s(compressBuffer->content + compressBuffer->offset, + MAX_PERSISTER_BUFFER_SIZE - compressBuffer->offset, zdata, dst_pos) != 0) { + delete zdata; + return -1; } - return -1; + compressBuffer->offset += dst_pos; + delete zdata; + return 0; } -int ZstdCompress::Compress(Bytef *data, uLong dlen) +int ZstdCompress::Compress(LogPersisterBuffer* &buffer, LogPersisterBuffer* &compressBuffer) { #ifdef USING_ZSTD_COMPRESS - zdlen = ZSTD_CStreamOutSize(); + uint32_t zdlen = ZSTD_CStreamOutSize(); zdata = new unsigned char [zdlen]; - - size_t const buffInSize = ZSTD_CStreamInSize(); - void* const buffIn = malloc(buffInSize); - size_t const buffOutSize = ZSTD_CStreamOutSize(); - void* const buffOut = malloc(buffOutSize); - + if (zdata == nullptr) { + cout << "no enough memory!" << endl; + return -1; + } + ZSTD_EndDirective mode; int compressionlevel = 1; - ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + cctx = ZSTD_createCCtx(); + if (cctx == nullptr) { + cout << "ZSTD_createCCtx() failed!" << endl; + delete zdata; + return -1; + } ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionlevel); - - size_t const toRead = buffInSize; + size_t const toRead = CHUNK; auto src_pos = 0; auto dst_pos = 0; - size_t read = dlen; - for (;;) { - if (read - src_pos < toRead) { - memcpy_s(buffIn, buffInSize - src_pos, data + src_pos, read); - src_pos += read; + size_t read = buffer->offset; + ZSTD_inBuffer input; + do { + bool flag = read - src_pos < toRead; + if (flag) { + memset_s(buffIn, CHUNK, 0, CHUNK); + if (memmove_s(buffIn, CHUNK, buffer->content + src_pos, read - src_pos) != 0) { + delete zdata; + return -1; + } + input = {buffIn, read - src_pos, 0}; + src_pos += read - src_pos; } else { - memcpy_s(buffIn, buffInSize - src_pos, data + src_pos, toRead); + memset_s(buffIn, CHUNK, 0, CHUNK); + if (memmove_s(buffIn, CHUNK, buffer->content + src_pos, toRead) != 0) { + delete zdata; + return -1; + } + input = {buffIn, toRead, 0}; src_pos += toRead; } - int const lastChunk = (read < toRead); - ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue; - - ZSTD_inBuffer input = {buffIn, read, 0}; + mode = flag ? ZSTD_e_end : ZSTD_e_continue; int finished; - do { - ZSTD_outBuffer output = {buffOut, buffOutSize, 0}; + ZSTD_outBuffer output = {buffOut, CHUNK, 0}; size_t const remaining = ZSTD_compressStream2(cctx, &output, &input, mode); - memcpy_s(zdata + dst_pos, zdlen - dst_pos, buffOut, output.pos); + if (memmove_s(zdata + dst_pos, zdlen, (Bytef *)buffOut, output.pos) != 0) { + delete zdata; + return -1; + } dst_pos += output.pos; - zdlen = dst_pos; - finished = lastChunk ? (remaining == 0) : (input.pos == input.size); + finished = flag ? (remaining == 0) : (input.pos == input.size); } while (!finished); - - if (lastChunk) { - break; - } - } - free(buffIn); - free(buffOut); + } while (mode != ZSTD_e_end); ZSTD_freeCCtx(cctx); + if (memcpy_s(compressBuffer->content + compressBuffer->offset, + MAX_PERSISTER_BUFFER_SIZE - compressBuffer->offset, zdata, dst_pos) != 0) { + delete zdata; + return -1; + } + compressBuffer->offset += dst_pos; + delete zdata; #endif // #ifdef USING_ZSTD_COMPRESS return 0; } diff --git a/services/hilogd/log_persister.cpp b/services/hilogd/log_persister.cpp index 441fc3cd1447b2c2fc51024e7fd36a85dd7631d7..1ac6fe48568a9666a2ed1e6158eb6c1bbdc18361 100644 --- a/services/hilogd/log_persister.cpp +++ b/services/hilogd/log_persister.cpp @@ -40,7 +40,6 @@ namespace HiviewDFX { using namespace std::literals::chrono_literals; using namespace std; - static std::list> logPersisters; static std::mutex g_listMutex; @@ -50,24 +49,47 @@ static std::mutex g_listMutex; (x) = nullptr; \ } while (0) -LogPersister::LogPersister(uint32_t id, string path, uint16_t compressType, - uint16_t compressAlg, int sleepTime, - LogPersisterRotator *rotator, HilogBuffer *_buffer) - : id(id), path(path), compressType(compressType), compressAlg(compressAlg), - sleepTime(sleepTime), rotator(rotator) +LogPersister::LogPersister(uint32_t id, string path, uint32_t fileSize, uint16_t compressAlg, int sleepTime, + LogPersisterRotator& rotator, HilogBuffer &_buffer) + : id(id), path(path), fileSize(fileSize), compressAlg(compressAlg), + sleepTime(sleepTime), rotator(&rotator) { toExit = false; hasExited = false; - hilogBuffer = _buffer; - LogCompress = nullptr; + hilogBuffer = &_buffer; + compressor = nullptr; fdinfo = nullptr; buffer = nullptr; + plainLogSize = 0; } LogPersister::~LogPersister() { SAFE_DELETE(rotator); - SAFE_DELETE(LogCompress); + SAFE_DELETE(compressor); + SAFE_DELETE(compressBuffer); +} + +int LogPersister::InitCompress() +{ + compressBuffer = new LogPersisterBuffer; + if (compressBuffer == NULL) { + return RET_FAIL; + } + switch (compressAlg) { + case COMPRESS_TYPE_NONE: + compressor = new NoneCompress(); + break; + case COMPRESS_TYPE_ZLIB: + compressor = new ZlibCompress(); + break; + case COMPRESS_TYPE_ZSTD: + compressor = new ZstdCompress(); + break; + default: + break; + } + return RET_SUCCESS; } int LogPersister::Init() @@ -76,7 +98,7 @@ int LogPersister::Init() if (nPos == RET_FAIL) { return RET_FAIL; } - mmapPath = path.substr(0, nPos) + "/." + to_string(id); + mmapPath = path.substr(0, nPos) + "/." + ANXILLARY_FILE_NAME + to_string(id); if (access(path.substr(0, nPos).c_str(), F_OK) != 0) { if (errno == ENOENT) { MkDirPath(path.substr(0, nPos).c_str()); @@ -93,8 +115,10 @@ int LogPersister::Init() if (hit) { return RET_FAIL; } + if (InitCompress() == RET_FAIL) { + return RET_FAIL; + } fd = open(mmapPath.c_str(), O_RDWR | O_CREAT | O_EXCL, 0); - bool restore = false; if (fd <= 0) { if (errno == EEXIST) { cout << "File already exists!" << endl; @@ -105,7 +129,7 @@ int LogPersister::Init() #ifdef DEBUG cout << "New log file: " << mmapPath << endl; #endif - lseek(fd, MAX_PERSISTER_BUFFER_SIZE - 1, SEEK_SET); + lseek(fd, sizeof(LogPersisterBuffer) - 1, SEEK_SET); write(fd, "", 1); } if (fd < 0) { @@ -114,10 +138,7 @@ int LogPersister::Init() #endif return RET_FAIL; } - fdinfo = fopen((mmapPath + ".info").c_str(), "r+"); - if (fdinfo == nullptr) { - fdinfo = fopen((mmapPath + ".info").c_str(), "w+"); - } + fdinfo = fopen((mmapPath + ".info").c_str(), "a+"); if (fdinfo == nullptr) { #ifdef DEBUG cout << "open loginfo file failed: " << strerror(errno) << endl; @@ -125,7 +146,7 @@ int LogPersister::Init() close(fd); return RET_FAIL; } - buffer = (LogPersisterBuffer *)mmap(nullptr, MAX_PERSISTER_BUFFER_SIZE, PROT_READ | PROT_WRITE, + buffer = (LogPersisterBuffer *)mmap(nullptr, sizeof(LogPersisterBuffer), PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); close(fd); if (buffer == MAP_FAILED) { @@ -136,14 +157,9 @@ int LogPersister::Init() return RET_FAIL; } if (restore == true) { - int moffset; - if (fscanf_s(fdinfo, "%04x", &moffset) == -1) { - return RET_FAIL; - } #ifdef DEBUG - cout << "Recovered persister, Offset=" << moffset << endl; + cout << "Recovered persister, Offset=" << buffer->offset << endl; #endif - SetBufferOffset(moffset); WriteFile(); } else { SetBufferOffset(0); @@ -169,8 +185,6 @@ int LogPersister::MkDirPath(const char *pMkdir) void LogPersister::SetBufferOffset(int off) { buffer->offset = off; - fseek(fdinfo, 0, SEEK_SET); - fprintf(fdinfo, "%04x\n", off); } int GenPersistLogHeader(HilogData *data, list& persistList) @@ -183,7 +197,6 @@ int GenPersistLogHeader(HilogData *data, list& persistList) showBuffer.domain = data->domain; showBuffer.tv_sec = data->tv_sec; showBuffer.tv_nsec = data->tv_nsec; - int offset = data->tag_len; char *dataCopy= (char*)calloc(data->len, sizeof(char)); memcpy_s(dataCopy, offset, data->tag, offset); @@ -222,17 +235,16 @@ bool LogPersister::writeUnCompressedBuffer(HilogData *data) int listSize = persistList.size(); if (persistList.empty()) { - listSize = GenPersistLogHeader(data, persistList); + listSize = GenPersistLogHeader(data, persistList); } while (listSize--) { string header = persistList.front(); uint16_t headerLen = header.length(); uint16_t size = headerLen + 1; - uint16_t orig_offset = buffer->offset; + uint32_t orig_offset = buffer->offset; int r = 0; if (buffer->offset + size > MAX_PERSISTER_BUFFER_SIZE) return false; - r = memcpy_s(buffer->content + buffer->offset, MAX_PERSISTER_BUFFER_SIZE - buffer->offset, header.c_str(), headerLen); if (r != 0) { @@ -253,39 +265,23 @@ int LogPersister::WriteData(HilogData *data) return -1; if (writeUnCompressedBuffer(data)) return 0; - switch (compressAlg) { - case COMPRESS_TYPE_OFF: - WriteFile(); - break; - case COMPRESS_TYPE_ZLIB: { - LogCompress = new ZlibCompress(); -#ifdef DEBUG - cout << buffer->content << endl; -#endif - LogCompress->Compress((Bytef *)buffer->content, buffer->offset); - rotator->Input((char *)LogCompress->zdata, LogCompress->zdlen); - rotator->FinishInput(); - SetBufferOffset(0); - } - break; -#ifdef USING_ZSTD_COMPRESS - case COMPRESS_TYPE_ZSTD: { - LogCompress = new ZstdCompress(); - LogCompress->Compress((Bytef *)buffer->content, buffer->offset); - rotator->Input((char *)LogCompress->zdata, LogCompress->zdlen); - rotator->FinishInput(); - SetBufferOffset(0); - } - break; -#endif // #ifdef USING_ZSTD_COMPRESS - default: - break; - } + if (compressor->Compress(buffer, compressBuffer) != 0) { + cout << "COMPRESS Error" << endl; + return -1; + }; + WriteFile(); return writeUnCompressedBuffer(data) ? 0 : -1; } void LogPersister::Start() { + if (!restore) { + std::cout << "Save Info file!" << std::endl; + fseek(fdinfo, 0, SEEK_SET); + fwrite(&info, sizeof(PersistRecoveryInfo), 1, fdinfo); + fsync(fileno(fdinfo)); + } + fclose(fdinfo); auto newThread = thread(&LogPersister::ThreadFunc, static_pointer_cast(shared_from_this())); newThread.detach(); @@ -294,10 +290,15 @@ void LogPersister::Start() inline void LogPersister::WriteFile() { - if (buffer->offset == 0) return; - if (compressAlg == 0) { - rotator->Input(buffer->content, buffer->offset); + if (buffer->offset == 0) + return; + rotator->Input((char *)compressBuffer->content, compressBuffer->offset); + plainLogSize += buffer->offset; + if (plainLogSize >= fileSize) { + plainLogSize = 0; + rotator->FinishInput(); } + compressBuffer->offset = 0; SetBufferOffset(0); } @@ -319,7 +320,6 @@ int LogPersister::ThreadFunc() WriteFile(); } } - cout << "running! " << compressAlg << endl; } WriteFile(); { @@ -354,7 +354,6 @@ void LogPersister::FillInfo(LogPersistQueryResult *response) if (strcpy_s(response->filePath, FILE_PATH_MAX_LEN, path.c_str())) { return; } - response->compressType = compressType; response->compressAlg = compressAlg; rotator->FillInfo(&response->fileSize, &response->fileNum); return; @@ -366,7 +365,9 @@ int LogPersister::Kill(const uint32_t id) std::lock_guard guard(g_listMutex); for (auto it = logPersisters.begin(); it != logPersisters.end(); ) { if ((*it)->Identify(id)) { +#ifdef DEBUG cout << "find a persister" << endl; +#endif (*it)->Exit(); it = logPersisters.erase(it); found = true; @@ -396,7 +397,6 @@ void LogPersister::Exit() cout << "removed mmap file" << endl; remove(mmapPath.c_str()); remove((mmapPath + ".info").c_str()); - fclose(fdinfo); return; } bool LogPersister::Identify(uint32_t id) @@ -413,5 +413,18 @@ uint8_t LogPersister::GetType() const { return TYPE_PERSISTER; } + +int LogPersister::SaveInfo(LogPersistStartMsg& pMsg) +{ + info.msg = pMsg; + info.types = queryCondition.types; + info.levels = queryCondition.levels; + if (strcpy_s(info.msg.filePath, FILE_PATH_MAX_LEN, pMsg.filePath) != 0) { + cout << "Failed to save persister file path" << endl; + return RET_FAIL; + } + cout << "Saved Path=" << info.msg.filePath << endl; + return RET_SUCCESS; +} } // namespace HiviewDFX } // namespace OHOS diff --git a/services/hilogd/log_persister_rotator.cpp b/services/hilogd/log_persister_rotator.cpp index 8426d2cc389d13df117130a9be869151c638d603..67e098206d701ae134c8f1fb2738bd7cd8d6b790 100644 --- a/services/hilogd/log_persister_rotator.cpp +++ b/services/hilogd/log_persister_rotator.cpp @@ -18,6 +18,9 @@ #include #include #include +#include +#include + namespace OHOS { namespace HiviewDFX { using namespace std; @@ -25,45 +28,34 @@ using namespace std; LogPersisterRotator::LogPersisterRotator(string path, uint32_t fileSize, uint32_t fileNum, string suffix) : fileNum(fileNum), fileSize(fileSize), fileName(path), fileSuffix(suffix) { - index = 0; - leftSize = 0; - needRotate = true; + index = -1; + needRotate = false; +} + +void LogPersisterRotator::Init() +{ + int nPos = fileName.find_last_of('/'); + std::string mmapPath = fileName.substr(0, nPos) + "/." + ANXILLARY_FILE_NAME + to_string(id); + if (access(fileName.substr(0, nPos).c_str(), F_OK) != 0) { + if (errno == ENOENT) { + mkdir(fileName.substr(0, nPos).c_str(), S_IRUSR | S_IWUSR | S_IXUSR | S_IRWXG | S_IRWXO); + } + } + fdinfo = fopen((mmapPath + ".info").c_str(), "w+"); } -int LogPersisterRotator::Input(const char *buf, int length) +int LogPersisterRotator::Input(const char *buf, uint32_t length) { cout << __func__ << " " << fileName << " " << index - << " " << length << " need: " << needRotate << endl; + << " " << length << " need: " << needRotate << endl; + if (length <= 0 || buf == nullptr) return -1; if (needRotate) { output.close(); Rotate(); needRotate = false; } - - if (length <= 0 || buf == nullptr) return -1; - - unsigned int offset = 0; - while (length >= leftSize) { - int pos = leftSize - 1; - while (pos >= 0 && buf[offset + pos] != '\n') - pos--; - int realSize = pos + 1; - if (realSize == 0 && (uint32_t)leftSize == fileSize) { - cout << "ERROR! some log line is too long" << endl; - break; - } else { - output.write(buf + offset, realSize); - offset += realSize; - length -= realSize; - output.close(); - } - Rotate(); - } - - output.write(buf + offset, length); - leftSize -= length; + output.write(buf, length); output.flush(); - return 0; } @@ -72,10 +64,10 @@ void LogPersisterRotator::InternalRotate() stringstream ss; ss << fileName << "."; int pos = ss.tellp(); - ss << 1 << fileSuffix; + ss << 0 << fileSuffix; remove(ss.str().c_str()); - for (uint32_t i = 2; i <= fileNum; ++i) { + for (uint32_t i = 1; i < fileNum; ++i) { ss.seekp(pos); ss << (i - 1) << fileSuffix; string newName = ss.str(); @@ -85,15 +77,13 @@ void LogPersisterRotator::InternalRotate() cout << "OLD NAME " << oldName << " NEW NAME " << newName << endl; rename(oldName.c_str(), newName.c_str()); } - output.open(ss.str(), ios::out); - leftSize = fileSize; } void LogPersisterRotator::Rotate() { cout << __func__ << endl; - if (index == (int)fileNum) { + if (index >= (int)(fileNum - 1)) { InternalRotate(); } else { index += 1; @@ -101,7 +91,9 @@ void LogPersisterRotator::Rotate() ss << fileName << "." << index << fileSuffix; cout << "THE FILE NAME !!!!!!! " << ss.str() << endl; output.open(ss.str(), ios::app); - leftSize = fileSize; + fseek(fdinfo, 0, SEEK_SET); + fwrite(&index, sizeof(uint8_t), 1, fdinfo); + fsync(fileno(fdinfo)); } } @@ -115,5 +107,15 @@ void LogPersisterRotator::FinishInput() { needRotate = true; } + +void LogPersisterRotator::SetIndex(int pIndex) +{ + index = pIndex; +} + +void LogPersisterRotator::SetId(uint32_t pId) +{ + id = pId; +} } // namespace HiviewDFX -} // namespace OHOS +} // namespace OHOS \ No newline at end of file diff --git a/services/hilogd/log_querier.cpp b/services/hilogd/log_querier.cpp index ea6d7eeb0430c9003bd72f9d8c6c320e4e70cf3e..e788b2d34898222437e4db65877c1eafbae8bc59 100644 --- a/services/hilogd/log_querier.cpp +++ b/services/hilogd/log_querier.cpp @@ -14,7 +14,6 @@ */ #include "log_querier.h" - #include #include #include @@ -25,11 +24,11 @@ #include #include #include +#include #include #include #include #include -#include "compressing_rotator.h" #include "hilog/log.h" #include "hilog_common.h" #include "log_data.h" @@ -38,16 +37,18 @@ #include "log_persister.h" #include "log_reader.h" + namespace OHOS { namespace HiviewDFX { using namespace std; - +namespace fs = std::filesystem; constexpr int MAX_DATA_LEN = 2048; string g_logPersisterDir = "/data/misc/logd/"; constexpr int DEFAULT_LOG_LEVEL = 1<version = 0; @@ -64,41 +65,21 @@ inline bool IsValidFileName(const std::string& strFileName) LogPersisterRotator* MakeRotator(LogPersistStartMsg& pLogPersistStartMsg) { string fileSuffix = ""; - - switch (pLogPersistStartMsg.compressType) { - case CompressStrategy::DIVIDED:{ - stringstream tmpFileSuffix; - tmpFileSuffix << pLogPersistStartMsg.filePath << "tmpLog" << pLogPersistStartMsg.jobId; - int tmpFileSize = 4096; - int tmpFileNum = 2; - return new CompressingRotator( - pLogPersistStartMsg.filePath, - pLogPersistStartMsg.fileSize, - pLogPersistStartMsg.fileNum, - tmpFileSize, tmpFileNum, tmpFileSuffix.str()); - } - case CompressStrategy::STREAM: - switch (pLogPersistStartMsg.compressAlg) { - case CompressAlg::COMPRESS_TYPE_ZSTD: - fileSuffix = ".zst"; - break; - case CompressAlg::COMPRESS_TYPE_ZLIB: - fileSuffix = ".gz"; - break; - default: - break; - }; - return new LogPersisterRotator( - pLogPersistStartMsg.filePath, - pLogPersistStartMsg.fileSize, - pLogPersistStartMsg.fileNum, - fileSuffix); + switch (pLogPersistStartMsg.compressAlg) { + case CompressAlg::COMPRESS_TYPE_ZSTD: + fileSuffix = ".zst"; + break; + case CompressAlg::COMPRESS_TYPE_ZLIB: + fileSuffix = ".gz"; + break; default: - return new LogPersisterRotator( - pLogPersistStartMsg.filePath, - pLogPersistStartMsg.fileSize, - pLogPersistStartMsg.fileNum); + break; }; + return new LogPersisterRotator( + pLogPersistStartMsg.filePath, + pLogPersistStartMsg.fileSize, + pLogPersistStartMsg.fileNum, + fileSuffix); } void HandleLogQueryRequest(std::shared_ptr logReader, HilogBuffer& buffer) @@ -114,7 +95,7 @@ void HandleNextRequest(std::shared_ptr logReader, HilogBuffer& buffer buffer.Query(logReader); } -void HandlePersistStartRequest(char* reqMsg, std::shared_ptr logReader, HilogBuffer* buffer) +void HandlePersistStartRequest(char* reqMsg, std::shared_ptr logReader, HilogBuffer& buffer) { char msgToSend[MAX_DATA_LEN]; const uint16_t sendMsgLen = sizeof(LogPersistStartResult); @@ -130,6 +111,14 @@ void HandlePersistStartRequest(char* reqMsg, std::shared_ptr logReade if (pLogPersistStartRst == nullptr) { return; } + if (pLogPersistStartMsg->fileSize < MAX_PERSISTER_BUFFER_SIZE) { + std::cout << "Persist log file size less than min size" << std::endl; + pLogPersistStartRst->jobId = pLogPersistStartMsg->jobId; + pLogPersistStartRst->result = RET_FAIL; + SetMsgHead(&pLogPersistStartRsp->msgHeader, MC_RSP_LOG_PERSIST_START, sendMsgLen); + logReader->hilogtoolConnectSocket->Write(msgToSend, sendMsgLen + sizeof(MessageHeader)); + return; + } string logPersisterPath; if (IsValidFileName(string(pLogPersistStartMsg->filePath)) == true) { logPersisterPath = (strlen(pLogPersistStartMsg->filePath) == 0) ? (g_logPersisterDir + "hilog") @@ -144,24 +133,26 @@ void HandlePersistStartRequest(char* reqMsg, std::shared_ptr logReade } strcpy_s(pLogPersistStartMsg->filePath, FILE_PATH_MAX_LEN, logPersisterPath.c_str()); rotator = MakeRotator(*pLogPersistStartMsg); + rotator->SetId(pLogPersistStartMsg->jobId); + rotator->Init(); std::shared_ptr persister = make_shared( pLogPersistStartMsg->jobId, pLogPersistStartMsg->filePath, - pLogPersistStartMsg->compressType, + pLogPersistStartMsg->fileSize, pLogPersistStartMsg->compressAlg, - SLEEP_TIME, rotator, buffer); - - pLogPersistStartRst->jobId = pLogPersistStartMsg->jobId; - pLogPersistStartRst->result = persister->Init(); + SLEEP_TIME, *rotator, buffer); persister->queryCondition.types = pLogPersistStartMsg->logType; persister->queryCondition.levels = DEFAULT_LOG_LEVEL; - if (pLogPersistStartRst->result == RET_FAIL) { + int saveInfoRes = persister->SaveInfo(*pLogPersistStartMsg); + pLogPersistStartRst->jobId = pLogPersistStartMsg->jobId; + pLogPersistStartRst->result = persister->Init(); + if (pLogPersistStartRst->result == RET_FAIL || saveInfoRes == RET_FAIL) { + cout << "Error initializing log persister!" << endl; persister.reset(); } else { persister->Start(); - buffer->AddLogReader(weak_ptr(persister)); + buffer.AddLogReader(weak_ptr(persister)); } - SetMsgHead(&pLogPersistStartRsp->msgHeader, MC_RSP_LOG_PERSIST_START, sendMsgLen); logReader->hilogtoolConnectSocket->Write(msgToSend, sendMsgLen + sizeof(MessageHeader)); } @@ -186,17 +177,29 @@ void HandlePersistDeleteRequest(char* reqMsg, std::shared_ptr logRead if (msgLen > sizeof(LogPersistStopMsg) * LOG_TYPE_MAX) { return; } - - while (pLogPersistStopMsg && recvMsgLen < msgLen) { - rst = LogPersister::Kill(pLogPersistStopMsg->jobId); - if (pLogPersistStopRst) { - pLogPersistStopRst->jobId = pLogPersistStopMsg->jobId; - pLogPersistStopRst->result = (rst < 0) ? RET_FAIL : RET_SUCCESS; - pLogPersistStopRst++; - } - pLogPersistStopMsg++; - recvMsgLen += sizeof(LogPersistStopMsg); - msgNum++; + list resultList; + list::iterator it; + rst = LogPersister::Query(DEFAULT_LOG_TYPE, resultList); + if (pLogPersistStopMsg && recvMsgLen < msgLen) { + if (pLogPersistStopMsg->jobId != JOB_ID_ALL) { + rst = LogPersister::Kill(pLogPersistStopMsg->jobId); + if (pLogPersistStopRst) { + pLogPersistStopRst->jobId = pLogPersistStopMsg->jobId; + pLogPersistStopRst->result = (rst < 0) ? RET_FAIL : RET_SUCCESS; + pLogPersistStopRst++; + msgNum++; + } + } else { + for (it = resultList.begin(); it != resultList.end(); ++it) { + rst = LogPersister::Kill((*it).jobId); + if (pLogPersistStopRst) { + pLogPersistStopRst->jobId = (*it).jobId; + pLogPersistStopRst->result = (rst < 0) ? RET_FAIL : RET_SUCCESS; + pLogPersistStopRst++; + msgNum++; + } + } + } } sendMsgLen = msgNum * sizeof(LogPersistStopResult); SetMsgHead(&pLogPersistStopRsp->msgHeader, MC_RSP_LOG_PERSIST_STOP, sendMsgLen); @@ -233,7 +236,6 @@ void HandlePersistQueryRequest(char* reqMsg, std::shared_ptr logReade pLogPersistQueryRst->result = (rst < 0) ? RET_FAIL : RET_SUCCESS; pLogPersistQueryRst->jobId = (*it).jobId; pLogPersistQueryRst->logType = (*it).logType; - pLogPersistQueryRst->compressType = (*it).compressType; pLogPersistQueryRst->compressAlg = (*it).compressAlg; if (strcpy_s(pLogPersistQueryRst->filePath, FILE_PATH_MAX_LEN, (*it).filePath)) { return; @@ -461,7 +463,7 @@ void LogQuerier::LogQuerierThreadFunc(std::shared_ptr logReader) } break; case MC_REQ_LOG_PERSIST_START: - HandlePersistStartRequest(g_tempBuffer, logReader, hilogBuffer); + HandlePersistStartRequest(g_tempBuffer, logReader, *hilogBuffer); break; case MC_REQ_LOG_PERSIST_STOP: HandlePersistDeleteRequest(g_tempBuffer, logReader); @@ -564,5 +566,57 @@ uint8_t LogQuerier::GetType() const return TYPE_CONTROL; } } + +int LogQuerier::RestorePersistJobs(HilogBuffer& _buffer) +{ + DIR *dir; + struct dirent *ent = nullptr; + if ((dir = opendir (g_logPersisterDir.c_str())) != NULL) { + while ((ent = readdir (dir)) != NULL) { + size_t length = strlen(ent->d_name); + std::string pPath(ent->d_name, length); + if (length >= INFO_SUFFIX && pPath.substr(length - INFO_SUFFIX, length) == ".info") { + if (pPath == "hilog.info") continue; + std::cout << "Found a persist job!" << std::endl; + FILE* infile = fopen((g_logPersisterDir + pPath).c_str(), "r"); + if (infile == NULL) { + std::cout << "Error opening recovery info file!" << std::endl; + continue; + } + PersistRecoveryInfo info; + fread(&info, sizeof(PersistRecoveryInfo), 1, infile); + fclose(infile); + LogPersisterRotator* rotator = rotator = MakeRotator(info.msg); + rotator->SetIndex(info.index + 1); + rotator->SetId(info.msg.jobId); + rotator->Init(); + printf("Recovery Info:\njobId=%u\nfilePath=%s\n", + info.msg.jobId, info.msg.filePath); + std::shared_ptr persister = make_shared( + info.msg.jobId, + info.msg.filePath, + info.msg.fileSize, + info.msg.compressAlg, + SLEEP_TIME, *rotator, _buffer); + int result = persister->Init(); + persister->queryCondition.types = info.types; + persister->queryCondition.levels = info.levels; + if (result == RET_FAIL) { + cout << "LogPersister Start Failed, result=" << result << endl; + persister.reset(); + } else { + persister->Start(); + _buffer.AddLogReader(weak_ptr(persister)); + } + } + } + closedir (dir); + } else { + perror ("Failed to open persister directory!"); + return EXIT_FAILURE; + } + cout << "Finished restoring persist jobs!" << endl; + return EXIT_SUCCESS; +} } // namespace HiviewDFX } // namespace OHOS diff --git a/services/hilogd/main.cpp b/services/hilogd/main.cpp index 12632927c5ba2e5777a0d3eb25f8ebe41d41dd14..2b54691772e76c83f801e1337ddb17638b335b13 100644 --- a/services/hilogd/main.cpp +++ b/services/hilogd/main.cpp @@ -19,6 +19,7 @@ #include #include "cmd_executor.h" +#include "log_querier.h" #include "hilog_input_socket_server.h" #include "log_collector.h" #include "flow_control_init.h" @@ -81,6 +82,12 @@ int HilogdEntry(int argc, char* argv[]) #endif server.RunServingThread(); } + + std::thread startupCheckThread([&hilogBuffer]() { + std::shared_ptr logQuerier = std::make_shared(nullptr, &hilogBuffer); + logQuerier->RestorePersistJobs(hilogBuffer); + }); + startupCheckThread.detach(); CmdExecutor cmdExecutor(&hilogBuffer); cmdExecutor.StartCmdExecutorThread(); diff --git a/services/hilogtool/include/hilogtool.h b/services/hilogtool/include/hilogtool.h index e2e8abfbf11242d13ce8a0401db82fe2ee7ce15c..67fa9e982c062c4d5c59d1a5f5f776204b4b60aa 100644 --- a/services/hilogtool/include/hilogtool.h +++ b/services/hilogtool/include/hilogtool.h @@ -42,7 +42,6 @@ typedef struct { std::string regexArgs; std::string buffSizeArgs; std::string logFileCtrlArgs; - std::string compressArgs; std::string fileSizeArgs; std::string fileNumArgs; std::string filePathArgs; @@ -62,4 +61,4 @@ typedef struct { } HilogArgs; } // namespace HiviewDFX } // namespace OHOS -#endif +#endif diff --git a/services/hilogtool/log_controller.cpp b/services/hilogtool/log_controller.cpp index eb585a965b64fb792cb7df835873c63513301a3e..e43801a5fc9925aeee7090232f975c90d2eb6af5 100644 --- a/services/hilogtool/log_controller.cpp +++ b/services/hilogtool/log_controller.cpp @@ -108,6 +108,18 @@ uint64_t GetBuffSize(const string& buffSizeStr) return buffSize; } +uint16_t GetCompressAlg(const std::string& pressAlg) +{ + if (pressAlg == "none") { + return COMPRESS_TYPE_NONE; + } else if (pressAlg == "zlib") { + return COMPRESS_TYPE_ZLIB; + } else if (pressAlg == "zstd") { + return COMPRESS_TYPE_ZSTD; + } + return COMPRESS_TYPE_ZLIB; +} + uint16_t GetLogLevel(const std::string& logLevelStr, std::string& logLevel) { if (logLevelStr == "debug" || logLevelStr == "DEBUG" || logLevelStr == "d" || logLevelStr == "D") { @@ -125,7 +137,7 @@ uint16_t GetLogLevel(const std::string& logLevelStr, std::string& logLevel) } else if (logLevelStr == "fatal" || logLevelStr == "FATAL" || logLevelStr == "f" || logLevelStr == "F") { logLevel = "F"; return LOG_FATAL; - } + } return 0xffff; } @@ -234,7 +246,7 @@ void LogQueryResponseOp(SeqPacketSocketClient& controller, char* recvBuffer, uin HilogShowLog(format, data, context, tailBuffer); NextRequestOp(controller, SENDIDA); break; - default: + default: NextRequestOp(controller, SENDIDA); break; } @@ -382,9 +394,6 @@ int32_t LogPersistOp(SeqPacketSocketClient& controller, uint8_t msgCmd, LogPersi Split(logPersistParam->jobIdStr, " ", vecJobId); logTypeNum = vecLogType.size(); jobIdNum = vecJobId.size(); - if (msgCmd == MC_REQ_LOG_PERSIST_STOP && logPersistParam->jobIdStr == "") { // support stop several jobs each time - return RET_FAIL; - } switch (msgCmd) { case MC_REQ_LOG_PERSIST_START: { LogPersistStartRequest* pLogPersistStartReq = reinterpret_cast(msgToSend); @@ -402,11 +411,9 @@ int32_t LogPersistOp(SeqPacketSocketClient& controller, uint8_t msgCmd, LogPersi } pLogPersistStartMsg->jobId = (logPersistParam->jobIdStr == "") ? DEFAULT_JOBID : stoi(logPersistParam->jobIdStr); - pLogPersistStartMsg->compressType = (logPersistParam->compressTypeStr == "") ? STREAM : stoi(logPersistParam - ->compressTypeStr); - pLogPersistStartMsg->compressAlg = (logPersistParam->compressAlgStr == "") ? COMPRESS_TYPE_ZLIB : stoi( - logPersistParam->compressAlgStr); - pLogPersistStartMsg->fileSize = (logPersistParam->fileSizeStr == "") ? fileSizeDefault : stoi( + pLogPersistStartMsg->compressAlg = (logPersistParam->compressAlgStr == "") ? COMPRESS_TYPE_ZLIB : + GetCompressAlg(logPersistParam->compressAlgStr); + pLogPersistStartMsg->fileSize = (logPersistParam->fileSizeStr == "") ? fileSizeDefault : GetBuffSize( logPersistParam->fileSizeStr); pLogPersistStartMsg->fileNum = (logPersistParam->fileNumStr == "") ? fileNumDefault : stoi(logPersistParam->fileNumStr); @@ -419,13 +426,18 @@ int32_t LogPersistOp(SeqPacketSocketClient& controller, uint8_t msgCmd, LogPersi SetMsgHead(&pLogPersistStartReq->msgHeader, msgCmd, sizeof(LogPersistStartRequest)); controller.WriteAll(msgToSend, sizeof(LogPersistStartRequest)); break; - } - + } case MC_REQ_LOG_PERSIST_STOP: { LogPersistStopRequest* pLogPersistStopReq = reinterpret_cast(msgToSend); LogPersistStopMsg* pLogPersistStopMsg = reinterpret_cast(&pLogPersistStopReq->logPersistStopMsg); + if (logPersistParam->jobIdStr == "") { + pLogPersistStopMsg->jobId = JOB_ID_ALL; + SetMsgHead(&pLogPersistStopReq->msgHeader, msgCmd, sizeof(LogPersistStopMsg)); + controller.WriteAll(msgToSend, sizeof(LogPersistStopMsg) + sizeof(MessageHeader)); + break; + } if (jobIdNum * sizeof(LogPersistStopMsg) + sizeof(MessageHeader) > MSG_MAX_LEN) { return RET_FAIL; } diff --git a/services/hilogtool/log_display.cpp b/services/hilogtool/log_display.cpp index 6bf63aac79f3a024098bbb16895970f9324db332..e79ba70baa211fdad75e8cf870adbfac6e14bceb 100644 --- a/services/hilogtool/log_display.cpp +++ b/services/hilogtool/log_display.cpp @@ -84,20 +84,6 @@ string GetOrigType(uint16_t shiftType) return logType; } -string GetPressTypeStr(uint16_t pressType) -{ - string pressTypeStr; - if (pressType == NONE) { - pressTypeStr = "none"; - } - if (pressType == STREAM) { - pressTypeStr = "stream"; - } - if (pressType == DIVIDED) { - pressTypeStr = "divided"; - } - return pressTypeStr; -} string GetPressAlgStr(uint16_t pressAlg) { @@ -271,13 +257,13 @@ int32_t ControlCmdResult(const char* message) (LogPersistStartResult*)&pLogPersistStartRsp->logPersistStartRst; while (pLogPersistStartRst && resultLen < msgLen) { if (pLogPersistStartRst->result == RET_FAIL) { + outputStr += "Persist task [jobid:"; outputStr += to_string(pLogPersistStartRst->jobId); - outputStr += " log file task start fail"; - outputStr += "\n"; + outputStr += "] start failed\n"; } else { + outputStr += "Persist task [jobid:"; outputStr += to_string(pLogPersistStartRst->jobId); - outputStr += " log file task start success"; - outputStr += "\n"; + outputStr += "] started successfully\n"; } pLogPersistStartRst++; resultLen += sizeof(LogPersistStartResult); @@ -292,13 +278,13 @@ int32_t ControlCmdResult(const char* message) LogPersistStopResult* pLogPersistStopRst = (LogPersistStopResult*)&pLogPersistStopRsp->logPersistStopRst; while (pLogPersistStopRst && resultLen < msgLen) { if (pLogPersistStopRst->result == RET_FAIL) { + outputStr += "Persist task [jobid:"; outputStr += to_string(pLogPersistStopRst->jobId); - outputStr += " log file task stop fail"; - outputStr += "\n"; + outputStr += "] stop failed\n"; } else { + outputStr += "Persist task [jobid:"; outputStr += to_string(pLogPersistStopRst->jobId); - outputStr += " log file task stop success"; - outputStr += "\n"; + outputStr += "] stopped successfully\n"; } pLogPersistStopRst++; resultLen += sizeof(LogPersistStopResult); @@ -314,16 +300,14 @@ int32_t ControlCmdResult(const char* message) (LogPersistQueryResult*)&pLogPersistQueryRsp->logPersistQueryRst; while (pLogPersistQueryRst && resultLen < msgLen) { if (pLogPersistQueryRst->result == RET_FAIL) { + outputStr = "Persist task [logtype:"; outputStr += GetLogTypeStr(pLogPersistQueryRst->logType); - outputStr = " log file task query fail"; - outputStr += "\n"; + outputStr += "] query failed\n"; } else { outputStr += to_string(pLogPersistQueryRst->jobId); outputStr += " "; outputStr += GetOrigType(pLogPersistQueryRst->logType); outputStr += " "; - outputStr += GetPressTypeStr(pLogPersistQueryRst->compressType); - outputStr += " "; outputStr += GetPressAlgStr(pLogPersistQueryRst->compressAlg); outputStr += " "; outputStr += pLogPersistQueryRst->filePath; @@ -398,7 +382,7 @@ bool HilogMatchByRegex(string context, string regExpArg) } } -void HilogShowLog(HilogShowFormat showFormat, HilogDataMessage* data, HilogArgs* context, +void HilogShowLog(HilogShowFormat showFormat, HilogDataMessage* data, HilogArgs* context, vector& tailBuffer) { if (data->sendId == SENDIDN) { @@ -436,7 +420,6 @@ void HilogShowLog(HilogShowFormat showFormat, HilogDataMessage* data, HilogArgs* showBuffer.tv_nsec = data->tv_nsec; int offset = data->tag_len; - char *dataBegin = data->data + offset; char *dataPos = data->data + offset; while (*dataPos != 0) { diff --git a/services/hilogtool/main.cpp b/services/hilogtool/main.cpp index 39de63e863c9562c12777bccf417546bfc0231f7..2444466771dd1ced9fb954d83eacc8cb6340f3cf 100644 --- a/services/hilogtool/main.cpp +++ b/services/hilogtool/main.cpp @@ -68,9 +68,6 @@ static void Helper() " specify the domain, no more than %d.\n" " -T , --Tag=\n" " specify the tag, no more than %d.\n" - " -c , --compress=\n" - " specify compress type of log files.\n" - " 0 plain text, 1 stream compression 2 compress per file.\n" " -a , --head= show n lines log on head.\n" " -z , --tail= show n lines log on tail.\n" " -G , --buffer-size=\n" @@ -79,7 +76,7 @@ static void Helper() " -e , --regex=\n" " show the logs which match the regular expression,\n" " is a regular expression.\n" - " -F , --filename=\n" + " -f , --filename=\n" " set log file name.\n" " -l , --length=\n" " set single log file size.\n" @@ -91,6 +88,10 @@ static void Helper() " query log file writing task query.\n" " start start a log file writing task, see -F -l -n -c for to set more configs,\n" " stop stop a log file writing task.\n" + " -m ,--stream=\n" + " none log file without compressing\n" + " zlib compress log file by the zlib algorithm\n" + " zstd compress log file by the zstd algorithm\n" " -v , --format= options:\n" " time display local time.\n" " color display colorful logs by log level.i.e. \x1B[38;5;231mVERBOSE\n" @@ -107,7 +108,7 @@ static void Helper() " \n Types, levels, domains, tags support exclusion query.\n" " Exclusion query can be done with parameters starting with \"^\" and delimiter \",\".\n" " Example: \"-t ^core,app\" excludes logs with types core and app.\n" - " Could be used along with other parameters.\n", + " Could be used along with other parameters.\n", MAX_DOMAINS, MAX_TAGS, MAX_PIDS ); } @@ -195,10 +196,9 @@ int HilogEntry(int argc, char* argv[]) { "buffer-size", required_argument, nullptr, 'G' }, { "jobid", required_argument, nullptr, 'j' }, { "flowctrl", required_argument, nullptr, 'Q' }, - { "compress", required_argument, nullptr, 'c' }, { "stream", required_argument, nullptr, 'm' }, { "number", required_argument, nullptr, 'n' }, - { "filename", required_argument, nullptr, 'F' }, + { "filename", required_argument, nullptr, 'f' }, { "private", required_argument, nullptr, 'p' }, { "domain", required_argument, nullptr, 'D' }, { "tag", required_argument, nullptr, 'T' }, @@ -209,7 +209,7 @@ int HilogEntry(int argc, char* argv[]) {nullptr, 0, nullptr, 0} }; - int choice = getopt_long(argc, argv, "hxz:grsSa:v:e:t:L:G:F:l:n:j:w:c:p:k:M:D:T:b:Q:m:P:", + int choice = getopt_long(argc, argv, "hxz:grsSa:v:e:t:L:G:f:l:n:j:w:p:k:M:D:T:b:Q:m:P:", longOptions, &optIndex); if (choice == -1) { break; @@ -310,16 +310,13 @@ int HilogEntry(int argc, char* argv[]) context.logFileCtrlArgs = optarg; noLogOption = true; break; - case 'c': - context.compressArgs = optarg; - break; case 'l': context.fileSizeArgs = optarg; break; case 'n': context.fileNumArgs = optarg; break; - case 'F': + case 'f': context.fileNameArgs = optarg; break; case 'j': @@ -480,7 +477,6 @@ int HilogEntry(int argc, char* argv[]) } else if (context.logFileCtrlArgs != "") { LogPersistParam logPersistParam; logPersistParam.logTypeStr = context.logTypeArgs; - logPersistParam.compressTypeStr = context.compressArgs; logPersistParam.compressAlgStr = context.algorithmArgs; logPersistParam.fileSizeStr = context.fileSizeArgs; logPersistParam.fileNumStr = context.fileNumArgs; @@ -584,7 +580,7 @@ int HilogEntry(int argc, char* argv[]) } default: - cout << "Invalid response from hilogd!" << endl; + cout << "Invalid response from hilogd! response: " << msgHeader->msgType << endl; break; } return 0;