diff --git a/ide/package.json b/ide/package.json index 6e9f13f524a3a9fa887ff6563afc5b753c191263..12be40e7b4e900b1103abb0b67f0663d93c550a9 100644 --- a/ide/package.json +++ b/ide/package.json @@ -4,6 +4,7 @@ "description": "Smart Perf", "main": "index.js", "scripts": { + "set-env":"set NODE_OPTIONS=\"--max-old-space-size=4096\"", "build": "webpack --mode=production --node-env=production", "build:dev": "webpack --mode=development", "build:prod": "webpack --mode=production --node-env=production", @@ -34,7 +35,6 @@ "useWb": true }, "setupFiles": [ - "jsdom-worker", "jest-canvas-mock" ], "setupFilesAfterEnv": [ @@ -56,11 +56,9 @@ "@types/jest": "*", "@types/node": "^17.0.10", "typescript": "^5.2.2", - "usb": "^2.4.2", "jest": "*", "jest-canvas-mock": "^2.3.1", "jest-environment-jsdom": "^28.1.0", - "jsdom-worker": "^0.2.1", "log4js": "^6.4.4", "node-fetch": "^2.6.7", "autoprefixer": "^10.4.14", @@ -75,7 +73,8 @@ "html-webpack-plugin": "^5.5.3", "webpack": "^5.89.0", "webpack-cli": "^5.1.4", - "webpack-dev-server": "^4.15.1" + "webpack-dev-server": "^4.15.1", + "jest-worker": "^29.5.0" }, "dependencies": { "@webcomponents/custom-elements": "^1.6.0" diff --git a/ide/src/hdc/HdcDeviceManager.ts b/ide/src/hdc/HdcDeviceManager.ts index 3aa0ca9bf2029857d0b421df8fa1449e6a2335c6..2715499856f023b3ffef34995d5a3d4bd92decf7 100644 --- a/ide/src/hdc/HdcDeviceManager.ts +++ b/ide/src/hdc/HdcDeviceManager.ts @@ -59,6 +59,7 @@ export class HdcDeviceManager { /** * findDevice */ + // @ts-ignore public static findDevice(): Promise { if (!('usb' in navigator)) { throw new Error('WebUSB not supported by the browser (requires HTTPS)'); diff --git a/ide/src/trace/SpApplication.ts b/ide/src/trace/SpApplication.ts index 5b760f70cfa593b7987758790998f6f7569c97d2..4ffbbf09c35f56de19a822acecaab717c561579e 100644 --- a/ide/src/trace/SpApplication.ts +++ b/ide/src/trace/SpApplication.ts @@ -1404,6 +1404,7 @@ export class SpApplication extends BaseElement { if (headerStr.indexOf('OHOSPROF') !== 0 && rowTraceStr.indexOf('49df') !== 0) { isAllowTrace = false; } + DbPool.sharedBuffer = null; } let index = 2; if (existFtrace.length > 0 && isAllowTrace) { @@ -1492,6 +1493,7 @@ export class SpApplication extends BaseElement { } let openFileInit = () => { + this.clearTraceFileCache(); SpStatisticsHttpUtil.addOrdinaryVisitAction({ event: 'open_trace', action: 'open_trace', @@ -2331,6 +2333,39 @@ export class SpApplication extends BaseElement { ); } + readTraceFileBuffer(): Promise { + return new Promise((resolve) => { + caches.match(DbPool.fileCacheKey).then(res => { + if (res) { + res.arrayBuffer().then(buffer => { + resolve(buffer); + }) + } else { + resolve(undefined); + } + }); + }); + }; + + clearTraceFileCache(): void { + caches.keys().then(keys => { + keys.forEach(key => { + if (key === DbPool.fileCacheKey) { + caches.delete(key).then(); + } else if (key.includes('/')) { + let splits = key.split('/'); + let fileDate = new Date(parseInt(splits[splits.length - 1])); + if (fileDate.toLocaleDateString() !== new Date().toLocaleDateString()) { + //如果不是当天的缓存则删去缓存文件 + caches.delete(key).then(); + } + } else { + caches.delete(key).then(); + } + }) + }); + } + private async download(mainMenu: LitMainMenu, fileName: string, isServer: boolean, dbName?: string) { let a = document.createElement('a'); if (isServer) { @@ -2341,7 +2376,10 @@ export class SpApplication extends BaseElement { return; } } else { - a.href = URL.createObjectURL(new Blob([DbPool.sharedBuffer!])); + let buffer = await this.readTraceFileBuffer(); + if (buffer) { + a.href = URL.createObjectURL(new Blob([buffer])); + } } a.download = fileName; a.click(); diff --git a/ide/src/trace/component/SpFlags.ts b/ide/src/trace/component/SpFlags.ts index 36775d0ef41889478c9da7628869c3bf2fbbb5e5..dd2332e39c3b64b907a8092d5bba6ab91300a374 100644 --- a/ide/src/trace/component/SpFlags.ts +++ b/ide/src/trace/component/SpFlags.ts @@ -253,6 +253,11 @@ export class FlagsConfig { switchOptions: [{ option: 'Enabled' }, { option: 'Disabled', selected: true }], describeContent: 'support Cpu State Binder-Runnable', }, + { + title: 'FfrtConvert', + switchOptions: [{ option: 'Enabled' }, { option: 'Disabled', selected: true }], + describeContent: 'Ffrt Convert templates', + }, ]; static getAllFlagConfig(): Array { diff --git a/ide/src/trace/component/chart/SpProcessChart.ts b/ide/src/trace/component/chart/SpProcessChart.ts index 90d940b0cf380d31a5a79303ea9c0aabb6e977f4..b4f2d2530e548e5bc5d154ed863de718ce0e32c6 100644 --- a/ide/src/trace/component/chart/SpProcessChart.ts +++ b/ide/src/trace/component/chart/SpProcessChart.ts @@ -20,7 +20,6 @@ import { queryAllActualData, queryAllExpectedData, queryAllJankProcess, - queryAllTaskPoolPid, queryEventCountMap, queryProcess, queryProcessAsyncFunc, @@ -35,6 +34,7 @@ import { queryProcessThreads, queryProcessThreadsByTable, queryStartupPidArray, + queryTaskPoolProcessIds, queryThreadData, } from '../../database/SqlLite'; import { Utils } from '../trace/base/Utils'; @@ -215,11 +215,10 @@ export class SpProcessChart { allExpectedProcess = await queryAllExpectedData(); allActualProcess = await queryAllActualData(); } - let allTaskPoolPid = await queryAllTaskPoolPid(); - let allTaskPoolProcess: Array = []; - allTaskPoolPid.forEach((value) => { - allTaskPoolProcess.push(value.pid); - }); + let allTaskPoolPid: Array<{ pid: number }> = []; + if (FlagsConfig.getFlagsConfigEnableStatus('TaskPool')) { + allTaskPoolPid = await queryTaskPoolProcessIds(); + } info('ProcessList Data size is: ', processList!.length); for (let i = 0; i < processList.length; i++) { const it = processList[i]; @@ -244,6 +243,9 @@ export class SpProcessChart { ) { processRow.addTemplateTypes('AppStartup'); } + if (allTaskPoolPid.find((process) => process.pid === it.pid) !== undefined) { + processRow.addTemplateTypes('TaskPool'); + } processRow.name = `${it.processName || 'Process'} ${it.pid}`; processRow.supplier = (): Promise> => queryProcessData(it.pid || -1, 0, TraceRow.range?.totalNS || 0); processRow.favoriteChangeHandler = this.trace.favoriteChangeHandler; @@ -517,7 +519,8 @@ export class SpProcessChart { if (linkNodeItem[0].rowEL.collect) { linkNodeItem[0].rowEL.translateY = linkNodeItem[0].rowEL.getBoundingClientRect().top - 195; } else { - linkNodeItem[0].rowEL.translateY = linkNodeItem[0].rowEL.offsetTop - this.trace.rowsPaneEL!.scrollTop; + linkNodeItem[0].rowEL.translateY = + linkNodeItem[0].rowEL.offsetTop - this.trace.rowsPaneEL!.scrollTop; } linkNodeItem[0].x = ns2xByTimeShaft(linkNodeItem[0].ns, this.trace.timerShaftEL!); linkNodeItem[0].offsetY = linkNodeItem[0].offsetY * 2; @@ -528,7 +531,8 @@ export class SpProcessChart { if (linkNodeItem[1].rowEL.collect) { linkNodeItem[1].rowEL.translateY = linkNodeItem[1].rowEL.getBoundingClientRect().top - 195; } else { - linkNodeItem[1].rowEL.translateY = linkNodeItem[1].rowEL.offsetTop - this.trace.rowsPaneEL!.scrollTop; + linkNodeItem[1].rowEL.translateY = + linkNodeItem[1].rowEL.offsetTop - this.trace.rowsPaneEL!.scrollTop; } linkNodeItem[1].x = ns2xByTimeShaft(linkNodeItem[1].ns, this.trace.timerShaftEL!); linkNodeItem[1].offsetY = linkNodeItem[1].offsetY * 2; @@ -548,7 +552,7 @@ export class SpProcessChart { linkProcessItem[0].rowEL.translateY = linkProcessItem[0].rowEL.offsetTop - this.trace.rowsPaneEL!.scrollTop; } - linkProcessItem[0].y = processRow!.translateY + linkProcessItem[0].offsetY;//11 + linkProcessItem[0].y = processRow!.translateY + linkProcessItem[0].offsetY; //11 if (linkProcessItem[1].rowEL.collect) { linkProcessItem[1].rowEL.translateY = linkProcessItem[1].rowEL.getBoundingClientRect().top - 195; } else { @@ -782,11 +786,6 @@ export class SpProcessChart { funcRow.style.height = `${maxHeight}px`; funcRow.name = `${thread.threadName || 'Thread'} ${thread.tid}`; funcRow.setAttribute('children', ''); - if (allTaskPoolProcess.indexOf(it.pid) !== -1) { - if (funcRow.name.startsWith('TaskWorkThread') || thread.is_main_thread === 1) { - funcRow.addTemplateTypes('Task Pool'); - } - } funcRow.supplier = (): Promise> => getFunDataByTid(thread.tid || 0, thread.upid || 0).then((funs: Array) => { if (funs.length > 0) { diff --git a/ide/src/trace/component/trace/base/TraceRowConfig.ts b/ide/src/trace/component/trace/base/TraceRowConfig.ts index 8073c8b56ed426076365d40cce924964355611c2..b72404c3e9d767640fd163621cd1bad3c7921a97 100644 --- a/ide/src/trace/component/trace/base/TraceRowConfig.ts +++ b/ide/src/trace/component/trace/base/TraceRowConfig.ts @@ -48,6 +48,7 @@ export class TraceRowConfig extends BaseElement { 'FrameTimeline', 'AnimationEffect', 'AppStartup', + 'TaskPool', 'HiSysEvent', 'EnergyEvent', 'Memory', diff --git a/ide/src/trace/component/trace/sheet/TabPaneCurrentSelection.ts b/ide/src/trace/component/trace/sheet/TabPaneCurrentSelection.ts index bc5383e2dfae6a1b14679a050d4e442952428239..0fb401712b5bf10202a21c297985fcd0178f1975 100644 --- a/ide/src/trace/component/trace/sheet/TabPaneCurrentSelection.ts +++ b/ide/src/trace/component/trace/sheet/TabPaneCurrentSelection.ts @@ -86,7 +86,7 @@ export function getTimeString(ns: number): string { currentTimeNs = currentTimeNs - Math.floor(currentTimeNs / microsecond1) * microsecond1; } if (currentTimeNs > 0) { - res += currentTimeNs + 'ns '; + res += currentTimeNs.toFixed(0) + 'ns '; } return res; } diff --git a/ide/src/trace/component/trace/sheet/TabPaneFilter.ts b/ide/src/trace/component/trace/sheet/TabPaneFilter.ts index d1eb77c1bfcc9610d06d514104cb4e450dd32c1f..ef3eb6fa0675e43c8fdec2f873c160f1b3540428 100644 --- a/ide/src/trace/component/trace/sheet/TabPaneFilter.ts +++ b/ide/src/trace/component/trace/sheet/TabPaneFilter.ts @@ -623,7 +623,7 @@ export class TabPaneFilter extends BaseElement { }; e.addEventListener('keyup', (event: any): void => { event.stopPropagation(); - if (event.keyCode === '13') { + if (event.keyCode === 13) { if (event?.target.value === '') { inputs[idx].value = idx === 0 ? '0' : '∞'; } diff --git a/ide/src/trace/database/Convert.ts b/ide/src/trace/database/Convert.ts index 72410531e2e7d05adfb328fdba5e5867b7ec5953..04eb973f7bd44d50a586551cd320437e2bfb3386 100644 --- a/ide/src/trace/database/Convert.ts +++ b/ide/src/trace/database/Convert.ts @@ -38,14 +38,20 @@ class ConvertThread { DbPool.sharedBuffer = res.buffer; handler(res.status, res.msg, res.results); }; - let pam = { - id: id, - action: 'getConvertData', - buffer: DbPool.sharedBuffer!, - }; - try { - this.worker!.postMessage(pam, [DbPool.sharedBuffer!]); - } catch (e: any) {} + caches.match(DbPool.fileCacheKey).then(resData => { + if (resData) { + resData.arrayBuffer().then(buffer => { + this.worker!.postMessage( + { + id: id, + action: 'getConvertData', + buffer: buffer!, + }, + [buffer!] + ); + }); + } + }); } } diff --git a/ide/src/trace/database/SqlLite.ts b/ide/src/trace/database/SqlLite.ts index 3a08c667679f29e8a9e3f76adc55527875336838..65ff4297cec2e5b3cc219171839912e01e0d35d4 100644 --- a/ide/src/trace/database/SqlLite.ts +++ b/ide/src/trace/database/SqlLite.ts @@ -164,16 +164,22 @@ class DbThread { handler(res.cutStatus, res.msg); } }; - this.worker!.postMessage( - { - id: id, - action: 'cut-file', - leftTs: leftTs, - rightTs: rightTs, - buffer: DbPool.sharedBuffer!, - }, - [DbPool.sharedBuffer!] - ); + caches.match(DbPool.fileCacheKey).then(resData => { + if (resData) { + resData.arrayBuffer().then(buffer => { + this.worker!.postMessage( + { + id: id, + action: 'cut-file', + leftTs: leftTs, + rightTs: rightTs, + buffer: buffer!, + }, + [buffer!] + ); + }); + } + }); } dbOpen = async ( @@ -184,6 +190,7 @@ class DbThread { msg: string; buffer: ArrayBuffer; sdkConfigMap: any; + fileKey: string; }> => { return new Promise((resolve, reject) => { let id = this.uuid(); @@ -194,6 +201,7 @@ class DbThread { msg: res.msg, sdkConfigMap: res.configSqlMap, buffer: res.buffer, + fileKey: res.fileKey }); } else { resolve({ status: res.init, msg: res.msg }); @@ -222,6 +230,7 @@ class DbThread { export class DbPool { static sharedBuffer: ArrayBuffer | null = null; + static fileCacheKey: string = 'null'; maxThreadNumber: number = 0; works: Array = []; progress: Function | undefined | null; @@ -341,18 +350,37 @@ export class DbPool { let configMap; for (let i = 0; i < this.works.length; i++) { let thread = this.works[i]; - let { status, msg, buffer, sdkConfigMap } = await thread.dbOpen(parseConfig, sdkWasmConfig); + let { status, msg, buffer, sdkConfigMap, fileKey } = await thread.dbOpen(parseConfig, sdkWasmConfig); if (!status) { DbPool.sharedBuffer = null; return { status, msg }; } else { configMap = sdkConfigMap; DbPool.sharedBuffer = buffer; + if (fileKey !== '-1') { + DbPool.fileCacheKey = fileKey; + } else { + DbPool.fileCacheKey = `trace/${new Date().getTime()}`; + this.saveTraceFileBuffer(DbPool.fileCacheKey, buffer); + } } } return { status: true, msg: 'ok', sdkConfigMap: configMap }; }; + saveTraceFileBuffer(key: string,buffer: ArrayBuffer): void { + caches.open(key).then(cache => { + let headers = new Headers(); + headers.append('Content-Length', `${buffer.byteLength}`); + headers.append('Content-Type', 'application/octet-stream'); + cache.put(key, new Response(buffer,{ + status: 200, + headers: headers + })).then(); + }); + } + + close = async () => { clearInterval(this.cutDownTimer); for (let i = 0; i < this.works.length; i++) { @@ -1345,6 +1373,26 @@ export const queryAppStartupProcessIds = (): Promise> => SELECT t.ipid FROM app_startup a LEFT JOIN thread t ON a.call_id = t.itid );` ); + +export const queryTaskPoolProcessIds = (): Promise> => + query( + 'queryAppStartupProcessIds', + `SELECT pid + FROM + process + WHERE + ipid IN ( + SELECT DISTINCT + ( ipid ) + FROM + thread + WHERE + itid IN ( SELECT DISTINCT ( callid ) FROM callstack WHERE name LIKE 'H:Task%' ) + AND name = 'TaskWorkThread' + )` + ); + + export const queryProcessContentCount = (): Promise> => query(`queryProcessContentCount`, `select pid,switch_count,thread_count,slice_count,mem_count from process;`); export const queryProcessThreadsByTable = (): Promise> => @@ -2784,7 +2832,7 @@ export const queryPerfThread = (): Promise> => a.process_id as pid, b.thread_name as processName from perf_thread a - left join (select distinct process_id, thread_name from perf_thread where process_id = thread_id ) b + left join (select distinct process_id, thread_name from perf_thread where process_id = thread_id) b on a.process_id = b.process_id order by pid;`, {} @@ -4785,13 +4833,6 @@ export const queryJsCpuProfilerData = (): Promise> => export const queryJsMemoryData = (): Promise> => query('queryJsMemoryData', `SELECT 1 WHERE EXISTS(SELECT 1 FROM js_heap_nodes)`); -export const queryAllTaskPoolPid = (): Promise> => - query( - 'queryAllTaskPoolPid', - `SELECT DISTINCT pid from task_pool LEFT JOIN callstack ON callstack.id = task_pool.execute_task_row - LEFT JOIN thread ON thread.id = callstack.callid LEFT JOIN process ON - process.id = thread.ipid WHERE task_pool.execute_task_row IS NOT NULL` - ); export const queryVmTrackerShmData = (iPid: number): Promise> => query( 'queryVmTrackerShmData', diff --git a/ide/src/trace/database/TraceWorker.ts b/ide/src/trace/database/TraceWorker.ts index 113a24fb8f0074b03db7d512b62378a72178ed68..3c8055508a1817d6906b6392c0989a4cb74d089b 100644 --- a/ide/src/trace/database/TraceWorker.ts +++ b/ide/src/trace/database/TraceWorker.ts @@ -38,6 +38,7 @@ let arkTsDataSize: number = 0; let currentAction: string = ''; let currentActionId: string = ''; +let ffrtFileCacheKey = '-1'; function clear() { if (Module != null) { @@ -167,6 +168,18 @@ let convertJSON = () => { } }; +function saveTraceFileBuffer(key: string,buffer: ArrayBuffer): void { + caches.open(key).then(cache => { + let headers = new Headers(); + headers.append('Content-Length', `${buffer.byteLength}`); + headers.append('Content-Type', 'application/octet-stream'); + cache.put(key, new Response(buffer,{ + status: 200, + headers: headers + })).then(); + }); +} + self.onmessage = async (e: MessageEvent) => { currentAction = e.data.action; currentActionId = e.data.id; @@ -174,6 +187,7 @@ self.onmessage = async (e: MessageEvent) => { clear(); } else if (e.data.action === 'open') { await initWASM(); + ffrtFileCacheKey = '-1'; // @ts-ignore self.postMessage({ id: e.data.id, @@ -190,8 +204,21 @@ self.onmessage = async (e: MessageEvent) => { bufferSlice.length = 0; } }; - let fn = Module.addFunction(callback, 'viii'); - reqBufferAddr = Module._Initialize(fn, REQ_BUF_SIZE); + let ffrtConvertCallback = (heapPtr: number, size: number, isEnd: number) => { + if (isEnd !== 1) { + let out: Uint8Array = Module.HEAPU8.slice(heapPtr, heapPtr + size); + bufferSlice.push(out); + } else { + arr = merged(); + bufferSlice.length = 0; + ffrtFileCacheKey = `ffrt/${new Date().getTime()}`; + saveTraceFileBuffer(ffrtFileCacheKey, arr.buffer); + } + }; + let fn1 = Module.addFunction(callback, 'viii'); + let fn2 = Module.addFunction(ffrtConvertCallback, 'viii'); + Module._TraceStreamer_Set_Log_Level(5); + reqBufferAddr = Module._Initialize(fn1, REQ_BUF_SIZE, fn2); let parseConfig = e.data.parseConfig; if (parseConfig !== '') { let parseConfigArray = enc.encode(parseConfig); @@ -330,7 +357,7 @@ self.onmessage = async (e: MessageEvent) => { const dataSlice = final.subarray(wrSize, wrSize + sliceLen); Module.HEAPU8.set(dataSlice, reqBufferAddr); wrSize += sliceLen; - r2 = Module._TraceStreamerParseDataEx(sliceLen); + r2 = Module._TraceStreamerParseDataEx(sliceLen, wrSize === final.length ? 1 : 0); if (r2 == -1) { break; } @@ -341,11 +368,7 @@ self.onmessage = async (e: MessageEvent) => { const dataSlice = uint8Array.subarray(wrSize, wrSize + sliceLen); Module.HEAPU8.set(dataSlice, reqBufferAddr); wrSize += sliceLen; - if (wrSize >= uint8Array.length) { - r2 = Module._TraceStreamerParseDataEx(sliceLen, 1); - } else { - r2 = Module._TraceStreamerParseDataEx(sliceLen, 0); - } + r2 = Module._TraceStreamerParseDataEx(sliceLen, wrSize === uint8Array.length ? 1 : 0); if (r2 == -1) { break; } @@ -355,7 +378,7 @@ self.onmessage = async (e: MessageEvent) => { for (let value of thirdWasmMap.values()) { value.model._TraceStreamer_In_ParseDataOver(); } - if (r2 == -1) { + if (r2 === -1) { // @ts-ignore self.postMessage({ id: e.data.id, @@ -370,6 +393,7 @@ self.onmessage = async (e: MessageEvent) => { // @ts-ignore self.postMessage({ id: e.data.id, ready: true, index: index + 1 }); }); + self.postMessage( { id: e.data.id, @@ -378,6 +402,7 @@ self.onmessage = async (e: MessageEvent) => { msg: 'ok', configSqlMap: thirdJsonResult, buffer: e.data.buffer, + fileKey: ffrtFileCacheKey }, // @ts-ignore [e.data.buffer] diff --git a/ide/webpack.config.js b/ide/webpack.config.js index 38d6d8ad723737e642c2c321ea04f63d857fe9c3..8ecc16731420df18e04e36b8c90b20a3f8f9f8d4 100644 --- a/ide/webpack.config.js +++ b/ide/webpack.config.js @@ -12,7 +12,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - // Generated using webpack-cli https://github.com/webpack/webpack-cli const path = require('path'); @@ -26,16 +25,6 @@ const childProcess = require('child_process'); const { exec } = require('child_process'); const fs = require('fs'); - -function directoryExists(path) { - try { - fs.accessSync(path); - return true; - } catch (error) { - return false; - } -} - function runCommand(command) { return new Promise((resolve, reject) => { exec(command, (error, stdout, stderr) => { @@ -48,6 +37,42 @@ function runCommand(command) { }); } +function cpFile(sourcePath, targetPath) { + fs.readdir(sourcePath, (err, files) => { + if (err) { + console.error('无法读取目录', err); + return; + } + files.forEach((file) => { + const source = `${sourcePath}/${file}`; + const target = `${targetPath}/${file}`; + fs.copyFile(source, target, (err) => { + if (err) { + console.error('无法复制文件', err); + return; + } + }); + }); + }); +} + +function clearDirectory(directoryPath) { + const isDirectoryExists = fs.existsSync(directoryPath); + + if (!isDirectoryExists) { + fs.mkdirSync(directoryPath); + } else { + fs.readdirSync(directoryPath).forEach((file) => { + const filePath = path.join(directoryPath, file); + if (fs.lstatSync(filePath).isDirectory()) { + return; + } else { + fs.unlinkSync(filePath); // 删除文件 + } + }); + } +} + const stylesHandler = isProduction ? MiniCssExtractPlugin.loader : 'style-loader'; //compile server ((flag) => { @@ -57,15 +82,9 @@ const stylesHandler = isProduction ? MiniCssExtractPlugin.loader : 'style-loader console.log('start compile server'); let outPath = path.normalize(path.join(__dirname, '/', 'dist')); let serverSrc = path.normalize(path.join(__dirname, '/server/main.go')); - if (!directoryExists(outPath)) { - runCommand(`mkdir ${outPath}`); - } else { - runCommand(`rm -rf ${outPath}/* `).then((result) => { - - }); - } - runCommand(`cp ./bin/* dist/`); - + let binPath = path.normalize(path.join(__dirname, '/', 'bin')); + clearDirectory(outPath); + cpFile(binPath, outPath); let rs; if (os.type() === 'Windows_NT') { rs = childProcess.spawnSync('go', ['build', '-o', outPath, serverSrc], { diff --git a/trace_streamer/src/BUILD.gn b/trace_streamer/src/BUILD.gn index 1b10184daf3ed753f5ef4563c54084306a0ae8e4..2ea4d50ed2e72f695bd1451d30822a8d50dc0657 100644 --- a/trace_streamer/src/BUILD.gn +++ b/trace_streamer/src/BUILD.gn @@ -98,6 +98,7 @@ ohos_source_set("trace_streamer_source") { part_name = "${OHOS_PROFILER_PART_NAME}" sources = [ "cfg/trace_streamer_config.cpp", + "rpc/ffrt_converter.cpp", "rpc/http_server.cpp", "rpc/http_socket.cpp", "rpc/rpc_server.cpp", diff --git a/trace_streamer/src/metrics/metrics.h b/trace_streamer/src/metrics/metrics.h index 39e736482112009f10f6e497bdc2edf14d0004d5..cf0ea7daed1125ee7579a3cb3614912b035ed300 100644 --- a/trace_streamer/src/metrics/metrics.h +++ b/trace_streamer/src/metrics/metrics.h @@ -27,6 +27,7 @@ #include "sysCallStrategy.h" #include "traceStateStrategy.h" #include "traceTaskStrategy.h" +#include "ts_common.h" enum METRICS_INDEX { METRICS_TRACE_MEM, diff --git a/trace_streamer/src/parser/ebpf_parser/bio_latency_data_parser.h b/trace_streamer/src/parser/ebpf_parser/bio_latency_data_parser.h index a18f6b7233ab54251c5bd65ea9b0ea975402e353..69720b489a79de901493457ec4273bdf38e699bc 100644 --- a/trace_streamer/src/parser/ebpf_parser/bio_latency_data_parser.h +++ b/trace_streamer/src/parser/ebpf_parser/bio_latency_data_parser.h @@ -15,7 +15,6 @@ #ifndef BIO_LATENCY_DATA_PARSER_H #define BIO_LATENCY_DATA_PARSER_H #include "clock_filter_ex.h" -#include "double_map.h" #include "ebpf_base.h" #include "htrace_plugin_time_parser.h" diff --git a/trace_streamer/src/parser/ebpf_parser/file_system_data_parser.h b/trace_streamer/src/parser/ebpf_parser/file_system_data_parser.h index a543a2c97283e580766248e6b3fb3f9abce729fd..9620918a8819b29904cec623c85d4c146c77b30c 100644 --- a/trace_streamer/src/parser/ebpf_parser/file_system_data_parser.h +++ b/trace_streamer/src/parser/ebpf_parser/file_system_data_parser.h @@ -15,7 +15,6 @@ #ifndef FILE_SYSTEM_DATA_PARSER_H #define FILE_SYSTEM_DATA_PARSER_H #include "clock_filter_ex.h" -#include "double_map.h" #include "ebpf_base.h" #include "htrace_plugin_time_parser.h" diff --git a/trace_streamer/src/parser/ebpf_parser/paged_memory_data_parser.h b/trace_streamer/src/parser/ebpf_parser/paged_memory_data_parser.h index 8f0ef45cd92d80534160560ef6634a83cd338e7c..e68991d139da645e8d1ccd2383801415d6e7adeb 100644 --- a/trace_streamer/src/parser/ebpf_parser/paged_memory_data_parser.h +++ b/trace_streamer/src/parser/ebpf_parser/paged_memory_data_parser.h @@ -15,7 +15,6 @@ #ifndef PAGED_MEMORY_DATA_PARSER_H #define PAGED_MEMORY_DATA_PARSER_H #include "clock_filter_ex.h" -#include "double_map.h" #include "ebpf_base.h" #include "ebpf_data_reader.h" #include "ebpf_stdtype.h" diff --git a/trace_streamer/src/parser/rawtrace_parser/ftrace_processor.h b/trace_streamer/src/parser/rawtrace_parser/ftrace_processor.h index cd10e4cef7d8c8ffd80c409c77b2ba3fc0ff4d14..8c79da2636474830dca4267f4aae7b1024b89798 100644 --- a/trace_streamer/src/parser/rawtrace_parser/ftrace_processor.h +++ b/trace_streamer/src/parser/rawtrace_parser/ftrace_processor.h @@ -20,9 +20,9 @@ #include #include "cpu_detail_parser.h" #include "ftrace_common_type.h" +#include "ftrace_event_processor.h" #include "ftrace_field_processor.h" #include "printk_formats_processor.h" -#include "ftrace_event_processor.h" namespace SysTuning { namespace TraceStreamer { diff --git a/trace_streamer/src/rpc/ffrt_converter.cpp b/trace_streamer/src/rpc/ffrt_converter.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3b205559b574598a7e6d859edd24cef695b6fcbe --- /dev/null +++ b/trace_streamer/src/rpc/ffrt_converter.cpp @@ -0,0 +1,573 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "ffrt_converter.h" +#include "log.h" +namespace SysTuning { +namespace TraceStreamer { +bool FfrtConverter::RecoverTraceAndGenerateNewFile(const std::string& ffrtFileName, std::ofstream& outFile) +{ + std::ifstream ffrtFile(ffrtFileName); + if (!ffrtFile.is_open() || !outFile.is_open()) { + TS_LOGE("ffrtFile or outFile is invalid."); + return false; + } + std::vector lines; + std::string line; + while (std::getline(ffrtFile, line)) + lines.push_back(std::move(line)); + TypeFfrtPid result = ClassifyLogsForFfrtWorker(lines); + ConvertFfrtThreadToFfrtTask(lines, result); + SupplementFfrtBlockAndWakeInfo(lines); + for (auto line : lines) { + outFile << line << std::endl; + } + return true; +} +int FfrtConverter::ExtractProcessId(const std::string& log) +{ + std::smatch match; + const std::regex pidPattern = std::regex("\\(\\s*\\d+\\) \\["); + if (std::regex_search(log, match, pidPattern)) { + for (size_t i = 0; i < match.size(); i++) { + if (match[i] == '-') { + return 0; + } + } + auto beginPos = match.str().find('(') + 1; + auto endPos = match.str().find(')'); + return std::stoi(match.str().substr(beginPos, endPos - beginPos)); + } else { + return 0; + } +} + +std::string FfrtConverter::ExtractTimeStr(const std::string& log) +{ + std::smatch match; + const std::regex timePattern = std::regex(" (\\d+)\\.(\\d+):"); + if (std::regex_search(log, match, timePattern)) { + return match.str().substr(1, match.str().size() - 2); + } else { + return ""; + } +} + +std::string FfrtConverter::ExtractCpuId(const std::string& log) +{ + std::smatch match; + const std::regex cpuIdPattern = std::regex("\\) \\[.*?\\]"); + if (std::regex_search(log, match, cpuIdPattern)) { + auto beginPos = match.str().find('[') + 1; + auto endPos = match.str().find(']'); + return match.str().substr(beginPos, endPos - beginPos); + } else { + return ""; + } +} + +std::string FfrtConverter::MakeBeginFakeLog(const std::string& mark, + const int pid, + const std::string& label, + const int gid, + const int tid, + const std::string& threadName, + const int prio) +{ + auto timestamp = ExtractTimeStr(mark); + auto cpuId = ExtractCpuId(mark); + std::unique_ptr result(new char[MAX_LEN]); + sprintf(result.get(), + "\n %s-%d (%7d) [%s] .... %s: sched_switch: prev_comm=%s prev_pid=%d prev_prio=%d prev_state=S ==> " + "next_comm=%s next_pid=%d0%d next_prio=%d\n", + threadName.c_str(), tid, pid, cpuId.c_str(), timestamp.c_str(), threadName.c_str(), tid, prio, + label.c_str(), pid, gid, prio); + return mark + result.get(); +} + +std::string FfrtConverter::MakeEndFakeLog(const std::string& mark, + const int pid, + const std::string& label, + const int gid, + const int tid, + const std::string& threadName, + const int prio) +{ + auto timestamp = ExtractTimeStr(mark); + auto cpuId = ExtractCpuId(mark); + std::unique_ptr result(new char[MAX_LEN]); + sprintf(result.get(), + " %s-%d0%d (%7d) [%s] .... %s: sched_switch: prev_comm=%s prev_pid=%d0%d prev_prio=%d prev_state=S " + "==> next_comm=%s next_pid=%d next_prio=%d\n", + label.c_str(), pid, gid, pid, cpuId.c_str(), timestamp.c_str(), label.c_str(), pid, gid, prio, + threadName.c_str(), tid, prio); + std::string fakeLog = result.get(); + memset(result.get(), 0, MAX_LEN); + if (mark.find("|B|") != std::string::npos || mark.find("|H:B ") != std::string::npos) { + sprintf(result.get(), " %s-%d0%d (%7d) [%s] .... %s: tracing_mark_write: E|%d\n", label.c_str(), pid, gid, + pid, cpuId.c_str(), timestamp.c_str(), pid); + fakeLog = result.get() + fakeLog; + } + return fakeLog; +} + +std::string FfrtConverter::ReplaceSchedSwitchLog(std::string& fakeLog, + const std::string& mark, + const int pid, + const std::string& label, + const int gid, + const int tid) +{ + std::unique_ptr result(new char[MAX_LEN]); + std::smatch match; + if (mark.find("prev_pid=" + std::to_string(tid)) != std::string::npos) { + if (regex_search(fakeLog, match, indexPattern_)) { + auto beginPos = fakeLog.find(match.str()); + sprintf(result.get(), " %s-%d0%d ", label.c_str(), pid, gid); + fakeLog = result.get() + fakeLog.substr(beginPos); + size_t pcommPos = fakeLog.find("prev_comm="); + size_t pPidPos = fakeLog.find("prev_pid="); + memset(result.get(), 0, MAX_LEN); + sprintf(result.get(), "prev_comm=%s ", label.c_str()); + fakeLog = fakeLog.substr(0, pcommPos) + result.get() + fakeLog.substr(pPidPos); + memset(result.get(), 0, MAX_LEN); + pPidPos = fakeLog.find("prev_pid="); + size_t pPrioPos = fakeLog.find("prev_prio="); + sprintf(result.get(), "prev_pid=%d0%d ", pid, gid); + fakeLog = fakeLog.substr(0, pPidPos) + result.get() + fakeLog.substr(pPrioPos); + memset(result.get(), 0, MAX_LEN); + } + } else if (mark.find("next_pid=" + std::to_string(tid)) != std::string::npos) { + sprintf(result.get(), "next_comm=%s ", label.c_str()); + size_t nCommPos = fakeLog.find("next_comm="); + size_t nPidPos = fakeLog.find("next_pid="); + fakeLog = fakeLog.substr(0, nCommPos) + result.get() + fakeLog.substr(nPidPos); + memset(result.get(), 0, MAX_LEN); + sprintf(result.get(), "next_pid=%d0%d ", pid, gid); + nPidPos = fakeLog.find("next_pid="); + size_t nPrioPos = fakeLog.find("next_prio="); + fakeLog = fakeLog.substr(0, nPidPos) + result.get() + fakeLog.substr(nPrioPos); + } + return fakeLog; +} + +std::string FfrtConverter::ReplaceSchedWakeLog(std::string& fakeLog, + const std::string& label, + const int pid, + const int gid) +{ + std::unique_ptr result(new char[MAX_LEN]); + sprintf(result.get(), "comm=%s ", label.c_str()); + size_t commPos = fakeLog.find("comm="); + size_t pidPos = fakeLog.find("pid="); + fakeLog = fakeLog.substr(0, commPos) + result.get() + fakeLog.substr(pidPos); + memset(result.get(), 0, MAX_LEN); + sprintf(result.get(), "pid=%d0%d ", pid, gid); + pidPos = fakeLog.find("pid="); + size_t prioPos = fakeLog.find("prio="); + fakeLog = fakeLog.substr(0, pidPos) + result.get() + fakeLog.substr(prioPos); + return fakeLog; +} + +std::string FfrtConverter::ReplaceSchedBlockLog(std::string& fakeLog, const int pid, const int gid) +{ + std::unique_ptr result(new char[MAX_LEN]); + sprintf(result.get(), "pid=%d0%d ", pid, gid); + size_t pidPos = fakeLog.find("pid"); + size_t ioPos = fakeLog.find("iowait="); + fakeLog = fakeLog.substr(0, pidPos) + result.get() + fakeLog.substr(ioPos); + return fakeLog; +} +std::string FfrtConverter::ReplaceTracingMarkLog(std::string& fakeLog, + const std::string& label, + const int pid, + const int gid) +{ + std::unique_ptr result(new char[MAX_LEN]); + std::smatch match; + if (regex_search(fakeLog, match, indexPattern_)) { + auto beginPos = fakeLog.find(match.str()); + sprintf(result.get(), " %s-%d0%d ", label.c_str(), pid, gid); + fakeLog = result.get() + fakeLog.substr(beginPos); + } + return fakeLog; +} +std::string FfrtConverter::ConvertWorkerLogToTask(const std::string& mark, + const int pid, + const std::string& label, + const int gid, + const int tid) +{ + std::string fakeLog = mark; + if (mark.find("sched_switch: ") != std::string::npos) { + return ReplaceSchedSwitchLog(fakeLog, mark, pid, label, gid, tid); + } + if (mark.find(": sched_wak") != std::string::npos) { + return ReplaceSchedWakeLog(fakeLog, label, pid, gid); + } + if (mark.find("sched_blocked_reason: ") != std::string::npos) { + return ReplaceSchedBlockLog(fakeLog, pid, gid); + } + return ReplaceTracingMarkLog(fakeLog, label, pid, gid); +} +int FfrtConverter::FindTid(std::string& log) +{ + std::string index = "prev_pid="; + auto beginPos = log.find(index); + auto endPos = log.find_first_of(" ", beginPos); + beginPos = beginPos + index.length(); + auto tid = stoi(log.substr(beginPos, endPos - beginPos)); + return tid; +} + +void FfrtConverter::ClassifySchedSwitchLogs(std::string& log, + size_t line, + std::unordered_map>& traceMap, + FfrtConverter::TypeFfrtPid& ffrtPidsMap) +{ + if (log.find("prev_comm=ffrt") != std::string::npos) { + auto pid = ExtractProcessId(log); + if (ffrtPidsMap.find(pid) == ffrtPidsMap.end()) { + ffrtPidsMap[pid] = {}; + } + std::string begin = "prev_comm="; + std::string end = " prev_pid="; + auto beginPos = log.find(begin) + begin.length(); + auto endPos = log.find(end); + auto tid = FindTid(log); + if (ffrtPidsMap[pid].find(tid) == ffrtPidsMap[pid].end()) { + ffrtPidsMap[pid][tid].name = log.substr(beginPos, endPos - beginPos); + } + } + auto prevTid = FindTid(log); + if (traceMap.find(prevTid) == traceMap.end()) { + traceMap[prevTid] = std::vector(); + } + traceMap[prevTid].push_back(line); + std::string begin = "next_pid="; + auto beginPos = log.find(begin) + begin.length(); + std::string end = " next_prio="; + auto endPos = log.find(end); + auto nextTid = stoi(log.substr(beginPos, endPos - beginPos)); + if (traceMap.find(nextTid) == traceMap.end()) { + traceMap[nextTid] = std::vector(); + } + traceMap[nextTid].push_back(line); + return; +} +void FfrtConverter::FindFfrtProcessAndClassifyLogs(std::string& log, + size_t line, + std::unordered_map>& traceMap, + FfrtConverter::TypeFfrtPid& ffrtPidsMap) +{ + if (log.find("sched_switch") != std::string::npos) { + ClassifySchedSwitchLogs(log, line, traceMap, ffrtPidsMap); + return; + } + if (log.find(": sched_wak") != std::string::npos || (log.find("sched_blocked_reason:") != std::string::npos)) { + std::string begin = "pid="; + auto beginPos = log.find(begin); + auto endPos = log.find_first_of(" ", beginPos); + beginPos = beginPos + begin.length(); + auto tid = stoi(log.substr(beginPos, endPos - beginPos)); + if (traceMap.find(tid) == traceMap.end()) { + traceMap[tid] = std::vector(); + } + traceMap[tid].push_back(line); + return; + } + static std::smatch match; + if (std::regex_search(log, match, indexPattern_)) { + auto endPos = log.find(match.str()); + std::string res = log.substr(0, endPos); + std::string begin = "-"; + auto beginPos = res.find_last_of(begin); + beginPos = beginPos + begin.length(); + auto tid = stoi(log.substr(beginPos, endPos - beginPos)); + if (traceMap.find(tid) == traceMap.end()) { + traceMap[tid] = std::vector(); + } + traceMap[tid].push_back(line); + } + return; +} + +bool FfrtConverter::IsDigit(const std::string& str) +{ + for (int i = 0; i < str.length(); i++) { + if (!std::isdigit(str[i])) { + return false; + } + } + return true; +} + +FfrtConverter::TypeFfrtPid FfrtConverter::ClassifyLogsForFfrtWorker(vector& results) +{ + TypeFfrtPid ffrtPidMap; + std::unordered_map> traceMap; + for (auto line = 0; line < results.size(); line++) { + FindFfrtProcessAndClassifyLogs(results[line], line, traceMap, ffrtPidMap); + } + for (auto& [pid, tids] : ffrtPidMap) { + for (auto& pair : tids) { + auto tid = pair.first; + ffrtPidMap[pid][tid].line = traceMap[tid]; + } + } + return ffrtPidMap; +} +void FfrtConverter::ConvertFfrtThreadToFfrtTask(vector& results, FfrtConverter::TypeFfrtPid& ffrtPidsMap) +{ + int prio; + std::unordered_map> taskLabels; + for (auto& [pid, tids] : ffrtPidsMap) { + taskLabels[pid] = {}; + for (auto& [tid, info] : ffrtPidsMap[pid]) { + auto& threadName = info.name; + auto switchInFakeLog = false; + auto switchOutFakeLog = false; + auto ffbkMarkRemove = false; + auto gid = WAKE_EVENT_DEFAULT_VALUE; + for (auto& line : info.line) { + auto mark = results[line]; + if (mark.find("sched_switch:") != std::string::npos) { + if (mark.find("prev_pid=" + std::to_string(tid)) != std::string::npos) { + static std::string beginPprio = "prev_prio="; + auto beginPos = mark.find(beginPprio); + beginPos = beginPos + beginPprio.length(); + auto endPos = mark.find_first_of(" ", beginPos); + prio = stoi(mark.substr(beginPos, endPos - beginPos)); + } else if (mark.find("next_pid=" + std::to_string(tid)) != std::string::npos) { + static std::string beginNprio = "next_prio="; + auto beginPos = mark.find(beginNprio); + beginPos = beginPos + beginNprio.length(); + prio = stoi(mark.substr(beginPos)); + } + } + if (mark.find("FFRT::[") != std::string::npos) { + std::string missLog; + auto beginPos = mark.rfind("["); + auto endPos = mark.rfind("]"); + auto label = mark.substr(beginPos + 1, endPos - beginPos - 1); + if (gid != WAKE_EVENT_DEFAULT_VALUE) { + missLog = MakeEndFakeLog(mark, pid, taskLabels[pid][gid], gid, tid, threadName, prio); + auto timestamp = ExtractTimeStr(mark); + auto cpuId = ExtractCpuId(mark); + std::unique_ptr result(new char[MAX_LEN]); + sprintf(result.get(), " %s-%d (%7d) [%s] .... %s: tracing_mark_write: E|%d\n", + threadName.c_str(), tid, pid, cpuId.c_str(), timestamp.c_str(), pid); + missLog = missLog + result.get(); + memset(result.get(), 0, MAX_LEN); + } + beginPos = mark.rfind("|"); + if (beginPos != std::string::npos && IsDigit(mark.substr(beginPos + 1))) { + gid = stoi(mark.substr(beginPos + 1)); + } else { + continue; + } + if (taskLabels[pid].find(gid) == taskLabels[pid].end()) { + taskLabels[pid][gid] = label; + } + results[line] = MakeBeginFakeLog(mark, pid, taskLabels[pid][gid], gid, tid, threadName, prio); + if (!missLog.empty()) { + results[line] = missLog + results[line]; + } + switchInFakeLog = true; + continue; + } + if (gid != WAKE_EVENT_DEFAULT_VALUE) { + const std::regex CoPattern = std::regex(" F\\|(\\d+)\\|Co\\|(\\d+)"); + const std::regex HCoPattern = std::regex(" F\\|(\\d+)\\|H:Co\\s(\\d+)"); + if (std::regex_search(mark, CoPattern) || std::regex_search(mark, HCoPattern)) { + results[line].clear(); + if (switchInFakeLog) { + switchInFakeLog = false; + continue; + } else { + switchOutFakeLog = true; + continue; + } + } + if (switchInFakeLog && (mark.find("tracing_mark_write: B") != std::string::npos)) { + results[line].clear(); + continue; + } + if (switchOutFakeLog && (mark.find("tracing_mark_write: E") != std::string::npos)) { + results[line].clear(); + continue; + } + const std::regex EndPattern = std::regex(" F\\|(\\d+)\\|[BF]\\|(\\d+)"); + const std::regex HEndPattern = std::regex(" F\\|(\\d+)\\|H:[BF]\\s(\\d+)"); + if (std::regex_search(mark, EndPattern) || std::regex_search(mark, HEndPattern)) { + results[line] = MakeEndFakeLog(mark, pid, taskLabels[pid][gid], gid, tid, threadName, prio); + gid = WAKE_EVENT_DEFAULT_VALUE; + switchOutFakeLog = false; + continue; + } + auto fakeLog = ConvertWorkerLogToTask(mark, pid, taskLabels[pid][gid], gid, tid); + if (fakeLog.find("FFBK[") != std::string::npos) { + if (fakeLog.find("[dep]") != std::string::npos) { + fakeLog = ""; + } else if (fakeLog.find("[chd]") != std::string::npos) { + auto beginPos = fakeLog.find("[chd]"); + fakeLog = fakeLog.replace(beginPos, 5, "[wait_child]"); + } else if (fakeLog.find("[dat]") != std::string::npos) { + auto beginPos = fakeLog.find("[dat]"); + fakeLog = fakeLog.replace(beginPos, 5, "[wait_data]"); + } else if (fakeLog.find("[fd]") != std::string::npos) { + auto beginPos = fakeLog.find("[fd]"); + fakeLog = fakeLog.replace(beginPos, 4, "[wait_fd]"); + } else if (fakeLog.find("[mtx]") != std::string::npos) { + auto beginPos = fakeLog.find("[mtx]"); + fakeLog = fakeLog.replace(beginPos, 5, "[mutex]"); + } else if (fakeLog.find("[slp]") != std::string::npos) { + auto beginPos = fakeLog.find("[slp]"); + fakeLog = fakeLog.replace(beginPos, 5, "[sleep]"); + } else if (fakeLog.find("[yld]") != std::string::npos) { + auto beginPos = fakeLog.find("[yld]"); + fakeLog = fakeLog.replace(beginPos, 5, "[yield]"); + } else if (fakeLog.find("[cnd]") != std::string::npos) { + auto beginPos = fakeLog.find("[cnd]"); + fakeLog = fakeLog.replace(beginPos, 5, "[cond_wait]"); + } else if (fakeLog.find("[cnt]") != std::string::npos) { + auto beginPos = fakeLog.find("[cnt]"); + fakeLog = fakeLog.replace(beginPos, 5, "[cond_timedwait]"); + } + ffbkMarkRemove = true; + } + if (ffbkMarkRemove && mark.find("tracing_mark_write: E") != std::string::npos) { + results[line].clear(); + ffbkMarkRemove = false; + continue; + } + results[line] = fakeLog; + continue; + } + } + } + } + return; +} +void FfrtConverter::SupplementFfrtBlockAndWakeInfo(vector& results) +{ + std::unordered_map> taskWak; + std::string readyEndLog; + std::unique_ptr result(new char[MAX_LEN]); + for (int line = 0; line < results.size(); line++) { + auto log = results[line]; + if (log.find("FFBK[") != std::string::npos) { + auto pid = ExtractProcessId(log); + auto beginPos = log.rfind("|"); + auto gid = stoi(log.substr(beginPos + 1)); + if (taskWak.find(pid) == taskWak.end()) { + taskWak[pid] = {}; + } + if (taskWak[pid].find(gid) == taskWak[pid].end()) { + taskWak[pid][gid] = {}; + } + readyEndLog = ""; + if (taskWak[pid][gid].state == "ready") { + auto timestamp = ExtractTimeStr(log); + auto cpuId = ExtractCpuId(log); + sprintf(result.get(), " <...>-%d0%d (%7d) [%s] .... %s: tracing_mark_write: E|%d\n", pid, gid, + pid, cpuId.c_str(), timestamp.c_str(), pid); + readyEndLog = result.get(); + memset(result.get(), 0, MAX_LEN); + } + taskWak[pid][gid].state = "block"; + auto endPos = results[line].rfind('|'); + results[line] = results[line].substr(0, endPos); + if (!readyEndLog.empty()) { + results[line] = readyEndLog + results[line]; + } + } else if (log.find("FFWK|") != std::string::npos) { + auto pid = ExtractProcessId(log); + auto beginPos = log.rfind('|'); + auto gid = stoi(log.substr(beginPos + 1)); + if (taskWak.find(pid) != taskWak.end() && taskWak[pid].find(gid) != taskWak[pid].end()) { + auto timestamp = ExtractTimeStr(log); + auto cpuId = ExtractCpuId(log); + std::string readyBeginLog; + if (log.find("H:FFWK") != std::string::npos) { + sprintf(result.get(), " <...>-%d0%d (%7d) [%s] .... %s: tracing_mark_write: B|%d|H:FFREADY\n", + pid, gid, pid, cpuId.c_str(), timestamp.c_str(), pid); + readyBeginLog = result.get(); + memset(result.get(), 0, MAX_LEN); + } else { + sprintf(result.get(), " <...>-%d0%d (%7d) [%s] .... %s: tracing_mark_write: B|%d|FFREADY\n", + pid, gid, pid, cpuId.c_str(), timestamp.c_str(), pid); + readyBeginLog = result.get(); + memset(result.get(), 0, MAX_LEN); + } + results[line] = readyBeginLog + results[line]; + if (taskWak[pid][gid].state == "ready") { + results[taskWak[pid][gid].prevWakLine] = results[taskWak[pid][gid].prevWakLine].substr( + results[taskWak[pid][gid].prevWakLine].find("FFREADY") + 8); + } + taskWak[pid][gid].state = "ready"; + taskWak[pid][gid].prevWakLine = line; + taskWak[pid][gid].prevWakeLog = log; + } + } else if (log.find("FFRT::[") != std::string::npos) { + auto pid = ExtractProcessId(log); + int gid; + auto beginPos = log.rfind('|'); + auto endPos = log.find_first_of('\n', beginPos + 1); + if (beginPos != std::string::npos && endPos != std::string::npos && + IsDigit(log.substr(beginPos + 1, endPos - beginPos - 1))) { + gid = stoi(log.substr(beginPos + 1, endPos - beginPos - 1)); + } else { + continue; + } + if (taskWak.find(pid) != taskWak.end() && taskWak[pid].find(gid) != taskWak[pid].end()) { + if (taskWak[pid][gid].state == "ready") { + auto timestamp = ExtractTimeStr(log); + auto cpuId = ExtractCpuId(log); + auto endPos = log.rfind('\n'); + auto beginPos = log.find_last_of('\n', endPos - 1); + auto switchLog = log.substr(beginPos + 1, endPos); + beginPos = switchLog.find("next_comm="); + endPos = switchLog.find("next_pid"); + auto taskComm = switchLog.substr(beginPos + 10, endPos - beginPos - 11); + beginPos = switchLog.find("next_pid="); + endPos = switchLog.find(" next_prio="); + auto taskPid = stoi(switchLog.substr(beginPos + 9, endPos - beginPos - 9)); + auto taskPrio = stoi(switchLog.substr(endPos + 11)); + auto cpuIdWake = ExtractCpuId(switchLog); + beginPos = taskWak[pid][gid].prevWakeLog.find("tracing_mark_write:"); + sprintf(result.get(), "sched_waking: comm=%s pid=%d prio=%d target_cpu=%s\n", taskComm.c_str(), + taskPid, taskPrio, cpuIdWake.c_str()); + auto wakingLog = taskWak[pid][gid].prevWakeLog.substr(0, beginPos) + result.get(); + memset(result.get(), 0, MAX_LEN); + sprintf(result.get(), "sched_wakeup: comm=%s pid=%d prio=%d target_cpu=%s", taskComm.c_str(), + taskPid, taskPrio, cpuIdWake.c_str()); + auto wakeupLog = taskWak[pid][gid].prevWakeLog.substr(0, beginPos) + result.get(); + memset(result.get(), 0, MAX_LEN); + results[taskWak[pid][gid].prevWakLine] = + results[taskWak[pid][gid].prevWakLine] + "\n" + wakingLog + wakeupLog; + sprintf(result.get(), " <...>-%d0%d (%7d) [%s] .... %s: tracing_mark_write: E|%d\n", pid, gid, + pid, cpuId.c_str(), timestamp.c_str(), pid); + readyEndLog = result.get(); + memset(result.get(), 0, MAX_LEN); + results[line] = readyEndLog + results[line]; + taskWak[pid][gid].state = "none"; + } + } + } + } + return; +} +} // namespace TraceStreamer +} // namespace SysTuning diff --git a/trace_streamer/src/rpc/ffrt_converter.h b/trace_streamer/src/rpc/ffrt_converter.h new file mode 100644 index 0000000000000000000000000000000000000000..2e2e5bdb5adc2584853e04345357fcd439deb7a1 --- /dev/null +++ b/trace_streamer/src/rpc/ffrt_converter.h @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef FFRT_CONVERTER_H +#define FFRT_CONVERTER_H +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +namespace SysTuning { +namespace TraceStreamer { +using namespace std; +#define WAKE_EVENT_DEFAULT_VALUE -1 +#define MAX_LEN 256 +struct ffrtContent { + std::string name; + std::vector line; +}; +struct WakeEvent { + std::string state = "none"; + int prevWakLine = WAKE_EVENT_DEFAULT_VALUE; + std::string prevWakeLog; +}; +class FfrtConverter { +public: + FfrtConverter() = default; + ~FfrtConverter() = default; + bool RecoverTraceAndGenerateNewFile(const std::string& ffrtFileName, std::ofstream& outFile); + +private: + using TypeFfrtPid = std::unordered_map>; + int ExtractProcessId(const std::string& log); + std::string ExtractTimeStr(const std::string& log); + std::string ExtractCpuId(const std::string& log); + TypeFfrtPid ClassifyLogsForFfrtWorker(vector& results); + void FindFfrtProcessAndClassifyLogs(std::string& log, + size_t line, + std::unordered_map>& traceMap, + TypeFfrtPid& ffrtPidsMap); + void ClassifySchedSwitchLogs(std::string& log, + size_t line, + std::unordered_map>& traceMap, + FfrtConverter::TypeFfrtPid& ffrtPidsMap); + int FindTid(string& log); + void ConvertFfrtThreadToFfrtTask(vector& results, TypeFfrtPid& ffrtPidsMap); + std::string MakeBeginFakeLog(const std::string& mark, + const int pid, + const std::string& label, + const int gid, + const int tid, + const std::string& tname, + const int prio); + std::string MakeEndFakeLog(const std::string& mark, + const int pid, + const std::string& label, + const int gid, + const int tid, + const std::string& tname, + const int prio); + std::string ReplaceSchedSwitchLog(std::string& fakeLog, + const std::string& mark, + const int pid, + const std::string& label, + const int gid, + const int tid); + std::string ReplaceSchedWakeLog(std::string& fakeLog, const std::string& label, const int pid, const int gid); + std::string ReplaceSchedBlockLog(std::string& fakeLog, const int pid, const int gid); + std::string ReplaceTracingMarkLog(std::string& fakeLog, const std::string& label, const int pid, const int gid); + std::string ConvertWorkerLogToTask(const std::string& mark, + const int pid, + const std::string& label, + const int gid, + const int tid); + void SupplementFfrtBlockAndWakeInfo(vector& results); + bool IsDigit(const std::string& str); + +private: + const std::regex indexPattern_ = std::regex("\\(.+\\)\\s+\\[\\d"); +}; +} // namespace TraceStreamer +} // namespace SysTuning +#endif // FFRT_CONVERTER_H diff --git a/trace_streamer/src/rpc/http_server.h b/trace_streamer/src/rpc/http_server.h index 3df412faae0981b5c742f54e3725d79e925b760d..bf04f04b96c52e76291c8daf5b8dd5835e2b53ea 100644 --- a/trace_streamer/src/rpc/http_server.h +++ b/trace_streamer/src/rpc/http_server.h @@ -23,6 +23,7 @@ #include #include "http_socket.h" #include "rpc_server.h" +#include "ts_common.h" namespace SysTuning { namespace TraceStreamer { class HttpServer { diff --git a/trace_streamer/src/rpc/rpc_server.cpp b/trace_streamer/src/rpc/rpc_server.cpp index f61e4b160ff4a158ca05976c2d3d9259a1ce62c2..1909894f3fccd61418e1695d65e667f397f74110 100644 --- a/trace_streamer/src/rpc/rpc_server.cpp +++ b/trace_streamer/src/rpc/rpc_server.cpp @@ -29,6 +29,7 @@ #include "log.h" #include "string_help.h" #include "trace_streamer_selector.h" +#include "ts_common.h" #include "version.h" #define UNUSED(expr) \ @@ -51,6 +52,7 @@ struct ParserConfig { int32_t appConfigValue; int32_t aniConfigValue; int32_t binderConfigValue; + int32_t ffrtConvertConfigValue; }; void from_json(const json& j, ParserConfig& v) { @@ -58,17 +60,104 @@ void from_json(const json& j, ParserConfig& v) j.at("AppStartup").get_to(v.appConfigValue); j.at("AnimationAnalysis").get_to(v.aniConfigValue); j.at("BinderRunnable").get_to(v.binderConfigValue); + j.at("FfrtConvert").get_to(v.ffrtConvertConfigValue); } } // namespace jsonns +#if IS_WASM +bool RpcServer::SaveAndParseFfrtData(const uint8_t* data, size_t len, ResultCallBack resultCallBack, bool isFinish) +{ + auto ffrtFileName = "ffrtFile.txt"; + static std::ofstream ffrtFile(ffrtFileName, std::ios::binary | std::ios::app); + if (!ffrtFile.is_open()) { + TS_LOGE("ffrtFile open filed!"); + return false; + } + ffrtFile.write(reinterpret_cast(data), len); + if (ffrtFile.fail() || ffrtFile.bad()) { + TS_LOGE("Failed to write data!"); + ffrtFile.close(); + return false; + } + if (!isFinish) { + return true; + } + ffrtFile.close(); + auto outTraceName = "outTrace.txt"; + std::ofstream outFile(outTraceName); + if (!outFile.is_open()) { + ffrtFile.close(); + std::filesystem::remove_all(ffrtFileName); + TS_LOGE("prepare outFile failed."); + return false; + } + FfrtConverter ffrtConverter; + auto ret = ffrtConverter.RecoverTraceAndGenerateNewFile(ffrtFileName, outFile); + outFile.close(); + std::filesystem::remove_all(ffrtFileName); + if (!ret) { + std::filesystem::remove_all(outTraceName); + return false; + } + outFile.close(); + if (ReadAndParseData(outTraceName) && SendConvertedFfrtFile(outTraceName, resultCallBack)) { + std::filesystem::remove_all(outTraceName); + return false; + } + std::filesystem::remove_all(outTraceName); + return true; +} +bool RpcServer::SendConvertedFfrtFile(const std::string& fileName, ResultCallBack resultCallBack) +{ + if (!resultCallBack) { + TS_LOGE("resultCallBack is nullptr!"); + return false; + } + std::ifstream inputFile(fileName); + if (!inputFile.is_open()) { + TS_LOGE("open file : %s failed!", fileName.c_str()); + return false; + } + char outData[G_CHUNK_SIZE]; + while (true) { + inputFile.read(outData, G_CHUNK_SIZE); + auto readSize = inputFile.gcount(); + resultCallBack(std::string(outData, readSize), SEND_CONTINUE); + if (inputFile.eof()) { + break; + } + } + resultCallBack("ok\r\n", SEND_FINISH); + return true; +} +bool RpcServer::ReadAndParseData(const std::string& filePath) +{ + std::ifstream inputFile(filePath); + if (!inputFile.is_open()) { + TS_LOGE("can not open %s.", filePath.c_str()); + return false; + } + while (true) { + std::unique_ptr buf = std::make_unique(G_CHUNK_SIZE); + inputFile.read(reinterpret_cast(buf.get()), G_CHUNK_SIZE); + auto readSize = inputFile.gcount(); + ts_->ParseTraceDataSegment(std::move(buf), readSize, false, inputFile.eof()); + if (inputFile.eof()) { + break; + } + } + ts_->WaitForParserEnd(); + inputFile.close(); + return true; +} +#endif bool RpcServer::ParseData(const uint8_t* data, size_t len, ResultCallBack resultCallBack, bool isFinish) { g_loadSize += len; - size_t blockSize = 1024 * 1024; do { - size_t parseSize = std::min(len, blockSize); + size_t parseSize = std::min(len, G_CHUNK_SIZE); std::unique_ptr buf = std::make_unique(parseSize); std::copy(data, data + parseSize, buf.get()); - if (!ts_->ParseTraceDataSegment(std::move(buf), parseSize, false, isFinish && (len <= blockSize))) { + if (!ts_->ParseTraceDataSegment(std::move(buf), parseSize, false, isFinish && (len <= G_CHUNK_SIZE))) { if (resultCallBack) { resultCallBack("formaterror\r\n", SEND_FINISH); } @@ -86,9 +175,8 @@ bool RpcServer::ParseData(const uint8_t* data, size_t len, ResultCallBack result bool RpcServer::ParseDataWithoutCallback(const uint8_t* data, size_t len, int32_t isFinish, bool isSplitFile) { g_loadSize += len; - size_t blockSize = 1024 * 1024; do { - size_t parseSize = std::min(len, blockSize); + size_t parseSize = std::min(len, G_CHUNK_SIZE); std::unique_ptr buf = std::make_unique(parseSize); std::copy(data, data + parseSize, buf.get()); if (!ts_->ParseTraceDataSegment(std::move(buf), parseSize, isSplitFile, isFinish && (len == parseSize))) { @@ -152,6 +240,25 @@ bool RpcServer::LongTraceSplitFile(const uint8_t* data, return true; } +bool RpcServer::DetermineSystrace(const uint8_t* data, size_t len) +{ + std::string startStr(reinterpret_cast(data), std::min(len, 20)); + if (startStr.find("# tracer") != std::string::npos) { + return true; + } + if (startStr.find("# TRACE") != std::string::npos) { + return true; + } + const std::regex systraceMatcher = std::regex(R"(-(\d+)\s+\(?\s*(\d+|-+)?\)?\s?\[(\d+)\]\s*)" + R"([a-zA-Z0-9.]{0,5}\s+(\d+\.\d+):\s+(\S+):)"); + std::smatch matcheLine; + std::string bytraceMode(reinterpret_cast(data), len); + if (std::regex_search(bytraceMode, matcheLine, systraceMatcher)) { + return true; + } + return false; +} + bool RpcServer::ParseSplitFileData(const uint8_t* data, size_t len, int32_t isFinish, @@ -525,6 +632,7 @@ bool RpcServer::ParserConfig(std::string parserConfigJson) ts_->UpdateAnimationTraceStatus(parserConfig.aniConfigValue); ts_->UpdateTaskPoolTraceStatus(parserConfig.taskConfigValue); ts_->UpdateBinderRunnableTraceStatus(parserConfig.binderConfigValue); + ffrtConvertEnabled_ = parserConfig.ffrtConvertConfigValue; return true; } } // namespace TraceStreamer diff --git a/trace_streamer/src/rpc/rpc_server.h b/trace_streamer/src/rpc/rpc_server.h index dde51580c7827c1d16db5f3e9eac7c5121b57a75..b0b596ad37ef288de0e6224b2a5acc69ba677e4f 100644 --- a/trace_streamer/src/rpc/rpc_server.h +++ b/trace_streamer/src/rpc/rpc_server.h @@ -19,11 +19,13 @@ #include #include #include "trace_streamer_selector.h" +#include "ffrt_converter.h" namespace SysTuning { namespace TraceStreamer { class RpcServer { public: using ResultCallBack = std::function; + using ExportDatabaseCallback = std::function; using ParseELFFileCallBack = std::function; using SendDataCallBack = std::function; using SplitFileCallBack = std::function; @@ -34,7 +36,6 @@ public: int32_t isFinish, SplitFileCallBack splitFileCallBack, bool isSplitFile); - bool ParserFileTimeSnap(const uint8_t* data, size_t len, ResultCallBack resultCallBack); bool ParseDataOver(const uint8_t* data, size_t len, ResultCallBack resultCallBack); bool SqlOperate(const uint8_t* data, size_t len, ResultCallBack resultCallBack); bool SqlQuery(const uint8_t* data, size_t len, ResultCallBack resultCallBack); @@ -60,7 +61,15 @@ public: uint32_t pageNum, SplitFileCallBack splitFileCallBack); bool GetTimeSnap(std::string dataString); + bool GetFfrtConvertStatus() + { + return ffrtConvertEnabled_; + }; + bool DetermineSystrace(const uint8_t* data, size_t len); #ifdef IS_WASM + bool SaveAndParseFfrtData(const uint8_t* data, size_t len, ResultCallBack resultCallBack, bool isFinish); + bool ReadAndParseData(const std::string& filePath); + bool SendConvertedFfrtFile(const std::string& fileName, ResultCallBack resultCallBack); int32_t DownloadELFCallback(const std::string& fileName, size_t totalLen, const uint8_t* data, @@ -72,11 +81,11 @@ public: private: void ProcPerfSplitResult(SplitFileCallBack splitFileCallBack, bool isLast); - std::unique_ptr ts_ = std::make_unique(); size_t lenParseData_ = 0; std::vector symbolsPathFiles_; std::vector> vTraceTimeSnap_; + bool ffrtConvertEnabled_ = false; }; } // namespace TraceStreamer } // namespace SysTuning diff --git a/trace_streamer/src/rpc/wasm_func.cpp b/trace_streamer/src/rpc/wasm_func.cpp index fb0aa4801e8a683738ecc0b9c9a378ff00c8aaad..8c14fd44b1cebf4143d6cff88e3e293faff4e070 100644 --- a/trace_streamer/src/rpc/wasm_func.cpp +++ b/trace_streamer/src/rpc/wasm_func.cpp @@ -24,6 +24,7 @@ RpcServer g_wasmTraceStreamer; extern "C" { using ReplyFunction = void (*)(const char* data, uint32_t len, int32_t finish); ReplyFunction g_reply; +ReplyFunction g_ffrtConvertedReply; uint8_t* g_reqBuf; uint32_t g_reqBufferSize; @@ -47,12 +48,17 @@ using ParseELFFunction = void (*)(const char* data, uint32_t len, int32_t finish ParseELFFunction g_parseELFCallback; uint8_t* g_FileNameBuf; uint32_t g_FileNameSize; +bool g_IsSystrace = false; +bool g_hasDeterminedSystrace = false; void ResultCallback(const std::string& jsonResult, int32_t finish) { g_reply(jsonResult.data(), jsonResult.size(), finish); } - +void FfrtConvertedResultCallback(const std::string& content, int32_t finish) +{ + g_ffrtConvertedReply(content.data(), content.size(), finish); +} void SplitFileCallback(const std::string& jsonResult, int32_t dataType, int32_t finish) { g_splitFile(jsonResult.data(), jsonResult.size(), dataType, finish); @@ -62,9 +68,12 @@ void ParseELFCallback(const std::string& SODataResult, int32_t finish) { g_parseELFCallback(SODataResult.data(), SODataResult.size(), finish); } -EMSCRIPTEN_KEEPALIVE uint8_t* Initialize(ReplyFunction replyFunction, uint32_t reqBufferSize) +EMSCRIPTEN_KEEPALIVE uint8_t* Initialize(ReplyFunction replyFunction, + uint32_t reqBufferSize, + ReplyFunction ffrtConvertedReply) { g_reply = replyFunction; + g_ffrtConvertedReply = ffrtConvertedReply; g_reqBuf = new uint8_t[reqBufferSize]; g_reqBufferSize = reqBufferSize; return g_reqBuf; @@ -205,7 +214,13 @@ EMSCRIPTEN_KEEPALIVE int32_t TraceStreamerParseData(const uint8_t* data, int32_t // return 0 while ok, -1 while failed EMSCRIPTEN_KEEPALIVE int32_t TraceStreamerParseDataEx(int32_t dataLen, bool isFinish) { - if (g_wasmTraceStreamer.ParseData(g_reqBuf, dataLen, nullptr, isFinish)) { + if (!g_hasDeterminedSystrace) { + g_IsSystrace = g_wasmTraceStreamer.DetermineSystrace(g_reqBuf, dataLen); + g_hasDeterminedSystrace = true; + } + if (g_wasmTraceStreamer.GetFfrtConvertStatus() && g_IsSystrace) { + return g_wasmTraceStreamer.SaveAndParseFfrtData(g_reqBuf, dataLen, &FfrtConvertedResultCallback, isFinish); + } else if (g_wasmTraceStreamer.ParseData(g_reqBuf, dataLen, nullptr, isFinish)) { return 0; } return -1; diff --git a/trace_streamer/src/trace_streamer/trace_streamer_selector.h b/trace_streamer/src/trace_streamer/trace_streamer_selector.h index 9688a9aa7910eabc3e23297a671246d4153f87a4..dd2d862d1610f675001a1add80191c63e72aaa6d 100644 --- a/trace_streamer/src/trace_streamer/trace_streamer_selector.h +++ b/trace_streamer/src/trace_streamer/trace_streamer_selector.h @@ -84,7 +84,6 @@ public: private: void InitFilter(); - void Remove_Folders(const char* dir); bool LoadQueryFile(const std::string& sqlOperator, std::vector& sqlStrings); TraceFileType fileType_; std::unique_ptr streamFilters_ = {};