diff --git a/source/lib/internal/ebpf/coolbpf b/source/lib/internal/ebpf/coolbpf index 0562b1397b8a8997b16d752d874dc5ad74149149..735ae6138430822502e1bd5ae2a366a8668ecd7e 160000 --- a/source/lib/internal/ebpf/coolbpf +++ b/source/lib/internal/ebpf/coolbpf @@ -1 +1 @@ -Subproject commit 0562b1397b8a8997b16d752d874dc5ad74149149 +Subproject commit 735ae6138430822502e1bd5ae2a366a8668ecd7e diff --git a/source/tools/monitor/ioMonitor/Makefile b/source/tools/monitor/ioMonitor/Makefile new file mode 100755 index 0000000000000000000000000000000000000000..1345670e82b6c09a9ae9dccd7af1ae9d1c7dfaea --- /dev/null +++ b/source/tools/monitor/ioMonitor/Makefile @@ -0,0 +1,4 @@ +mods = ioMon +target := ioMonitor + +include $(SRC)/mk/sh.mk diff --git a/source/tools/monitor/ioMonitor/README.md b/source/tools/monitor/ioMonitor/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4856ff901df85de226f1ff34314d68dfd311d54f --- /dev/null +++ b/source/tools/monitor/ioMonitor/README.md @@ -0,0 +1,2 @@ +# 功能说明 +监控服务主程序，收集系统监控指标，支持查看历史监控数据 diff --git a/source/tools/monitor/ioMonitor/ioMon/__init__.py b/source/tools/monitor/ioMonitor/ioMon/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cb8e4b62acc8012add0dc0175a04b6b4f51940ec --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- + +if __name__ == "__main__": + pass diff --git a/source/tools/monitor/ioMonitor/ioMon/displayClass.py b/source/tools/monitor/ioMonitor/ioMon/displayClass.py new file mode 100755 index 0000000000000000000000000000000000000000..cc51352ecf36ffa2b488a22b5e1c94961e544d00 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/displayClass.py @@ -0,0 +1,510 @@ +# -*- coding: utf-8 -*- + +import os +import sys +import string +import time +import re +import json +import threading +from collections import OrderedDict +from nfPut import CnfPut + + +def bwToValue(bw): + units = ["B", "KB", "MB", "GB", "TB", "PB"] + if str(bw) == '0': + return 0 + for i in range(5, -1, -1): + if units[i] in bw: + return float(bw.split(units[i])[0]) * pow(1024, i) + + +def humConvert(value): + units = ["B", "KB", "MB", "GB", "TB", "PB"] + size = 1024.0 + + if value == 0: + return value + for i in range(len(units)): + if (value / size) < 1: + return "%.1f%s/s" % (value, units[i]) + value = value / size + + +def iolatencyResultReport(*argvs): + result = [] + nf = argvs[1] + nfPutPrefix = str(argvs[2]) + statusReportDicts = argvs[3] + ioburst = False + nfPrefix = [] + iolatStartT = statusReportDicts['iolatency']['startT'] + iolatEndT = statusReportDicts['iolatency']['endT'] + ioutilStartT = statusReportDicts['ioutil']['startT'] + ioutilEndT = statusReportDicts['ioutil']['endT'] + lastIOburstT = statusReportDicts['iolatency']['lastIOburstT'] + + # If IO burst occurs in the short term(first 180 secs or next 60secs) or + # during IO delay diagnosis, it should be considered as one of + # the delay factors + if iolatStartT - lastIOburstT < 300 or \ + (ioutilStartT >= (iolatStartT - 300) and ioutilEndT <= (iolatEndT + 60)): + statusReportDicts['iolatency']['lastIOburstT'] = iolatStartT + ioburst = True + + os.system('ls -rtd '+argvs[0]+'/../* | head -n -5 | '\ + 'xargs --no-run-if-empty rm {} -rf') + if os.path.exists(argvs[0]+'/result.log.stat'): + with open(argvs[0]+'/result.log.stat') as logF: + data = logF.readline() + else: + return + try: + stat = json.loads(data, object_pairs_hook=OrderedDict) + except Exception: + return + + for ds in stat['summary']: + delays = sorted(ds['delays'], + key=lambda e: (float(e['percent'].strip('%'))), + reverse=True) + maxDelayComp = delays[0]['component'] + maxDelayPercent = float(delays[0]['percent'].strip('%')) + avgLat = format(sum([d['avg'] for d in delays])/1000.0, '.3f') + diagret = 'diagret=\"IO delay(AVG %sms) detected in %s\"' % ( + avgLat, str(ds['diskname'])) + nfPrefix.append(',diag_type=IO-Delay,devname='+str(ds['diskname'])) + + if ioburst and {maxDelayComp, delays[1]['component']}.issubset( + ['disk', 'os(block)']): + if (delays[0]['avg'] / delays[1]['avg']) < 10: + suggest = 'solution=\"reduce IO pressure. Refer to the '\ + 'diagnosis of IO-Burst and optimize some tasks\"' + diskIdx = 0 + if maxDelayComp == 'os(block)': + diskIdx = 1 + reason = ( + 'reason=\"IO burst occurs, too mang IO backlogs'\ + '(disk avg/max lat:%s/%s ms, lat percent:%s,'\ + ' OS dispatch avg/max lat:%s/%s ms, lat percent:%s\"' % + (str(delays[diskIdx]['avg'] / 1000.000), + str(delays[diskIdx]['max'] / 1000.000), + str(delays[diskIdx]['percent']), + str(delays[1 - diskIdx]['avg'] / 1000.000), + str(delays[1 - diskIdx]['max'] / 1000.000), + str(delays[1 - diskIdx]['percent']))) + result.append(diagret+','+reason+','+suggest) + continue + else: + statusReportDicts['iolatency']['lastIOburstT'] = lastIOburstT + + suggest = 'solution=\"Please ask the OS kernel expert\"' + maxDelayLog = 'avg/max lat:%s/%s ms, lat percent:%s' %( + str(delays[0]['avg']/1000.000), + str(delays[0]['max']/1000.000), + str(delays[0]['percent'])) + if maxDelayComp == 'disk': + reason = ( + 'reason=\"Disk delay(processing IO slowly, %s)\"' %(maxDelayLog)) + suggest = 'solution=\"Please confirm whether the disk is normal\"' + elif maxDelayComp == 'os(block)': + if delays[1]['component'] == 'disk' and \ + float(delays[1]['percent'].strip('%')) > 20: + with open(argvs[0]+'/resultCons.log') as logF: + data = filter(lambda l : 'F' in l, logF.readlines()) + flushIO = False + if len(data) > 0: + for d in data: + if 'F' in d.split()[-6]: + flushIO = True + break + if flushIO: + suggest = ( + 'Disable flush IO dispatch(echo \"write through\" > '\ + '/sys/class/block/%s/queue/write_cache;'\ + 'echo 0 > /sys/class/block/%s/queue/fua)}' % ( + str(ds['diskname']), str(ds['diskname']))) + suggest += '; Notes: Flush IO is a special instruction to '\ + 'ensure that data is stored persistently on the disk '\ + 'in time, and not saved in the internal cache of the disk.'\ + ' Before disabling, please confirm with the disk FAE '\ + '\"Whether it is necessary to rely on the software to issue'\ + ' flush instructions to ensure data persistent storage\",'\ + ' And avoid data loss due to crash or disk power down' + suggest = 'solution=\"'+suggest+'\"' + else: + suggest = 'solution=\"Please confirm whether the disk is normal\"' + reason = ( + 'reason=\"Disk delay(processing %s slowly, avg/max lat:'\ + '%s/%s ms, lat percent:%s)\"' %( + 'Flush IO' if flushIO else 'IO', + str(delays[1]['avg']/1000.000), + str(delays[1]['max']/1000.000), + str(delays[1]['percent']))) + result.append(diagret+','+reason+','+suggest) + continue + reason = ( + 'reason=\"OS delay(Issuing IO slowly at os(block), %s)\"' %( + maxDelayLog)) + else: + reason = ( + 'reason=\"OS delay(processing IO slowly at %s, %s)\"' %( + str(maxDelayComp), maxDelayLog)) + result.append(diagret+','+reason+','+suggest) + + for e, p in zip(result, nfPrefix): + # print(e+'\n') + #nf.put(nfPutPrefix, p+' '+e) + nf.puts(nfPutPrefix+p+' '+e) + statusReportDicts['iolatency']['valid'] = True + + +def iohangResultReport(*argvs): + abnormalDicts={} + firstioDicts={} + result=[] + nf=argvs[1] + nfPutPrefix=str(argvs[2]) + statusReportDicts = argvs[3] + nfPrefix=[] + + os.system('ls -rtd '+argvs[0]+'/../* | head -n -5 |'\ + ' xargs --no-run-if-empty rm {} -rf') + if os.path.exists(argvs[0]+'/result.log'): + with open(argvs[0]+'/result.log') as logF: + data=logF.readline() + else: + return + try: + stat=json.loads(data, object_pairs_hook = OrderedDict) + except Exception: + return + + for ds in stat['summary']: + maxDelay = 0 + hungIO = None + if ds['diskname'] not in abnormalDicts.keys(): + abnormalDicts.setdefault(ds['diskname'], {}) + firstioDicts.setdefault( + ds['diskname'], + {'time':0, 'iotype':0, 'sector':0}) + for hi in ds['hung ios']: + key=hi['abnormal'].split('hang')[0] + delay = float(hi['abnormal'].split('hang')[1].split()[0]) + if delay > maxDelay: + maxDelay = delay + hungIO = hi + if key not in abnormalDicts[ds['diskname']].keys(): + abnormalDicts[ds['diskname']].setdefault(key, 0) + abnormalDicts[ds['diskname']][key] += 1 + t = hungIO['time'].split('.')[0] + tStamp = float(time.mktime(time.strptime(t,'%Y-%m-%d %H:%M:%S'))) + tStamp -= maxDelay + firstioDicts[ds['diskname']]['time'] = \ + time.strftime('%Y/%m/%d %H:%M:%S', time.localtime(tStamp+8*3600)) + firstioDicts[ds['diskname']]['iotype'] = hungIO['iotype'] + firstioDicts[ds['diskname']]['sector'] = hungIO['sector'] + for diskname, val in abnormalDicts.items(): + abnormalDicts[diskname] = OrderedDict( + sorted(val.items(), key=lambda e: e[1], reverse=True)) + + with open(argvs[0]+'/result.log.stat') as logF: + data = logF.readline() + try: + stat = json.loads(data, object_pairs_hook=OrderedDict) + except Exception: + return + + for ds in stat['summary']: + hungIOS = sorted(ds['hung ios'], + key = lambda e: (float(e['percent'].strip('%'))), + reverse = True) + maxDelayComp=hungIOS[0]['component'] + maxDelayPercent=float(hungIOS[0]['percent'].strip('%')) + maxDelay=format(hungIOS[0]['max']/1000.0, '.3f') + diagret='diagret=\"IO hang %sms detected in %s' % ( + maxDelay, ds['diskname'])+'\"' + nfPrefix.append(',diag_type=IO-Hang,devname='+str(ds['diskname'])) + for key in abnormalDicts[ds['diskname']].keys(): + if maxDelayComp in key: + detail = str( + ''.join(re.findall(re.compile(r'[(](.*?)[)]', re.S), key))) + break + reason = ('reason=\"%s hang(%s, avg/max delay:%s/%s ms), first hang['\ + 'time:%s, iotype:%s, sector:%d]\"' %( + maxDelayComp, detail, + str(hungIOS[0]['avg']/1000.000), + str(hungIOS[0]['max']/1000.000), + firstioDicts[ds['diskname']]['time'], + firstioDicts[ds['diskname']]['iotype'], + firstioDicts[ds['diskname']]['sector'])) + if maxDelayComp == 'Disk' or maxDelayComp == 'OS': + suggest = 'solution=\"Please confirm whether the disk is normal\"' + if maxDelayComp == 'OS': + suggest = 'solution=\"Please ask the OS kernel expert\"' + result.append(diagret+','+reason+','+suggest) + + for e, p in zip(result, nfPrefix): + nf.puts(nfPutPrefix+p+' '+e) + #nf.put(nfPutPrefix, p+' '+e) + statusReportDicts['iohang']['valid'] = True + + +def ioutilDataParse(data, resultInfo): + tUnit = None + totalBw = totalIops = 0 + for ds in data['mstats']: + iops = ds['iops_rd'] + ds['iops_wr'] + bps = bwToValue(ds['bps_wr']) + bwToValue(ds['bps_rd']) + totalBw += bps + totalIops += iops + key = ds['comm']+':'+ds['pid']+':'+ds['cid'][0:20]+':'+ds['device'] + if not tUnit: + if ds['bps_wr'] != '0': + tUnit = ds['bps_wr'].split('/')[1] + else: + tUnit = ds['bps_rd'].split('/')[1] + if key not in resultInfo.keys(): + resultInfo.setdefault(key, + {'disk':ds['device'], 'maxIops':0, 'maxBps':0, 'file':ds['file']}) + resultInfo[key]['maxBps'] = max(bps, resultInfo[key]['maxBps']) + resultInfo[key]['maxIops'] = max(iops, resultInfo[key]['maxIops']) + if resultInfo[key]['maxBps'] != bps or resultInfo[key]['maxIops'] != iops: + resultInfo[key]['file'] = ds['file'] + if 'bufferio' in resultInfo.keys(): + del resultInfo[key]['bufferio'] + if 'bufferio' in ds.keys() and 'bufferio' not in resultInfo[key].keys(): + resultInfo[key].setdefault('bufferio', ds['bufferio']) + return totalIops,totalBw,tUnit + + +def ioutilReport(nf, nfPutPrefix, resultInfo, tUnit, diagret): + top = 1 + suggestPS = reason = '' + resultInfo = \ + sorted(resultInfo.items(), key=lambda e: e[1]['maxBps'], reverse=True) + for key, val in resultInfo: + if val['maxIops'] < 50 or val['maxBps'] < 1024 * 1024 * 5: + continue + file = ', target file:'+str(val['file']) if val['file'] != '-' else '' + if 'kworker' in str(key): + kTasklist = [] + if 'bufferio' in val.keys(): + for i in val["bufferio"]: + if 'KB' in i["Wrbw"]: + continue + kTasklist.append(i['task']) + file += ('%s Wrbw %s disk %s file %s;' % + (i['task'], i["Wrbw"], i["device"], i["file"])) + if len(kTasklist): + file = '(Write bio from: '+file+')' + if top == 1: + suggestPS = '(Found \'kworker\' flush dirty pages, Try to reduce'\ + ' the buffer-IO write?%s or check the config /proc/sys/vm/'\ + '{dirty_ratio,dirty_background_ratio} too small?)' %( + '('+';'.join(kTasklist)+')' if len(kTasklist) else '') + maxBps = humConvert(val['maxBps']).replace('s', tUnit) + reason += ('%d. task[%s], access disk %s with iops:%s, bps:%s%s; ' %( + top, str(key.rsplit(':',1)[0]), str(val['disk']), + str(val['maxIops']), maxBps, file)) + if top == 1 and suggestPS == '': + suggestPS = '(Found task \'%s\')' %(str(key.rsplit(':',1)[0])) + top += 1 + suggest = \ + 'Optimize the tasks that contributes the most IO flow%s' % suggestPS + putIdx = ',diag_type=IO-Burst ' + putField = 'diagret=\"%s\",reason=\"%s\",solution=\"%s\"' %( + diagret, reason, suggest) + #nf.put(nfPutPrefix, + if reason != '': + nf.puts(nfPutPrefix+putIdx+putField) + # print(prefix+reason+suggest+'\n') + + +def ioutilResultReport(*argvs): + resultInfo= {} + nf= argvs[1] + nfPutPrefix= str(argvs[2]) + statusReportDicts = argvs[3] + totalBw = 0 + maxIops = maxBw = 0 + minIops = minBw = sys.maxsize + tUnit = None + + os.system('ls -rtd '+os.path.dirname(argvs[0])+'/../* | head -n -5 |'\ + ' xargs --no-run-if-empty rm {} -rf') + if os.path.exists(argvs[0]): + with open(argvs[0]) as logF: + dataList = logF.readlines() + else: + return + for data in dataList: + try: + stat = json.loads(data, object_pairs_hook =OrderedDict) + except Exception: + return + iops,bw,tUnit = ioutilDataParse(stat, resultInfo) + maxIops = max(maxIops, iops) + minIops = min(minIops, iops) + maxBw = max(maxBw, bw) + minBw = min(minBw, bw) + totalBw += bw + if totalBw < 1024 * 1024 * 10: + return + + if resultInfo: + content = 'Iops:'+str(minIops)+'~'+str(maxIops)+\ + ', Bps:'+humConvert(minBw).replace('s', tUnit)+\ + '~'+humConvert(maxBw).replace('s', tUnit) + diagret = 'IO-Burst('+content+') detected' + ioutilReport(nf, nfPutPrefix, resultInfo, tUnit, diagret) + statusReportDicts['ioutil']['valid'] = True + + +def iowaitDataParse(data, resultInfo): + unkownDisable = False + for io in data['iowait']: + if 'many dirty' in io['reason'] or 'queue full' in io['reason']: + unkownDisable = True + if 'Unkown' in io['reason'] and unkownDisable == True: + continue + key = io['comm']+':'+io['tgid']+':'+io['pid'] + if key not in resultInfo.keys(): + resultInfo.setdefault( + key, {'timeout': 0, 'maxIowait': 0, 'reason': ''}) + if float(io['iowait']) > float(resultInfo[key]['maxIowait']): + resultInfo[key]['maxIowait'] = io['iowait'] + resultInfo[key]['timeout'] = io['timeout'] + resultInfo[key]['reason'] = io['reason'] + return data['global iowait'],unkownDisable + + +def iowaitReport(nf, nfPutPrefix, unkownDisable, resultInfo, diagret): + top = 0 + reason = '' + resDicts = { + 'Too many dirty pages':False, + 'Device queue full':False, + 'Ioscheduler queue full':False} + + for key, val in resultInfo.items(): + if unkownDisable == True and 'Unkown' in val['reason']: + del resultInfo[key] + + resultInfo = OrderedDict( + sorted(resultInfo.items(), key=lambda e: float(e[1]['maxIowait']), + reverse=True)[:3]) + for key, val in resultInfo.items(): + if unkownDisable == True: + resDicts[val['reason']] = True + top += 1 + reason += ( + '%d. task[%s], wait %sms, contribute iowait %s due to \'%s\'; ' %( + top, str(key), str(val['timeout']), str(val['maxIowait'])+'%', + str(val['reason']))) + + if unkownDisable == True: + if resDicts['Too many dirty pages'] == True: + suggest = 'Reduce io-write pressure or Adjust /proc/sys/vm/'\ + '{dirty_ratio,dirty_bytes} larger carefully' + else: + if resDicts['Device queue full'] and resDicts['Ioscheduler queue full']: + suggest = \ + 'Device queue full -> Disk busy due to disk queue full, '\ + 'Please reduce io pressure;'\ + 'Ioscheduler queue full -> Io scheduler busy due to '\ + 'scheduler queue full, '\ + 'Please reduce io pressure or Adjust '\ + '/sys/block//queue/nr_requests larger carefully' + elif resDicts['Device queue full']: + suggest = 'Disk busy due to disk queue full, '\ + 'Please reduce io pressure' + elif resDicts['Ioscheduler queue full']: + suggest = 'Io scheduler busy due to scheduler queue full, '\ + 'Please reduce io pressure or Adjust '\ + '/sys/block//queue/nr_requests larger carefully' + else: + suggest = 'Report stacktrace to OS kernel specialist' + + putIdx = ',diag_type=IOwait-high ' + putField = 'diagret=\"%s\",reason=\"%s\",solution=\"%s\"' %( + diagret, reason, suggest) + #nf.put(nfPutPrefix, + nf.puts(nfPutPrefix+putIdx+putField) + + +def iowaitResultReport(*argvs): + resultInfo = {} + nf = argvs[1] + nfPutPrefix = str(argvs[2]) + statusReportDicts = argvs[3] + maxGiowait = 0 + minGiowait = sys.maxsize + unkownDisable = None + + os.system('ls -rtd '+os.path.dirname(argvs[0])+'/../* | head -n -5 |'\ + ' xargs --no-run-if-empty rm {} -rf') + if os.path.exists(argvs[0]): + with open(argvs[0]) as logF: + dataList = logF.readlines() + else: + return + + for data in dataList: + try: + stat = json.loads(data, object_pairs_hook=OrderedDict) + except Exception: + return + gIowait,disable = iowaitDataParse(stat, resultInfo) + if not unkownDisable: + unkownDisable = disable + maxGiowait = max(maxGiowait, gIowait) + minGiowait = min(minGiowait, gIowait) + + if resultInfo: + content = str(minGiowait)+'%~'+str(maxGiowait)+'%' + diagret = 'IOwait high('+content+') detected' + iowaitReport(nf, nfPutPrefix, unkownDisable, resultInfo, diagret) + statusReportDicts['iowait']['valid'] = True + # print(diagret+reason+solution+'\n') + + +class displayClass(object): + def __init__(self, sender): + self.funcResultReportDicts = { + 'iohang': iohangResultReport, + 'ioutil': ioutilResultReport, + 'iolatency': iolatencyResultReport, + 'iowait': iowaitResultReport} + self.statusReportDicts = { + 'iohang': {'startT': 0, 'endT': 0, 'valid': False}, + 'ioutil': {'startT': 0, 'endT': 0, 'valid': False, + 'iopsThresh': 0, 'bpsThresh': 0}, + 'iolatency': {'startT': 0, 'endT': 0, 'valid': False, + 'lastIOburstT': 0}, + 'iowait': {'startT': 0, 'endT': 0, 'valid': False}, + } + self._sender = sender + self._nfPutPrefix = 'IOMonDiagLog' + + def markIoburst(self, now): + self.statusReportDicts['iolatency']['lastIOburstT'] = now + + def setIoburstThresh(self, iopsThresh, bpsThresh): + self.statusReportDicts['ioutil']['iopsThresh'] = iopsThresh + self.statusReportDicts['ioutil']['bpsThresh'] = bpsThresh + + def diagnoseValid(self, diagType): + return self.statusReportDicts[diagType]['valid'] + + def start(self, timeout, diagType, filepath, startTime, endTime): + self.statusReportDicts[diagType]['startT'] = startTime + self.statusReportDicts[diagType]['endT'] = endTime + self.statusReportDicts[diagType]['valid'] = False + argvs = [ + filepath, self._sender, self._nfPutPrefix, self.statusReportDicts] + timer = threading.Timer(timeout, + self.funcResultReportDicts[diagType], + argvs) + timer.start() diff --git a/source/tools/monitor/ioMonitor/ioMon/exceptCheckClass.py b/source/tools/monitor/ioMonitor/ioMon/exceptCheckClass.py new file mode 100755 index 0000000000000000000000000000000000000000..23b2a9ec13fd4b3872677b05098132d376520f2b --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/exceptCheckClass.py @@ -0,0 +1,187 @@ +# -*- coding: utf-8 -*- + +import sys +import string + + +class exceptCheckClass(): + def __init__(self, window): + self.window = int(window) if window is not None else 100 + self._exceptChkDicts = {} + + def addItem(self, key): + exceptChkItem = { + 'baseThresh': { + 'nrSample': 0, + 'moveWinData': [], + 'curWinMinVal': sys.maxsize, + 'curWinMaxVal': 0, + 'moveAvg': 0, + 'thresh': 0}, + 'compensation': { + 'thresh': 0, + 'shouldUpdThreshComp': True, + 'decRangeThreshAvg': 0, + 'decRangeCnt': 0, + 'minStableThresh': sys.maxsize, + 'maxStableThresh': 0, + 'stableThreshAvg': 0, + 'nrStableThreshSample': 0}, + 'dynTresh': sys.maxsize, + 'usedWin': 0} + self._exceptChkDicts.setdefault(key, exceptChkItem) + + # The sliding window calculates the basic threshold, through which the spikes + # and burrs in the IO indicators can be screened. The calculation idea is as + # follows: + # 1. take 100 data as a group for calculation (calculate 1 ~ 100 data for the + # first time, 2 ~ 101 for the second time, 3 ~ 102 for the third time, and + # so on), and calculate the average value mavg of 100 data in the current + # window + # 2. obtain the maximum value Max and minimum value min of 100 data, then record + # the thresh (MAX((max-mavg),(mavg-min))) each time, and calculate the average + # value(threshavg) of all thresh at this time each time, taking threshavg as + # the basic threshold for this time + # 3. The next basic threshold follows steps 1, 2, and so on + def _calcBaseThresh(self, key, e): + exceptChkDict = self._exceptChkDicts[key] + bt = exceptChkDict['baseThresh'] + thresh = None + + bt['nrSample'] += 1 + if bt['nrSample'] >= self.window: + if len(bt['moveWinData']) < self.window: + bt['moveWinData'].append(e) + else: + bt['moveWinData'][exceptChkDict['usedWin'] % self.window] = e + moveAvg = float( + format(sum(bt['moveWinData']) / float(self.window), '.1f')) + + # Find the min and max values of this window so far + maxVal = max(bt['curWinMaxVal'], e) + minVal = min(bt['curWinMinVal'], e) + nrThreshSample = bt['nrSample'] + 1 - self.window + thresh = float( + format(max(maxVal - moveAvg, moveAvg - minVal), '.1f')) + # Calculate base threshold + threshAvg = float(format( + (bt['thresh'] * (nrThreshSample - 1) + thresh) / nrThreshSample, + '.3f')) + bt['thresh'] = threshAvg + bt['moveAvg'] = moveAvg + bt['curWinMaxVal'] = maxVal + bt['curWinMinVal'] = minVal + + exceptChkDict['usedWin'] += 1 + if exceptChkDict['usedWin'] >= self.window: + # the next window, set min and Max to 0 + bt['curWinMaxVal'] = 0 + bt['curWinMinVal'] = sys.maxsize + exceptChkDict['usedWin'] = 0 + else: + # Here, only the first window will enter to ensure that + # the data in one window is accumulated + bt['moveWinData'].append(e) + bt['curWinMaxVal'] = max(bt['curWinMaxVal'], e) + bt['curWinMinVal'] = min(bt['curWinMinVal'], e) + exceptChkDict['usedWin'] += 1 + return thresh + + # Called by _calcCompThresh to calculate the compensation value + # under normal steady state + def _calcStableThresh(self, ct, curBaseThresh, curThresh): + # Discard points exceeding (base-threshold / 10) + avg = ct['decRangeThreshAvg'] + if (curThresh - avg) < ((curBaseThresh - avg) / 10.0): + tSum = ct['stableThreshAvg'] * \ + ct['nrStableThreshSample'] + curThresh + ct['nrStableThreshSample'] += 1 + ct['stableThreshAvg'] = tSum / ct['nrStableThreshSample'] + ct['minStableThresh'] = min(ct['minStableThresh'], curThresh) + ct['maxStableThresh'] = max(ct['maxStableThresh'], curThresh) + # 1.5 windows of stable data have been counted, + # which can be used as normal threshold compensation value + if ct['nrStableThreshSample'] >= (self.window * 1.5): + ct['thresh'] = \ + max(ct['stableThreshAvg'] - ct['minStableThresh'], + ct['maxStableThresh'] - ct['stableThreshAvg']) + ct['shouldUpdThreshComp'] = False + ct['minStableThresh'] = sys.maxsize + ct['maxStableThresh'] = 0 + ct['stableThreshAvg'] = ct['decRangeThreshAvg'] = 0 + ct['nrStableThreshSample'] = ct['decRangeCnt'] = 0 + + # Calculate the threshold compensation value and superimpose this value + # on the basic threshold to eliminate false alarms + def _calcCompThresh(self, key, lastBaseThresh, curThresh): + exceptChkDict = self._exceptChkDicts[key] + curBaseThresh = exceptChkDict['baseThresh']['thresh'] + ct = exceptChkDict['compensation'] + + # It is not confirmed whether the current state is constant + # (constant state is defined as IO index fluctuation, which is stable) + # 1. the max basic threshold of this window is the compensation value + # 2. enter a new window to reset to the current basic threshold + if ct['shouldUpdThreshComp'] == True and \ + (ct['thresh'] < curBaseThresh or exceptChkDict['usedWin'] == 0): + ct['thresh'] = curBaseThresh + + # Continuous monotonic decreasing, constant steady state, + # constant compensation threshold inferred + if curBaseThresh < lastBaseThresh: + tSum = ct['decRangeThreshAvg'] * ct['decRangeCnt'] + curThresh + ct['decRangeCnt'] += 1 + ct['decRangeThreshAvg'] = tSum / ct['decRangeCnt'] + # The monotonic decline has continued for 1.5 windows, + # indicating that IO pressure may return to normality + if ct['decRangeCnt'] >= (self.window * 1.5): + self._calcStableThresh(ct, curBaseThresh, curThresh) + else: + # As long as the basic threshold curve is not + # continuously monotonically decreasing, + # reset to 0 and make statistics again + ct['minStableThresh'] = sys.maxsize + ct['maxStableThresh'] = 0 + ct['stableThreshAvg'] = ct['decRangeThreshAvg'] = 0 + ct['nrStableThreshSample'] = ct['decRangeCnt'] = 0 + + # Update the dynamic threshold of the corresponding indicator type + # and call it after collecting the IO indicators. The key is await, + # util, IOPs, BPS, etc + def updateDynThresh(self, key, e): + exceptChkDict = self._exceptChkDicts[key] + bt = exceptChkDict['baseThresh'] + ct = exceptChkDict['compensation'] + lastBaseThresh = bt['thresh'] + + curThresh = self._calcBaseThresh(key, e) + if curThresh is not None: + self._calcCompThresh(key, lastBaseThresh, curThresh) + exceptChkDict['dynTresh'] = \ + bt['thresh'] + bt['moveAvg'] + ct['thresh'] + + # Turn off the threshold compensation of the corresponding indicators. + # Generally, when it is detected that the IO util exceeds 20%, + # it will be disabled according to the situation of each indicator + def disableThreshComp(self, key): + exceptChkDict = self._exceptChkDicts[key] + ct = exceptChkDict['compensation'] + bt = exceptChkDict['baseThresh'] + + #if exceptChkDict['dynTresh'] == sys.maxsize: + # return + + if ct['shouldUpdThreshComp'] == True: + ct['shouldUpdThreshComp'] = False + exceptChkDict['dynTresh'] = bt['thresh'] + bt['moveAvg'] + ct['thresh'] = 0.000001 + + + def getNrDataSample(self, key): + return self._exceptChkDicts[key]['baseThresh']['nrSample'] + + # Get the dynamic threshold of the corresponding indicator type, + # call it after collecting the IO indicators, and judge whether + # the indicators are abnormal. The key is await, util, IOPs, BPS, etc + def getDynThresh(self, key): + return self._exceptChkDicts[key]['dynTresh'] diff --git a/source/tools/monitor/ioMonitor/ioMon/exceptDiagnoseClass.py b/source/tools/monitor/ioMonitor/ioMon/exceptDiagnoseClass.py new file mode 100755 index 0000000000000000000000000000000000000000..98f912ce69cd5e17357bc36987c1a67031582763 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/exceptDiagnoseClass.py @@ -0,0 +1,267 @@ +# -*- coding: utf-8 -*- + +import os +import string +import time +from collections import OrderedDict +import threading +from tools.iofstool.iofsstat import iofsstatStart +from tools.iowaitstat.iowaitstat import iowaitstatStart +from displayClass import displayClass + + +class runDiag(object): + def __init__(self, logRootPath, sender): + self.funcDicts = { + 'iohang': self.startIohangDiagnose, + 'ioutil': self.startIoutilDiagnose, + 'iolatency': self.startIolatencyDiagnose, + 'iowait': self.startIowaitDiagnose} + self.lastDiagTimeDicts = \ + {'iohang': 0, 'ioutil': 0, 'iolatency': 0, 'iowait': 0} + self.display = displayClass(sender) + self.sysakPath = 'sysak' + self.logRootPath = logRootPath + + + def _recentDiagnoseValid(self, diagType): + return self.display.diagnoseValid(diagType) + + + def startIohangDiagnose(self, *argv): + devname = argv[0] + now = time.time() + if now - self.lastDiagTimeDicts['iohang'] <= 60: + return + startTime = time.strftime('%Y-%m-%d-%H:%M:%S', time.localtime(now)) + logdir = self.logRootPath+'/iosdiag/hangdetect/'+startTime + file = logdir+'/result.log.seq' + outlog = logdir+'/resultCons.log' + if not os.path.exists(logdir): + try: + os.makedirs(logdir) + except Exception: + return + self.lastDiagTimeDicts['iohang'] = now + if devname is not None: + os.system(self.sysakPath+' -g iosdiag hangdetect -o -t 3000 -T 10 -f '+ + file+' '+devname+' > '+outlog+' &') + else: + os.system(self.sysakPath+' -g iosdiag hangdetect -o -t 3000 -T 10 -f '+ + file+' > '+outlog+' &') + self.display.start(20, 'iohang', logdir, now, now+60) + + + def startIolatencyDiagnose(self, *argv): + devname = argv[0] + thresh = argv[1] + ioburst = argv[2] + now = time.time() + if now - self.lastDiagTimeDicts['iolatency'] <= 60: + return + startTime = time.strftime('%Y-%m-%d-%H:%M:%S', time.localtime(now)) + logdir = self.logRootPath+'/iosdiag/latency/'+startTime + file = logdir+'/result.log.seq' + outlog = logdir+'/resultCons.log' + if not os.path.exists(logdir): + try: + os.makedirs(logdir) + except Exception: + return + self.lastDiagTimeDicts['iolatency'] = now + if devname is not None: + os.system(self.sysakPath+' -g iosdiag latency -t '+str(thresh) + + ' -T 45 -f '+file+' '+devname+' > '+outlog+' &') + else: + os.system(self.sysakPath+' -g iosdiag latency -t '+str(thresh) + + ' -T 45 -f '+file+' > '+outlog+' &') + if ioburst: + self.display.markIoburst(now) + self.display.start(60, 'iolatency', logdir, now, now+60) + + + def startIoutilDiagnose(self, *argv): + devname = argv[0] + bwThresh = argv[1] + iopsThresh = argv[2] + now = time.time() + if now - self.lastDiagTimeDicts['ioutil'] <= 60: + return + startTime = time.strftime('%Y-%m-%d-%H:%M:%S', time.localtime(now)) + logdir = self.logRootPath+'/iosdiag/iofsstat/'+startTime + outlog = logdir+'/resultCons.log' + if not os.path.exists(logdir): + try: + os.makedirs(logdir) + except Exception: + return + self.lastDiagTimeDicts['ioutil'] = now + #self.display.setIoburstThresh(iopsThresh, bwThresh) + argvs = ['-j',outlog,'-n','-m','-c','1','-t','5','-T','40', + '-i',str(iopsThresh),'-b',str(bwThresh)] + threading.Thread(target=iofsstatStart, args=(argvs,)).start() + self.display.start(55, 'ioutil', outlog, now, now+60) + + + def startIowaitDiagnose(self, *argv): + iowaitThresh = argv[0] + now = time.time() + if now - self.lastDiagTimeDicts['iowait'] <= 60: + return + startTime = time.strftime('%Y-%m-%d-%H:%M:%S', time.localtime(now)) + logdir = self.logRootPath+'/iosdiag/iowaitstat/'+startTime + outlog = logdir+'/resultCons.log' + if not os.path.exists(logdir): + try: + os.makedirs(logdir) + except Exception: + return + self.lastDiagTimeDicts['iowait'] = now + argvs = ['-j', outlog, '-t', '5', '-w', str(iowaitThresh), '-T', '45'] + threading.Thread(target=iowaitstatStart, args=(argvs,)).start() + self.display.start(55, 'iowait', outlog, now, now+60) + + + def runDiagnose(self, diagType, argv): + self.funcDicts[diagType](*list(argv)) + + +class diagnoseClass(runDiag): + def __init__(self, window, logRootPath, sender): + super(diagnoseClass, self).__init__(logRootPath, sender) + self.window = window + self.diagnoseDicts = OrderedDict() + self._diagStat = OrderedDict( + {'iohang': {'run': False, 'argv': [0, 0, 0, 0, 0, 0, 0, 0]}, + 'ioutil': {'run': False, 'argv': [0, 0, 0, 0, 0, 0, 0, 0]}, + 'iowait': {'run': False, 'argv': [0, 0, 0, 0, 0, 0, 0, 0]}, + 'iolatency': {'run': False, 'argv': [0, 0, 0, 0, 0, 0, 0, 0]}}) + + + def addItem(self, devname, key, reportInterval, triggerInterval): + diagRecord = { + 'statWindow': self.window, + 'trigger': False, + 'lastReport': 0, + 'reportInterval': reportInterval, + 'reportCnt': 0, + 'lastDiag': 0, + 'triggerInterval': triggerInterval, + 'diagArgs': [0, 0, 0, 0, 0, 0, 0, 0]} + if devname not in self.diagnoseDicts.keys(): + self.diagnoseDicts.setdefault(devname, {key: diagRecord}) + else: + self.diagnoseDicts[devname].setdefault(key, diagRecord) + + + def setUpDiagnose(self, devname, key, nrSample, *argv): + diagnoseDicts = self.diagnoseDicts[devname][key] + lastDiag = diagnoseDicts['lastDiag'] + lastReport = diagnoseDicts['lastReport'] + statWindow = diagnoseDicts['statWindow'] + reportInterval = diagnoseDicts['reportInterval'] + triggerInterval = diagnoseDicts['triggerInterval'] + + if reportInterval != 0: + if lastReport == 0 or (nrSample-lastReport) > statWindow: + diagnoseDicts['lastReport'] = nrSample + diagnoseDicts['reportCnt'] = 1 + else: + diagnoseDicts['reportCnt'] += 1 + if diagnoseDicts['reportCnt'] > reportInterval: + if lastDiag == 0 or (nrSample-lastDiag) > triggerInterval: + diagnoseDicts['trigger'] = True + diagnoseDicts['reportCnt'] = 0 + diagnoseDicts['lastDiag'] = nrSample + else: + diagnoseDicts['lastReport'] = nrSample + diagnoseDicts['reportCnt'] = 0 + elif triggerInterval != 0: + if lastDiag == 0 or (nrSample-lastDiag) >= triggerInterval: + diagnoseDicts['lastDiag'] = nrSample + diagnoseDicts['trigger'] = True + else: + diagnoseDicts['trigger'] = True + + for idx, val in enumerate(argv): + diagnoseDicts['diagArgs'][idx] = val + + + def isException(self, devname, key): + diagnoseDicts = self.diagnoseDicts[devname][key] + reportInterval = diagnoseDicts['reportInterval'] + triggerInterval = diagnoseDicts['triggerInterval'] + + if reportInterval != 0: + if (diagnoseDicts['reportCnt'] + 1) >= reportInterval: + return True + elif triggerInterval != 0: + return True + else: + return True + return False + + + def clearDiagStat(self): + for diagType, stat in self._diagStat.items(): + stat['run'] = False + stat['argv'][0:] = [0, 0, 0, 0, 0, 0, 0, 0] + + + def checkDiagnose(self): + diagnoseDicts = self.diagnoseDicts + diagInfo = {'iohang': [], 'iolatency': [], 'ioutil': []} + diagStat = self._diagStat + ioburst = False + + for devname, diagDict in diagnoseDicts.items(): + if devname == 'system': + if diagDict['iowait']['trigger'] == True: + diagStat['iowait']['run'] = True + diagStat['iowait']['argv'][0] = \ + diagDict['iowait']['diagArgs'][0] + diagDict['iowait']['trigger'] = False + continue + + for diagType in ['iohang', 'iolatency', 'ioutil']: + if diagDict[diagType]['trigger'] == True: + if diagType == 'iolatency': + ioburst = diagDict['iolatency']['diagArgs'][1] + diagInfo[diagType].append(devname) + diagDict[diagType]['trigger'] = False + + for diagType, value in diagInfo.items(): + diagStat[diagType]['run'] = True + if len(value) > 1: + diagStat[diagType]['argv'][0] = None + elif len(value) == 1: + diagStat[diagType]['argv'][0] = value[0] + else: + diagStat[diagType]['run'] = False + + if diagStat['ioutil']['run'] == True: + for idx in [1,2]: + val = sorted( + [diagnoseDicts[dev]['ioutil']['diagArgs'][idx-1] + for dev in diagInfo['ioutil']], + reverse=True) + diagStat['ioutil']['argv'][idx] = val[-1] + + if diagStat['iolatency']['run'] == True: + diagStat['iolatency']['argv'][1] = sorted( + [diagnoseDicts[dev]['iolatency']['diagArgs'][0] + for dev in diagInfo['iolatency']], + reverse=True)[-1] + diagStat['iolatency']['argv'][2] = ioburst + + for diagType, stat in diagStat.items(): + if stat['run'] == True: + self.runDiagnose(diagType, stat['argv']) + stat['run'] = False + + + # In displayClass, after the diagnostic log is reported to the remote end, + # it will be marked as a valid diagnosis, and In exceptDiagnoseClass, + # clear the valid mark before each diagnosis + def recentDiagnoseValid(self, diagType): + return self._recentDiagnoseValid(diagType) diff --git a/source/tools/monitor/ioMonitor/ioMon/ioMonCfgClass.py b/source/tools/monitor/ioMonitor/ioMon/ioMonCfgClass.py new file mode 100755 index 0000000000000000000000000000000000000000..0610135c7c287a052bbe8edbf6bf061d0d1c5457 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/ioMonCfgClass.py @@ -0,0 +1,137 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import signal +import string +import time +import json +from collections import OrderedDict + +globalCfgDicts = {} +globalCfgPath = '' +def loadCfg(cfgPath): + with open(cfgPath) as f: + data = f.read() + return json.loads(data) + + +def loadCfgHander(signum, frame): + global globalCfgDicts + globalCfgDicts = loadCfg(globalCfgPath) + + +class ioMonCfgClass(object): + def __init__(self, cfgArg, resetCfg, logRootPath): + global globalCfgPath + self.cfgPath = logRootPath+'/ioMon/ioMonCfg.json' + globalCfgPath = self.cfgPath + cfg = self._paserCfg(cfgArg) + hasArgs = any(list(cfg.values())) + if not os.path.exists(self.cfgPath) or resetCfg: + cfg['iowait'] = int(cfg['iowait']) if cfg['iowait'] else 5 + cfg['await'] = int(cfg['await']) if cfg['await'] else 10 + cfg['util'] = int(cfg['util']) if cfg['util'] else 20 + cfg['iops'] = int(cfg['iops']) if cfg['iops'] else 150 + cfg['bps'] = int(cfg['bps']) if cfg['bps'] else 31457280 + cfg['cycle'] = int(cfg['cycle']) if cfg['cycle'] else 1000 + cfg['diagIowait'] = cfg['diagIowait'] if cfg['diagIowait'] else 'on' + cfg['diagIoburst'] = cfg['diagIoburst'] if cfg['diagIoburst'] else 'on' + cfg['diagIolat'] = cfg['diagIolat'] if cfg['diagIolat'] else 'on' + cfg['diagIohang'] = cfg['diagIohang'] if cfg['diagIohang'] else 'off' + self._updateCfg(cfg) + return + else: + self._loadCfg() + if hasArgs: + self._updateCfg(cfg) + + + def _paserCfg(self, cfgArg): + cfgDicts = { + 'iowait':None, 'await':None, 'util':None, 'iops':None, 'bps':None, + 'cycle':None, 'diagIowait':None, 'diagIoburst':None, + 'diagIolat':None, 'diagIohang':None} + try: + cfgList = \ + cfgArg.split(',') if cfgArg is not None and len(cfgArg) > 0 else [] + for cfg in cfgList: + errstr = None + c = cfg.split('=') + if c[0] not in cfgDicts.keys() or len(c[1]) == 0: + errstr = "bad cfg item: %s, must be in %s" %( + cfg, str(cfgDicts.keys())) + elif 'diag' not in c[0] and not c[1].isdigit(): + errstr = "monitor cfg argv must be digit: %s." %cfg + elif 'diag' in c[0] and c[1] not in ['on', 'off']: + errstr = \ + "diagnose cfg argv must be [\'on\', \'off\']: %s." %cfg + if errstr: + print(errstr) + sys.exit(0) + cfgDicts[c[0]] = c[1] + except Exception: + print "bad cfg: %s." %cfg + sys.exit(0) + return cfgDicts + + + def _setGlobalCfgDicts(self, CfgDicts): + global globalCfgDicts + globalCfgDicts = CfgDicts + + + def _getGlobalCfgDicts(self): + global globalCfgDicts + return globalCfgDicts + + + def _updateCfg(self, cfgDicts): + oldCfg = {} + if not os.path.exists(self.cfgPath): + if not os.path.exists(os.path.dirname(self.cfgPath)): + os.mkdir(os.path.dirname(self.cfgPath)) + else: + oldCfg = loadCfg(self.cfgPath) + f = open(self.cfgPath, 'w+') + newCfg = json.loads(json.dumps(cfgDicts)) + if oldCfg: + for key,val in newCfg.items(): + if val is not None: + oldCfg[key] = val + newCfg = oldCfg + s = json.dumps(newCfg, indent=4) + f.write(s) + f.close() + self._setGlobalCfgDicts(newCfg) + + + def _loadCfg(self): + self._setGlobalCfgDicts(loadCfg(self.cfgPath)) + + + def createCfgFlagFile(self): + f = open(os.path.dirname(self.cfgPath)+'/.ioMonCfgFlag', 'w+') + f.write(str(os.getpid())) + f.close() + signal.signal(signal.SIGUSR2, loadCfgHander) + + + def notifyIoMon(self): + try: + with open(os.path.dirname(self.cfgPath)+'/.ioMonCfgFlag') as f: + pid = f.read() + with open('/proc/'+str(pid)+'/cmdline') as f: + cmdline = f.read().strip() + except Exception: + sys.exit(0) + if 'ioMonEntry' in cmdline: + os.system('kill -USR2 '+str(pid)) + + + def getCfgItem(self, key): + val = str(self._getGlobalCfgDicts()[key]) + if val.isdigit(): + val = int(val) + return val diff --git a/source/tools/monitor/ioMonitor/ioMon/ioMonitorClass.py b/source/tools/monitor/ioMonitor/ioMon/ioMonitorClass.py new file mode 100755 index 0000000000000000000000000000000000000000..2a9dec0cd56e16e4bb2243e96f171e17c15b821b --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/ioMonitorClass.py @@ -0,0 +1,372 @@ +# -*- coding: utf-8 -*- + +import os +import sys +import signal +import string +import argparse +import time +from exceptDiagnoseClass import diagnoseClass +from exceptCheckClass import exceptCheckClass +from ioMonCfgClass import ioMonCfgClass +from collections import OrderedDict +from nfPut import CnfPut + +class ioMonitorClass(object): + def __init__(self, logRootPath, cfg, pipeFile): + self.window = 60 + self.cfg = cfg + self.cfg.createCfgFlagFile() + self.diagSwitch = { + 'diagIowait': {'sw': self.cfg.getCfgItem('diagIowait'), + 'esi':'IOwait-High'}, + 'diagIoburst': {'sw': self.cfg.getCfgItem('diagIoburst'), + 'esi':'IO-Delay'}, + 'diagIolat': {'sw': self.cfg.getCfgItem('diagIolat'), + 'esi':'IO-Burst'}, + 'diagIohang': {'sw': self.cfg.getCfgItem('diagIohang'), + 'esi':'IO-Hang'} + } + self._sender = CnfPut(pipeFile) + self._nfPutTlb = 'IOMonIndForDisksIO' + self._nfPutTlb4System = 'IOMonIndForSystemIO' + self.fieldDicts = OrderedDict() + self.exceptChkDicts = {'system': exceptCheckClass(self.window)} + self.exceptChkDicts['system'].addItem('iowait') + self.diagnose = diagnoseClass(self.window, logRootPath, self._sender) + self.diagnose.addItem('system', 'iowait', 0, 60) + self.fDiskStats = open("/proc/diskstats") + self.cpuStatIowait = {'sum': 0, 'iowait': 0} + self.uploadInter = 0 + self.exceptionStat = {'system': {'IOwait-High': {'cur':0,'max':0}}} + self.dataStat = {'system': {'iowait': 0}} + + + def _addMonitorAttrForDisk(self, disk): + dataStat = self.dataStat + exceptChkDicts = self.exceptChkDicts + diagnose = self.diagnose + exceptionStat = self.exceptionStat + + # used for reporting per-index to database + dataStat.setdefault( + disk, {'await': 0, 'util': 0, 'iops': 0, 'bps': 0, 'qusize': 0}) + + # add exception-check attr for per-index on per-disk + exceptChkDicts.setdefault(disk, exceptCheckClass(self.window)) + for key in ['util', 'await', 'iops', 'bps']: + exceptChkDicts[disk].addItem(key) + + # add diagnose attr for per-index on per-disk + diagnoseDict = { + 'iohang': {'triggerInterval': self.window * 5, + 'reportInterval': 10}, + 'ioutil': {'triggerInterval': 60, 'reportInterval': 0}, + 'iolatency': {'triggerInterval': 60, 'reportInterval': 0} + } + for key, item in diagnoseDict.items(): + diagnose.addItem( + disk, key, item['reportInterval'], item['triggerInterval']) + # used for reporting exception to database + exceptionStat.setdefault( + disk, + {'IO-Delay':{'cur':0,'max':0}, 'IO-Burst':{'cur':0,'max':0}, + 'IO-Hang':{'cur':0,'max':0}}) + + + def _removeDiskMonitor(self, disk): + del self.fieldDicts[disk] + del self.dataStat[disk] + del self.exceptChkDicts[disk] + del self.diagnose[disk] + del self.exceptionStat[disk] + + + def _disableThreshComp(self, devname, qusize): + exceptChkDicts = self.exceptChkDicts + exceptChkDicts[devname].disableThreshComp('util') + exceptChkDicts[devname].disableThreshComp('iops') + exceptChkDicts[devname].disableThreshComp('bps') + if qusize > 1: + exceptChkDicts[devname].disableThreshComp('await') + + + def _calcIowait(self): + with open("/proc/stat") as fStat: + statList = map(long, fStat.readline().split()[1:]) + iowait = float(format( + (statList[4] - self.cpuStatIowait['iowait']) * 100.0 / + (sum(statList) - self.cpuStatIowait['sum']), '.2f')) + return iowait + + + def _calcIoIndex(self, devname, field, secs): + ds = self.dataStat[devname] + uploadInter = self.uploadInter + + rws = field['1'][1] + field['5'][1] - field['1'][0] - field['5'][0] + iops = round(rws / secs, 1) + ds['iops'] = (ds['iops'] * (uploadInter - 1) + iops) / uploadInter + + rwSecs = field['3'][1] + field['7'][1] - field['3'][0] - field['7'][0] + bps = rwSecs / secs * 512 + ds['bps'] = (ds['bps'] * (uploadInter - 1) + bps) / uploadInter + + qusize = round((field['11'][1] - field['11'][0]) / secs / 1000, 2) + ds['qusize'] = (ds['qusize'] * (uploadInter - 1) + qusize) / uploadInter + + rwTiks = field['4'][1] + field['8'][1] - field['4'][0] - field['8'][0] + wait = round(rwTiks / (iops * secs), 2) if iops else 0 + ds['await'] = (ds['await'] * (uploadInter - 1) + wait) / uploadInter + + times = field['10'][1] - field['10'][0] + util = round(times * 100.0 / (secs * 1000), 2) + util = util if util <= 100 else 100.0 + ds['util'] = (ds['util'] * (uploadInter - 1) + util) / uploadInter + + return {'iops': iops*secs, 'bps': bps*secs, 'qusize': qusize*secs, + 'wait': wait, 'util': util} + + + def _checkDiagSwitchChange(self, t): + s = self.diagSwitch[t] + newSW = self.cfg.getCfgItem(t) + if newSW != s['sw'] and newSW == 'on': + for k,v in self.exceptionStat.items(): + if s['esi'] in v.keys(): + v[s['esi']]['max'] = 0 + s['sw'] = newSW + return newSW + + + def _checkIOwaitException(self, iowait): + exceptChk = self.exceptChkDicts['system'] + dataStat = self.dataStat['system'] + es = self.exceptionStat['system']['IOwait-High'] + diagnose = self.diagnose + uploadInter = self.uploadInter + + if iowait >= self.cfg.getCfgItem('iowait'): + exceptChk.disableThreshComp('iowait') + + dataStat['iowait'] = \ + (dataStat['iowait'] * (uploadInter - 1) + iowait) / uploadInter + + # Detect iowait exception + minThresh = self.cfg.getCfgItem('iowait') + iowaitThresh = max(exceptChk.getDynThresh('iowait'), minThresh) + if iowait >= iowaitThresh: + es['cur'] += 1 + diagSW = self._checkDiagSwitchChange('diagIowait') + rDiagValid = diagnose.recentDiagnoseValid('iowait') + # Configure iowait diagnosis + if diagSW == 'on' and (es['cur'] > es['max'] or not rDiagValid): + nrSample = exceptChk.getNrDataSample('iowait') + iowaitArg = max(int(iowait * 0.25), minThresh) + diagnose.setUpDiagnose('system', 'iowait', nrSample, iowaitArg) + + exceptChk.updateDynThresh('iowait', iowait) + + def _checkIoburstException(self, devname, es, bps, iops, exceptChk): + bpsLowW = self.cfg.getCfgItem('bps') + bpsHighW = max(exceptChk.getDynThresh('bps'), bpsLowW) + bpsMiddW = max(bpsLowW, bpsHighW / 2) + iopsLowW = self.cfg.getCfgItem('iops') + iopsHighW = max(exceptChk.getDynThresh('iops'), iopsLowW) + iopsMiddW = max(iopsLowW, iopsHighW / 2) + ioburst = exception = False + + if iops >= iopsMiddW or bps >= bpsMiddW: + ioburst = True + bpsOver = True if bps >= bpsHighW else False + iopsOver = True if iops >= iopsHighW else False + + if iopsOver or bpsOver: + es['cur'] += 1 + diagSW = self._checkDiagSwitchChange('diagIoburst') + rDiagValid = self.diagnose.recentDiagnoseValid('ioutil') + # Configure IO load diagnosis + if diagSW == 'on' and (es['cur'] > es['max'] or not rDiagValid): + bpsArg = iopsArg = 0 + if bpsOver == True: + bpsArg = max(int(bps * 0.25), bpsLowW) + if iopsOver == True: + iopsArg = max(int(iops * 0.7), iopsLowW) + nrSample = exceptChk.getNrDataSample('util') + self.diagnose.setUpDiagnose( + devname, 'ioutil', nrSample, bpsArg, iopsArg) + return ioburst + + def _checkIohangException( + self, devname, es, util, qusize, iops, exceptChk): + # Detect IO hang + if util >= 100 and qusize >= 1 and iops < 50: + # Configure IO hang diagnosis + if self.diagnose.isException(devname, 'iohang') == True: + es['cur'] += 1 + diagSW = self._checkDiagSwitchChange('diagIohang') + rDiagValid = self.diagnose.recentDiagnoseValid('iohang') + if diagSW == 'on' and (es['cur'] > es['max'] or not rDiagValid): + nrSample = exceptChk.getNrDataSample('util') + self.diagnose.setUpDiagnose(devname, 'iohang', nrSample) + + def _checkUtilException(self, devname, util, iops, bps, qusize): + exceptChk = self.exceptChkDicts[devname] + exceptionStat = self.exceptionStat[devname] + diagnose = self.diagnose + ioburst = False + + utilMinThresh = self.cfg.getCfgItem('util') + utilThresh = max(exceptChk.getDynThresh('util'), utilMinThresh) + if util >= utilThresh: + es = exceptionStat['IO-Burst'] + ioburst = \ + self._checkIoburstException(devname, es, bps, iops, exceptChk) + if not ioburst: + es = exceptionStat['IO-Hang'] + self._checkIohangException( + devname, es, util, qusize, iops, exceptChk) + exceptChk.updateDynThresh('util', util) + exceptChk.updateDynThresh('iops', iops) + exceptChk.updateDynThresh('bps', bps) + return ioburst + + + def _checkAwaitException(self, devname, wait, ioburst): + exceptChk = self.exceptChkDicts[devname] + es = self.exceptionStat[devname]['IO-Delay'] + diagnose = self.diagnose + + awaitMinThresh = self.cfg.getCfgItem('await') + awaitThresh = max(exceptChk.getDynThresh('await'), awaitMinThresh) + if wait >= awaitThresh: + es['cur'] += 1 + diagSW = self._checkDiagSwitchChange('diagIolat') + rDiagValid = diagnose.recentDiagnoseValid('iolatency') + # Configuring IO delay diagnostics + if diagSW == 'on' and (es['cur'] > es['max'] or not rDiagValid): + nrSample = exceptChk.getNrDataSample('await') + waitArg = max(int(wait * 0.4), awaitMinThresh) + diagnose.setUpDiagnose( + devname, 'iolatency', nrSample, waitArg, ioburst) + exceptChk.updateDynThresh('await', wait) + + + def _reportDataToRemote(self, devList): + # report datastat&&exception to database + nCycle = 1000.0 / float(self.cfg.getCfgItem('cycle')) + dataStat = self.dataStat['system'] + es = self.exceptionStat['system']['IOwait-High'] + + putIdx = ',idx_type=system_Indicator,devname=system ' + putField = 'iowait=%f,iowaithighCnt=%f' %( + dataStat['iowait'], es['cur'] / nCycle) + self._sender.puts(self._nfPutTlb4System + putIdx + putField) + + es['max'] = max(es['max'] if es['cur'] else 0, es['cur']) + es['cur'] = 0 + cur = {'IO-Delay':0, 'IO-Burst':0, 'IO-Hang':0} + for devname in devList: + dataStat = self.dataStat[devname] + es = self.exceptionStat[devname] + for type in cur.keys(): + cur[type] = int(es[type]['cur']) / nCycle + es[type]['max'] = \ + max(es[type]['max'] if cur[type] else 0, cur[type]) + es[type]['cur'] = 0 + + putIdx = ',idx_type=iostat_Indicator,devname=%s ' % devname + putField = 'await=%f,util=%f,iops=%f,bps=%f,qusize=%f' %( + dataStat['await'], dataStat['util'], dataStat['iops'], + dataStat['bps'] / 1024.0, dataStat['qusize'] + ) + putField += ',iodelayCnt=%f,ioburstCnt=%f,iohangCnt=%f' %( + cur['IO-Delay'], cur['IO-Burst'], cur['IO-Hang']) + self._sender.puts(self._nfPutTlb + putIdx + putField) + + + def _collectBegin(self): + fieldDicts = self.fieldDicts + + # collect iowait begin + with open("/proc/stat") as fStat: + cpuStatList = map(long, fStat.readline().split()[1:]) + self.cpuStatIowait['sum'] = sum(cpuStatList) + self.cpuStatIowait['iowait'] = cpuStatList[4] + + # collect iostat begin + self.fDiskStats.seek(0) + for stat in self.fDiskStats.readlines(): + stat = stat.split() + if os.path.exists('/sys/block/'+stat[2]) == False: + if stat[2] in fieldDicts.keys(): + self._removeDiskMonitor(stat[2]) + continue + + if stat[2] in fieldDicts.keys(): + field = fieldDicts[stat[2]] + else: + field = { + '1': [0, 0], '3': [0, 0], '4': [0, 0], + '5': [0, 0], '7': [0, 0], '8': [0, 0], + '10': [0, 0], '11': [0, 0]} + # add data staticsis for per-disk + fieldDicts.setdefault(stat[2], field) + self._addMonitorAttrForDisk(stat[2]) + + for idx, value in field.items(): + value[0] = long(stat[int(idx) + 2]) + + + def _collectEnd(self, secs): + fieldDicts = self.fieldDicts + exceptChkDicts = self.exceptChkDicts + uploadInter = self.uploadInter + + self.uploadInter = \ + 1 if ((uploadInter * secs) % 60) == 0 else (uploadInter + 1) + + # Calculate iowait + iowait = self._calcIowait() + # Detect iowait exception + self._checkIOwaitException(iowait) + + # collect iostat end + self.fDiskStats.seek(0) + for stat in self.fDiskStats.readlines(): + stat = stat.split() + if os.path.exists('/sys/block/'+stat[2]) == False: + if stat[2] in fieldDicts.keys(): + self._removeDiskMonitor(stat[2]) + continue + try: + for idx, value in fieldDicts[stat[2]].items(): + value[1] = long(stat[int(idx) + 2]) + except Exception: + continue + + for devname, field in fieldDicts.items(): + io = self._calcIoIndex(devname, field, secs) + if io['util'] >= self.cfg.getCfgItem('util'): + # There is IO Burst at present, turn off threshold compensation + self._disableThreshComp(devname, io['qusize']) + # Detect util exception + ioburst = self._checkUtilException( + devname, io['util'], io['iops'], io['bps'], io['qusize']) + # Detect await exception + self._checkAwaitException(devname, io['wait'], ioburst) + + if ((self.uploadInter * secs) % 60) == 0: + self._reportDataToRemote(fieldDicts.keys()) + + + def monitor(self): + while True: + secs = self.cfg.getCfgItem('cycle') / 1000.0 + self._collectBegin() + time.sleep(secs) + self._collectEnd(secs) + # Check if it is necessary to start the diagnosis + self.diagnose.checkDiagnose() + self.fDiskStats.close() + diff --git a/source/tools/monitor/ioMonitor/ioMon/ioMonitorMain.py b/source/tools/monitor/ioMonitor/ioMon/ioMonitorMain.py new file mode 100755 index 0000000000000000000000000000000000000000..921a343550f23126ec91c5ea91147adbb4ac6ce9 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/ioMonitorMain.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +import argparse +import signal +from ioMonCfgClass import ioMonCfgClass +from ioMonitorClass import ioMonitorClass + +setcfg_descripton = """set monitor cfg, like -s \'xxxx=xxx,xxx=xxx\' +iowait, The min cpu-iowait not report exceptions(1~100, default 5). +await, The min partition-await not report exceptions(N ms, default 10). +util, The min partition-util not report exceptions(1~100, default 20). +bps, The min partition-bps not report exceptions(bytes, default 30MB). +iops, The min partition-iops not report exceptions(default 150). +cycle, The cycle of Monitoring data collection(default 500ms). +diagIowait, Disable or enable diagnose while reporting iowait exceptions(default on). +diagIolat, Disable or enable diagnose while reporting latency exceptions(default on). +diagIoburst, Disable or enable diagnose while reporting ioburst exceptions(default on). +diagIohang, Disable or enable diagnose while reporting iohang exceptions(default on). +""" + +def getRunArgsFromYaml(yamlF): + logRootPath = '' + pipeFile = '' + with open(yamlF) as f: + lines = f.readlines() + for l in lines: + if not l.startswith('#'): + if 'proc_path:' in l: + logRootPath = l.split()[1].strip('\n') + elif 'outline:' in l: + pipeFile = lines[lines.index(l) + 1].split()[1].strip('\n') + if logRootPath and pipeFile: + break + if not logRootPath or not pipeFile: + raise ValueError( + 'Unable to get labels \"proc_path\" and \"outline\" in %s' % yamlF) + return logRootPath+'/run',pipeFile + + +def main(): + examples = """e.g. + ./ioMonitorMain.py -y [yaml_file] + Start ioMonitor + ./ioMonitorMain.py -y [yaml_file] --reset_cfg --only_set_cfg + Only reset cfg to default + ./ioMonitorMain.py -y [yaml_file] -s 'iowait=10,iops=200,diagIowait=on' --only_set_cfg + Only set min-iowait&&min-iops and disable iowait diagnose to config. + """ + parser = argparse.ArgumentParser( + description="start ioMonitor.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=examples) + parser.add_argument('-y','--yaml_file', + help='Specify the socket pipe for data upload'\ + ' and exception log path') + parser.add_argument('-s','--set_cfg', help=setcfg_descripton) + parser.add_argument('-r','--reset_cfg', action='store_true', + help='Reset cfg to default') + parser.add_argument('-o','--only_set_cfg', action='store_true', + help='Only set cfg') + args = parser.parse_args() + + signal.signal(signal.SIGCHLD, signal.SIG_IGN) + logRootPath,pipeFile = getRunArgsFromYaml(args.yaml_file) + if args.only_set_cfg: + if not os.path.exists(logRootPath+'/ioMon/ioMonCfg.json'): + print("%s" % ("config fail, not found ioMonCfg.json")) + return + if args.set_cfg is None and not args.reset_cfg: + print("%s" % ("--set_cfg or --reset_cfg not found.")) + return + ioMonCfg = ioMonCfgClass(args.set_cfg, args.reset_cfg, logRootPath) + ioMonCfg.notifyIoMon() + return + + ioMonCfg = ioMonCfgClass(args.set_cfg, args.reset_cfg, logRootPath) + ioMon = ioMonitorClass(logRootPath, ioMonCfg, pipeFile) + ioMon.monitor() + +if __name__ == "__main__": + main() diff --git a/source/tools/monitor/ioMonitor/ioMon/nfPut.py b/source/tools/monitor/ioMonitor/ioMon/nfPut.py new file mode 100755 index 0000000000000000000000000000000000000000..5fcaea9604eb7b9afd2e867f9e63d12163c3170a --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/nfPut.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： nfPut + Description : + Author : liaozhaoyan + date： 2022/4/28 +------------------------------------------------- + Change Activity: + 2022/4/28: +------------------------------------------------- +""" +__author__ = 'liaozhaoyan' + +import os +import socket +import requests +MAX_BUFF = 128 * 1024 + + +class CnfPut(object): + def __init__(self, pipeFile): + self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + self._path = pipeFile + if not os.path.exists(self._path): + raise ValueError("pipe path is not exist. please check unity is running.") + + + def puts(self, s): + if len(s) > MAX_BUFF: + raise ValueError("message len %d, is too long ,should less than%d" % (len(s), MAX_BUFF)) + return self._sock.sendto(s, self._path) + + +if __name__ == "__main__": + nf = CnfPut("/tmp/sysom") + pass diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/__init__.py b/source/tools/monitor/ioMonitor/ioMon/tools/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cb8e4b62acc8012add0dc0175a04b6b4f51940ec --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- + +if __name__ == "__main__": + pass diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/__init__.py b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cb8e4b62acc8012add0dc0175a04b6b4f51940ec --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- + +if __name__ == "__main__": + pass diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/common.py b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/common.py new file mode 100755 index 0000000000000000000000000000000000000000..f01dbf271205c9063832d2f83f2aace435caa5f1 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/common.py @@ -0,0 +1,129 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import string +import re +from subprocess import PIPE, Popen + + +def execCmd(cmd): + p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) + return p.stdout.read().decode('utf-8') + + +def echoFile(filename, txt): + execCmd("echo \'"+txt+"\' > "+filename) + + +def echoFileAppend(filename, txt): + execCmd("echo \'"+txt+"\' >> "+filename) + + +def humConvert(value, withUnit): + units = ["B", "KB", "MB", "GB", "TB", "PB"] + size = 1024.0 + + if value == 0: + return value + + for i in range(len(units)): + if (value / size) < 1: + if withUnit: + return "%.1f%s/s" % (value, units[i]) + else: + return "%.1f" % (value) + value = value / size + + +def getDevt(devname): + try: + with open('/sys/class/block/' + devname + '/dev') as f: + dev = f.read().split(':') + return ((int(dev[0]) << 20) + int(dev[1])) + except Exception: + return -1 + + +def getDevtRegion(devname): + if os.path.exists('/sys/block/'+devname): + isPart = False + elif os.path.exists('/sys/class/block/'+devname): + isPart = True + else: + return [-1, -1] + + master = devname if not isPart else \ + os.readlink('/sys/class/block/'+devname).split('/')[-2] + partList = list( + filter(lambda x: master in x, + os.listdir('/sys/class/block/'+master))) + if not partList: + partList = [] + partList.append(master) + return [getDevt(p) for p in partList] + + +def getTgid(pid): + try: + with open("/proc/"+str(pid)+"/status") as f: + return ''.join(re.findall(r'Tgid:(.*)', f.read())).lstrip() + except IOError: + return '-' + return '-' + + +def fixComm(comm, pid): + try: + if ".." in comm: + with open("/proc/"+str(pid)+"/comm") as f: + return f.read().rstrip('\n') + except IOError: + return comm + return comm + + +def getContainerId(pid): + try: + piddir = "/proc/"+str(pid) + with open(piddir+"/cgroup") as f: + # ... + # cpuset,cpu,cpuacct:/docker/e2afa607d8f13e5b1f89d38ee86d86.... + # memory:/docker/e2afa607d8f13e5b1f89d38ee86..... + # ... + cid = f.read().split('memory:')[1].split('\n')[0].rsplit('/',1)[1] + if not cid: + cid = '-' + except Exception: + cid = '-' + return cid + + +def getFullNameFromProcPid(pid, ino): + try: + piddir = "/proc/"+str(pid) + # list the open files of the task + fdList = os.listdir(piddir+"/fd") + for f in fdList: + try: + path = os.readlink(piddir+"/fd/"+f) + if '/dev/' in path or '/proc/' in path or '/sys/' in path: + continue + + if os.path.isfile(path) and os.stat(path).st_ino == int(ino): + return path + except (IOError, EOFError) as e: + continue + except Exception: + pass + return "-" + + +def supportKprobe(name): + file = '/sys/kernel/debug/tracing/available_filter_functions' + with open(file) as f: + ss = f.read() + if ss.find(name) > 0: + return True + return False diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/diskstatClass.py b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/diskstatClass.py new file mode 100755 index 0000000000000000000000000000000000000000..c9755bfed0b770fc029d7a55a1e8a8153e0b2dac --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/diskstatClass.py @@ -0,0 +1,219 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import time +import json +import string +from collections import OrderedDict +from common import humConvert + + +class diskstatClass(object): + def __init__(self, devname, utilThresh, json, nodiskStat, Pattern): + self.devname = devname + self.json = json + self.cycle = 1 + self.started = False + self.Pattern = Pattern + self.nodiskStat = nodiskStat + self.utilThresh = int(utilThresh) if utilThresh is not None else 0 + self.fieldDicts = OrderedDict() + self.diskInfoDicts = {} + self.deviceStatDicts = {} + self.f = open("/proc/diskstats") + if json: + self.fJson = open(json, 'w+') + + + def getDevNameByDevt(self, devt): + try: + return self.diskInfoDicts[str(devt)]['partName'] + except Exception: + return '-' + + def getMasterDev(self, devt): + try: + return self.diskInfoDicts[str(devt)]['master'] + except Exception: + return '-' + + def getDiskStatInd(self, disk, key): + return self.deviceStatDicts[disk][key] + + def start(self): + fieldDicts = self.fieldDicts + diskInfoDicts = self.diskInfoDicts + deviceStatDicts = self.deviceStatDicts + + if self.started: + return + self.started = True + self.cycle = time.time() + self.f.seek(0) + for stat in self.f.readlines(): + stat = stat.split() + if self.devname is not None and self.devname not in stat[2] and \ + stat[2] not in self.devname: + continue + + field = {\ + '1':[0,0], '2':[0,0], '3':[0,0], '4':[0,0],\ + '5':[0,0], '6':[0,0], '7':[0,0], '8':[0,0],\ + '10':[0,0], '11':[0,0]} + for idx,value in field.items(): + value[0] = long(stat[int(idx)+2]) + if stat[2] not in fieldDicts.keys(): + fieldDicts.setdefault(stat[2], field) + path = os.readlink('/sys/class/block/'+stat[2]).split('/') + master = path[-2] + if master not in path[-1]: + master = path[-1] + diskInfoDicts.setdefault( + str((int(stat[0])<<20)+int(stat[1])), + {'partName': stat[2], 'master': master}) + deviceStat = { + 'r_rqm':0, 'w_rqm':0, 'r_iops':0, 'w_iops':0, 'r_bps':0, + 'w_bps':0, 'wait':0, 'r_wait':0, 'w_wait':0, 'util%':-1} + deviceStatDicts.setdefault(stat[2], deviceStat) + else: + deviceStatDicts[stat[2]]['util%'] = -1 + fieldDicts[stat[2]].update(field) + + def stop(self): + fieldDicts = self.fieldDicts + deviceStatDicts = self.deviceStatDicts + self.cycle = max(int(time.time()-self.cycle), 1) + + if not self.started: + return + self.started = False + + self.f.seek(0) + for stat in self.f.readlines(): + stat = stat.split() + if self.devname is not None and self.devname not in stat[2] and \ + stat[2] not in self.devname: + continue + for idx,value in fieldDicts[stat[2]].items(): + value[1] = long(stat[int(idx)+2]) + + for devname,field in fieldDicts.items(): + if self.devname is not None and devname not in self.devname and \ + self.devname not in devname: + continue + util = round((field['10'][1]-field['10'][0])*100.0/(self.cycle*1000),2) + util = util if util <= 100 else 100.0 + if util < self.utilThresh: + continue + deviceStatDicts[devname]['util%'] = util + r_iops = field['1'][1]-field['1'][0] + deviceStatDicts[devname]['r_iops'] = r_iops + r_rqm = field['2'][1]-field['2'][0] + deviceStatDicts[devname]['r_rqm'] = r_rqm + w_iops = field['5'][1]-field['5'][0] + deviceStatDicts[devname]['w_iops'] = w_iops + w_rqm = field['6'][1]-field['6'][0] + deviceStatDicts[devname]['w_rqm'] = w_rqm + r_bps = (field['3'][1]-field['3'][0]) * 512 + deviceStatDicts[devname]['r_bps'] = r_bps + w_bps = (field['7'][1]-field['7'][0]) * 512 + deviceStatDicts[devname]['w_bps'] = w_bps + r_ticks = field['4'][1]-field['4'][0] + w_ticks = field['8'][1]-field['8'][0] + wait = round((r_ticks+w_ticks)/(r_iops+w_iops), 2) if (r_iops+w_iops) else 0 + deviceStatDicts[devname]['wait'] = wait + r_wait = round(r_ticks / r_iops, 2) if r_iops else 0 + deviceStatDicts[devname]['r_wait'] = r_wait + w_wait = round(w_ticks / w_iops, 2) if w_iops else 0 + deviceStatDicts[devname]['w_wait'] = w_wait + + + def __showJson(self): + deviceStatDicts = self.deviceStatDicts + + statJsonStr = '{\ + "time":"",\ + "diskstats":[]}' + dstatDicts = json.loads(statJsonStr, object_pairs_hook=OrderedDict) + dstatDicts['time'] = time.strftime('%Y/%m/%d %H:%M:%S', time.localtime()) + for devname,stat in deviceStatDicts.items(): + if stat['util%'] < 0: + continue + dstatJsonStr = '{\ + "diskname":"","r_rqm":0,"w_rqm":0,"r_iops":0,"w_iops":0,\ + "r_bps":0,"w_bps":0,"wait":0,"r_wait":0,"w_wait":0,"util%":0}' + dstatDict = json.loads(dstatJsonStr, object_pairs_hook=OrderedDict) + dstatDict["diskname"] = devname + for key,val in stat.items(): + dstatDict[key] = val + dstatDicts["diskstats"].append(dstatDict) + if len(dstatDicts["diskstats"]) > 0: + data = json.dumps(dstatDicts) + self.writeDataToJson(data) + return + + def show(self): + secs = self.cycle + deviceStatDicts = self.deviceStatDicts + if self.nodiskStat: + return + + if self.enableJsonShow() == True: + self.__showJson() + return + + if self.Pattern: + WrTotalIops = 0 + RdTotalIops = 0 + WrTotalBw = 0 + RdTotalBw = 0 + print('%-20s%-8s%-8s%-8s%-8s%-12s%-12s%-8s%-8s%-8s%-8s' %\ + (("device-stat:"),"r_rqm","w_rqm","r_iops","w_iops","r_bps",\ + "w_bps","wait","r_wait","w_wait","util%")) + stSecs = str(secs)+'s' if secs > 1 else 's' + for devname,stat in deviceStatDicts.items(): + if (not self.devname and not os.path.exists('/sys/block/'+devname)) or \ + stat['util%'] < 0: + continue + if self.Pattern: + WrTotalIops += stat['w_iops'] + RdTotalIops += stat['r_iops'] + WrTotalBw += stat['w_bps'] + RdTotalBw += stat['r_bps'] + stWbps = humConvert(stat['w_bps'], True).replace('s', stSecs) if stat['w_bps'] else 0 + stRbps = humConvert(stat['r_bps'], True).replace('s', stSecs) if stat['r_bps'] else 0 + print('%-20s%-8s%-8s%-8s%-8s%-12s%-12s%-8s%-8s%-8s%-8s' %\ + (devname, str(stat['r_rqm']), str(stat['w_rqm']), str(stat['r_iops']), + str(stat['w_iops']), stRbps, stWbps, str(stat['wait']), str(stat['r_wait']), + str(stat['w_wait']), str(stat['util%']))) + if self.Pattern: + print('totalIops:%d(r:%d, w:%d), totalBw:%s(r:%s, w:%s)' % + ((WrTotalIops+RdTotalIops), RdTotalIops, WrTotalIops, + (humConvert((WrTotalBw+RdTotalBw), True).replace('s', stSecs) if (WrTotalBw+RdTotalBw > 0) else 0), + (humConvert(RdTotalBw, True).replace('s', stSecs) if RdTotalBw else 0), + (humConvert(WrTotalBw, True).replace('s', stSecs) if WrTotalBw else 0))) + print("") + + def clear(self): + self.f.close() + if self.enableJsonShow(): + self.fJson.close() + + def notCareDevice(self, devname): + if not self.nodiskStat and self.deviceStatDicts[devname]['util%'] < 0: + return True + return False + + def disableShow(self): + deviceStatDicts = self.deviceStatDicts + for devname,stat in deviceStatDicts.items(): + if self.deviceStatDicts[devname]['util%'] >= 0: + return False + return True + + def enableJsonShow(self): + return True if self.json else False + + def writeDataToJson(self, data): + self.fJson.write(data+'\n') diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/fsstatClass.py b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/fsstatClass.py new file mode 100755 index 0000000000000000000000000000000000000000..763de3024c7032bb148aa6238e99b77fc49814f9 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/fsstatClass.py @@ -0,0 +1,418 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import signal +import string +import time +import re +import json +from collections import OrderedDict +from diskstatClass import diskstatClass +from common import getDevt,getDevtRegion +from common import humConvert,supportKprobe +from common import execCmd,echoFile,echoFileAppend +from common import getTgid,fixComm,getContainerId,getFullNameFromProcPid +from mmap import PAGESIZE + + +def getMntPath(fileInfoDict): + mntfname = fileInfoDict['mntfname'] + fsmountInfo = fileInfoDict['fsmountinfo'] + + if len(fsmountInfo) <= 0: + return '-' + + if mntfname.isspace() or len(mntfname) == 0: + return fsmountInfo[0].split()[1] + try: + for l in fsmountInfo: + if l.find(mntfname) > -1: + return l.split()[1] + return '-' + except IndexError: + return fsmountInfo[0].split()[1] + + +def getFullName(fileInfoDict): + fileSuffix = '' + + mntdir = getMntPath(fileInfoDict) + if mntdir == '/': + mntdir = '' + + for f in [ + fileInfoDict['d3fname'], fileInfoDict['d2fname'], + fileInfoDict['d1fname'], fileInfoDict['bfname']]: + if f != '/' and f != '(fault)': + fileSuffix += ('/' + f) + if fileInfoDict['d3fname'] != '/' and fileInfoDict['d3fname'] != '(fault)': + fileSuffix = '/...' + fileSuffix + filename = mntdir + fileSuffix + + if '...' in filename: + f = getFullNameFromProcPid( + fileInfoDict['pid'], fileInfoDict['ino']) + if f != '-': + filename = f + return filename + + +class fsstatClass(diskstatClass): + def __init__( + self, devname, pid, utilThresh, bwThresh, top, + json, nodiskStat, miscStat, Pattern): + super(fsstatClass, self).__init__( + devname, utilThresh, json, nodiskStat, Pattern) + self.expression = [] + self.pid = pid + self.miscStat = miscStat + self.devname = devname + self.top = int(top) if top is not None else 99999999 + self.bwThresh = int(bwThresh) if bwThresh is not None else 0 + self.devt = getDevtRegion(devname) if devname is not None else [-1, -1] + tracingBaseDir = "/sys/kernel/debug/tracing" + self.kprobeEvent = tracingBaseDir+"/kprobe_events" + self.tracingDir = tracingBaseDir+'/instances/iofsstat4fs' + self.kprobeDir = self.tracingDir+"/events/kprobes" + self.kprobe = [] + self.kprobeArgsFormat = 'dev=+0x10(+0x28(+0x20(%s))):u32 '\ + 'inode_num=+0x40(+0x20(%s)):u64 len=%s:u64 '\ + 'mntfname=+0x0(+0x28(+0x0(+0x10(%s)))):string '\ + 'bfname=+0x0(+0x28(+0x18(%s))):string '\ + 'd1fname=+0x0(+0x28(+0x18(+0x18(%s)))):string '\ + 'd2fname=+0x0(+0x28(+0x18(+0x18(+0x18(%s))))):string '\ + 'd3fname=+0x0(+0x28(+0x18(+0x18(+0x18(+0x18(%s)))))):string %s' + + kprobeArgs = self._getKprobeArgs('None') + self.ftracePaserCommArgs = ' comm=(.*)' if 'comm=' in kprobeArgs else '' + mmapKprobeArgs = self._getKprobeArgs('mmap') + self.fsmountInfo = self._getFsMountInfo() + for entry in self.fsmountInfo: + fstype = entry.split()[2] + self._kprobeReadWrite(fstype, kprobeArgs) + self._kprobeMmap(fstype, mmapKprobeArgs) + if len(self.kprobe) <= 0: + print("%s" % ("error: not available kprobe")) + sys.exit(-1) + self.outlogFormatBase = 10 + + def _kprobeReadWrite(self, fstype, kprobeArgs): + for op in ['write', 'read']: + kPoints = [ + fstype+"_file_"+op+"_iter", fstype+"_file_"+op, + fstype+"_file_aio_"+op, "generic_file_aio_"+op] + if list(set(self.kprobe) & set(kPoints)): + continue + kprobe = None + for k in kPoints: + if supportKprobe(k): + kprobe = k + break + if not kprobe: + if self.enableJsonShow() == False: + print("warnning: not available %s kprobe" % op) + continue + pointKprobe = 'p '+kprobe+' '+kprobeArgs + self.kprobe.append(kprobe) + self.expression.append(pointKprobe) + + def _kprobeMmap(self, fstype, kprobeArgs): + for kprobe in [fstype+"_page_mkwrite", 'filemap_fault']: + if kprobe in self.kprobe: + continue + if not supportKprobe(kprobe): + if self.enableJsonShow() == False: + print("not support kprobe %s" % kprobe) + continue + pointKprobe = 'p '+kprobe+' '+kprobeArgs + self.kprobe.append(kprobe) + self.expression.append(pointKprobe) + + def _getKprobeArgs(self, type): + commArgs = '' + vinfo = execCmd('uname -r') + version = vinfo.split('.') + + if type == 'mmap': + offFile = '0xa0(+0x0%s)' if int(version[0]) > 4 or ( + int(version[0]) == 4 and int(version[1]) > 10) else '0xa0%s' + offLen = '0x0(+0x0%s)' if int(version[0]) > 4 or ( + int(version[0]) == 4 and int(version[1]) > 10) else '0x0%s' + else: + offLen = '0x10' + offFile = '0x0' if int(version[0]) > 3 or ( + int(version[0]) == 3 and int(version[1]) > 10) else '0x8' + if int(version[0]) <= 3: + offLen = '0x8' if int(version[1]) < 13 else '0x18' + + if int(version[0]) > 3: + commArgs = 'comm=$comm' + + re= execCmd('lscpu | grep -E \"Architecture|架构\" | sed \"s/：/:/g\"') + arch = re.split(':')[1].strip() + regs = { + "arm":['(%r0)','(%r1)'], + "x86":['(%di)', '(%si)'], + "aarch64":['(%x0)','(%x1)']} + argv0 = argv1 = '' + for key,val in regs.items(): + if arch.startswith(key): + if type == 'mmap': + argv0 = '+' + (offFile % val[0]) + argv1 = '+' + (offLen % val[0]) + argv2 = argv1 + else: + argv2 = argv0 = '+' + offFile + val[0] + argv1 = '+' + offLen + val[1] + break + if argv0 == '': + raise ValueError('arch %s not support' % arch) + + kprobeArgs = self.kprobeArgsFormat % ( + argv0, argv0, argv1, argv0, argv0, argv0, argv0, argv2, commArgs) + return kprobeArgs + + def _getFsMountInfo(self): + devList = [] + if self.devname is not None: + devList.append('/dev/'+self.devname) + else: + sysfsBlockDirList = os.listdir("/sys/block") + for dev in sysfsBlockDirList: + devList.append('/dev/'+dev) + with open("/proc/mounts") as f: + fsmountInfo = list(filter(lambda x: any( + e in x for e in devList), f.readlines())) + return fsmountInfo + + def config(self): + devt = self.devt + + if not os.path.exists(self.tracingDir): + os.mkdir(self.tracingDir) + for exp in self.expression: + probe = 'p_'+exp.split()[1]+'_0' + enableKprobe = self.kprobeDir+"/"+probe+"/enable" + filterKprobe = self.kprobeDir+"/"+probe+"/filter" + if os.path.exists(enableKprobe): + echoFile(enableKprobe, "0") + if devt[0] > 0: + echoFile(filterKprobe, "0") + echoFileAppend(self.kprobeEvent, '-:%s' % probe) + + echoFileAppend(self.kprobeEvent, exp) + if devt[0] > 0: + dev = getDevt(self.devname) + if dev == min(devt): + echoFile(filterKprobe, + "dev>="+str(min(devt))+"&&dev<="+str(max(devt))) + else: + echoFile(filterKprobe, "dev=="+str(dev)) + echoFile(enableKprobe, "1") + fmt = execCmd("grep print "+self.kprobeDir+"/"+probe+"/format") + matchObj = re.match(r'(.*) dev=(.*) inode_num=(.*)', fmt) + if 'x' in matchObj.group(2): + self.outlogFormatBase = 16 + + def start(self): + echoFile(self.tracingDir+"/trace", "") + echoFile(self.tracingDir+"/tracing_on", "1") + super(fsstatClass, self).start() + + def stop(self): + echoFile(self.tracingDir+"/tracing_on", "0") + super(fsstatClass, self).stop() + + def clear(self): + for exp in self.expression: + probe = 'p_'+exp.split()[1]+'_0' + enableKprobe = self.kprobeDir+"/"+probe+"/enable" + if not os.path.exists(enableKprobe): + continue + echoFile(enableKprobe, "0") + if self.devt[0] > 0: + filterKprobe = self.kprobeDir+"/"+probe+"/filter" + echoFile(filterKprobe, "0") + echoFileAppend(self.kprobeEvent, '-:%s' % probe) + super(fsstatClass, self).clear() + + def _paserTraceToStat(self, traceText): + bwTotal = 0 + stat = {} + mStat = {} + fileInfoDict = { + 'device': 0, 'mntfname': '', 'bfname': '', 'd1fname': '', + 'd2fname': '', 'd3fname': '', 'fsmountinfo': '', 'ino': 0, + 'pid': 0} + commArgs = self.ftracePaserCommArgs + hasCommArgs = True if len(commArgs) else False + + # pool-1-thread-2-5029 [002] .... 5293018.252338: p_ext4_file_write_iter_0:\ + # (ext4_file_write_iter+0x0/0x6d0 [ext4]) dev=265289729 inode_num=530392 len=38 + # ... + for entry in traceText: + if ('dev=' not in entry) or ('.so' in entry and 'lib' in entry) or ( + '=\"etc\"' in entry) or ('=\"usr\"' in entry and ( + '=\"bin\"' in entry or '=\"sbin\"' in entry)): + continue + + matchObj = re.match( + r'(.*) \[([^\[\]]*)\] (.*) dev=(.*) inode_num=(.*) len=(.*)'+ + ' mntfname=(.*) bfname=(.*) d1fname=(.*) d2fname=(.*)'+ + ' d3fname=(.*)'+commArgs, entry) + if matchObj is None: + continue + + pid = (matchObj.group(1).rsplit('-', 1))[1].strip() + dev = int(matchObj.group(4), self.outlogFormatBase) + if (self.pid is not None and int(pid) != self.pid) or \ + str(dev) == '0': + continue + + if hasCommArgs: + comm = matchObj.group(12).strip("\"") + else: + comm = (matchObj.group(1).rsplit('-', 1))[0].strip() + comm = fixComm(comm, pid) + if '..' in comm: + continue + + device = self.getDevNameByDevt(dev) + if device == '-': + continue + if self.miscStat is not None: + disk = self.getMasterDev(dev) + if not mStat.has_key(disk): + mStat.setdefault(disk, {}) + stat = mStat[disk] + + ino = int(matchObj.group(5), self.outlogFormatBase) + inoTask = str(ino)+':'+str(comm)+':'+device + if not stat.has_key(inoTask): + fsmountinfo = [f for f in self.fsmountInfo if ('/dev/'+device) in f] + fileInfoDict['device'] = device + fileInfoDict['mntfname'] = matchObj.group(7).strip("\"") + fileInfoDict['bfname'] = matchObj.group(8).strip("\"") + fileInfoDict['d1fname'] = matchObj.group(9).strip("\"") + fileInfoDict['d2fname'] = matchObj.group(10).strip("\"") + fileInfoDict['d3fname'] = matchObj.group(11).strip("\"") + fileInfoDict['fsmountinfo'] = fsmountinfo + fileInfoDict['ino'] = ino + fileInfoDict['pid'] = pid + stat.setdefault(inoTask, + {"inode":str(ino), "comm": comm, "tgid": getTgid(pid), "pid": pid, + "cnt_wr": 0, "bw_wr": 0, "cnt_rd": 0, "bw_rd": 0, "device": device, + "cid":getContainerId(pid), "file": getFullName(fileInfoDict)}) + + size = int(matchObj.group(6), self.outlogFormatBase) + if 'filemap_fault' in entry or 'page_mkwrite' in entry: + size = PAGESIZE + if 'write' in entry or 'page_mkwrite' in entry: + stat[inoTask]["cnt_wr"] += 1 + stat[inoTask]["bw_wr"] += int(size) + if 'read' in entry or 'filemap_fault' in entry: + stat[inoTask]["cnt_rd"] += 1 + stat[inoTask]["bw_rd"] += int(size) + if pid != stat[inoTask]["pid"]: + stat[inoTask]["pid"] = pid + stat[inoTask]["tgid"] = getTgid(pid) + if stat[inoTask]["cid"] == '-': + stat[inoTask]["cid"] = getContainerId(pid) + bwTotal += int(size) + return bwTotal,stat,mStat + + def _joinMiscStat(self, mStat): + for d,val in self.miscStat: + if d not in mStat.keys(): + mStat.setdefault(d, {}) + mStat[d].update(dict(val)) + tmpStat = [] + for d,val in mStat.items(): + idxSort = 'bw_wr' + if self.getDiskStatInd(d, 'w_iops') < self.getDiskStatInd(d, 'r_iops'): + idxSort = 'bw_rd' + s = sorted( + val.items(), key=lambda e: (e[1][idxSort]), reverse=True)[:self.top] + tmpStat.append((d, s)) + del self.miscStat[:] + self.miscStat.extend(tmpStat) + return 0 + + def showJson(self, stat): + secs = self.cycle + statJsonStr = '{"time":"","fsstats":[]}' + fstatDicts = json.loads(statJsonStr, object_pairs_hook=OrderedDict) + fstatDicts['time'] = time.strftime( + '%Y/%m/%d %H:%M:%S', time.localtime()) + stSecs = str(secs)+'s' if secs > 1 else 's' + for key, item in stat.items(): + if (item["cnt_wr"] + item["cnt_rd"]) == 0: + continue + item["bw_wr"] = \ + humConvert(item["bw_wr"], True).replace('s', stSecs) if item["bw_wr"] else 0 + item["bw_rd"] = \ + humConvert(item["bw_rd"], True).replace('s', stSecs) if item["bw_rd"] else 0 + fsstatJsonStr = '{\ + "inode":0,"comm":"","tgid":0,"pid":0,"cnt_rd":0,\ + "bw_rd":0,"cnt_wr":0,"bw_wr":0,"device":0,"cid":0,"file":0}' + fsstatDict = json.loads( + fsstatJsonStr, object_pairs_hook=OrderedDict) + for key, val in item.items(): + fsstatDict[key] = val + fstatDicts["fsstats"].append(fsstatDict) + if len(fstatDicts["fsstats"]) > 0: + self.writeDataToJson(json.dumps(fstatDicts)) + + def printStat(self, stat): + secs = self.cycle + print("%-20s%-8s%-8s%-24s%-8s%-12s%-8s%-12s%-12s%-12s%-32s%s" + % ("comm", "tgid", "pid", "cid", "cnt_rd", "bw_rd", "cnt_wr", + "bw_wr", "inode", "device", "filepath")) + stSecs = str(secs)+'s' if secs > 1 else 's' + for key, item in stat: + if (item["cnt_wr"] + item["cnt_rd"]) == 0: + continue + item["bw_wr"] = \ + humConvert(item["bw_wr"], True).replace('s', stSecs) if item["bw_wr"] else 0 + item["bw_rd"] = \ + humConvert(item["bw_rd"], True).replace('s', stSecs) if item["bw_rd"] else 0 + print("%-20s%-8s%-8s%-24s%-8d%-12s%-8d%-12s%-12s%-12s%s" + % (item["comm"], item["tgid"], item["pid"], item["cid"][0:20], + item["cnt_rd"], item["bw_rd"], item["cnt_wr"], item["bw_wr"], + item["inode"], item["device"], item["file"])) + print("") + + def show(self): + secs = self.cycle + with open(self.tracingDir+"/trace") as f: + traceText = f.read().split('\n') + #traceText = f.readlines() + #traceText = \ + # list(filter(lambda x: any(e in x for e in self.kprobe), f.readlines())) + bwTotal,stat,mStat = self._paserTraceToStat(traceText) + + if self.miscStat is not None: + return self._joinMiscStat(mStat) + elif (self.bwThresh and (bwTotal/secs) < self.bwThresh): + return + + stat = sorted(stat.items(), key=lambda e: ( + e[1]["bw_wr"]+e[1]["bw_rd"]), reverse=True)[:self.top] + + if self.enableJsonShow() == False: + print(time.strftime('%Y/%m/%d %H:%M:%S', time.localtime())) + if self.disableShow() == False: + super(fsstatClass, self).show() + + if self.enableJsonShow() == True: + self.showJson(stat) + else: + self.printStat(stat) + + def entry(self, interval): + self.start() + time.sleep(float(interval)) + self.stop() + self.show() diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/iofsstat.py b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/iofsstat.py new file mode 100755 index 0000000000000000000000000000000000000000..1829031599d4ce4f642b484e4bb2c75f16e3e683 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/iofsstat.py @@ -0,0 +1,107 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import signal +import string +import argparse +import threading +from collections import OrderedDict +from iostatClass import iostatClass +from fsstatClass import fsstatClass +from promiscClass import promiscClass +import time + +global_iofsstat_stop = False +def signal_exit_handler(signum, frame): + global global_iofsstat_stop + global_iofsstat_stop = True + +def exit_handler(): + global global_iofsstat_stop + global_iofsstat_stop = True + +def iofsstatStart(argv): + global global_iofsstat_stop + global_iofsstat_stop = False + if os.geteuid() != 0: + print("%s" % ("This program must be run as root. Aborting.")) + sys.exit(0) + examples = """e.g. + ./iofsstat.py -d vda -c 1 + Report iops and bps of process for vda per 1secs + ./iofsstat.py -d vda1 --fs -c 1 + Report fs IO-BW and file of process for vda1(must be parttion mounted by filesystem) per 1secs + ./iofsstat.py -m -c 5 -t 5 + Report top5 iops&&bps&&file of process with misc mode per 5secs + ./iofsstat.py -d vda -c 1 -b 1048576 -i 350 + Report process that iops over 350 or bps over 1048576 for vda per 1secs + ./iofsstat.py -u 90 + Report disk that io-util over %90 + """ + parser = argparse.ArgumentParser( + description="Report IO statistic for partitions.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=examples) + parser.add_argument('-T','--Timeout', help='Specify the timeout for program exit(secs).') + parser.add_argument('-t','--top', help='Report the TopN with the largest IO resources.') + parser.add_argument('-u','--util_thresh', help='Specify the util-thresh to report.') + parser.add_argument('-b','--bw_thresh', help='Specify the BW-thresh to report.') + parser.add_argument('-i','--iops_thresh', help='Specify the IOPS-thresh to report.') + parser.add_argument('-c','--cycle', help='Specify refresh cycle(secs).') + parser.add_argument('-d','--device', help='Specify the disk name.') + parser.add_argument('-p','--pid', help='Specify the process id.') + parser.add_argument('-j','--json', help='Specify the json-format output.') + parser.add_argument('-f','--fs', action='store_true', + help='Report filesystem statistic for partitions.') + parser.add_argument('-P','--Pattern', action='store_true', + help='Report IO pattern(--fs not support).') + parser.add_argument('-n','--nodiskStat', action='store_true', + help='Not report disk stat.') + parser.add_argument('-m','--misc', action='store_true', + help='Promiscuous mode.') + args = parser.parse_args(argv) if argv else parser.parse_args() + + secs = float(args.cycle) if args.cycle is not None else 0 + devname = args.device + pid = int(args.pid) if args.pid else None + if argv is None: + signal.signal(signal.SIGINT, signal_exit_handler) + signal.signal(signal.SIGHUP, signal_exit_handler) + signal.signal(signal.SIGTERM, signal_exit_handler) + if args.Timeout is not None: + timeoutSec = args.Timeout if args.Timeout > 0 else 10 + secs = secs if secs > 0 else 1 + if argv is None: + signal.signal(signal.SIGALRM, signal_exit_handler) + signal.alarm(int(timeoutSec)) + else: + timer = threading.Timer(int(timeoutSec), exit_handler) + timer.start() + loop = True if secs > 0 else False + interval = secs if loop == True else 1 + if args.misc: + c = promiscClass(devname, args.util_thresh, args.iops_thresh, + args.bw_thresh, args.top, args.json, args.nodiskStat, + args.Pattern) + else: + if args.fs: + c = fsstatClass(devname, pid, args.util_thresh, args.bw_thresh, + args.top, args.json, args.nodiskStat, None, args.Pattern) + else: + c = iostatClass(devname, pid, args.util_thresh, args.iops_thresh, + args.bw_thresh, args.top, args.json, args.nodiskStat, None, + args.Pattern) + c.config() + while global_iofsstat_stop != True: + c.entry(interval) + if loop == False: + break + c.clear() + +def main(): + iofsstatStart(None) + +if __name__ == "__main__": + main() diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/iostatClass.py b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/iostatClass.py new file mode 100755 index 0000000000000000000000000000000000000000..e7d03e85dfdd7fe0499f838dd1f427dbf79461d4 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/iostatClass.py @@ -0,0 +1,244 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import signal +import string +import time +import re +import json +from collections import OrderedDict +from diskstatClass import diskstatClass +from common import getDevtRegion +from common import humConvert,echoFile +from common import getContainerId + + +class iostatClass(diskstatClass): + def __init__( + self, devname, pid, utilThresh, iopsThresh, bwThresh, + top, json, nodiskStat, miscStat, Pattern): + super(iostatClass, self).__init__( + devname, utilThresh, json, nodiskStat, Pattern) + self.pid = pid + self.miscStat = miscStat + self.top = int(top) if top is not None else 99999999 + self.iopsThresh = int(iopsThresh) if iopsThresh is not None else 0 + self.bwThresh = int(bwThresh) if bwThresh is not None else 0 + self.devt = min(getDevtRegion(devname)) if devname is not None else -1 + self.tracingDir = "/sys/kernel/debug/tracing/instances/iofsstat4io" + self.blkTraceDir = self.tracingDir+"/events/block" + + def config(self): + devt = self.devt + if not os.path.exists(self.tracingDir): + os.mkdir(self.tracingDir) + if devt > 0: + echoFile(self.blkTraceDir+"/block_getrq/filter", "dev=="+str(devt)) + else: + echoFile(self.blkTraceDir+"/block_getrq/filter", "") + echoFile(self.blkTraceDir+"/block_getrq/enable", "1") + + def start(self): + echoFile(self.tracingDir+"/trace", "") + echoFile(self.tracingDir+"/tracing_on", "1") + super(iostatClass, self).start() + + def stop(self): + echoFile(self.tracingDir+"/tracing_on", "0") + super(iostatClass, self).stop() + + def clear(self): + echoFile(self.blkTraceDir+"/block_getrq/enable", "0") + if self.devt > 0: + echoFile(self.blkTraceDir+"/block_getrq/filter", "0") + super(iostatClass, self).clear() + + def showJson(self, stat): + secs = self.cycle + statJsonStr = '{"time":"","iostats":[]}' + iostatDicts = json.loads(statJsonStr, object_pairs_hook=OrderedDict) + iostatDicts['time'] = time.strftime( + '%Y/%m/%d %H:%M:%S', time.localtime()) + stSecs = str(secs)+'s' if secs > 1 else 's' + for key, item in stat.items(): + if (item["iops_rd"] + item["iops_wr"]) == 0: + continue + item["bps_rd"] = \ + humConvert(item["bps_rd"], True).replace('s', stSecs) if item["bps_rd"] else 0 + item["bps_wr"] = \ + humConvert(item["bps_wr"], True).replace('s', stSecs) if item["bps_wr"] else 0 + iostatJsonStr = '{\ + "comm":"","pid":0,"bps_rd":0,"iops_rd":0,"iops_wr":0,"bps_wr":0,"device":0}' + iostatDict = json.loads(iostatJsonStr, object_pairs_hook=OrderedDict) + for key in ['comm', 'pid', 'bps_rd', 'iops_rd', 'iops_wr', 'bps_wr', 'device']: + iostatDict[key] = item[key] + iostatDicts["iostats"].append(iostatDict) + if len(iostatDicts["iostats"]) > 0: + self.writeDataToJson(json.dumps(iostatDicts)) + + def patternIdx(self, size): + dp = [ + ("pat_W4K", (4*1024)), ("pat_W16K", (16*1024)), + ("pat_W32K", (32*1024)), ("pat_W64K",(64*1024)), + ("pat_W128K", (128*1024)), ("pat_W256K", (256*1024)), + ("pat_W512K", (512*1024))] + for d in dp: + if size <= d[1]: + return d[0] + return 'pat_Wlarge' + + def patternPercent(self, pat, total): + if total == 0 or pat == 0: + return '0' + return format(pat / (total * 1.0) * 100, '.2f') + '%' + + def show(self): + iopsTotal = 0 + WrIopsTotal = 0 + RdIopsTotal = 0 + bwTotal = 0 + WrBwTotal = 0 + RdBwTotal = 0 + stat = {} + mStat = {} + secs = self.cycle + with open(self.tracingDir+"/trace") as f: + traceText = list( + filter(lambda x: 'block_getrq' in x, f.readlines())) + # jbd2/vda1-8-358 ... : block_getrq: 253,0 WS 59098136 + 120 [jbd2/vda1-8] + for entry in traceText: + oneIO = entry.split() + matchObj = re.match( + r'(.*) \[([^\[\]]*)\] (.*) \[([^\[\]]*)\]\n', entry) + comm = matchObj.group(4) + pid = matchObj.group(1).rsplit('-', 1)[1].strip() + if self.pid is not None and int(pid) != self.pid: + continue + devinfo = oneIO[-6-comm.count(' ')].split(',') + dev = ((int(devinfo[0]) << 20) + int(devinfo[1])) + if str(dev) == '0': + continue + device = self.getDevNameByDevt(dev) + if device == '-' or self.notCareDevice(device) == True: + continue + if self.miscStat is not None: + if not mStat.has_key(device): + mStat.setdefault(device, {}) + stat = mStat[device] + iotype = oneIO[-5-comm.count(' ')] + sectors = oneIO[-2-comm.count(' ')] + task = str(pid)+':'+device + if bool(stat.has_key(task)) != True: + stat.setdefault(task, + {"comm":"", "pid": pid, "iops_rd": 0, + "iops_wr": 0, "bps_rd": 0, "bps_wr": 0, + "flushIO": 0, "device": device, + "cid":getContainerId(pid), + "pat_W4K":0, "pat_W16K":0, "pat_W32K":0, + "pat_W64K":0, "pat_W128K":0, "pat_W256K":0, + "pat_W512K":0, "pat_Wlarge":0}) + size = int(sectors) * 512 + if len(comm) > 0: + stat[task]["comm"] = comm + if 'R' in iotype: + stat[task]["iops_rd"] += 1 + stat[task]["bps_rd"] += size + bwTotal += size + iopsTotal += 1 + if 'W' in iotype: + stat[task]["iops_wr"] += 1 + stat[task]["bps_wr"] += size + bwTotal += size + iopsTotal += 1 + if self.Pattern and size > 0 and size < 1024 * 1024 * 100: + stat[task][self.patternIdx(size)] += 1 + if 'F' in iotype: + stat[task]["flushIO"] += 1 + + if self.iopsThresh or self.bwThresh: + if (self.bwThresh and bwTotal >= self.bwThresh) or \ + (self.iopsThresh and iopsTotal >= self.iopsThresh): + pass + else: + return + + if self.enableJsonShow() == False: + print(time.strftime('%Y/%m/%d %H:%M:%S', time.localtime())) + super(iostatClass, self).show() + + if self.miscStat is not None: + tmpStat = [] + for d,val in mStat.items(): + s = sorted(val.items(), + key=lambda e: (e[1]["bps_wr"]+e[1]["bps_rd"]), + reverse=True)[:self.top] + tmpStat.append((d, s)) + del self.miscStat[:] + self.miscStat.extend(tmpStat) + return + + stat = sorted(stat.items(), + key=lambda e: (e[1]["iops_rd"] + e[1]["iops_wr"]), + reverse=True)[:self.top] + + if self.enableJsonShow() == True: + self.showJson(stat) + return + + tPattern = '' + if self.Pattern: + WrIopsTotal = 0 + RdIopsTotal = 0 + WrBwTotal = 0 + RdBwTotal = 0 + tPattern = ('%-12s%-12s%-12s%-12s%-12s%-12s%-12s%-12s' % ( + "pat_W4K", "pat_W16K", "pat_W32K", "pat_W64K", "pat_W128K", + "pat_W256K", "pat_W512K", "pat_Wlarge" + )) + print('%-20s%-8s%-24s%-12s%-16s%-12s%-12s%-12s%s' % + ("comm", "pid", "cid", "iops_rd", "bps_rd", "iops_wr", "bps_wr", + "device", tPattern)) + stSecs = str(secs)+'s' if secs > 1 else 's' + for key, item in stat: + if (item["iops_rd"] + item["iops_wr"]) == 0: + continue + patPercent = '' + if self.Pattern: + WrIopsTotal += item["iops_wr"] + RdIopsTotal += item["iops_rd"] + WrBwTotal += item["bps_wr"] + RdBwTotal += item["bps_rd"] + patPercent = ('%-12s%-12s%-12s%-12s%-12s%-12s%-12s%-12s' % ( + self.patternPercent(item["pat_W4K"], item["iops_wr"]), + self.patternPercent(item["pat_W16K"], item["iops_wr"]), + self.patternPercent(item["pat_W32K"], item["iops_wr"]), + self.patternPercent(item["pat_W64K"], item["iops_wr"]), + self.patternPercent(item["pat_W128K"], item["iops_wr"]), + self.patternPercent(item["pat_W256K"], item["iops_wr"]), + self.patternPercent(item["pat_W512K"], item["iops_wr"]), + self.patternPercent(item["pat_Wlarge"], item["iops_wr"]) + )) + item["bps_rd"] = \ + humConvert(item["bps_rd"], True).replace('s', stSecs) if item["bps_rd"] else 0 + item["bps_wr"] = \ + humConvert(item["bps_wr"], True).replace('s', stSecs) if item["bps_wr"] else 0 + patPercent += item["cid"] + print('%-20s%-8s%-24s%-12s%-16s%-12s%-12s%-12s%s' % (item["comm"], + str(item["pid"]), item["cid"][0:20], str(item["iops_rd"]), + item["bps_rd"], str(item["iops_wr"]), item["bps_wr"], + item["device"], patPercent)) + if self.Pattern: + print('totalIops:%d(r:%d, w:%d), totalBw:%s(r:%s, w:%s)' % + (iopsTotal, RdIopsTotal, WrIopsTotal, + (humConvert(bwTotal, True).replace('s', stSecs) if bwTotal else 0), + (humConvert(RdBwTotal, True).replace('s', stSecs) if RdBwTotal else 0), + (humConvert(WrBwTotal, True).replace('s', stSecs) if WrBwTotal else 0))) + print("") + + def entry(self, interval): + self.start() + time.sleep(float(interval)) + self.stop() + self.show() diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/promiscClass.py b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/promiscClass.py new file mode 100755 index 0000000000000000000000000000000000000000..b3f90fc5ecaeb4861d5e77ee5c2092af254a5dda --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/promiscClass.py @@ -0,0 +1,215 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import signal +import string +import time +import re +import json +from collections import OrderedDict +from common import humConvert +from iostatClass import iostatClass +from fsstatClass import fsstatClass + + +class promiscClass(): + def __init__( + self, devname, utilThresh, iopsThresh, bwThresh, top, json, + nodiskStat, Pattern): + self._iostat = [] + self._fsstat = [] + self.fs = fsstatClass(devname, None, utilThresh, bwThresh, + top, json, nodiskStat, self._fsstat, Pattern) + self.io = iostatClass(devname, None, utilThresh, iopsThresh, + bwThresh, top, json, nodiskStat, self._iostat, + Pattern) + + + def _selectKworker(self, iostat, fsItem, kworker): + select = None + largeFound = False + diff = sys.maxsize + for k in kworker: + fsRestBw = fsItem["bw_wr"] + if 'restBW' in fsItem.keys(): + fsRestBw = fsItem["restBW"] + if fsRestBw > (iostat[k]["bps_wr"] * 15) or \ + (fsRestBw * 50) < iostat[k]["bps_wr"]: + continue + d = abs(fsItem["bw_wr"] - iostat[k]["bps_wr"]) + diff = min(d, diff) + if iostat[k]["bps_wr"] > fsItem["bw_wr"]: + if not largeFound or diff == d: + select = k + largeFound = True + continue + if not largeFound and diff == d: + select = k + return select + + + def _addBioToKworker(self, iostat, kworker, fsItem): + repeated = False + k = self._selectKworker(iostat, fsItem, kworker) + if not k: + return False, 0 + if 'bufferio' not in iostat[k].keys(): + iostat[k].setdefault('bufferio', []) + task = fsItem["comm"]+':'+str(fsItem["tgid"])+':'+str(fsItem["pid"])+\ + ':'+fsItem["cid"][0:20] + bio = {'task': task, 'Wrbw': fsItem["bw_wr"], 'file': fsItem["file"], + 'device': fsItem["device"]} + for d in iostat[k]["bufferio"]: + if task == d['task'] and d['file'] == bio['file'] and \ + d['device'] == bio['device']: + d['Wrbw'] = max(d['Wrbw'], bio["Wrbw"]) + repeated = True + break + if not repeated: + iostat[k]["bufferio"].append(bio) + return True, iostat[k]["bps_wr"] + + + def _checkDeleteItem(self, addOK, costBW, item): + now = time.time() + # After 10 secs without adding to any kworker, we will delete the fsItem + agingTime = 10 + if 'restBW' not in item.keys(): + item.setdefault('restBW', item["bw_wr"]) + item.setdefault('startAging', now) + if addOK: + item["startAging"] = time.time() + item["restBW"] = item["restBW"] - costBW if addOK else item["restBW"] + if item["restBW"] <= 0 or (item["restBW"] < item["bw_wr"] and \ + (now - item["startAging"]) >= agingTime): + return True + return False + + + def _miscIostatFromFsstat(self): + fsstats = self._fsstat + iostats = dict(self._iostat) + for disk, fsItems in fsstats: + if disk not in iostats.keys(): + continue + rmList = [] + iostat = dict(iostats[disk]) + kworker = [key for key,val in iostat.items() if 'kworker' in val['comm']] + for key, item in fsItems: + taskI = item["pid"]+':'+disk + if taskI in iostat.keys(): + if 'file' not in iostat[taskI].keys(): + iostat[taskI].setdefault('file', []) + iostat[taskI]['cid'] = item['cid'] + iostat[taskI]["file"].append(item["file"]) + if item["bw_wr"] <= (iostat[taskI]["bps_wr"] * 15): + rmList.append((key, item)) + continue + if kworker: + if item["bw_wr"] < item["bw_rd"]: + rmList.append((key, item)) + continue + addOK,cost = self._addBioToKworker(iostat, kworker, item) + deleted = self._checkDeleteItem(addOK, cost, item) + if deleted: + rmList.append((key, item)) + for key, item in rmList: + fsItems.remove((key, item)) + iostats[disk] = iostat + return iostats + + + def _miscShowJson(self, iostats): + secs = self.io.cycle + statJsonStr = '{"time":"","mstats":[]}' + mstatDicts = json.loads(statJsonStr, object_pairs_hook=OrderedDict) + mstatDicts['time'] = time.strftime('%Y/%m/%d %H:%M:%S', time.localtime()) + stSecs = str(secs)+'s' if secs > 1 else 's' + + for key, item in iostats: + if (item["iops_rd"]+item["iops_wr"]) == 0 or (item["bps_rd"]+item["bps_wr"]) == 0: + continue + item["bps_rd"] = humConvert( + item["bps_rd"], True).replace('s', stSecs) if item["bps_rd"] else '0' + item["bps_wr"] = humConvert( + item["bps_wr"], True).replace('s', stSecs) if item["bps_wr"] else '0' + if 'file' not in item.keys(): + item.setdefault('file', '-') + if 'kworker' in item["comm"] and 'bufferio' in item.keys(): + for i in item["bufferio"]: + i["Wrbw"] = humConvert(i["Wrbw"], True).replace('s', stSecs) + mstatDicts["mstats"].append(item) + if len(mstatDicts["mstats"]) > 0: + self.io.writeDataToJson(json.dumps(mstatDicts)) + + + def miscShow(self): + secs = self.io.cycle + if not self._fsstat and not self._iostat: + return + + iostats = self._miscIostatFromFsstat() + if not iostats: + return + tmp = {} + for d in iostats.values(): + tmp.update(dict(d)) + iostats = sorted( + tmp.items(), + key=lambda e: (int(e[1]["bps_rd"])+int(e[1]["bps_wr"])), + reverse=True) + if self.io.enableJsonShow() == True: + self._miscShowJson(iostats) + return + + print('%-20s%-8s%-24s%-12s%-16s%-12s%-12s%-8s%s' % + ("comm", "pid", "cid", "iops_rd", "bps_rd", "iops_wr", "bps_wr", + "device", "file")) + stSecs = str(secs)+'s' if secs > 1 else 's' + for key, item in iostats: + if (item["iops_rd"]+item["iops_wr"]) == 0 or (item["bps_rd"]+item["bps_wr"]) == 0: + continue + item["bps_rd"] = humConvert( + item["bps_rd"], True).replace('s', stSecs) if item["bps_rd"] else '0' + item["bps_wr"] = humConvert( + item["bps_wr"], True).replace('s', stSecs) if item["bps_wr"] else '0' + file = str(item["file"]) if 'file' in item.keys() else '-' + print('%-20s%-8s%-24s%-12s%-16s%-12s%-12s%-8s%s' % + (item["comm"], str(item["pid"]), item["cid"][0:20], str(item["iops_rd"]), + item["bps_rd"], str(item["iops_wr"]), item["bps_wr"], item["device"], file)) + if 'kworker' in item["comm"] and 'bufferio' in item.keys(): + for i in item["bufferio"]: + i["Wrbw"] = humConvert(i["Wrbw"], True).replace('s', stSecs) + print(' |----%-32s WrBw:%-12s Device:%-8s File:%s' % + (i['task'], i["Wrbw"], i["device"], i["file"])) + print("") + + + def config(self): + self.fs.config() + self.io.config() + + def start(self): + self.clear() + self.fs.start() + self.io.start() + + def stop(self): + self.fs.stop() + self.io.stop() + + def clear(self): + del self._iostat[:] + + def show(self): + self.fs.show() + self.io.show() + self.miscShow() + + def entry(self, interval): + self.start() + time.sleep(float(interval)) + self.stop() + self.show() diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iowaitstat/__init__.py b/source/tools/monitor/ioMonitor/ioMon/tools/iowaitstat/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cb8e4b62acc8012add0dc0175a04b6b4f51940ec --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iowaitstat/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- + +if __name__ == "__main__": + pass diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iowaitstat/iowaitstat.py b/source/tools/monitor/ioMonitor/ioMon/tools/iowaitstat/iowaitstat.py new file mode 100755 index 0000000000000000000000000000000000000000..69f473c87947f220f2376a1dba130d05eb1ef316 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iowaitstat/iowaitstat.py @@ -0,0 +1,354 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import signal +import string +import argparse +import time +import re +import json +import threading +from collections import OrderedDict +from subprocess import PIPE, Popen +import shlex + +global_iowaitstat_stop = False + + +def signal_exit_handler(signum, frame): + global global_iowaitstat_stop + global_iowaitstat_stop = True + +def exit_handler(): + global global_iowaitstat_stop + global_iowaitstat_stop = True + +def execCmd(cmd): + p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) + return p.stdout.read().decode('utf-8') + + +def getTgid(pid): + try: + with open("/proc/"+str(pid)+"/status") as f: + return ''.join(re.findall(r'Tgid:(.*)', f.read())).lstrip() + except IOError: + return '-' + return '-' + + +def fixComm(comm, pid): + try: + if ".." in comm: + with open("/proc/"+str(pid)+"/comm") as f: + return f.read().rstrip('\n') + except IOError: + return comm + return comm + + +def echoFile(filename, txt): + os.system("echo \""+txt+"\" > "+filename) + + +def echoFileAppend(filename, txt): + os.system("echo \""+txt+"\" >> "+filename) + + +def supportKprobe(name): + cmd = "cat /sys/kernel/debug/tracing/available_filter_functions | grep " + name + ss = execCmd(cmd).strip() + for res in ss.split('\n'): + if ':' in res: + res = res.split(":", 1)[1] + if ' [' in res: # for ko symbol + res = res.split(" [", 1)[0] + if res == name: + return True + return False + +class iowaitClass(): + def __init__(self, pid, cycle, top, json, iowait_thresh): + self.pid = pid + self.top = int(top) if top is not None else 99999999 + self.json = json + self.cycle = cycle + self.iowait_thresh = int(iowait_thresh) if iowait_thresh is not None else 0 + self.kprobeEvent = "/sys/kernel/debug/tracing/kprobe_events" + self.tracingDir = "/sys/kernel/debug/tracing/instances/iowait" + self.kprobeDir = self.tracingDir+"/events/kprobes" + self.expression = [] + self.kprobe = [] + self.cpuStatIowait = {'sum': 0, 'iowait': 0} + if json: + self.fJson = open(json, 'w+') + + for kprobe,retProbe in {'io_schedule_timeout':True, 'io_schedule':True}.items(): + if supportKprobe(kprobe) == False: + print("not available %s kprobe" % kprobe) + continue + self.expression.append('p:p_%s_0 %s' % (kprobe, kprobe)) + self.kprobe.append('p_%s_0' % kprobe) + if retProbe == True: + self.expression.append('r:r_%s_0 %s' % (kprobe, kprobe)) + self.kprobe.append('r_%s_0' % kprobe) + if len(self.kprobe) == 0: + print "not available kprobe" + sys.exit(0) + + def config(self): + if not os.path.exists(self.tracingDir): + os.mkdir(self.tracingDir) + for exp in self.expression: + probe = exp.split()[0].split(':')[1] + enableKprobe = self.kprobeDir+"/"+probe+"/enable" + if os.path.exists(enableKprobe): + echoFile(enableKprobe, "0") + echoFileAppend(self.kprobeEvent, '-:%s' % probe) + + echoFileAppend(self.kprobeEvent, exp) + echoFile(enableKprobe, "1") + + def start(self): + echoFile(self.tracingDir+"/trace", "") + echoFile(self.tracingDir+"/tracing_on", "1") + with open("/proc/stat") as fStat: + cpuStatList = map(long, fStat.readline().split()[1:]) + self.cpuStatIowait['sum'] = sum(cpuStatList) + self.cpuStatIowait['iowait'] = cpuStatList[4] + + def stop(self): + echoFile(self.tracingDir+"/tracing_on", "0") + + def clear(self): + for exp in self.expression: + probe = exp.split()[0].split(':')[1] + enableKprobe = self.kprobeDir+"/"+probe+"/enable" + if os.path.exists(enableKprobe): + echoFile(enableKprobe, "0") + echoFileAppend(self.kprobeEvent, '-:%s' % probe) + if self.json: + self.fJson.close() + + def writeDataToJson(self, data): + self.fJson.write(data+'\n') + + def showJson(self, stat, totalTimeout, gloabIowait): + top = 0 + statJsonStr = '{"time":"", "global iowait":0,"iowait":[]}' + iowaitStatDicts = json.loads(statJsonStr, object_pairs_hook=OrderedDict) + iowaitStatDicts['time'] = time.strftime('%Y/%m/%d %H:%M:%S', time.localtime()) + iowaitStatDicts['global iowait'] = gloabIowait + for pid, item in stat.items(): + if item["timeout"] == 0: + continue + if top >= self.top: + break + top += 1 + iowait = str(round(item["timeout"] / totalTimeout * gloabIowait, 2)) + item["timeout"] = str(round(item["timeout"]*1000, 3)) + reason = '' + maxCnt = 0 + for key, val in item['reason'].items(): + if 'balance_dirty' in key: + reason = 'Too many dirty pages' + break + elif 'blk_mq_get_tag' in key: + reason = 'Device queue full' + break + elif 'get_request' in key: + reason = 'Ioscheduler queue full' + break + else: + if val > maxCnt: + reason = 'Unkown[stacktrace:'+key.replace('<-', '->')+']' + maxCnt = val + iowaitStatJsonStr = '{"comm":"","pid":0,"tgid":0,"timeout":0,"iowait":0,"reason":0}' + iowaitStatDict = json.loads( + iowaitStatJsonStr, object_pairs_hook=OrderedDict) + iowaitStatDict["comm"] = item["comm"] + iowaitStatDict["pid"] = pid + iowaitStatDict["tgid"] = item["tgid"] + iowaitStatDict["timeout"] = item["timeout"] + iowaitStatDict["iowait"] = iowait + iowaitStatDict["reason"] = reason + iowaitStatDicts["iowait"].append(iowaitStatDict) + if len(iowaitStatDicts["iowait"]) > 0: + self.writeDataToJson(json.dumps(iowaitStatDicts)) + + def show(self): + top = 0 + totalTimeout = 0 + stat = {} + secs = self.cycle + traceText = [] + + with open("/proc/stat") as fStat: + statList = map(long, fStat.readline().split()[1:]) + gloabIowait = float(format( + (statList[4]-self.cpuStatIowait['iowait'])*100.0 / + (sum(statList)-self.cpuStatIowait['sum']), '.2f')) + if gloabIowait < self.iowait_thresh: + return + + with open(self.tracingDir+"/trace") as f: + traceLoglist = list(filter(lambda x: any(e in x for e in self.kprobe), f.readlines())) + traceText = traceLoglist + + # jbd2/vda2-8-605 [001] .... 38890020.539912: p_io_schedule_0: (io_schedule+0x0/0x40) + # jbd2/vda2-8-605 [002] d... 38890020.540633: r_io_schedule_0: (bit_wait_io+0xd/0x50 <- io_schedule) + # <...>-130620 [002] .... 38891029.116442: p_io_schedule_timeout_0: (io_schedule_timeout+0x0/0x40) + # <...>-130620 [002] d... 38891029.123657: r_io_schedule_timeout_0: (balance_dirty_pages+0x270/0xc60 <- io_schedule_timeout) + for entry in traceText: + matchObj = re.match(r'(.*) \[([^\[\]]*)\] (.*) (.*): (.*): (.*)\n', entry) + if matchObj is None: + continue + commInfo = matchObj.group(1).rsplit('-', 1) + pid = commInfo[1].strip() + if self.pid is not None and pid != self.pid: + continue + if bool(stat.has_key(pid)) != True: + comm = fixComm(commInfo[0].lstrip(), pid) + if '..' in comm: + continue + stat.setdefault(pid, + {"comm": comm, "tgid": getTgid(pid), + "timeout": 0, "reason": {}, "entry": []}) + stat[pid]["entry"].append({ + 'time':matchObj.group(4), + 'point':matchObj.group(5), + 'trace':matchObj.group(6)}) + + if stat: + for key,item in stat.items(): + item["entry"] = sorted(item["entry"], key=lambda e: float(e["time"]), reverse=False) + count = 0 + startT = 0 + for entry in item["entry"]: + count += 1 + if (count % 2 != 0 and 'p_' not in entry['point']) or \ + (count % 2 == 0 and 'r_' not in entry['point']): + count = 0 + startT = 0 + continue + + if count % 2 != 0: + startT = float(entry['time']) + continue + + if startT > 0 and float(entry['time']) > startT: + if re.split('[(,+]', entry['trace'])[1] in re.split('[-,)]', entry['trace'])[1]: + count = 0 + startT = 0 + continue + item['timeout'] += (float(entry['time']) - startT) + totalTimeout += (float(entry['time']) - startT) + startT = 0 + if entry['trace'] not in item['reason'].keys(): + item['reason'].setdefault(entry['trace'], 0) + item['reason'][entry['trace']] += 1 + + if stat: + stat = OrderedDict(sorted(stat.items(), key=lambda e: e[1]["timeout"], reverse=True)) + if self.json: + self.showJson(stat, totalTimeout, gloabIowait) + return + else: + head = str(time.strftime('%Y/%m/%d %H:%M:%S', time.localtime()))+' -> global iowait%: '+str(gloabIowait) + print head + + print("%-32s%-8s%-8s%-16s%-12s%s" % ("comm", "tgid", "pid", "waitio(ms)", "iowait(%)", "reasons")) + for pid, item in stat.items(): + if item["timeout"] == 0: + continue + if top >= self.top: + break + top += 1 + iowait = str(round(item["timeout"] / totalTimeout * gloabIowait, 2)) + item["timeout"] = str(round(item["timeout"]*1000, 3)) + reason = '' + maxCnt = 0 + for key, val in item['reason'].items(): + if 'balance_dirty' in key: + reason = 'Too many dirty pages' + break + elif 'blk_mq_get_tag' in key: + reason = 'Device queue full' + break + elif 'get_request' in key: + reason = 'Ioscheduler queue full' + break + else: + if val > maxCnt: + reason = 'Unkown[stacktrace:'+key.replace('<-', '->')+']' + maxCnt = val + print("%-32s%-8s%-8s%-16s%-12s%s" + % (item["comm"], item["tgid"], pid, item["timeout"], iowait, str(reason))) + print("") + + def entry(self, interval): + self.start() + time.sleep(float(interval)) + self.stop() + self.show() + +def iowaitstatStart(argv): + global global_iowaitstat_stop + global_iowaitstat_stop = False + if os.geteuid() != 0: + print("%s" % ("This program must be run as root. Aborting.")) + sys.exit(0) + examples = """e.g. + ./iowaitstat.py + Report iowait for tasks + ./iowaitstat.py -c 1 + Report iowait for tasks per secs + ./iowaitstat.py -p [PID] -c 1 + Report iowait for task with [PID] per 1secs + """ + parser = argparse.ArgumentParser( + description="Report iowait for tasks.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=examples) + parser.add_argument('-p', '--pid', help='Specify the process id.') + parser.add_argument('-T', '--Timeout', + help='Specify the timeout for program exit(secs).') + parser.add_argument( + '-t', '--top', help='Report the TopN with the largest iowait.') + parser.add_argument('-c', '--cycle', help='Specify refresh cycle(secs).') + parser.add_argument('-j', '--json', help='Specify the json-format output.') + parser.add_argument('-w','--iowait_thresh', help='Specify the iowait-thresh to report.') + args = parser.parse_args(argv) if argv else parser.parse_args() + + pid = int(args.pid) if args.pid else None + secs = float(args.cycle) if args.cycle is not None else 0 + if argv is None: + signal.signal(signal.SIGINT, signal_exit_handler) + signal.signal(signal.SIGHUP, signal_exit_handler) + signal.signal(signal.SIGTERM, signal_exit_handler) + if args.Timeout is not None: + timeoutSec = args.Timeout if args.Timeout > 0 else 10 + secs = secs if secs > 0 else 1 + if argv is None: + signal.signal(signal.SIGALRM, signal_exit_handler) + signal.alarm(int(timeoutSec)) + else: + timer = threading.Timer(int(timeoutSec), exit_handler) + timer.start() + loop = True if secs > 0 else False + c = iowaitClass(pid, secs, args.top, args.json, args.iowait_thresh) + c.config() + interval = secs if loop == True else 1 + while global_iowaitstat_stop != True: + c.entry(interval) + if loop == False: + break + c.clear() + +def main(): + iowaitstatStart(None) + +if __name__ == "__main__": + main() diff --git a/source/tools/monitor/ioMonitor/ioMonitor.sh b/source/tools/monitor/ioMonitor/ioMonitor.sh new file mode 100755 index 0000000000000000000000000000000000000000..dfd3d4689472b7715c3cbce077915fa054ee7f88 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMonitor.sh @@ -0,0 +1,11 @@ +#!/bin/sh +#****************************************************************# +# ScriptName: ioMonitor.sh +# Author: $SHTERM_REAL_USER@alibaba-inc.com +# Create Date: 2021-06-06 16:53 +# Modify Author: $SHTERM_REAL_USER@alibaba-inc.com +# Modify Date: 2021-06-06 16:53 +# Function: +#***************************************************************# +TOOLS_ROOT="$SYSAK_WORK_PATH/tools" +python $TOOLS_ROOT/ioMon/ioMonitorMain.py $* diff --git a/source/tools/monitor/unity/beaver/beaver.c b/source/tools/monitor/unity/beaver/beaver.c index 293cbb4b36895da35d203cad7717dd5dc29709da..9885f0567b3893d087c3fe89c49caf169e20fa16 100644 --- a/source/tools/monitor/unity/beaver/beaver.c +++ b/source/tools/monitor/unity/beaver/beaver.c @@ -12,22 +12,19 @@ #include #include -LUALIB_API void luaL_traceback (lua_State *L, lua_State *L1, const char *msg, int level); +extern int lua_reg_errFunc(lua_State *L); +extern int lua_check_ret(int ret); +int lua_load_do_file(lua_State *L, const char* path); -static void report_lua_failed(lua_State *L) { - fprintf(stderr, "\nFATAL ERROR:%s\n\n", lua_tostring(L, -1)); -} - -static int call_init(lua_State *L, char *fYaml) { +static int call_init(lua_State *L, int err_func, char *fYaml) { int ret; lua_Number lret; lua_getglobal(L, "init"); lua_pushstring(L, fYaml); - ret = lua_pcall(L, 1, 1, 0); + ret = lua_pcall(L, 1, 1, err_func); if (ret) { - perror("luaL_call init func error"); - report_lua_failed(L); + lua_check_ret(ret); goto endCall; } @@ -67,6 +64,7 @@ void LuaAddPath(lua_State *L, char *name, char *value) { static lua_State * echos_init(char *fYaml) { int ret; + int err_func; /* create a state and load standard library. */ lua_State *L = luaL_newstate(); @@ -77,22 +75,15 @@ static lua_State * echos_init(char *fYaml) { /* opens all standard Lua libraries into the given state. */ luaL_openlibs(L); - LuaAddPath(L, "path", "../beaver/?.lua"); + err_func = lua_reg_errFunc(L); - ret = luaL_loadfile(L, "../beaver/beaver.lua"); - ret = lua_pcall(L, 0, LUA_MULTRET, 0); + ret = lua_load_do_file(L, "../beaver/beaver.lua"); if (ret) { - const char *msg = lua_tostring(L, -1); - perror("luaL_dofile error"); - if (msg) { - luaL_traceback(L, L, msg, 0); - fprintf(stderr, "FATAL ERROR:%s\n\n", msg); - } goto endLoad; } - ret = call_init(L, fYaml); + ret = call_init(L, err_func, fYaml); if (ret < 0) { goto endCall; } @@ -107,13 +98,14 @@ static lua_State * echos_init(char *fYaml) { static int echos(lua_State *L) { int ret; + int err_func; lua_Number lret; + err_func = lua_gettop(L); lua_getglobal(L, "echo"); - ret = lua_pcall(L, 0, 1, 0); + ret = lua_pcall(L, 0, 1, err_func); if (ret) { - perror("lua call error"); - report_lua_failed(L); + lua_check_ret(ret); goto endCall; } @@ -147,7 +139,6 @@ int beaver_init(char *fYaml) { } ret = echos(L); lua_close(L); - sleep(5); // to release port } exit(1); } diff --git a/source/tools/monitor/unity/beaver/export.lua b/source/tools/monitor/unity/beaver/export.lua index e1436c60c8c6a2f51227d5cba9ed833fc26f434e..b6f3b52e6b9b619cf98cb94516619d7c5b09fda7 100644 --- a/source/tools/monitor/unity/beaver/export.lua +++ b/source/tools/monitor/unity/beaver/export.lua @@ -24,11 +24,13 @@ local function qFormData(from, tData) local res = {} local len = #tData local last = 0 + local c = 0 for i = len, 1, -1 do local line = tData[i] if from == line.title then if last == 0 or last == line.time then - table.insert(res, line) + c = c + 1 + res[c] = line last = line.time else break @@ -40,8 +42,10 @@ end local function packLine(title, ls, v) local tLs = {} + local c = 0 for k, v in pairs(ls) do - table.insert(tLs, string.format("%s=\"%s\"", k , v)) + c = c + 1 + tLs[c] = string.format("%s=\"%s\"", k , v) end local label = "" if #tLs then @@ -56,15 +60,18 @@ function Cexport:export() self._fox:resize() self._fox:qlast(self._freq, qs) local res = {} + local c = 0 for _, line in ipairs(self._tDescr) do local from = line.from local tFroms = qFormData(from, qs) if #tFroms then local title = line.title local help = string.format("# HELP %s %s", title, line.help) - table.insert(res, help) + c = c + 1 + res[c] = help local sType = string.format("# TYPE %s %s", title, line.type) - table.insert(res, sType) + c = c + 1 + res[c] = sType for _, tFrom in ipairs(tFroms) do local labels = system:deepcopy(tFrom.labels) @@ -74,7 +81,8 @@ function Cexport:export() labels.instance = self._instance for k, v in pairs(tFrom.values) do labels[line.head] = k - table.insert(res, packLine(title, labels, v)) + c = c + 1 + res[c] = packLine(title, labels, v) end end end diff --git a/source/tools/monitor/unity/beaver/frame.lua b/source/tools/monitor/unity/beaver/frame.lua index 0c243edd09cf599b54b9ddd2ab20b4b79e022402..29b98103ceb12e23419d87e1c5c23290c408ec4d 100644 --- a/source/tools/monitor/unity/beaver/frame.lua +++ b/source/tools/monitor/unity/beaver/frame.lua @@ -6,27 +6,28 @@ -- refer to https://blog.csdn.net/zx_emily/article/details/83024065 -local unistd = require("posix.unistd") -local poll = require("posix.poll") - require("common.class") local ChttpComm = require("httplib.httpComm") local pystring = require("common.pystring") +local system = require("common.system") local Cframe = class("frame", ChttpComm) function Cframe:_init_() ChttpComm._init_(self) self._objs = {} + self._obj_res = {} end local function waitDataRest(fread, rest, tReq) local len = 0 local tStream = {tReq.data} + local c = #tStream while len < rest do local s = fread() if s then len = len + #s - table.insert(tStream, s) + c = c + 1 + tStream[c] = s else return -1 end @@ -123,6 +124,14 @@ function Cframe:echo404() return pystring:join("\r\n", tHttp) end +function Cframe:findObjRes(path) + for k, v in pairs(self._obj_res) do + if string.find(path, k) then + return v + end + end +end + function Cframe:proc(fread) local stream = waitHttpHead(fread) if stream == nil then -- read return stream or error code or nil @@ -135,13 +144,15 @@ function Cframe:proc(fread) local obj = self._objs[tReq.path] local res, keep = obj:call(tReq) return res, keep - else - print("show all path.") - for k, _ in pairs(self._objs) do - print("path:", k) - end - return self:echo404(), false end + + local obj = self:findObjRes(tReq.path) + if obj then + local res, keep = obj:calls(tReq) + return res, keep + end + + return self:echo404(), false end end @@ -150,4 +161,9 @@ function Cframe:register(path, obj) self._objs[path] = obj end +function Cframe:registerRe(path, obj) + assert(self._obj_res[path] == nil, "the " .. path .. " is already registered.") + self._obj_res[path] = obj +end + return Cframe diff --git a/source/tools/monitor/unity/beaver/guide/bpf.md b/source/tools/monitor/unity/beaver/guide/bpf.md index 81b67b622aa772e25ad6b4412c4d342eb67bfb97..9edada61b147fa8193c70ab36f45e65d4f5473d7 100644 --- a/source/tools/monitor/unity/beaver/guide/bpf.md +++ b/source/tools/monitor/unity/beaver/guide/bpf.md @@ -1,13 +1,13 @@ -## 基于 eBPF 的监控开发手册 +## 基于 eBPF 的周期性采样监控开发手册 -我们在 `source/tools/monitor/unity/collector/plugin/bpfsample2` 路径提供了一个基于 eBPF 的监控开发样例。其主要包含三个部分： +我们在 `source/tools/monitor/unity/collector/plugin/bpfsample` 路径提供了一个基于 eBPF 的周期性采样监控开发样例。其主要包含三个部分： 1. Makefile: 用于编译该工具； -2. bpfsample2.bpf.c: 此处编写 eBPF 程序 -3. bpfsmaple2.c: 此处编写用户态程序 +2. bpfsample.bpf.c: 此处编写 eBPF 程序 +3. bpfsmaple.c: 此处编写用户态程序 接下分别介绍这三个部分。 @@ -16,9 +16,9 @@ ```Makefile newdirs := $(shell find ./ -type d) -bpfsrcs := bpfsample2.bpf.c -csrcs := bpfsample2.c -so := libbpfsample2.so +bpfsrcs := bpfsample.bpf.c +csrcs := bpfsample.c +so := libbpfsample.so include ../bpfso.mk ``` @@ -30,36 +30,32 @@ include ../bpfso.mk 开发者只需要关注上述三个变量的修改即可。 -### bpfsample2.bpf.c: eBPF 程序的编写 +### bpfsample.bpf.c: eBPF 程序的编写 ```c #include #include -#include "bpfsample2.h" +#include "bpfsample.h" -BPF_PERF_OUTPUT(perf, 1024); +BPF_ARRAY(count, u64, 200); SEC("kprobe/netstat_seq_show") int BPF_KPROBE(netstat_seq_show, struct sock *sk, struct msghdr *msg, size_t size) { - struct event e = {}; - - e.ns = ns(); - e.cpu = cpu(); - e.pid = pid(); - comm(e.comm); - - bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &e, sizeof(struct event)); + int default_key = 0; + u64 *value = bpf_map_lookup_elem(&count, &default_key); + if (value) { + __sync_fetch_and_add(value, 1); + } return 0; } - ``` -1. `vmlinux.h` 和 `coolbpf.h` 是coolbpf框架提供的两个头文件，里面包含了类似 `BPF_PERF_OUTPUT` 的helper函数，以及内核结构体的定义 -2. `bpfsample2.h` 是开发者自定义的头文件 +1. `vmlinux.h` 和 `coolbpf.h` 是coolbpf框架提供的两个头文件，里面包含了类似 `BPF_ARRAY` 的helper函数，以及内核结构体的定义 +2. `bpfsample.h` 是开发者自定义的头文件 -### bpfsample2.c: 用户态程序的编写 +### bpfsample.c: 用户态程序的编写 unity 监控框架提供了三个函数，分别是： @@ -79,22 +75,45 @@ void deinit(void) } ``` -在 `init` 函数里，需要去 load, attach eBPF程序，如有需要可能还会创建用于接收perf事件的线程。为了开发方便，coolbpf提供了简单的宏定义去完成这一系列的操作，即 `LOAD_SKEL_OBJECT(skel_name, perf);` 。因此，一般 `init` 函数具体形式如下： +在 `init` 函数里，需要去 load, attach eBPF程序。为了开发方便，coolbpf提供了简单的宏定义去完成这一系列的操作，即 `LOAD_SKEL_OBJECT(skel_name);` 。因此，一般 `init` 函数具体形式如下： ```c int init(void *arg) { - return LOAD_SKEL_OBJECT(bpf_sample2, perf);; + return LOAD_SKEL_OBJECT(bpf_sample); +} +``` + +对于 `call` 函数，我们需要周期性去读取 `map` 数据。本样例，在 `call` 函数读取 `count` map里面的数据，去统计事件触发的频次。 + + +```c +int call(int t, struct unity_lines *lines) +{ + int countfd = bpf_map__fd(bpfsample->maps.count); + int default_key = 0; + uint64_t count = 0; + uint64_t default_count = 0; + struct unity_line* line; + + bpf_map_lookup_elem(countfd, &default_key, &count); + bpf_map_update_elem(countfd, &default_key, &default_count, BPF_ANY); + + unity_alloc_lines(lines, 1); + line = unity_get_line(lines, 0); + unity_set_table(line, "bpfsample"); + unity_set_value(line, 0, "value", count); + + return 0; } ``` -对于 `call` 函数，我们保持不变，即直接 `return 0`。 对于 `deinit` 函数，同 `init` 函数里提供的 `LOAD_SKEL_OBJECT` 宏定义一样，我们也提供了类似的销毁宏定义，即：`DESTORY_SKEL_BOJECT`。 因此，一般 `deinit` 函数具体形式如下： ```c int deinit(void *arg) { - return DESTORY_SKEL_BOJECT(bpf_sample2); + return DESTORY_SKEL_BOJECT(bpf_sample); } ``` \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/guide/bpf_perf.md b/source/tools/monitor/unity/beaver/guide/bpf_perf.md new file mode 100644 index 0000000000000000000000000000000000000000..2614c60c4401f9498bbceda764cd8d0e0f0fd362 --- /dev/null +++ b/source/tools/monitor/unity/beaver/guide/bpf_perf.md @@ -0,0 +1,100 @@ + + +## 基于 eBPF 的事件监控开发手册 + + +我们在 `source/tools/monitor/unity/collector/plugin/bpfsample2` 路径提供了一个基于 eBPF 的监控开发样例。其主要包含三个部分： + +1. Makefile: 用于编译该工具； +2. bpfsample2.bpf.c: 此处编写 eBPF 程序 +3. bpfsmaple2.c: 此处编写用户态程序 + +接下分别介绍这三个部分。 + +### Makfile + +```Makefile +newdirs := $(shell find ./ -type d) + +bpfsrcs := bpfsample2.bpf.c +csrcs := bpfsample2.c +so := libbpfsample2.so + +include ../bpfso.mk +``` + +1. `bpfsrcs`: 用来指定需要编译的 eBPF 程序源文件 +2. `csrcs`: 用来指定需要编译的用户态程序源文件 +3. `so`: 用来指定生成目标动态库名称 + +开发者只需要关注上述三个变量的修改即可。 + + +### bpfsample2.bpf.c: eBPF 程序的编写 + +```c +#include +#include +#include "bpfsample2.h" + +BPF_PERF_OUTPUT(perf, 1024); + +SEC("kprobe/netstat_seq_show") +int BPF_KPROBE(netstat_seq_show, struct sock *sk, struct msghdr *msg, size_t size) +{ + struct event e = {}; + + e.ns = ns(); + e.cpu = cpu(); + e.pid = pid(); + comm(e.comm); + + bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &e, sizeof(struct event)); + return 0; +} + +``` + +1. `vmlinux.h` 和 `coolbpf.h` 是coolbpf框架提供的两个头文件，里面包含了类似 `BPF_PERF_OUTPUT` 的helper函数，以及内核结构体的定义 +2. `bpfsample2.h` 是开发者自定义的头文件 + + +### bpfsample2.c: 用户态程序的编写 + +unity 监控框架提供了三个函数，分别是： + +```c +int init(void *arg) +{ + return 0; +} + +int call(int t, struct unity_lines *lines) +{ + return 0; +} + +void deinit(void) +{ +} +``` + +在 `init` 函数里，需要去 load, attach eBPF程序，如有需要可能还会创建用于接收perf事件的线程。为了开发方便，coolbpf提供了简单的宏定义去完成这一系列的操作，即 `LOAD_SKEL_OBJECT(skel_name, perf);` 。因此，一般 `init` 函数具体形式如下： + +```c +int init(void *arg) +{ + return LOAD_SKEL_OBJECT(bpf_sample2, perf);; +} +``` + +对于 `call` 函数，我们保持不变，即直接 `return 0`。 + +对于 `deinit` 函数，同 `init` 函数里提供的 `LOAD_SKEL_OBJECT` 宏定义一样，我们也提供了类似的销毁宏定义，即：`DESTORY_SKEL_BOJECT`。 因此，一般 `deinit` 函数具体形式如下： + +```c +int deinit(void *arg) +{ + return DESTORY_SKEL_BOJECT(bpf_sample2); +} +``` \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/guide/dev.md b/source/tools/monitor/unity/beaver/guide/dev.md new file mode 100644 index 0000000000000000000000000000000000000000..5868f3a479e25b63ff871d960551d20cc0eb0ed7 --- /dev/null +++ b/source/tools/monitor/unity/beaver/guide/dev.md @@ -0,0 +1,9 @@ +245 上开发 + +``` +docker pull registry.cn-hangzhou.aliyuncs.com/sysom/unity:v1.1 +docker run --net=host --privileged=true -v /:/mnt/host:ro --name unity -it -d registry.cn-hangzhou.aliyuncs.com/sysom/unity:v1.1 /bin/bash +docker exec -it unity bash +cd build/sysak/source/tools/monitor/unity/test/bees/ +./run.sh +``` \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/guide/dev_proc.md b/source/tools/monitor/unity/beaver/guide/dev_proc.md index 3c006d0dfb0e7bca223ec63064099788ffe118e1..b21b886396687cebc482298f38717f879a80677e 100644 --- a/source/tools/monitor/unity/beaver/guide/dev_proc.md +++ b/source/tools/monitor/unity/beaver/guide/dev_proc.md @@ -304,4 +304,4 @@ return CkvProc -[返回目录](/guide) \ No newline at end of file +[返回目录](/guide/guide.md) \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/guide/develop.md b/source/tools/monitor/unity/beaver/guide/develop.md new file mode 100644 index 0000000000000000000000000000000000000000..119451798142785dc30375114255098daaf2482d --- /dev/null +++ b/source/tools/monitor/unity/beaver/guide/develop.md @@ -0,0 +1,681 @@ +# 1、unity 监控框架概述 + +unity 监控框架以插件化开发为主，支持coolbpf 应用，以及多种数据发布方式。具有配置灵活，资源占用率低等优点，适合在服务器监控等领域部署。 + +![frame](image/frame.png) + +# 2、开发构建流程 + +## 2.1 clone 代码 + +开发机可以访问gitee和registry.cn-hangzhou.aliyuncs.com，并且已经安装了docker 和 git。 + +``` +git clone -b unity https://gitee.com/anolis/sysak.git +``` + +## 2.2 拉起容器 + +``` +docker run -v /root/1ext/code/:/root/code -v /:/mnt/host:ro -v /var/run/docker.sock:/var/run/docker.sock --net=host --name unity --privileged -itd registry.cn-hangzhou.aliyuncs.com/sysom/sysom:v1.0 /bin/sh +``` + +docker 参数说明： + +* /root/1ext/code/:/root/code -> 将代码目录挂载到容器目录下，方便代码同步 +* /:/mnt/host:ro/:/mnt/host:ro ->将host根目录的以只读方式挂载进来 +* /var/run/docker.sock:/var/run/docker.sock -> 挂载host 侧的docker 接口，可以根据自己开发机的实际情况进行选择 +* --name unity docker 名，可以自定义命名 +* --privileged 特权容器模式，如果要在容器里面进行调试，该选项不能省略 + +启动编译 + +``` +./configure --enable-libbpf --enable-target-unity + make +``` + +编译后在 sysak/out/.sysak_components/tools/dist 目录下会生成目标包文件。 + +## 2.3 准备 plugin.yaml 配置文件 + +unity 监控启动脚本默认会从 /etc/sysak/plugin.yaml 读取配置。典型的配置表说明： + +``` +config: + freq: 20 # 采集间隔 + port: 8400 # 监听端口 + bind_addr: 0.0.0.0 # 监听ip + backlog: 32 # 服务监听对队列长度， + identity: # 实例id配置模式，当前支持以下五种模式 + # hostip: 获取主机IP + # curl: 通过网络请求获取，需要指定url 参数，适合ECS场景 + # file: 从文件读取，需要指定path 参数 + # specify: 指定id，需要指定name参数 + mode: specify + name: test_specify + proc_path: /mnt/host/ # proc 文件路径，在host侧，为 / 在容器侧，如配置 -v /:/mnt/host 则配置为 /mnt/host + +outline: # 外部数据入口，适合接入外部数据场景 + - /tmp/sysom # 外部unix socket 路径，可以指定多个 + +luaPlugins: ["proc_buddyinfo", "proc_diskstats", "proc_meminfo", "proc_mounts", "proc_netdev", + "proc_snmp_stat", "proc_sockstat", "proc_stat", "proc_statm", "proc_vmstat"] # 控制lua 插件加载 + + +plugins: # 插件列表 对应 /collector/plugin 路径下编译出来的c库文件。 + - so: kmsg # 库名 + description: "collect dmesg info." # 描述符 + …… + +metrics: # export 导出的 metrics 列表 + - + title: sysak_proc_cpu_total # 显示的表名 + from: cpu_total # 数据源头，对应collector生成的数据表 + head: mode # 字段名，在prometheus 中以label 方式呈现 + help: "cpu usage info for total." # help 说明 + type: "gauge" # 数据类型 + …… +``` + +## 2.4 启动监控 + +进入 sysak/out/.sysak_components/tools/dist/app/beeQ 目录下， 执行run.sh 脚本，启动监控 +执行 curl 即可以查询到实时数据 + +``` +curl 127.0.0.1:8400/metrics +``` + +# 3、监控开发 + +## 3.1、监控指标采集 by lua + +本节将描述讲解如何基于lua 开发proc 数据采集。 + +### 3.1.1、纯pystring 处理方法 + +预备知识，lua + +* [pystring](https://gitee.com/chuyansz/sysak/blob/opensource_branch/source/tools/monitor/unity/beaver/guide/pystring.md) 库，处理字符串 +* [面向对象设计](https://gitee.com/chuyansz/sysak/blob/opensource_branch/source/tools/monitor/unity/beaver/guide/oop.md) + +以提取 /proc/net/sockstat 数据为例，原始的信息如下： + +``` +#cat /proc/net/sockstat +sockets: used 83 +TCP: inuse 6 orphan 0 tw 0 alloc 33 mem 2 +UDP: inuse 6 mem 12 +UDPLITE: inuse 0 +RAW: inuse 0 +FRAG: inuse 0 memory 0 +``` + +#### 3.1.1.1、数据处理策略 +sockstat 接口导出的数据非常有规律，基本上是 + +``` +[大标题]: [小标题] [值] …… +[大标题]: [小标题] [值] …… +``` + +这种方法进行组合，可以针对以上方式进行处理。 + +#### 3.1.1.2、数据格式 + +监控使用 [protobuf](https://www.jianshu.com/p/a24c88c0526a) 来序列化和存取数据，标准数据.proto 文件描述如下： + +``` + message labels { + required string name = 1; + required string index = 2; + } + message values { + required string name = 1; + required double value = 2; + } + message logs { + required string name = 1; + required string log = 2; + } + message dataLine{ + required string line = 1; + repeated labels ls = 2; + repeated values vs = 3; + repeated logs log = 4; + } + message dataLines{ + repeated dataLine lines = 1; + } + } +``` + +想了解监控 对 protobuf的处理，可以参考 [这个通用库](https://gitee.com/chuyansz/sysak/blob/opensource_branch/source/tools/monitor/unity/common/protoData.lua) + +#### 3.1.1.3、 vproc 虚基础类 +vproc 是所有 proc 接口数据采集的基础类，提供了通用的数据封装函数。根据前面的proto 文件描述，存储数据实质就是一堆数据表行组成的，在[vproc](https://gitee.com/chuyansz/sysak/blob/opensource_branch/source/tools/monitor/unity/collector/vproc.lua) 声明如下： + +``` +function CvProc:_packProto(head, labels, vs, log) + return {line = head, ls = labels, vs = vs, log = log} +end +``` + +添加数据行： + +``` +function CvProc:appendLine(line) + table.insert(self._lines["lines"], line) +end +``` + +将生成好的数据往外部table 中推送并清空本地数据： + +``` +function CvProc:push(lines) + for _, v in ipairs(self._lines["lines"]) do + table.insert(lines["lines"], v) + end + self._lines = nil + return lines +end +``` + +#### 3.1.1.4、整体代码实现 +了解了vproc 类后，就可以从vproc 实现一个 /proc/net/sockstat 数据采集接口。代码 实现和注释如下： + +``` +require("class") -- 面向对象 class 声明 +local pystring = require("common.pystring") +local CvProc = require("collector.vproc") + +local CprocSockStat = class("procsockstat", CvProc) -- 从vproc 继承 + +function CprocSockStat:_init_(proto, pffi, pFile) -- 调用构造函数 + CvProc._init_(self, proto, pffi, pFile or "/proc/net/sockstat") +end + +function CprocSockStat:proc(elapsed, lines) -- 在主循环中会周期性调用proc 函数进行收集数据 + CvProc.proc(self) -- 新建本地表 + local vs = {} -- 用于暂存有效数据 + for line in io.lines(self.pFile) do -- 读取文件内容 + local cells = pystring:split(line, ":", 1) -- 按: 分割标题和内容 + if #cells > 1 then -- 防止 空行产生无效数据 + local head, body = cells[1], cells[2] + head = string.lower(head) -- 标题统一小写 + body = pystring:lstrip(body, " ") -- 去除开头的空格 + local bodies = pystring:split(body, " ") -- 按空格分割内容 + local len = #bodies / 2 + for i = 1, len do + local title = string.format("%s_%s", head, bodies[2 * i - 1]) -- 组合数值标题 + local v = { + name=title, + value=tonumber(bodies[2 * i]) + } + table.insert(vs, v) -- 添加到暂存表中 + end + end + end + self:appendLine(self:_packProto("sock_stat", nil, vs)) -- 保存到本地表中 + return self:push(lines) --推送到全局表，并发送出去 +end + +return CprocSockStat -- 这一行不能少 +``` + +#### 3.1.1.5、注册到主循环中 + +[loop.lua](https://gitee.com/chuyansz/sysak/blob/opensource_branch/source/tools/monitor/unity/collector/loop.lua) 是周期性采样所有数据的循环实现。首先将文件引入： + +``` +local CprocSockStat = require("collector.proc_sockstat") +``` + +然后添加到collector 表中 + +``` +CprocSockStat.new(self._proto, procffi), +``` + +此时数据已经保存在本地 + +#### 3.1.1.6、导出到export + +要将采集到的指标采集到export，只需要在 [plugin.yaml](https://gitee.com/chuyansz/sysak/blob/opensource_branch/source/tools/monitor/unity/collector/plugin.yaml) 中添加以下行做配置即可： + +``` + - title: sysak_sock_stat + from: sock_stat # 代码中声明的表行 + head: value + help: "sock stat counters from /proc/net/sockstat" + type: "gauge" +``` + +#### 3.1.1.7、 数据呈现 +用浏览器打开本地8400端口，到指标链接中，就可以提取到以下新增数据 + +``` +# HELP sysak_sock_stat sock stat counters. +# TYPE sysak_sock_stat gauge +sysak_sock_stat{value="frag_inuse",instance="12345abdc"} 0.0 +sysak_sock_stat{value="udplite_inuse",instance="12345abdc"} 0.0 +sysak_sock_stat{value="udp_mem",instance="12345abdc"} 8.0 +sysak_sock_stat{value="tcp_mem",instance="12345abdc"} 1.0 +sysak_sock_stat{value="tcp_alloc",instance="12345abdc"} 32.0 +sysak_sock_stat{value="frag_memory",instance="12345abdc"} 0.0 +sysak_sock_stat{value="sockets_used",instance="12345abdc"} 80.0 +sysak_sock_stat{value="raw_inuse",instance="12345abdc"} 0.0 +sysak_sock_stat{value="tcp_tw",instance="12345abdc"} 0.0 +sysak_sock_stat{value="tcp_orphan",instance="12345abdc"} 0.0 +sysak_sock_stat{value="tcp_inuse",instance="12345abdc"} 5.0 +``` + +### 3.1.2、FFI 处理方式 +关于lua ffi 说明，可以先参考[lua扩展ffi](https://luajit.org/ext_ffi.html)，本质是lua 可以通过ffi 接口直接调用C库参数，无需经过中间栈上传参等操作。 + +ffi的注意点： + +* ffi 数组下标是从0开始，和lua下标从1开始不一样； +* 可以直接引用ffi 中的数据结构，效率要比原生lua 高很多； +* ffi 是luajit 的功能，原生lua 并不支持； + +#### 3.1.2.1、 为什么要使用ffi？ +pystring 虽然可以高效处理字符串数据，但是相比c语言中的scanf 接口来说效率还是要低很多。因此按行读取proc 数据，可以采用 ffi 接口来显著提升数据处理效率 + +#### 3.1.2.2、 ffi 数据结构和api 说明 + +proc 数据以变参为主，下面的结构体主要用于scanf 获取变参， 用于上层数据处理 + +``` +#define VAR_INDEX_MAX 64 + +// 变参整数类型，用于收集纯整数类型的数据 +typedef struct var_long { + int no; // 收集到参数数量 + long long value[VAR_INDEX_MAX]; //参数列表 +}var_long_t; + +// 变参字符串类型 +typedef struct var_string { + int no; // 收集到参数数量 + char s[VAR_INDEX_MAX][32]; //参数列表 +}var_string_t; + +// 变参 k vs 类型 +typedef struct var_kvs { + int no; // 收集到参数数量 + char s[32]; // 标题 + long long value[VAR_INDEX_MAX]; // 参数列表 +}var_kvs_t; +``` + +导出的c api + +``` +int var_input_long(const char * line, struct var_long *p); +int var_input_string(const char * line, struct var_string *p); +int var_input_kvs(const char * line, struct var_kvs *p); +``` + +综合来说： + +* var\_long\_t 适合纯整数数字输出的场景 +* var\_string\_t 适合纯字符串输出的场景 +* var\_kvs\_t 适合单字符串 + 多整形数字 组合的场景，如 /proc/stat的内容输出 + +其它重复组合场景可以先按照 var\_string\_t 来收集，然后对指定位置的数字字符串通过tonumber 进行转换。 + +#### 3.1.2.3 实际应用例子 +以[kvProc.lua](https://gitee.com/chuyansz/sysak/blob/opensource_branch/source/tools/monitor/unity/collector/kvProc.lua) 为例，它实现了一个通用kv组合的proc接口数据的数据高效的处理方法。如经常使用到的 /proc/meminfo ，是典型的kv值例子 + +``` +#cat /proc/meminfo +MemTotal: 2008012 kB +MemFree: 104004 kB +MemAvailable: 1060412 kB +Buffers: 167316 kB +Cached: 877672 kB +SwapCached: 0 kB +Active: 1217032 kB +Inactive: 522236 kB +Active(anon): 694948 kB +Inactive(anon): 236 kB +Active(file): 522084 kB +Inactive(file): 522000 kB +…… +``` +对应处理代码说明，重点需要关注**readKV**函数实现。 + +``` +local system = require("common.system") +require("common.class") +local CvProc = require("collecotor.vproc") + +local CkvProc = class("kvProc", CvProc) + +function CkvProc:_init_(proto, pffi, mnt, pFile, tName) + CvProc._init_(self, proto, pffi, pFile) -- 从基础类继承 + self._protoTable = { + line = tName, -- 表名 如/proc/meminfo 可以取 meminfo 为表名 + ls = nil, + vs = {} + } +end + +function CkvProc:checkTitle(title) -- 去除label中的保留字符，防止数据保存失败 + local res = string.gsub(title, ":", "") --去除 :和) + res = string.gsub(res, "%)", "") + res = string.gsub(res, "%(", "_") --（替换为_ + return res +end + +function CkvProc:readKV(line) -- 处理单行数据 + local data = self._ffi.new("var_kvs_t") -- 新增一个 var_kvs_t 结构体 + assert(self._cffi.var_input_kvs(self._ffi.string(line), data) == 0) --调用c api 进行读取 + assert(data.no >= 1) --确保访问成功 + + local name = self._ffi.string(data.s) -- 标题处理 + name = self:checkTitle(name) + local value = tonumber(data.value[0]) + + local cell = {name=name, value=value} -- 生存一段数据 + table.insert(self._protoTable["vs"], cell) -- 将数据存入表中 +end + +function CkvProc:proc(elapsed, lines) --处理数据 + self._protoTable.vs = {} + CvProc.proc(self) + for line in io.lines(self.pFile) do --遍历行 + self:readKV(line) -- 处理数据 + end + self:appendLine(self._protoTable) -- 添加到大表中 + return self:push(lines) --往外推送 +end + +return CkvProc +``` + +## 3.2、C 插件开发 + +在collector/plugin/sample 目录下有一个示例工程，它的本质其实就是一个so文件的编译项目。首先要看下sample 同级目录下的公共头文件 plugin_head.h，该头文件提供了数据生成的API，降低开发者实现难度。 + +``` +/// \brief 申请数据行数量，在填入数据前统一申请，根据实际情况填入 + /// \param lines 数据结构体 + /// \param num 申请行号数量 + /// \return 成功返回 0 + inline int unity_alloc_lines(struct unity_lines * lines, unsigned int num) __attribute__((always_inline)); + /// \brief 获取对应行数据，用于填入数据 + /// \param lines 数据结构体 + /// \param i 对应行下标 + /// \return 返回对应的数据行 + inline struct unity_line * unity_get_line(struct unity_lines * lines, unsigned int i) __attribute__((always_inline)); + /// \brief 设置数据行 表名 + /// \param line 行指针 + /// \param table 表名 + /// \return 成功返回 0 + inline int unity_set_table(struct unity_line * line, const char * table) __attribute__((always_inline)); + /// \brief 设置数据行 索引信息 + /// \param line 行指针 + /// \param i 索引下标 + /// \param name 索引名 + /// \param index 索引内容 + /// \return 成功返回 0 + inline int unity_set_index(struct unity_line * line, unsigned int i, const char * name, const char * index) __attribute__((always_inline)); + /// \brief 设置数据行 指标信息 + /// \param line 行指针 + /// \param i 指标下标 + /// \param name 指标名 + /// \param value 指标内容 + /// \return 成功返回 0 + inline int unity_set_value(struct unity_line * line, unsigned int i, const char * name, double value) __attribute__((always_inline)); + /// \brief 设置数据行 日志信息 + /// \param line 行指针 + /// \param name 日志名 + /// \param value 日志内容 + /// \return 成功返回 0 + inline int unity_set_log(struct unity_line * line, const char * name, const char * log) __attribute__((always_inline)); + /// \brief 设置数据行 日志信息 + /// \return 返回mount 目录 + char* get_unity_proc(void); +``` + +**数据规格限制** + +1. unity\_set\_table 中 table 参数长度应该小于32（不含） +2. unity\_set\_index 中 name、index和unity\_set\_value 中 name 参数长度应该要小于16（不含） +3. unity\_set\_index 下标从0开始，并小于 4，即最多4个索引。而且下标数值应该连续，否则数据会从留白处截断 +4. unity\_set\_index 下标从0开始，并小于 32，即最多32个数值。而且下标数值应该连续，否则数据会从留白处截断； +5. unity\_set\_log 中的log 指针需要开发者进行释放； +6. get\_unity\_proc参考2.3节中 proc_path 中的内容； + +### 3.2.1、sample 用例代码 + +适合周期性数据采集的场景，通过周期性调用call 函数来收集数据 + +参考 sample.c + +``` + + /// \brief 插件构造函数，在加载so的时候，会调用一次init + /// \param arg 当前未使用，为NULL + /// \return 成功返回 0 + int init(void * arg) { + printf("sample plugin install.\n"); + return 0; + } + + /// \brief 插件调用函数，通过调用在函数来收集要采集的指标 + /// \param t，间隔周期，如15s的采样周期，则该值为15 + /// \param lines 数值指针，用于填充采集到的数据。 + /// \return 成功返回 0 + int call(int t, struct unity_lines* lines) { + static double value = 0.0; + struct unity_line* line; + + unity_alloc_lines(lines, 2); + line = unity_get_line(lines, 0); + unity_set_table(line, "sample_tbl1"); + unity_set_index(line, 0, "mode", "sample1"); + unity_set_value(line, 0, "value1", 1.0 + value); + unity_set_value(line, 1, "value2", 2.0 + value); + + line = unity_get_line(lines, 1); + unity_set_table(line, "sample_tbl2"); + unity_set_value(line, 0, "value1", 3.0 + value); + unity_set_value(line, 1, "value2", 4.0 + value); + unity_set_value(line, 2, "value3", 3.1 + value); + unity_set_value(line, 3, "value4", 4.1 + value); + + value += 0.1; + return 0; + } + + /// \brief 插件析构函数，调用完该函数时，必须要确保该插件已申请的资源已经全部释放完毕。 + /// \return 成功返回 0 + void deinit(void) { + printf("sample plugin uninstall\n"); + } +``` + +### 3.2.3、threads 代码 + +sample 适合常规数据采集，周期性遍历插件拉取指标的场景。但在实际实践中，还存在数据主动推送的场景。如下图紫线路径所示： + +![dataflow](image/queue.png) + +这种场景下，可以通过创建thread 方式进行进行数据推送，相关参考代码在 collector/plugin/thread 目录 + +``` +#include "sample_threads.h" +#include +#include + +static volatile pthread_t sample_thread_id = 0; //进程id，停止的时候使用 + +static int sample_thread_func(struct beeQ* q, void * arg); //线程回调函数声明，可以通过arg 向 线程回调函数传参 +int init(void * arg) { + struct beeQ* q = (struct beeQ *)arg; + sample_thread_id = beeQ_send_thread(q, NULL, sample_thread_func); // 创建线程 + printf("start sample_thread_id: %lu\n", sample_thread_id); + return 0; +} + +static int sample_thread_func(struct beeQ* q, void * arg) { + unsigned int ret; + while (plugin_is_working()) { + static double value = 1.0; + struct unity_line* line; + struct unity_lines * lines = unity_new_lines(); + + unity_alloc_lines(lines, 1); + line = unity_get_line(lines, 0); + unity_set_table(line, "sample_tbl3"); + unity_set_value(line, 0, "value1", 1.0 + value); + unity_set_value(line, 1, "value2", 2.0 + value); + unity_set_log(line, "log", "hello world."); + beeQ_send(q, lines); // 往队列里面推送数据 + ret = sleep(5); + if (ret > 0) { // interrupt by signal + break; + } + } + return 0; +} + +int call(int t, struct unity_lines* lines) { + static double value = 0.0; + struct unity_line* line; + + unity_alloc_lines(lines, 1); + line = unity_get_line(lines, 0); + unity_set_table(line, "sample_tbl1"); + unity_set_index(line, 0, "mode", "threads"); + unity_set_value(line, 0, "value1", 1.0 + value); + unity_set_value(line, 1, "value2", 2.0 + value); + + value += 0.1; + return 0; +} + +void deinit(void) { + plugin_thread_stop(sample_thread_id); + printf("thread plugin uninstall\n"); +} + +``` + +**在线程回调函数中，必须要判断所有调用到的函数是否被信号打断，用于决定是否需要退出并释放相应资源。** + +如实例代码中需要获取sleep 函数的返回值，根据[sleep函数](https://man7.org/linux/man-pages/man3/sleep.3.html)的返回值说明： + +``` +Zero if the requested time has elapsed, or the number of seconds + left to sleep, if the call was interrupted by a signal handler. +``` + +需要判断是否存在sleep 函数被打断的场景。 + +## 3.3、coolbpf 插件开发 + +关于coolbpf，可以参考[这里](https://gitee.com/anolis/coolbpf) + +`/collector/plugin/bpfsample2` 路径提供了一个基于 eBPF 的监控开发样例。其主要包含三个部分： + +1. Makefile: 用于编译该工具； +2. bpfsample2.bpf.c: 此处编写 eBPF 程序 +3. bpfsmaple2.c: 此处编写用户态程序 + +接下分别介绍这三个部分。 + +### 3.3.1、Makfile + +```Makefile +newdirs := $(shell find ./ -type d) + +bpfsrcs := bpfsample2.bpf.c +csrcs := bpfsample2.c +so := libbpfsample2.so + +include ../bpfso.mk +``` + +1. `bpfsrcs`: 用来指定需要编译的 eBPF 程序源文件 +2. `csrcs`: 用来指定需要编译的用户态程序源文件 +3. `so`: 用来指定生成目标动态库名称 + +开发者只需要关注上述三个变量的修改即可。 + + +### 3.3.2、bpfsample2.bpf.c: eBPF 程序的编写 + +```c +#include +#include +#include "bpfsample2.h" + +BPF_PERF_OUTPUT(perf, 1024); + +SEC("kprobe/netstat_seq_show") +int BPF_KPROBE(netstat_seq_show, struct sock *sk, struct msghdr *msg, size_t size) +{ + struct event e = {}; + + e.ns = ns(); + e.cpu = cpu(); + e.pid = pid(); + comm(e.comm); + + bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &e, sizeof(struct event)); + return 0; +} + +``` + +1. `vmlinux.h` 和 `coolbpf.h` 是coolbpf框架提供的两个头文件，里面包含了类似 `BPF_PERF_OUTPUT` 的helper函数，以及内核结构体的定义 +2. `bpfsample2.h` 是开发者自定义的头文件 + + +### 3.3.3、bpfsample2.c: 用户态程序的编写 + +unity 监控框架提供了三个函数，分别是： + +```c +int init(void *arg) +{ + return 0; +} + +int call(int t, struct unity_lines *lines) +{ + return 0; +} + +void deinit(void) +{ +} +``` + +在 `init` 函数里，需要去 load, attach eBPF程序，如有需要可能还会创建用于接收perf事件的线程。为了开发方便，coolbpf提供了简单的宏定义去完成这一系列的操作，即 `LOAD_SKEL_OBJECT(skel_name, perf);` 。因此，一般 `init` 函数具体形式如下： + +```c +int init(void *arg) +{ + return LOAD_SKEL_OBJECT(bpf_sample2, perf);; +} +``` + +对于 `call` 函数，我们保持不变，即直接 `return 0`。 + +对于 `deinit` 函数，同 `init` 函数里提供的 `LOAD_SKEL_OBJECT` 宏定义一样，我们也提供了类似的销毁宏定义，即：`DESTORY_SKEL_BOJECT`。 因此，一般 `deinit` 函数具体形式如下： + +```c +int deinit(void *arg) +{ + return DESTORY_SKEL_BOJECT(bpf_sample2); +} +``` + + + diff --git a/source/tools/monitor/unity/beaver/guide/guide.md b/source/tools/monitor/unity/beaver/guide/guide.md index 0ac809e7a0ffbd462335bac5b3112c8bed718682..654a28c5af42083d61ef6c24db516d92bc689bc3 100644 --- a/source/tools/monitor/unity/beaver/guide/guide.md +++ b/source/tools/monitor/unity/beaver/guide/guide.md @@ -1,8 +1,8 @@ # 目录 - -1. [插件化与热更新](/guide/hotplugin) -2. [面向对象设计](/guide/oop) -3. [字符串处理](/guide/pystring) -4. [页面开发](/guide/webdevel) -5. [proc和probe记录表](/guide/proc_probe) -6. [采集proc 接口指标](/guide/dev_proc) \ No newline at end of file +1. [开发手册](/guide/guide.md) +2. [插件化与热更新](/guide/hotplugin.md) +3. [面向对象设计](/guide/oop.md) +4. [字符串处理](/guide/pystring.md) +5. [页面开发](/guide/webdevel.md) +6. [proc和probe记录表](/guide/proc_probe.md) +7. [采集proc 接口指标](/guide/dev_proc.md) \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/guide/hotplugin.md b/source/tools/monitor/unity/beaver/guide/hotplugin.md index 5eb67541b94d1cc9e7bed100142e18127f55adb6..5754909c921f73ce0f972a97a86d3a3c35ed42e7 100644 --- a/source/tools/monitor/unity/beaver/guide/hotplugin.md +++ b/source/tools/monitor/unity/beaver/guide/hotplugin.md @@ -111,4 +111,4 @@ unity监控采用[yaml](http://yaml.org/)对插件进行管理，当前插件分 此时数据只是已经更新入库了，但是要在nodexport上面显示，需要配置beaver/export.yaml 文件，才能将查询从数据表中更新。 -[返回目录](/guide) \ No newline at end of file +[返回目录](/guide/guide.md) \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/guide/image/frame.png b/source/tools/monitor/unity/beaver/guide/image/frame.png new file mode 100644 index 0000000000000000000000000000000000000000..3892a4cb3f167ffb3c1a32be0ad7e1003e66f262 Binary files /dev/null and b/source/tools/monitor/unity/beaver/guide/image/frame.png differ diff --git a/source/tools/monitor/unity/beaver/guide/image/python.png b/source/tools/monitor/unity/beaver/guide/image/python.png new file mode 100644 index 0000000000000000000000000000000000000000..fb588d4672af570eef744f1e4d39ea3f866e1a4b Binary files /dev/null and b/source/tools/monitor/unity/beaver/guide/image/python.png differ diff --git a/source/tools/monitor/unity/beaver/guide/image/queue.png b/source/tools/monitor/unity/beaver/guide/image/queue.png new file mode 100644 index 0000000000000000000000000000000000000000..01fc076bf35716308fccc70cc0c37e9718810a9c Binary files /dev/null and b/source/tools/monitor/unity/beaver/guide/image/queue.png differ diff --git a/source/tools/monitor/unity/beaver/guide/oop.md b/source/tools/monitor/unity/beaver/guide/oop.md index 90c265bc6927e9b562df770a247f56fceb3498c3..87c4a6754e752649827246147bd32a627a11a880 100644 --- a/source/tools/monitor/unity/beaver/guide/oop.md +++ b/source/tools/monitor/unity/beaver/guide/oop.md @@ -113,4 +113,4 @@ Ctwo 继承于Cone，这里重新实现并复用了父类的say方法。 function Cone:say() function Cone.say(self) -[返回目录](/guide) +[返回目录](/guide/guide.md) diff --git a/source/tools/monitor/unity/beaver/guide/proc_probe.md b/source/tools/monitor/unity/beaver/guide/proc_probe.md index ac412ac6c83db8d42dad220b6049c8d67c0e860e..3283c9bc5e96e4a90a094158e29eb80b9f7bddea 100644 --- a/source/tools/monitor/unity/beaver/guide/proc_probe.md +++ b/source/tools/monitor/unity/beaver/guide/proc_probe.md @@ -22,4 +22,4 @@ libbpf kprobe/kretprobe/trace\_event/perf event 等事件记录在这里 | ----- | --------- | | xxx | xxx | -[返回目录](/guide) +[返回目录](/guide/guide.md) diff --git a/source/tools/monitor/unity/beaver/guide/pystring.md b/source/tools/monitor/unity/beaver/guide/pystring.md index 5e795f4bcb036c979bf099737770d1ce889ffa77..c783d0b2d50f219eef093c0e8c5ed329ce01683f 100644 --- a/source/tools/monitor/unity/beaver/guide/pystring.md +++ b/source/tools/monitor/unity/beaver/guide/pystring.md @@ -1,4 +1,5 @@ # 字符串处理 +![pystring](image/python.png) 同为脚本语言，lua 默认的字符串处理并不像python 那么完善。但只要通过拓展，也可以像python 一样对字符串进行处理。当前已经实现了split/strip 等高频使用函数。参考[Python字符串处理](https://www.jianshu.com/p/b758332c44bb) @@ -98,4 +99,4 @@ find 用于子串查找，成功返回首次开始的位置，如果不包含， assert(pystring:find("hello world.", "hello") == 1) ``` -[返回目录](/guide) \ No newline at end of file +[返回目录](/guide/guide.md) \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/guide/webdevel.md b/source/tools/monitor/unity/beaver/guide/webdevel.md index 1fa5b829a1f845e248a2b7a65bbdbdcbc222c868..b23efd6d3c2e76031c8073e73a6215e04964ca25 100644 --- a/source/tools/monitor/unity/beaver/guide/webdevel.md +++ b/source/tools/monitor/unity/beaver/guide/webdevel.md @@ -58,11 +58,11 @@ end return CurlGuide ``` -这里采用了面向对象方法实现，关于面向对象，可以[参考这里](/guide/oop) +这里采用了面向对象方法实现，关于面向对象，可以[参考这里](/guide/oop.md) ## 热更新 * 如果仅修改了markdown文件，直接更新文件刷新页面即可； * 如果修改了lua文件，给主进程发送1号信号，进程会重新装载，新页面也会立即生效； -[返回目录](/guide) \ No newline at end of file +[返回目录](/guide/guide.md) \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/identity.lua b/source/tools/monitor/unity/beaver/identity.lua index 27582ac6379291c6ded83aea88a6f86f7ed603e5..1c4e93e392e340f807bc7a2f6a37fc58731da4ab 100644 --- a/source/tools/monitor/unity/beaver/identity.lua +++ b/source/tools/monitor/unity/beaver/identity.lua @@ -25,8 +25,8 @@ end local function getAdd(hostName) local _, resolved = socket.dns.toip(hostName) local listTab = {} - for _, v in pairs(resolved.ip) do - table.insert(listTab, v) + for i, v in pairs(resolved.ip) do + listTab[i] = v end return listTab end diff --git a/source/tools/monitor/unity/beaver/index.lua b/source/tools/monitor/unity/beaver/index.lua index c3ce944ce09128543ae7765c8adc1984cbed0d5e..b6884e0b14aaa2dc193eff0db8e8d38b0f495b22 100644 --- a/source/tools/monitor/unity/beaver/index.lua +++ b/source/tools/monitor/unity/beaver/index.lua @@ -52,7 +52,7 @@ function CurlIndex:show(tReq) ### Tips - This page is rendered directly via markdown, for [guide](/guide) + This page is rendered directly via markdown, for [guide](/guide/guide.md) ]] local content2 = string.format("\n thread id is:%d\n", unistd.getpid()) local title = "welcome to visit SysAk Agent server." diff --git a/source/tools/monitor/unity/beaver/localBeaver.lua b/source/tools/monitor/unity/beaver/localBeaver.lua index 5671ce87c5b841982eee678acf7d0ed77f31fdf3..cb062006e0c744c55551e4758b6ed5378771633c 100644 --- a/source/tools/monitor/unity/beaver/localBeaver.lua +++ b/source/tools/monitor/unity/beaver/localBeaver.lua @@ -34,6 +34,13 @@ function CLocalBeaver:_init_(frame, fYaml) end function CLocalBeaver:_del_() + for fd in pairs(self._cos) do + socket.shutdown(fd, socket.SHUT_RDWR) + local res = self._cffi.del_fd(self._efd, fd) + print("close fd: " .. fd) + assert(res >= 0) + end + if self._efd then self._cffi.deinit(self._efd) end @@ -42,11 +49,6 @@ function CLocalBeaver:_del_() end end -local function posixError(msg, err, errno) - local s = msg .. string.format(": %s, errno: %d", err, errno) - error(s) -end - function CLocalBeaver:_installTmo(fd) self._tmos[fd] = os.time() end @@ -57,7 +59,7 @@ function CLocalBeaver:_checkTmo() -- ! coroutine will del self._tmos cell in loop, so create a mirror table for safety local tmos = system:dictCopy(self._tmos) for fd, t in pairs(tmos) do - if now - t >= 60 then + if now - t >= 10 * 60 then local e = self._ffi.new("native_event_t") e.ev_close = 1 e.fd = fd @@ -81,25 +83,51 @@ function CLocalBeaver:_installFFI() return efd end +local function localBind(fd, tPort) + local try = 0 + local res, err, errno + + -- can reuse for time wait socket. + res, err, errno = socket.setsockopt(fd, socket.SOL_SOCKET, socket.SO_REUSEADDR, 1); + if not res then + system:posixError("set sock opt failed."); + end + + while try < 120 do + res, err, errno = socket.bind(fd, tPort) + if res then + return 0 + elseif errno == 98 then -- port already in use? try 30s; + unistd.sleep(1) + try = try + 1 + else + break + end + end + system:posixError(string.format("bind port %d failed.", tPort.port), err, errno) +end + function CLocalBeaver:_install_fd(port, ip, backlog) local fd, res, err, errno fd, err, errno = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) if fd then -- for socket - res, err, errno = socket.bind(fd, {family=socket.AF_INET, addr=ip, port=port}) - if res then -- for bind + local tPort = {family=socket.AF_INET, addr=ip, port=port} + local r, msg = pcall(localBind, fd, tPort) + if r then res, err, errno = socket.listen(fd, backlog) if res then -- for listen return fd else - posixError("socket listen failed", err, errno) + unistd.close(fd) + system:posixError("socket listen failed", err, errno) end - else -- for bind failed + else + print(msg) unistd.close(fd) - posixError("socket bind failed", err, errno) os.exit(1) end else -- socket failed - posixError("create socket failed", err, errno) + system:posixError("create socket failed", err, errno) end end @@ -120,7 +148,7 @@ function CLocalBeaver:read(fd, maxLen) return nil end else - posixError("socket recv error", err, errno) + system:posixError("socket recv error", err, errno) end else print(system:dump(e)) @@ -137,7 +165,6 @@ function CLocalBeaver:write(fd, stream) sent, err, errno = socket.send(fd, stream) if sent then if sent < #stream then -- send buffer may full - print("need to send buffer for " .. (#stream - sent)) res = self._cffi.mod_fd(self._efd, fd, 1) -- epoll write ev assert(res == 0) @@ -149,19 +176,23 @@ function CLocalBeaver:write(fd, stream) stream = string.sub(stream, sent + 1) sent, err, errno = socket.send(fd, stream) if sent == nil then - posixError("socket send error.", err, errno) + if errno == 11 then -- EAGAIN ? + goto continue + end + system:posixError("socket send error.", err, errno) return nil end else -- need to read ? may something error or closed. return nil end + ::continue:: end res = self._cffi.mod_fd(self._efd, fd, 0) -- epoll read ev only assert(res == 0) end return 1 else - posixError("socket send error.", err, errno) + system:posixError("socket send error.", err, errno) return nil end end @@ -211,12 +242,12 @@ function CLocalBeaver:accept(fd, e) self:co_add(nfd) self:_installTmo(nfd) else - posixError("accept new socket failed", err, errno) + system:posixError("accept new socket failed", err, errno) end end end -function CLocalBeaver:_poll(bfd, nes) +function CLocalBeaver:_pollFd(bfd, nes) for i = 0, nes.num - 1 do local e = nes.evs[i]; local fd = e.fd @@ -233,10 +264,7 @@ function CLocalBeaver:_poll(bfd, nes) self:_checkTmo() end -function CLocalBeaver:poll() - assert(self._once, "poll loop only run once time.") - self._once = false - +function CLocalBeaver:_poll() local bfd = self._bfd local efd = self._efd while true do @@ -244,17 +272,21 @@ function CLocalBeaver:poll() local res = self._cffi.poll_fds(efd, 10, nes) if res < 0 then - break + return "end poll." end - self:_poll(bfd, nes) + self:_pollFd(bfd, nes) end +end + +function CLocalBeaver:poll() + assert(self._once, "poll loop only run once time.") + self._once = false + + local _, msg = pcall(self._poll, self) + print(msg) - for fd in pairs(self._cos) do - local res = self._cffi.del_fd(self._efd, fd) - assert(res >= 0) - end return 0 end -return CLocalBeaver \ No newline at end of file +return CLocalBeaver diff --git a/source/tools/monitor/unity/beaver/url_export_raw.lua b/source/tools/monitor/unity/beaver/url_export_raw.lua index b6c904bf177105d9f26c650d81090f0133d0348f..2629fba8a12cb52f146c31a4f191d4ff9511a726 100644 --- a/source/tools/monitor/unity/beaver/url_export_raw.lua +++ b/source/tools/monitor/unity/beaver/url_export_raw.lua @@ -14,6 +14,7 @@ function CurlExportRaw:_init_(frame, export) self._export = export self._urlCb["/export/metrics"] = function(tReq) return self:show(tReq) end + self._urlCb["/metrics"] = function(tReq) return self:show(tReq) end self:_install(frame) end diff --git a/source/tools/monitor/unity/beaver/url_guide.lua b/source/tools/monitor/unity/beaver/url_guide.lua index 1362b01e3c9530432e360dcadbee6c942fe7330a..fcb6cd5ca7bef90c427d6055e4254e1dedcce6e4 100644 --- a/source/tools/monitor/unity/beaver/url_guide.lua +++ b/source/tools/monitor/unity/beaver/url_guide.lua @@ -5,56 +5,20 @@ --- require("common.class") +local pystring = require("common.pystring") local ChttpHtml = require("httplib.httpHtml") local CurlGuide = class("CurlIndex", ChttpHtml) function CurlGuide:_init_(frame) ChttpHtml._init_(self) - self._urlCb["/guide"] = function(tReq) return self:guide(tReq) end - self._urlCb["/guide/hotplugin"] = function(tReq) return self:hotplugin(tReq) end - self._urlCb["/guide/oop"] = function(tReq) return self:oop(tReq) end - self._urlCb["/guide/pystring"] = function(tReq) return self:pystring(tReq) end - self._urlCb["/guide/webdevel"] = function(tReq) return self:webdevel(tReq) end - self._urlCb["/guide/proc_probe"] = function(tReq) return self:proc_probe(tReq) end - self._urlCb["/guide/dev_proc"] = function(tReq) return self:dev_proc(tReq) end - self:_install(frame) + self:_installRe("^/guide*", frame) + self._head = "/" -- need to strip + self._filePath = "../beaver/" end -local function loadFile(fPpath) - local path = "../beaver/guide/" .. fPpath - local f = io.open(path,"r") - local s = f:read("*all") - f:close() - return s -end - -function CurlGuide:guide(tReq) - return {title="guide", content=self:markdown(loadFile("guide.md"))} -end - -function CurlGuide:hotplugin(tReq) - return {title="hotplugin", content=self:markdown(loadFile("hotplugin.md"))} -end - -function CurlGuide:oop(tReq) - return {title="oop", content=self:markdown(loadFile("oop.md"))} -end - -function CurlGuide:pystring(tReq) - return {title="pystring", content=self:markdown(loadFile("pystring.md"))} -end - -function CurlGuide:webdevel(tReq) - return {title="webdevel", content=self:markdown(loadFile("webdevel.md"))} -end - -function CurlGuide:proc_probe(tReq) - return {title="proc and probes", content=self:markdown(loadFile("proc_probe.md"))} -end - -function CurlGuide:dev_proc(tReq) - return {title="develop proc interface.", content=self:markdown(loadFile("dev_proc.md"))} +function CurlGuide:callRe(tReq, keep) + return self:reSource(tReq, keep, self._head, self._filePath) end return CurlGuide diff --git a/source/tools/monitor/unity/beeQ/apps.c b/source/tools/monitor/unity/beeQ/apps.c index a0cdc50ae2c5b6047d35800182378278645b4cdb..2e522bc996dce9bdc8c7d90974c2866dae340c48 100644 --- a/source/tools/monitor/unity/beeQ/apps.c +++ b/source/tools/monitor/unity/beeQ/apps.c @@ -15,22 +15,63 @@ static int sample_period = 0; extern char *g_yaml_file; -LUALIB_API void luaL_traceback (lua_State *L, lua_State *L1, const char *msg, int level); +static int lua_traceback(lua_State *L) +{ + const char *errmsg = lua_tostring(L, -1); + lua_getglobal(L, "debug"); + lua_getfield(L, -1, "traceback"); + lua_call(L, 0, 1); + printf("%s \n%s\n", errmsg, lua_tostring(L, -1)); + return 1; +} + +int lua_reg_errFunc(lua_State *L) { + lua_pushcfunction(L, lua_traceback); + return lua_gettop(L); +} -static void report_lua_failed(lua_State *L) { - fprintf(stderr, "\nFATAL ERROR:%s\n\n", lua_tostring(L, -1)); +int lua_check_ret(int ret) { + switch (ret) { + case 0: + break; + case LUA_ERRRUN: + printf("lua runtime error.\n"); + break; + case LUA_ERRMEM: + printf("lua memory error.\n"); + case LUA_ERRERR: + printf("lua exec error.\n"); + case LUA_ERRSYNTAX: + printf("file syntax error.\n"); + case LUA_ERRFILE: + printf("load lua file error.\n"); + default: + printf("bad res for %d\n", ret); + exit(1); + } + return ret; } -static int call_init(lua_State *L) { +int lua_load_do_file(lua_State *L, const char* path) { + int err_func = lua_gettop(L); + int ret; + + ret = luaL_loadfile(L, path); + if (ret) { + return lua_check_ret(ret); + } + ret = lua_pcall(L, 0, LUA_MULTRET, err_func); + return lua_check_ret(ret); +} + +static int call_init(lua_State *L, int err_func) { int ret; lua_Number lret; lua_getglobal(L, "init"); lua_pushinteger(L, (int)gettidv1()); - ret = lua_pcall(L, 1, 1, 0); + ret = lua_pcall(L, 1, 1, err_func); if (ret) { - perror("luaL_call init func error"); - report_lua_failed(L); goto endCall; } @@ -56,7 +97,7 @@ static int call_init(lua_State *L) { static lua_State * app_recv_init(void) { int ret; - + int err_func; /* create a state and load standard library. */ lua_State *L = luaL_newstate(); if (L == NULL) { @@ -65,19 +106,14 @@ static lua_State * app_recv_init(void) { } /* opens all standard Lua libraries into the given state. */ luaL_openlibs(L); + err_func = lua_reg_errFunc(L); - ret = luaL_dofile(L, "bees.lua"); + ret = lua_load_do_file(L, "bees.lua"); if (ret) { - const char *msg = lua_tostring(L, -1); - perror("luaL_dofile error"); - if (msg) { - luaL_traceback(L, L, msg, 0); - fprintf(stderr, "FATAL ERROR:%s\n\n", msg); - } goto endLoad; } - ret = call_init(L); + ret = call_init(L, err_func); if (ret < 0) { goto endCall; } @@ -112,6 +148,7 @@ int app_recv_proc(void* msg, struct beeQ* q) { int lret; lua_State *L = (lua_State *)(q->qarg); char *body; + int err_func; if (counter != sighup_counter) { // check counter for signal. lua_close(L); @@ -134,13 +171,13 @@ int app_recv_proc(void* msg, struct beeQ* q) { goto endMem; } memcpy(body, &pMsg->body[0], len); + err_func = lua_gettop(L); lua_getglobal(L, "proc"); lua_pushlstring(L, body, len); - ret = lua_pcall(L, 1, 1, 0); + ret = lua_pcall(L, 1, 1, err_func); free(body); if (ret) { - perror("lua call error"); - report_lua_failed(L); + lua_check_ret(ret); goto endCall; } @@ -165,6 +202,7 @@ int app_recv_proc(void* msg, struct beeQ* q) { endReturn: endCall: free(msg); + exit(1); return ret; } @@ -190,6 +228,7 @@ int collector_qout(lua_State *L) { static lua_State * app_collector_init(void* q, void* proto_q) { int ret; + int err_func; lua_Number lret; /* create a state and load standard library. */ @@ -199,15 +238,10 @@ static lua_State * app_collector_init(void* q, void* proto_q) { goto endNew; } luaL_openlibs(L); + err_func = lua_reg_errFunc(L); - ret = luaL_dofile(L, "collectors.lua"); + ret = lua_load_do_file(L, "collectors.lua"); if (ret) { - const char *msg = lua_tostring(L, -1); - perror("luaL_dofile error"); - if (msg) { - luaL_traceback(L, L, msg, 0); - fprintf(stderr, "FATAL ERROR:%s\n\n", msg); - } goto endLoad; } @@ -218,10 +252,9 @@ static lua_State * app_collector_init(void* q, void* proto_q) { lua_pushlightuserdata(L, q); lua_pushlightuserdata(L, proto_q); lua_pushstring(L, g_yaml_file); - ret = lua_pcall(L, 3, 1, 0); + ret = lua_pcall(L, 3, 1, err_func); if (ret < 0) { - perror("luaL_call init func error"); - report_lua_failed(L); + lua_check_ret(ret); goto endCall; } @@ -252,6 +285,7 @@ static lua_State * app_collector_init(void* q, void* proto_q) { static int app_collector_work(lua_State **pL, void* q, void* proto_q) { int ret; + int err_func; lua_Number lret; static int counter = 0; @@ -268,12 +302,12 @@ static int app_collector_work(lua_State **pL, void* q, void* proto_q) { counter = sighup_counter; } + err_func = lua_gettop(L); lua_getglobal(L, "work"); lua_pushinteger(L, sample_period); - ret = lua_pcall(L, 1, 1, 0); + ret = lua_pcall(L, 1, 1, err_func); if (ret) { - perror("luaL_call init func error"); - report_lua_failed(L); + lua_check_ret(ret); goto endCall; } diff --git a/source/tools/monitor/unity/beeQ/pack.sh b/source/tools/monitor/unity/beeQ/pack.sh index 7595760e157e44dc302e348ac11c6216ad01ce3a..2b00214122932a589882eb03d885b821e872be9d 100755 --- a/source/tools/monitor/unity/beeQ/pack.sh +++ b/source/tools/monitor/unity/beeQ/pack.sh @@ -35,9 +35,11 @@ cp beeQ/run.sh ${APP}/beeQ/ mkdir ${APP}/collector mkdir ${APP}/collector/native +mkdir ${APP}/collector/outline cp collector/native/*.so* ${APP}/collector/native/ cp collector/native/*.lua ${APP}/collector/native/ cp collector/*.lua ${APP}/collector/ +cp collector/outline/*.lua ${APP}/collector/outline cp collector/plugin.yaml ${APP}/collector/ mkdir ${APP}/common diff --git a/source/tools/monitor/unity/beeQ/proto_queue.lua b/source/tools/monitor/unity/beeQ/proto_queue.lua index f1988d89f251ea5a18d6e96a06de31248f91ac7a..ecece0f1ab04fbcc11a73fb2cec5fb63f4cfdceb 100644 --- a/source/tools/monitor/unity/beeQ/proto_queue.lua +++ b/source/tools/monitor/unity/beeQ/proto_queue.lua @@ -21,12 +21,14 @@ function CprotoQueue:que() end function CprotoQueue:load_label(unity_line, line) + local c = #line.ls for i=0, 4 - 1 do local name = self._ffi.string(unity_line.indexs[i].name) local index = self._ffi.string(unity_line.indexs[i].index) if #name > 0 then - table.insert(line.ls, {name = name, index = index}) + c = c + 1 + line.ls[c] = {name = name, index = index} else return end @@ -34,12 +36,14 @@ function CprotoQueue:load_label(unity_line, line) end function CprotoQueue:load_value(unity_line, line) + local c = #line.vs for i=0, 32 - 1 do local name = self._ffi.string(unity_line.values[i].name) local value = unity_line.values[i].value if #name > 0 then - table.insert(line.vs, {name = name, value = value}) + c = c + 1 + line.vs[c] = {name = name, value = value} else return end @@ -56,7 +60,8 @@ function CprotoQueue:load_log(unity_line, line) end function CprotoQueue:_proc(unity_lines, lines) - for i=0, unity_lines.num - 1 do + local c = #lines["lines"] + for i = 0, unity_lines.num - 1 do local unity_line = unity_lines.line[i] local line = {line = self._ffi.string(unity_line.table), ls = {}, @@ -66,7 +71,8 @@ function CprotoQueue:_proc(unity_lines, lines) self:load_label(unity_line, line) self:load_value(unity_line, line) self:load_log(unity_line, line) - table.insert(lines["lines"], line) + c = c + 1 + lines["lines"][c] = line end end diff --git a/source/tools/monitor/unity/beeQ/run.sh b/source/tools/monitor/unity/beeQ/run.sh index 0221f07c45f9aa1daae62dcbc37ceec5d5341950..9c23dcdd510933a747a26ef07e111ea320f80d48 100755 --- a/source/tools/monitor/unity/beeQ/run.sh +++ b/source/tools/monitor/unity/beeQ/run.sh @@ -9,4 +9,8 @@ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../install/ export LUA_PATH="../../lua/?.lua;../../lua/?/init.lua;" export LUA_CPATH="../../lib/?.so;../../lib/loadall.so;" -./unity-mon +yaml_path=$1 +[ ! $yaml_path ] && yaml_path="/etc/sysak/plugin.yaml" + +echo $yaml_yaml_path +./unity-mon $yaml_path diff --git a/source/tools/monitor/unity/collector/loop.lua b/source/tools/monitor/unity/collector/loop.lua index 1b2781815c62048f500f0497c54ee623e985d429..7e970dc864b91d1ce96cea6961599a7925ea3043 100644 --- a/source/tools/monitor/unity/collector/loop.lua +++ b/source/tools/monitor/unity/collector/loop.lua @@ -7,19 +7,7 @@ require("common.class") local CprotoData = require("common.protoData") local procffi = require("collector.native.procffi") - -local CprocStat = require("collector.proc_stat") -local CprocMeminfo = require("collector.proc_meminfo") -local CprocVmstat = require("collector.proc_vmstat") -local CprocNetdev = require("collector.proc_netdev") -local CprocDiskstats = require("collector.proc_diskstats") -local CprocSockStat = require("collector.proc_sockstat") -local CprocSnmpStat = require("collector.proc_snmp_stat") -local CprocMounts = require("collector.proc_mounts") -local CprocStatm = require("collector.proc_statm") -local CprocBuddyinfo = require("collector.proc_buddyinfo") local Cplugin = require("collector.plugin") - local system = require("common.system") local Cloop = class("loop") @@ -27,21 +15,23 @@ local Cloop = class("loop") function Cloop:_init_(que, proto_q, fYaml) local res = system:parseYaml(fYaml) self._proto = CprotoData.new(que) - self._procs = { - CprocStat.new(self._proto, procffi, res.config.proc_path), - CprocMeminfo.new(self._proto, procffi, res.config.proc_path), - CprocVmstat.new(self._proto, procffi, res.config.proc_path), - CprocNetdev.new(self._proto, procffi, res.config.proc_path), - CprocDiskstats.new(self._proto, procffi, res.config.proc_path), - CprocSockStat.new(self._proto, procffi, res.config.proc_path), - CprocSnmpStat.new(self._proto, procffi, res.config.proc_path), - CprocMounts.new(self._proto, procffi, res.config.proc_path), - CprocStatm.new(self._proto, procffi, res.config.proc_path), - CprocBuddyinfo.new(self._proto, procffi, res.config.proc_path), - } + self:loadLuaPlugin(res, res.config.proc_path) self._plugin = Cplugin.new(self._proto, procffi, que, proto_q, fYaml) end +function Cloop:loadLuaPlugin(res, proc_path) + local luas = res.luaPlugins + + self._procs = {} + if res.luaPlugins then + for i, plugin in ipairs(luas) do + local CProcs = require("collector." .. plugin) + self._procs[i] = CProcs.new(self._proto, procffi, proc_path) + end + end + print("add " .. #self._procs .. " lua plugin.") +end + function Cloop:work(t) local lines = self._proto:protoTable() for k, obj in pairs(self._procs) do diff --git a/source/tools/monitor/unity/collector/native/Makefile b/source/tools/monitor/unity/collector/native/Makefile index 03d3d2f896ef24e04cd05bbb42b30b30553beed7..d185cf7f20c1ca39f6ff8063572fd9a927038207 100644 --- a/source/tools/monitor/unity/collector/native/Makefile +++ b/source/tools/monitor/unity/collector/native/Makefile @@ -1,7 +1,7 @@ CC := gcc CFLAG := -g -fpic LDFLAG := -g -fpic -shared -OBJS := procffi.o sig_stop.o unity_interface.o +OBJS := procffi.o sig_stop.o unity_interface.o fastKsym.o SO := libprocffi.so all: $(SO) diff --git a/source/tools/monitor/unity/collector/native/fastKsym.c b/source/tools/monitor/unity/collector/native/fastKsym.c new file mode 100644 index 0000000000000000000000000000000000000000..ea0e2db8600e05242507d114258aac1e7d3f9e11 --- /dev/null +++ b/source/tools/monitor/unity/collector/native/fastKsym.c @@ -0,0 +1,166 @@ +// +// Created by 廖肇燕 on 2022/12/18. +// + +#include "fastKsym.h" +#include +#include +#include +#include +#include +#include +#include +#include + +static int tfd = 0; +static int sym_cnt = 0; +static struct ksym_cell * gCell = NULL; + +static int load_ksyms(int fd, int stack_only) { + int ret = 0; + int count = 0; + struct ksym_cell cell; + void * addr; + char buf[128]; + + FILE *pf = fopen("/proc/kallsyms", "r"); + + if (pf == NULL) { + ret = -errno; + fprintf(stderr, "open /proc/kallsyms failed, errno, %d, %s", errno, strerror(errno)); + goto endOpen; + } + + while (!feof(pf)) { + if (!fgets(buf, sizeof(buf), pf)) + break; + + ret = sscanf(buf, "%p %c %64s %31s", &addr, &cell.type, cell.func, cell.module); + if (ret == 3) { + cell.module[0] = '\0'; + } else if (ret < 3) { + fprintf(stderr, "bad kallsyms line: %s", buf); + goto endRead; + } + + if (!addr) + continue; + + if (stack_only && (cell.type != 't') && (cell.type != 'T')) { + continue; + } + cell.addr = (addr_t) addr; + + ret = write(fd, &cell, sizeof (cell)); + if (ret < 0) { + fprintf(stderr, "write file failed, errno, %d, %s", errno, strerror(errno)); + goto endWrite; + } + count ++; + } + + fclose(pf); + return count; + + endWrite: + endRead: + fclose(pf); + endOpen: + return ret; +} + +static int sym_cmp(const void *p1, const void *p2) +{ + return ((struct ksym_cell *)p1)->addr > ((struct ksym_cell *)p2)->addr; +} + +static int sort_ksym(int fd, int count) { + int ret = 0 ; + struct stat sb; + void *pmmap; + + ret = fstat(fd, &sb); + if (ret < 0) { + fprintf(stderr, "fstat file failed, errno, %d, %s", errno, strerror(errno)); + goto endStat; + } + + pmmap = mmap(NULL, sb.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (pmmap == NULL) { + fprintf(stderr, "mmap file failed, errno, %d, %s", errno, strerror(errno)); + ret = -EACCES; + goto endMmap; + } + + qsort(pmmap, count, sizeof (struct ksym_cell), sym_cmp); + + gCell = (struct ksym_cell*)pmmap; + + return ret; + endMmap: + endStat: + return ret; +} + +int ksym_setup(int stack_only) { + int ret; + + FILE *pf = tmpfile(); + if (pf == NULL) { + ret = -errno; + fprintf(stderr, "open file failed, errno, %d, %s", errno, strerror(errno)); + goto endTmpfile; + } + + tfd = fileno(pf); + + ret = load_ksyms(tfd, stack_only); + if (ret < 0) { + goto endLoad; + } + sym_cnt = ret; + + ret = sort_ksym(tfd, ret); + if (ret < 0) { + goto endSort; + } + + return ret; + endSort: + endLoad: + close(tfd); + endTmpfile: + return ret; +} + +struct ksym_cell* ksym_search(addr_t key) { + int start = 0, end = sym_cnt; + int mid; + + if (sym_cnt <= 0) { + printf("sym_cnt: %d", sym_cnt); + return NULL; + } + + while (start < end) { + mid = start + (end - start) / 2; + + if (key < gCell[mid].addr) { + end = mid; + } else if (key > gCell[mid].addr) { + start = mid + 1; + } else { + return &gCell[mid]; + } + } + + if (start > 0) { + if ((gCell[start - 1].addr < key) && (key < gCell[start].addr)) { + return &gCell[start - 1]; + } + } + if (start == sym_cnt) { + return &gCell[end - 1]; + } + return NULL; +} diff --git a/source/tools/monitor/unity/collector/native/fastKsym.h b/source/tools/monitor/unity/collector/native/fastKsym.h new file mode 100644 index 0000000000000000000000000000000000000000..57bf8be2a73eaa4bc6e196a4525856c5edcff605 --- /dev/null +++ b/source/tools/monitor/unity/collector/native/fastKsym.h @@ -0,0 +1,20 @@ +// +// Created by 廖肇燕 on 2022/12/18. +// + +#ifndef FASTKSYM_FASTKSYM_H +#define FASTKSYM_FASTKSYM_H + +typedef unsigned long addr_t; + +struct ksym_cell { + addr_t addr; + char func[64]; + char module[31]; + char type; +}; + +int ksym_setup(int stack_only); +struct ksym_cell* ksym_search(addr_t key); + +#endif //FASTKSYM_FASTKSYM_H diff --git a/source/tools/monitor/unity/collector/native/sig_stop.c b/source/tools/monitor/unity/collector/native/sig_stop.c index ec3d865b26dc7de4cdbfc5c1f4090505880811fa..c006072d2e61083ba2c9c45f0349d9c1b8016b9e 100644 --- a/source/tools/monitor/unity/collector/native/sig_stop.c +++ b/source/tools/monitor/unity/collector/native/sig_stop.c @@ -4,8 +4,10 @@ #include "sig_stop.h" #include - #include +#include +#include +#include "fastKsym.h" static volatile int working = 1; @@ -38,7 +40,22 @@ static void sig_register(void) { sigaction(SIGQUIT, &action, NULL); } +static void bump_memlock_rlimit1(void) +{ + struct rlimit rlim_new = { + .rlim_cur = RLIM_INFINITY, + .rlim_max = RLIM_INFINITY, + }; + + if (setrlimit(RLIMIT_MEMLOCK, &rlim_new)) { + fprintf(stderr, "Failed to increase RLIMIT_MEMLOCK limit!\n"); + exit(1); + } +} + void plugin_init(void) { + bump_memlock_rlimit1(); + ksym_setup(1); sig_register(); working = 1; } diff --git a/source/tools/monitor/unity/collector/outline/outline.c b/source/tools/monitor/unity/collector/outline/outline.c index 753049affaf550a81e25baad459c30047caf1b11..eae253a42ba97f34900077b08531592c32321b5c 100644 --- a/source/tools/monitor/unity/collector/outline/outline.c +++ b/source/tools/monitor/unity/collector/outline/outline.c @@ -5,23 +5,19 @@ #include "outline.h" #include -LUALIB_API void luaL_traceback (lua_State *L, lua_State *L1, const char *msg, int level); +extern int lua_reg_errFunc(lua_State *L); +extern int lua_check_ret(int ret); +int lua_load_do_file(lua_State *L, const char* path); -static void report_lua_failed(lua_State *L) { - fprintf(stderr, "\nFATAL ERROR:%s\n\n", lua_tostring(L, -1)); -} - -static int call_init(lua_State *L, void* q, char *fYaml) { +static int call_init(lua_State *L, int err_func, void* q, char *fYaml) { int ret; lua_Number lret; lua_getglobal(L, "init"); lua_pushlightuserdata(L, q); lua_pushstring(L, fYaml); - ret = lua_pcall(L, 2, 1, 0); + ret = lua_pcall(L, 2, 1, err_func); if (ret) { - perror("luaL_call init func error"); - report_lua_failed(L); goto endCall; } @@ -48,6 +44,7 @@ static int call_init(lua_State *L, void* q, char *fYaml) { extern int collector_qout(lua_State *L); static lua_State * pipe_init(void* q, char *fYaml) { int ret; + int err_func; lua_Number lret; /* create a state and load standard library. */ @@ -57,21 +54,17 @@ static lua_State * pipe_init(void* q, char *fYaml) { goto endNew; } luaL_openlibs(L); + err_func = lua_reg_errFunc(L); - ret = luaL_dofile(L, "outline.lua"); + ret = lua_load_do_file(L, "outline.lua"); if (ret) { - const char *msg = lua_tostring(L, -1); - perror("luaL_dofile error"); - if (msg) { - luaL_traceback(L, L, msg, 0); - fprintf(stderr, "FATAL ERROR:%s\n\n", msg); - } goto endLoad; } lua_register(L, "collector_qout", collector_qout); - ret = call_init(L, q, fYaml); - if (ret < 0) { + ret = call_init(L, err_func, q, fYaml); + if (ret) { + lua_check_ret(ret); goto endCall; } return L; @@ -85,13 +78,14 @@ static lua_State * pipe_init(void* q, char *fYaml) { static int work(lua_State *L) { int ret; + int err_func; lua_Number lret; + err_func = lua_gettop(L); lua_getglobal(L, "work"); - ret = lua_pcall(L, 0, 1, 0); + ret = lua_pcall(L, 0, 1, err_func); if (ret) { - perror("lua call error"); - report_lua_failed(L); + lua_check_ret(ret); goto endCall; } diff --git a/source/tools/monitor/unity/collector/outline/pipeMon.lua b/source/tools/monitor/unity/collector/outline/pipeMon.lua index 1298192d7326ae3241c5523890517b17cea40be2..81f2069698ef18cb43d89eff15d38ae33415f5bf 100644 --- a/source/tools/monitor/unity/collector/outline/pipeMon.lua +++ b/source/tools/monitor/unity/collector/outline/pipeMon.lua @@ -35,11 +35,11 @@ end function CpipeMon:setupPipe(fYaml) local res = system:parseYaml(fYaml) - for _, path in ipairs(res.outline) do + for i, path in ipairs(res.outline) do if unistd.access(path) then unistd.unlink(path) end - table.insert(self._paths, path) + self._paths[i] = path socket.unix = require("socket.unix") local s = socket.unix.udp() @@ -58,14 +58,20 @@ local function trans(title, ls, vs, log) local values = {} local logs = {} + local c = 0 for k, v in pairs(ls) do - table.insert(labels, {name=k, index=v}) + c = c + 1 + labels[c] = {name=k, index=v} end + c = 0 for k, v in pairs(vs) do - table.insert(values, {name=k, value=v}) + c = c + 1 + values[c] = {name=k, value=v} end + c = 0 for k, v in pairs(log) do - table.insert(logs, {name=k, log=v}) + c = c + 1 + logs[c] = {name=k, log=v} end return {line = title, ls = labels, vs = values, log = logs} end diff --git a/source/tools/monitor/unity/collector/plugin.lua b/source/tools/monitor/unity/collector/plugin.lua index 41b217465bd1bb10dc5815c4375feeb7d9468a0b..4ccf946be3f5f76c4402d45b18b229a0504aea0f 100644 --- a/source/tools/monitor/unity/collector/plugin.lua +++ b/source/tools/monitor/unity/collector/plugin.lua @@ -55,12 +55,14 @@ function Cplugin:setup(plugins, proto_q) end function Cplugin:load_label(unity_line, line) + local c = #line.ls for i=0, 4 - 1 do local name = self._ffi.string(unity_line.indexs[i].name) local index = self._ffi.string(unity_line.indexs[i].index) if #name > 0 then - table.insert(line.ls, {name = name, index = index}) + c = c + 1 + line.ls[c] = {name = name, index = index} else return end @@ -68,12 +70,14 @@ function Cplugin:load_label(unity_line, line) end function Cplugin:load_value(unity_line, line) + local c = #line.vs for i=0, 32 - 1 do local name = self._ffi.string(unity_line.values[i].name) local value = unity_line.values[i].value if #name > 0 then - table.insert(line.vs, {name = name, value = value}) + c = c + 1 + line.vs[c] = {name = name, value = value} else return end @@ -90,6 +94,7 @@ function Cplugin:load_log(unity_line, line) end function Cplugin:_proc(unity_lines, lines) + local c = #lines["lines"] for i=0, unity_lines.num - 1 do local unity_line = unity_lines.line[i] local line = {line = self._ffi.string(unity_line.table), @@ -100,7 +105,8 @@ function Cplugin:_proc(unity_lines, lines) self:load_label(unity_line, line) self:load_value(unity_line, line) self:load_log(unity_line, line) - table.insert(lines["lines"], line) + c = c + 1 + lines["lines"][c] = line end end diff --git a/source/tools/monitor/unity/collector/plugin.yaml b/source/tools/monitor/unity/collector/plugin.yaml index d622f53a952933331db79fac1998cb4ffaaa295a..3c5ae4a137c4636ac7f0e1fbafcaef7a6da005f5 100644 --- a/source/tools/monitor/unity/collector/plugin.yaml +++ b/source/tools/monitor/unity/collector/plugin.yaml @@ -7,12 +7,15 @@ config: mode: specify name: test_specify # mode: hostip - proc_path: /mnt/home/ # in container mode, like -v /:/mnt/host , should use /mnt/host/ + proc_path: /mnt/host/ # in container mode, like -v /:/mnt/host , should use /mnt/host/ # proc_path: / # in container mode, like -v /:/mnt/host , should use /mnt/host/ outline: - /tmp/sysom +luaPlugins: ["proc_buddyinfo", "proc_diskstats", "proc_meminfo", "proc_mounts", "proc_netdev", + "proc_snmp_stat", "proc_sockstat", "proc_stat", "proc_statm", "proc_vmstat"] + plugins: - so: kmsg description: "collect dmesg info." @@ -32,7 +35,16 @@ plugins: - so: proc_loadavg description: "collect load avg" - + - + so: unity_nosched + description: "nosched:sys hold cpu and didn't scheduling" + - so: net_health + description: "tcp net health." + - so: net_retrans + description: "tcp retrans monitor." + - + so: unity_irqoff + description: "irqoff:detect irq turned off and can't response" metrics: - title: sysak_proc_cpu_total @@ -125,3 +137,43 @@ metrics: head: value help: "buddyinfo of system from /proc/loadavg" type: "gauge" + - title: sysak_IOMonIndForDisksIO + from: IOMonIndForDisksIO + head: value + help: "Disk IO indicators and abnormal events" + type: "gauge" + - title: sysak_IOMonIndForSystemIO + from: IOMonIndForSystemIO + head: value + help: "System indicators and abnormal events about IO" + type: "gauge" + - title: sysak_IOMonDiagLog + from: IOMonDiagLog + head: value + help: "Diagnose log for IO exception" + type: "gauge" + - title: sched_moni_jitter + from: sched_moni_jitter + head: value + help: "nosched/irqoff:sys and irqoff hold cpu and didn't scheduling" + type: "gauge" + - title: sysak_cpu_dist + from: cpu_dist + head: value + help: "task cpu sched dist." + type: "gauge" + - title: sysak_net_health_hist + from: net_health_hist + head: value + help: "net_health_hist" + type: "gauge" + - title: sysak_net_health_count + from: net_health_count + head: value + help: "net_health_count" + type: "gauge" + - title: sysak_net_retrans_count + from: net_retrans_count + head: value + help: "net_retrans_count" + type: "gauge" diff --git a/source/tools/monitor/unity/collector/plugin/Makefile b/source/tools/monitor/unity/collector/plugin/Makefile index 94f8322c6261e7c71f1c302624498a493644575f..32744415fad96e51e06e765e3f752a6389eacaeb 100644 --- a/source/tools/monitor/unity/collector/plugin/Makefile +++ b/source/tools/monitor/unity/collector/plugin/Makefile @@ -4,7 +4,7 @@ LDFLAG := -g -fpic -shared OBJS := proto_sender.o LIB := libproto_sender.a -DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample2 +DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample2 bpfsample unity_nosched unity_irqoff cpudist net_health net_retrans netlink all: $(LIB) $(DEPMOD) diff --git a/source/tools/monitor/unity/collector/plugin/bpf_head.h b/source/tools/monitor/unity/collector/plugin/bpf_head.h index 6acf256f68c6902a7ea1604f3df1d4a5586788ad..99adf37f268966e809a4102d36d13f8333af0374 100644 --- a/source/tools/monitor/unity/collector/plugin/bpf_head.h +++ b/source/tools/monitor/unity/collector/plugin/bpf_head.h @@ -4,15 +4,22 @@ #ifndef UNITY_BPF_HEAD_H #define UNITY_BPF_HEAD_H +#include -#define DEFINE_SEKL_OBJECT(skel_name) \ - struct skel_name##_bpf *skel_name = NULL; \ - static pthread_t perf_thread = 0; +#ifdef COOLBPF_PERF_THREAD -#define DESTORY_SKEL_BOJECT(skel_name) \ - if (perf_thread > 0) \ - kill_perf_thread(perf_thread); \ - skel_name##_bpf__destroy(skel_name) +#define DEFINE_SEKL_OBJECT(skel_name) \ + struct skel_name##_bpf *skel_name = NULL; \ + static pthread_t perf_thread = 0; \ + int thread_worker(struct beeQ *q, void *arg) \ + { \ + perf_thread_worker(arg); \ + return 0; \ + } \ + void handle_lost_events(void *ctx, int cpu, __u64 lost_cnt) \ + { \ + printf("Lost %llu events on CPU #%d!\n", lost_cnt, cpu); \ + } #define LOAD_SKEL_OBJECT(skel_name, perf) \ ( \ @@ -40,7 +47,7 @@ DESTORY_SKEL_BOJECT(skel_name); \ goto load_bpf_skel_out; \ } \ - struct perf_thread_arguments *perf_args = malloc(sizeof(struct perf_thread_arguments)); \ + struct perf_thread_arguments *perf_args = calloc(1, sizeof(struct perf_thread_arguments)); \ if (!perf_args) \ { \ __ret = -ENOMEM; \ @@ -57,4 +64,51 @@ __ret; \ }) +#define DESTORY_SKEL_BOJECT(skel_name) \ + if (perf_thread > 0) \ + kill_perf_thread(perf_thread); \ + skel_name##_bpf__destroy(skel_name); +#else +#define DEFINE_SEKL_OBJECT(skel_name) \ + struct skel_name##_bpf *skel_name = NULL; + +#define LOAD_SKEL_OBJECT(skel_name, perf) \ + ( \ + { \ + __label__ load_bpf_skel_out; \ + int __ret = 0; \ + skel_name = skel_name##_bpf__open(); \ + if (!skel_name) \ + { \ + printf("failed to open BPF object\n"); \ + __ret = -1; \ + goto load_bpf_skel_out; \ + } \ + __ret = skel_name##_bpf__load(skel_name); \ + if (__ret) \ + { \ + printf("failed to load BPF object: %d\n", __ret); \ + DESTORY_SKEL_BOJECT(skel_name); \ + goto load_bpf_skel_out; \ + } \ + __ret = skel_name##_bpf__attach(skel_name); \ + if (__ret) \ + { \ + printf("failed to attach BPF programs: %s\n", strerror(-__ret)); \ + DESTORY_SKEL_BOJECT(skel_name); \ + goto load_bpf_skel_out; \ + } \ + load_bpf_skel_out: \ + __ret; \ + }) + +#define DESTORY_SKEL_BOJECT(skel_name) \ + skel_name##_bpf__destroy(skel_name); +#endif + +#define coobpf_map_find(OBJ, NAME) bpf_object__find_map_fd_by_name(OBJ, NAME) +#define coobpf_key_next(FD, KEY, NEXT) bpf_map_get_next_key(FD, KEY, NEXT) +#define coobpf_key_value(FD, KEY, VALUE) bpf_map_lookup_elem(FD, KEY, VALUE) + +#include "plugin_head.h" #endif //UNITY_BPF_HEAD_H diff --git a/source/tools/monitor/unity/collector/plugin/bpfsample/Makefile b/source/tools/monitor/unity/collector/plugin/bpfsample/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..0b8e79a20ed2d65dd829f77bf3be3032ea35e280 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/bpfsample/Makefile @@ -0,0 +1,8 @@ + +newdirs := $(shell find ./ -type d) + +bpfsrcs := bpfsample.bpf.c +csrcs := bpfsample.c +so := libbpfsample.so + +include ../bpfso.mk \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.bpf.c b/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.bpf.c new file mode 100644 index 0000000000000000000000000000000000000000..12556f2a0323c461d9396571defc7ddaa135ccf5 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.bpf.c @@ -0,0 +1,23 @@ + + +#include +#include +#include "bpfsample.h" + + + +BPF_ARRAY(count, u64, 200); + +SEC("kprobe/netstat_seq_show") +int BPF_KPROBE(netstat_seq_show, struct sock *sk, struct msghdr *msg, size_t size) +{ + int default_key = 0; + u64 *value = bpf_map_lookup_elem(&count, &default_key); + if (value) { + __sync_fetch_and_add(value, 1); + } + return 0; +} + + + diff --git a/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.c b/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.c new file mode 100644 index 0000000000000000000000000000000000000000..18eff585168214cba7e0e413d41b6cfbd9d97f8e --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.c @@ -0,0 +1,41 @@ + + +#include "bpfsample.h" +#include "bpfsample.skel.h" + +#include +#include +#include "../../../../unity/beeQ/beeQ.h" + +DEFINE_SEKL_OBJECT(bpfsample); + +int init(void *arg) +{ + printf("bpfsample plugin install.\n"); + return LOAD_SKEL_OBJECT(bpfsample); +} + +int call(int t, struct unity_lines *lines) +{ + int countfd = bpf_map__fd(bpfsample->maps.count); + int default_key = 0; + uint64_t count = 0; + uint64_t default_count = 0; + struct unity_line* line; + + bpf_map_lookup_elem(countfd, &default_key, &count); + bpf_map_update_elem(countfd, &default_key, &default_count, BPF_ANY); + + unity_alloc_lines(lines, 1); + line = unity_get_line(lines, 0); + unity_set_table(line, "bpfsample"); + unity_set_value(line, 0, "value", count); + + return 0; +} + +void deinit(void) +{ + printf("bpfsample plugin uninstall.\n"); + DESTORY_SKEL_BOJECT(bpfsample); +} diff --git a/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.h b/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.h new file mode 100644 index 0000000000000000000000000000000000000000..9bd981fccbabbfd0b482fd7e123b493ce839e9e4 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.h @@ -0,0 +1,52 @@ + + +#ifndef BPF_SAMPLE_H +#define BPF_SAMPLE_H + +#ifndef __VMLINUX_H__ + +#include "../plugin_head.h" + +#define DEFINE_SEKL_OBJECT(skel_name) \ + struct skel_name##_bpf *skel_name = NULL; + +#define LOAD_SKEL_OBJECT(skel_name) \ + ( \ + { \ + __label__ load_bpf_skel_out; \ + int __ret = 0; \ + skel_name = skel_name##_bpf__open(); \ + if (!skel_name) \ + { \ + printf("failed to open BPF object\n"); \ + __ret = -1; \ + goto load_bpf_skel_out; \ + } \ + __ret = skel_name##_bpf__load(skel_name); \ + if (__ret) \ + { \ + printf("failed to load BPF object: %d\n", __ret); \ + DESTORY_SKEL_BOJECT(skel_name); \ + goto load_bpf_skel_out; \ + } \ + __ret = skel_name##_bpf__attach(skel_name); \ + if (__ret) \ + { \ + printf("failed to attach BPF programs: %s\n", strerror(-__ret)); \ + DESTORY_SKEL_BOJECT(skel_name); \ + goto load_bpf_skel_out; \ + } \ + load_bpf_skel_out: \ + __ret; \ + }) + +#define DESTORY_SKEL_BOJECT(skel_name) \ + skel_name##_bpf__destroy(skel_name); + +int init(void *arg); +int call(int t, struct unity_lines *lines); +void deinit(void); + +#endif + +#endif diff --git a/source/tools/monitor/unity/collector/plugin/cpudist/Makefile b/source/tools/monitor/unity/collector/plugin/cpudist/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..9fc693eb4513feaebc49b5ccf1143651a8a192a3 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/cpudist/Makefile @@ -0,0 +1,8 @@ + +newdirs := $(shell find ./ -type d) + +bpfsrcs := cpudist.bpf.c +csrcs := cpudist.c +so := libcpudist.so + +include ../bpfso.mk \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.bpf.c b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.bpf.c new file mode 100644 index 0000000000000000000000000000000000000000..1dc01053bd26eb1e96b899b233c1b17012c8b2db --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.bpf.c @@ -0,0 +1,40 @@ +// +// Created by 廖肇燕 on 2023/2/23. +// + +#include +#include + +BPF_ARRAY(cpudist, u64, 20); +BPF_HASH(start, u32, u64, 128 * 1024); + +struct sched_switch_args { + u16 type; + u8 flag; + u8 preeempt; + u32 c_pid; + char prev_comm[16]; + u32 prev_pid; + u32 prev_prio; + u64 prev_state; + char next_comm[16]; + u32 next_pid; + u32 next_prio; +}; +SEC("tracepoint/sched/sched_switch") +int sched_switch_hook(struct sched_switch_args *args){ + u64 ts = ns(); + u64 *pv; + u32 prev = args->prev_pid; + u32 next = args->next_pid; + + if (next > 0) { + bpf_map_update_elem(&start, &next, &ts, BPF_ANY); + } + pv = bpf_map_lookup_elem(&start, &prev); + if (pv && ts > *pv) { + hist10_push((struct bpf_map_def *)&cpudist, (ts - *pv) / 1000); + } + return 0; +} + diff --git a/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.c b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.c new file mode 100644 index 0000000000000000000000000000000000000000..5ea28bacb5c54b54cf2852f898c8352843461953 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.c @@ -0,0 +1,86 @@ +// +// Created by 廖肇燕 on 2023/2/23. +// + + +#include "cpudist.h" +#include "../bpf_head.h" +#include "cpudist.skel.h" + +#define CPU_DIST_INDEX 8 +#define DIST_ARRAY_SIZE 20 +DEFINE_SEKL_OBJECT(cpudist); +static int dist_fd = 0; + +int init(void *arg) +{ + int ret; + printf("cpudist plugin install.\n"); + ret = LOAD_SKEL_OBJECT(cpudist, perf); + dist_fd = coobpf_map_find(cpudist->obj, "cpudist"); + return ret; +} + +static int get_dist(unsigned long *locals) { + int i = 0; + unsigned long value = 0; + int key, key_next; + + key = 0; + while (coobpf_key_next(dist_fd, &key, &key_next) == 0) { + coobpf_key_value(dist_fd, &key_next, &value); + locals[i ++] = value; + if (i > DIST_ARRAY_SIZE) { + break; + } + key = key_next; + } + return i; +} + +static int cal_dist(unsigned long* values) { + int i, j; + int size; + static unsigned long rec[DIST_ARRAY_SIZE] = {0}; + unsigned long locals[DIST_ARRAY_SIZE]; + + size = get_dist(locals); + for (i = 0; i < CPU_DIST_INDEX - 1; i ++) { + values[i] = locals[i] - rec[i]; + rec[i] = locals[i]; + } + j = i; + values[j] = 0; + for (; i < size; i ++) { + values[j] += locals[i] - rec[i]; + rec[i] = locals[i]; + } + return 0; +} + + +int call(int t, struct unity_lines *lines) +{ + int i; + unsigned long values[CPU_DIST_INDEX]; + const char *names[] = {"us1", "us10", "us100", "ms1", "ms10", "ms100", "s1", "so"}; + struct unity_line* line; + + unity_alloc_lines(lines, 1); // 预分配好 + line = unity_get_line(lines, 0); + unity_set_table(line, "cpu_dist"); + + cal_dist(values); + for (i = 0; i < CPU_DIST_INDEX; i ++ ) { + unity_set_value(line, i, names[i], values[i]); + } + + return 0; +} + +void deinit(void) +{ + printf("cpudist plugin uninstall.\n"); + DESTORY_SKEL_BOJECT(cpudist); +} + diff --git a/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.h b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.h new file mode 100644 index 0000000000000000000000000000000000000000..1bcac9a4871119def3f042e6becc1a343e2ce298 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.h @@ -0,0 +1,8 @@ +// +// Created by 廖肇燕 on 2023/2/23. +// + +#ifndef UNITY_CPUDIST_H +#define UNITY_CPUDIST_H + +#endif //UNITY_CPUDIST_H diff --git a/source/tools/monitor/unity/collector/plugin/net_health/Makefile b/source/tools/monitor/unity/collector/plugin/net_health/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..55801edc4973f417eed7f907603430c89156f8c8 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/net_health/Makefile @@ -0,0 +1,8 @@ + +newdirs := $(shell find ./ -type d) + +bpfsrcs := net_health.bpf.c +csrcs := net_health.c +so := libnet_health.so + +include ../bpfso.mk \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/net_health/net_health.bpf.c b/source/tools/monitor/unity/collector/plugin/net_health/net_health.bpf.c new file mode 100644 index 0000000000000000000000000000000000000000..5efee8bb68c82250d2d089014bd4aebdc71cae07 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/net_health/net_health.bpf.c @@ -0,0 +1,29 @@ +// +// Created by 廖肇燕 on 2023/2/24. +// +#include +#include + +BPF_ARRAY(outCnt, u64, 2); +BPF_ARRAY(netHist, u64, 20); + +static inline void addCnt(int k, u64 val) { + u64 *pv = bpf_map_lookup_elem(&outCnt, &k); + if (pv) { + __sync_fetch_and_add(pv, val); + } +} + +SEC("kprobe/tcp_validate_incoming") +int j_tcp_validate_incoming(struct pt_regs *ctx) { + struct tcp_sock *tp = (struct tcp_sock *)PT_REGS_PARM1(ctx); + u64 ts = BPF_CORE_READ(tp, srtt_us) >> 3; + u64 ms = ts / 1000; + if (ms > 0) { + addCnt(0, ms); + addCnt(1, 1); + hist10_push((struct bpf_map_def *)&netHist, ms); + } + return 0; +} + diff --git a/source/tools/monitor/unity/collector/plugin/net_health/net_health.c b/source/tools/monitor/unity/collector/plugin/net_health/net_health.c new file mode 100644 index 0000000000000000000000000000000000000000..ea63d2344209ab3bef0130eeda85e1559d3327b1 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/net_health/net_health.c @@ -0,0 +1,108 @@ +// +// Created by 廖肇燕 on 2023/2/24. +// + +#include "net_health.h" +#include "../bpf_head.h" +#include "net_health.skel.h" + +#define NET_DIST_INDEX 4 +#define DIST_ARRAY_SIZE 20 + +DEFINE_SEKL_OBJECT(net_health); +static int cnt_fd = 0; +static int dist_fd = 0; + +int init(void *arg) +{ + int ret; + printf("net_health plugin install.\n"); + ret = LOAD_SKEL_OBJECT(net_health, perf); + cnt_fd = coobpf_map_find(net_health->obj, "outCnt"); + dist_fd = coobpf_map_find(net_health->obj, "netHist"); + return ret; +} + +static int get_dist(unsigned long *locals) { + int i = 0; + unsigned long value = 0; + int key, key_next; + + key = 0; + while (coobpf_key_next(dist_fd, &key, &key_next) == 0) { + coobpf_key_value(dist_fd, &key_next, &value); + locals[i ++] = value; + if (i > DIST_ARRAY_SIZE) { + break; + } + key = key_next; + } + return i; +} + +static int cal_dist(unsigned long* values) { + int i, j; + int size; + static unsigned long rec[DIST_ARRAY_SIZE] = {0}; + unsigned long locals[DIST_ARRAY_SIZE]; + + size = get_dist(locals); + for (i = 0; i < NET_DIST_INDEX - 1; i ++) { + values[i] = locals[i] - rec[i]; + rec[i] = locals[i]; + } + j = i; + values[j] = 0; + for (; i < size; i ++) { + values[j] += locals[i] - rec[i]; + rec[i] = locals[i]; + } + return 0; +} + +static int get_count(unsigned long* values) { + int key; + static unsigned long rec[2]; + unsigned long now[2]; + + key = 0; + coobpf_key_value(cnt_fd, &key, &now[0]); + key = 1; + coobpf_key_value(cnt_fd, &key, &now[1]); + + values[0] = now[0] - rec[0]; rec[0] = now[0]; + values[1] = now[1] - rec[1]; rec[1] = now[1]; + return 0; +} + +int call(int t, struct unity_lines *lines) +{ + int i; + unsigned long values[NET_DIST_INDEX]; + const char *names[] = { "ms10", "ms100", "s1", "so"}; + struct unity_line* line; + + unity_alloc_lines(lines, 2); // 预分配好 + line = unity_get_line(lines, 0); + unity_set_table(line, "net_health_hist"); + + cal_dist(values); + for (i = 0; i < NET_DIST_INDEX; i ++ ) { + unity_set_value(line, i, names[i], values[i]); + } + + get_count(values); + line = unity_get_line(lines, 1); + unity_set_table(line, "net_health_count"); + unity_set_value(line, 0, "sum", values[0]); + unity_set_value(line, 1, "count", values[1]); + return 0; +} + +void deinit(void) +{ + printf("net_health plugin uninstall.\n"); + DESTORY_SKEL_BOJECT(net_health); +} + + diff --git a/source/tools/monitor/unity/collector/plugin/net_health/net_health.h b/source/tools/monitor/unity/collector/plugin/net_health/net_health.h new file mode 100644 index 0000000000000000000000000000000000000000..dd3cebc2333a98d3765226f6aa95566843829310 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/net_health/net_health.h @@ -0,0 +1,8 @@ +// +// Created by 廖肇燕 on 2023/2/24. +// + +#ifndef UNITY_NET_HEALTH_H +#define UNITY_NET_HEALTH_H + +#endif //UNITY_NET_HEALTH_H diff --git a/source/tools/monitor/unity/collector/plugin/net_retrans/Makefile b/source/tools/monitor/unity/collector/plugin/net_retrans/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..09638c6ef30c5b23189d99750dda9e86fed08183 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/net_retrans/Makefile @@ -0,0 +1,8 @@ + +newdirs := $(shell find ./ -type d) + +bpfsrcs := net_retrans.bpf.c +csrcs := net_retrans.c +so := libnet_retrans.so + +include ../bpfso.mk \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.bpf.c b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.bpf.c new file mode 100644 index 0000000000000000000000000000000000000000..70769cbaaba8b8d68ea24e32916aa175a3f418ea --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.bpf.c @@ -0,0 +1,283 @@ +// +// Created by 廖肇燕 on 2023/2/24. +// + +#include +#include +#include "net_retrans.h" + +struct liphdr { + __u8 ver_hdl; + __u8 tos; + __be16 tot_len; + __be16 id; + __be16 frag_off; + __u8 ttl; + __u8 protocol; + __sum16 check; + __be32 saddr; + __be32 daddr; +}; + +#define MAX_ENTRY 128 +#define BPF_F_FAST_STACK_CMP (1ULL << 9) +#define KERN_STACKID_FLAGS (0 | BPF_F_FAST_STACK_CMP) +#define _(P) ({typeof(P) val = 0; bpf_probe_read(&val, sizeof(val), &P); val;}) + +BPF_PERF_OUTPUT(perf, 1024); +BPF_STACK_TRACE(stack, MAX_ENTRY); +BPF_ARRAY(outCnt, u64, NET_RETRANS_TYPE_MAX); + +static inline void addCnt(int k, u64 val) { + u64 *pv = bpf_map_lookup_elem(&outCnt, &k); + if (pv) { + __sync_fetch_and_add(pv, val); + } +} + +static inline int get_tcp_info(struct data_t* pdata, struct tcp_sock *ts) +{ + pdata->rcv_nxt = BPF_CORE_READ(ts, rcv_nxt); + pdata->rcv_wup = BPF_CORE_READ(ts, rcv_wup); + pdata->snd_nxt = BPF_CORE_READ(ts, snd_nxt); + pdata->snd_una = BPF_CORE_READ(ts, snd_una); + pdata->copied_seq = BPF_CORE_READ(ts, copied_seq); + pdata->snd_wnd = BPF_CORE_READ(ts, snd_wnd); + pdata->rcv_wnd = BPF_CORE_READ(ts, rcv_wnd); + + pdata->lost_out = BPF_CORE_READ(ts, lost_out); + pdata->packets_out = BPF_CORE_READ(ts, packets_out); + pdata->retrans_out = BPF_CORE_READ(ts, retrans_out); + pdata->sacked_out = BPF_CORE_READ(ts, sacked_out); + pdata->reordering = BPF_CORE_READ(ts, reordering); + return 0; +} + +static inline int get_skb_info(struct data_t* pdata, struct sk_buff *skb, u32 type) +{ + u16 offset; + u8 ihl; + void* head; + struct liphdr *piph; + struct tcphdr *ptcph; + + addCnt(type, 1); + pdata->type = type; + pdata->sk_state = 0; + + head = (void*)BPF_CORE_READ(skb, head); + offset = BPF_CORE_READ(skb, network_header); + piph = (struct liphdr *)(head + offset); + ihl = _(piph->ver_hdl) & 0x0f; + ptcph = (struct tcphdr *)((void *)piph + ihl * 4); + + pdata->ip_dst = _(piph->daddr); + pdata->dport = BPF_CORE_READ(ptcph, dest); + pdata->ip_src = _(piph->saddr); + pdata->sport = BPF_CORE_READ(ptcph, source); + return 0; +} + +static inline void get_list_task(struct list_head* phead, struct data_t* e) { + struct list_head *next = BPF_CORE_READ(phead, next); + if (next) { + wait_queue_entry_t *entry = container_of(next, wait_queue_entry_t, entry); + struct poll_wqueues *pwq = (struct poll_wqueues *)BPF_CORE_READ(entry, private); + if (pwq) + { + struct task_struct* tsk = (struct task_struct*)BPF_CORE_READ(pwq, polling_task); + if (tsk) { + e->pid = BPF_CORE_READ(tsk, pid); + bpf_probe_read(&e->comm[0], TASK_COMM_LEN, &tsk->comm[0]); + } + } + } +} + +static inline void get_sock_task(struct sock *sk, struct data_t* e) { + struct socket_wq *wq = BPF_CORE_READ(sk, sk_wq); + if (wq) { + struct list_head* phead = (struct list_head*)((char *)wq + offsetof(struct socket_wq, wait.head)); + get_list_task(phead, e); + } +} + +static inline void get_task(struct data_t* pdata, struct sock *sk) { + pdata->pid = 0; + pdata->comm[0] = '\0'; + + get_sock_task(sk, pdata); +} + +static inline void get_socket_task(struct data_t* e, struct sock *sk) { +// struct socket* psocket; +// struct socket_wq *wq; + + e->pid = 0; + e->comm[0] = '\0'; + +// psocket = BPF_CORE_READ(sk, sk_socket); +// wq = BPF_CORE_READ(psocket, wq); +// if (wq) { +// struct list_head* phead = (struct list_head*)((char *)wq + offsetof(struct socket_wq, wait.head)); +// get_list_task(phead, e); +// } +} + +static inline int get_info(struct data_t* pdata, struct sock *sk, u32 type) +{ + struct inet_sock *inet = (struct inet_sock *)sk; + + addCnt(type, 1); + pdata->type = type; + pdata->ip_dst = BPF_CORE_READ(sk, __sk_common.skc_daddr); + pdata->dport = BPF_CORE_READ(sk, __sk_common.skc_dport); + pdata->ip_src = BPF_CORE_READ(sk, __sk_common.skc_rcv_saddr); + pdata->sport = BPF_CORE_READ(inet, inet_sport); + pdata->sk_state = BPF_CORE_READ(sk, __sk_common.skc_state); + return 0; +} + +static inline int check_inner(unsigned int ip) +{ + int i; + const unsigned int array[3][2] = { + {0x0000000A, 0x000000ff}, + {0x000010AC, 0x0000f0ff}, + {0x0000A8C0, 0x0000ffff}, + }; + + if (ip == 0) { + return 1; + } +#pragma unroll 3 + for (i =0; i < 3; i ++) { + if ((ip & array[i][1]) == array[i][0]) { + return 1; + } + } + return 0; +} + +static inline int check_ip(struct data_t* pdata) { + return check_inner(pdata->ip_src) && check_inner(pdata->ip_dst); +} + +SEC("kprobe/tcp_enter_loss") +int j_tcp_enter_loss(struct pt_regs *ctx) +{ + struct sock *sk; + struct data_t data = {}; + u32 stat; + + sk = (struct sock *)PT_REGS_PARM1(ctx); + stat = BPF_CORE_READ(sk, __sk_common.skc_state); + if (stat != 1) { + return 0; + } + get_task(&data, sk); + get_info(&data, sk, NET_RETRANS_TYPE_RTO); + data.stack_id = 0; + get_tcp_info(&data, (struct tcp_sock *)sk); + if (check_ip(&data)) { + bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &data, sizeof(data)); + } + return 0; +} + +SEC("kprobe/tcp_send_probe0") +int j_tcp_send_probe0(struct pt_regs *ctx) +{ + struct sock *sk; + struct data_t data = {}; + u32 stat; + + sk = (struct sock *)PT_REGS_PARM1(ctx); + stat = BPF_CORE_READ(sk, __sk_common.skc_state); + if (stat == 0) { + return 0; + } + + get_info(&data, sk, NET_RETRANS_TYPE_ZERO); + data.stack_id = 0; + get_task(&data, sk); + get_tcp_info(&data, (struct tcp_sock *)sk); + + bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &data, sizeof(data)); + return 0; +} + +SEC("kprobe/tcp_v4_send_reset") +int j_tcp_v4_send_reset(struct pt_regs *ctx) +{ + struct sock *sk; + struct data_t data = {}; + + sk = (struct sock *)PT_REGS_PARM1(ctx); + if (sk == NULL) { + struct sk_buff *skb = (struct sk_buff *)PT_REGS_PARM2(ctx); + get_skb_info(&data, skb, NET_RETRANS_TYPE_RST); + get_task(&data, NULL); + data.stack_id = 0; + } + else { + get_info(&data, sk, NET_RETRANS_TYPE_RST_SK); + get_task(&data, sk); + data.stack_id = bpf_get_stackid(ctx, &stack, KERN_STACKID_FLAGS); + } + if (check_ip(&data)) { + bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &data, sizeof(data)); + } + return 0; +} + +SEC("kprobe/tcp_send_active_reset") +int j_tcp_send_active_reset(struct pt_regs *ctx) +{ + struct sock *sk; + struct data_t data = {}; + + sk = (struct sock *)PT_REGS_PARM1(ctx); + get_info(&data, sk, NET_RETRANS_TYPE_RST_ACTIVE); + data.stack_id = bpf_get_stackid(ctx, &stack, KERN_STACKID_FLAGS); + + get_task(&data, sk); + if (check_ip(&data)) { + bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &data, sizeof(data)); + } + return 0; +} + +#define TCP_SYN_SENT 2 +#define TCPF_SYN_SENT (1 << TCP_SYN_SENT) +SEC("kprobe/tcp_retransmit_skb") +int j_tcp_retransmit_skb(struct pt_regs *ctx){ + struct sock *sk; + unsigned char stat; + + sk = (struct sock *)PT_REGS_PARM1(ctx); + + stat = BPF_CORE_READ(sk, __sk_common.skc_state); + if (stat == TCP_SYN_SENT) + { + struct data_t data = {}; + + get_info(&data, sk, NET_RETRANS_TYPE_SYN); + get_task(&data, sk); + bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &data, sizeof(data)); + } + return 0; +} + +SEC("kprobe/tcp_rtx_synack") +int j_tcp_rtx_synack(struct pt_regs *ctx) +{ + struct sock *sk; + struct data_t data = {}; + + sk = (struct sock *)PT_REGS_PARM1(ctx); + get_info(&data, sk, NET_RETRANS_TYPE_SYN_ACK); + get_task(&data, sk); + bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &data, sizeof(data)); + return 0; +} diff --git a/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c new file mode 100644 index 0000000000000000000000000000000000000000..2f5a196901111c3297ef74ad4cb9e8a87346afbc --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c @@ -0,0 +1,215 @@ +// +// Created by 廖肇燕 on 2023/2/24. +// + +#include "net_retrans.h" +#define COOLBPF_PERF_THREAD +#include "../bpf_head.h" +#include "net_retrans.skel.h" + +#include +#include + +#include +#include +#include + +static volatile int budget = 0; // for log budget +static int cnt_fd = 0; +static int stack_fd = 0; + +const char *net_title[] = {"rto_retrans", "zero_probe", \ + "noport_reset", "bad_sync", \ + "net_proc", "syn_send", "syn_ack"}; + +int proc(int stack_fd, struct data_t *e, struct unity_line *line); +void handle_event(void *ctx, int cpu, void *data, __u32 data_sz) +{ + int ret; + if (budget > 0) { + struct data_t *e = (struct data_t *)data; + struct beeQ *q = (struct beeQ *)ctx; + struct unity_line *line; + struct unity_lines *lines = unity_new_lines(); + + printf("receive: %d msg.\n", data_sz); + + unity_alloc_lines(lines, 1); + line = unity_get_line(lines, 0); + ret = proc(stack_fd, e, line); + if (ret >= 0) { + beeQ_send(q, lines); + } + budget --; + } +} + +DEFINE_SEKL_OBJECT(net_retrans); +int init(void *arg) +{ + int ret; + printf("net_retrans plugin install.\n"); + + ret = LOAD_SKEL_OBJECT(net_retrans, perf); + cnt_fd = coobpf_map_find(net_retrans->obj, "outCnt"); + stack_fd = coobpf_map_find(net_retrans->obj, "stack"); + return ret; +} + +static int get_count(unsigned long *locals) { + int i = 0; + + for (i = 0; i < NET_RETRANS_TYPE_MAX; i ++) { + coobpf_key_value(cnt_fd, &i, &locals[i]); + } + return i; +} + +static int cal_retrans(unsigned long *values) { + int i; + static unsigned long rec[NET_RETRANS_TYPE_MAX] = {0}; + unsigned long locals[NET_RETRANS_TYPE_MAX]; + + get_count(locals); + for (i = 0; i < NET_RETRANS_TYPE_MAX; i ++) { + values[i] = locals[i] - rec[i]; + rec[i] = locals[i]; + } + return 0; +} + +int call(int t, struct unity_lines *lines) { + int i; + unsigned long values[NET_RETRANS_TYPE_MAX]; + struct unity_line* line; + + budget = t; //release log budget + + unity_alloc_lines(lines, 1); // 预分配好 + line = unity_get_line(lines, 0); + unity_set_table(line, "net_retrans_count"); + + cal_retrans(values); + for (i = 0; i < NET_RETRANS_TYPE_MAX; i ++) { + unity_set_value(line, i, net_title[i], values[i]); + } + + return 0; +} + +void deinit(void) +{ + printf("net_retrans plugin uninstall.\n"); + DESTORY_SKEL_BOJECT(net_retrans); +} + +#define LOG_MAX 256 +static char log[LOG_MAX]; + +static char * transIP(unsigned long lip) { + struct in_addr addr; + memcpy(&addr, &lip, sizeof(lip)); + return inet_ntoa(addr); +} + +static const char * resetSock(int stack_fd, struct data_t *e){ + unsigned long addr; + int i = 1; //last stack + struct ksym_cell* cell; + + coobpf_key_value(stack_fd, &i, &addr); + if (addr) { + cell = ksym_search(addr); + if (cell) { + if (strcmp(cell->func, "tcp_v4_rcv") == 0) { + if (e->sk_state == 12) { + return "bad_ack"; // TCP_NEW_SYN_REC + } else { + return "tw_rst"; + } + } else if (strcmp(cell->func, "tcp_check_req") == 0) { + return "bad_syn"; + } else if (strcmp(cell->func, "tcp_v4_do_rcv") == 0) { + return "tcp_stat"; + } else { + return "unknown_sock"; + } + } + } + return "failure_sock"; +} + +static const char * resetActive(int stack_fd, struct data_t *e){ + unsigned long addr; + int i = 1; //last stack + struct ksym_cell* cell; + + coobpf_key_value(stack_fd, &i, &addr); + if (addr) { + cell = ksym_search(addr); + if (cell) { + if (strcmp(cell->func, "tcp_out_of_resources") == 0) { + return "tcp_oom"; + } else if (strcmp(cell->func, "tcp_keepalive_timer") == 0) { + return "keep_alive"; + } else if (strcmp(cell->func, "inet_release") == 0) { + return "bad_close"; + } else if (strcmp(cell->func, "tcp_close") == 0) { + return "bad_close"; + } else if (strcmp(cell->func, "tcp_disconnect") == 0) { + return "tcp_abort"; + } else if (strcmp(cell->func, "tcp_abort") == 0) { + return "tcp_abort"; + } else { + return "unknown_active"; + } + } + } + return "failure_active"; +} + +int proc(int stack_fd, struct data_t *e, struct unity_line *line) { + snprintf(log, LOG_MAX, "task:%d|%s, tcp:%s:%d->%s:%d, state:%d, ", e->pid, e->comm, \ + transIP(e->ip_src), htons(e->sport), \ + transIP(e->ip_src), htons(e->sport), \ + e->sk_state); + switch (e->type) { + case NET_RETRANS_TYPE_RTO: + case NET_RETRANS_TYPE_ZERO: + case NET_RETRANS_TYPE_SYN: + case NET_RETRANS_TYPE_SYN_ACK: + { + char buf[LOG_MAX]; + snprintf(buf, LOG_MAX, "rcv_nxt:%d, rcv_wup:%d, snd_nxt:%d, snd_una:%d, copied_seq:%d, " + "snd_wnd:%d, rcv_wnd:%d, lost_out:%d, packets_out:%d, retrans_out:%d, " + "sacked_out:%d, reordering:%d", + e->rcv_nxt, e->rcv_wup, e->snd_nxt, e->snd_una, e->copied_seq, + e->snd_wnd, e->rcv_wnd, e->lost_out, e->packets_out, e->retrans_out, + e->sacked_out, e->reordering + ); + strncat(log, buf, LOG_MAX); + } + break; + case NET_RETRANS_TYPE_RST: + strncat(log, "noport", LOG_MAX); + break; + case NET_RETRANS_TYPE_RST_SK: + { + const char *type = resetSock(stack_fd, e); + strncat(log, type, LOG_MAX); + } + break; + case NET_RETRANS_TYPE_RST_ACTIVE: + { + const char *type = resetActive(stack_fd, e); + strncat(log, type, LOG_MAX); + } + break; + default: + break; + } + unity_set_table(line, "net_retrans_log"); + unity_set_index(line, 0, "type", net_title[e->type]); + unity_set_log(line, "log", log); + return 0; +} diff --git a/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.h b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.h new file mode 100644 index 0000000000000000000000000000000000000000..5d9539240341700a6d9cde9671a8ea7851d76b92 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.h @@ -0,0 +1,48 @@ +// +// Created by 廖肇燕 on 2023/2/24. +// + +#ifndef UNITY_NET_RETRANS_H +#define UNITY_NET_RETRANS_H + +#define TASK_COMM_LEN 16 + +enum { + NET_RETRANS_TYPE_RTO, + NET_RETRANS_TYPE_ZERO, + NET_RETRANS_TYPE_RST, + NET_RETRANS_TYPE_RST_SK, + NET_RETRANS_TYPE_RST_ACTIVE, + NET_RETRANS_TYPE_SYN, + NET_RETRANS_TYPE_SYN_ACK, + NET_RETRANS_TYPE_MAX, +}; + + +struct data_t { + char comm[TASK_COMM_LEN]; + unsigned int pid; + unsigned int type; + unsigned int ip_src; + unsigned int ip_dst; + unsigned short sport; + unsigned short dport; + unsigned short sk_state; + unsigned short stack_id; + + unsigned int rcv_nxt; + unsigned int rcv_wup; + unsigned int snd_nxt; + unsigned int snd_una; + unsigned int copied_seq; + unsigned int snd_wnd; + unsigned int rcv_wnd; + + unsigned int lost_out; + unsigned int packets_out; + unsigned int retrans_out; + unsigned int sacked_out; + unsigned int reordering; +}; + +#endif //UNITY_NET_RETRANS_H diff --git a/source/tools/monitor/unity/collector/plugin/netlink/Makefile b/source/tools/monitor/unity/collector/plugin/netlink/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..cfe0e64946181e6062b0ed9991f6bad85b9e3d2a --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/netlink/Makefile @@ -0,0 +1,19 @@ +CC := gcc +CFLAG := -g -fpic +LDFLAG := -g -fpic -shared +OBJS := netlink.o +SO := libnetlink.so + +all: $(SO) install + +%.o: %.c + $(CC) -c $< -o $@ $(CFLAG) + +$(SO): $(OBJS) + $(CC) -o $@ $(OBJS) $(LDFLAG) + +install: $(SO) + cp $(SO) ../../native/ + +clean: + rm -f $(SO) $(OBJS) \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/netlink/netlink.c b/source/tools/monitor/unity/collector/plugin/netlink/netlink.c new file mode 100644 index 0000000000000000000000000000000000000000..4bae1997d23274c7096ea8b10843d27bd0062cfd --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/netlink/netlink.c @@ -0,0 +1,90 @@ +#include "netlink.h" +#include +#include +#include + +static char buffer[4096]; + +int get_conntrack_drop() +{ + int total_drop = 0, i; + FILE *fp = NULL; + fp = popen("conntrack -S", "r"); + + if (!fp) + return -1; + + while (fgets(buffer, 4096, fp) != NULL) + { + char *buf = buffer; + while ((buf = strstr(buf, " drop=")) != NULL) + { + buf += strlen(" drop="); + for (i = 0;; i++) + { + if (buf[i] > '9' || buf[i] < '0') + { + buf[i] = 0; + break; + } + } + total_drop += atoi(buf); + buf += i + 1; + } + } + pclose(fp); + return total_drop; +} + +int get_tc_drop() +{ + int total_drop = 0, i; + FILE *fp = NULL; + fp = popen("tc -s qdisc", "r"); + + if (!fp) + return -1; + + while (fgets(buffer, 4096, fp) != NULL) + { + char *buf = buffer; + while ((buf = strstr(buf, "dropped ")) != NULL) + { + buf += strlen("dropped "); + for (i = 0;; i++) + { + if (buf[i] > '9' || buf[i] < '0') + { + buf[i] = 0; + break; + } + } + total_drop += atoi(buf); + buf += i + 1; + } + } + pclose(fp); + return total_drop; +} + + +int init(void * arg) { + printf("netlink plugin install\n"); + return 0; +} + +int call(int t, struct unity_lines* lines) { + struct unity_line* line; + + unity_alloc_lines(lines, 1); + line = unity_get_line(lines, 0); + unity_set_table(line, "netlink"); + unity_set_value(line, 0, "conntrack_drop", get_conntrack_drop()); + unity_set_value(line, 1, "tc_drop", get_tc_drop()); + + return 0; +} + +void deinit(void) { + printf("netlink plugin uninstall\n"); +} diff --git a/source/tools/monitor/unity/collector/plugin/netlink/netlink.h b/source/tools/monitor/unity/collector/plugin/netlink/netlink.h new file mode 100644 index 0000000000000000000000000000000000000000..4a55af11577dee646f55ce4d1807cb077609983f --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/netlink/netlink.h @@ -0,0 +1,11 @@ +#ifndef __NET_LINK_H +#define __NET_LINK_H + +#include "../plugin_head.h" + +int init(void * arg); +int call(int t, struct unity_lines* lines); +void deinit(void); + + +#endif \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/plugin_head.h b/source/tools/monitor/unity/collector/plugin/plugin_head.h index 202a110f9f165042b701ff84d177f55aaaeed683..ee00783df7e7b076a5a985f2e6d2967736010358 100644 --- a/source/tools/monitor/unity/collector/plugin/plugin_head.h +++ b/source/tools/monitor/unity/collector/plugin/plugin_head.h @@ -39,6 +39,7 @@ struct unity_lines { #include "../../beeQ/beeQ.h" #include "../native/sig_stop.h" #include "../native/unity_interface.h" +#include "../native/fastKsym.h" inline struct unity_lines *unity_new_lines(void) __attribute__((always_inline)); inline int unity_alloc_lines(struct unity_lines * lines, unsigned int num) __attribute__((always_inline)); diff --git a/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c b/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c index 7494e32523b568e3b7a618958b6e3b30507ca031..d06dd79219c19bbbd89cba88e66d90d1deeb12be 100644 --- a/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c +++ b/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c @@ -4,6 +4,7 @@ #include "proc_loadavg.h" #define LOADAVG_PATH "/proc/loadavg" +char *real_proc_path; struct stats_load { unsigned long nr_running; @@ -15,6 +16,14 @@ struct stats_load { int init(void * arg) { + int i, lenth; + char *mntpath = get_unity_proc(); + + lenth = strlen(mntpath)+strlen(LOADAVG_PATH); + real_proc_path = calloc(lenth+2, 1); + if (!real_proc_path) + return -errno; + snprintf(real_proc_path, lenth+1, "%s%s", mntpath, LOADAVG_PATH); printf("proc_loadavg plugin install.\n"); return 0; } @@ -28,7 +37,7 @@ int full_line(struct unity_line *uline) fp = NULL; errno = 0; - if ((fp = fopen(LOADAVG_PATH, "r")) == NULL) { + if ((fp = fopen(real_proc_path, "r")) == NULL) { ret = errno; printf("WARN: proc_loadavg install FAIL fopen\n"); return ret; @@ -67,12 +76,14 @@ int call(int t, struct unity_lines* lines) { unity_alloc_lines(lines, 1); line = unity_get_line(lines, 0); - unity_set_table(line, "proc_loadavg"); + unity_set_table(line, "sched_moni"); full_line(line); return 0; } void deinit(void) { + if (real_proc_path) + free(real_proc_path); printf("proc_loadavg plugin uninstall\n"); } diff --git a/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c b/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c index 41850778615f534a1118f8b5d295eb9800234511..5d1075f834219d802ae3a42115d113baacc99b46 100644 --- a/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c +++ b/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c @@ -19,13 +19,23 @@ struct sched_stats { }; long nr_cpus; +char *real_proc_path; struct unity_line **lines1; +struct sched_stats curr_min, curr_max; struct sched_stats *schstats, *schstats2, *delta, *curr, *oldp; int init(void * arg) { int ret; + int i, lenth; + char *mntpath = get_unity_proc(); + lenth = strlen(mntpath)+strlen(SCHEDSTAT_PATH); + real_proc_path = calloc(lenth+2, 1); + if (!real_proc_path) + return -errno; + snprintf(real_proc_path, lenth+1, "%s%s", mntpath, SCHEDSTAT_PATH); + printf("path=%s\n", real_proc_path); errno = 0; lines1 = NULL; @@ -64,6 +74,9 @@ int init(void * arg) printf("WARN: proc_schedstat install FAIL calloc 4\n"); return ret; } + + curr_min.delay = -1ULL; + curr_max.delay = 0; printf("proc_schedstat plugin install.\n"); return 0; } @@ -87,7 +100,7 @@ int full_line(struct unity_line **uline1, struct unity_line *uline2) fp = NULL; errno = 0; idx = 0; - if ((fp = fopen(SCHEDSTAT_PATH, "r")) == NULL) { + if ((fp = fopen(real_proc_path, "r")) == NULL) { ret = errno; printf("WARN: proc_schedstat install FAIL fopen\n"); return ret; @@ -134,6 +147,16 @@ int full_line(struct unity_line **uline1, struct unity_line *uline2) unity_set_index(uline1[cpu], 0, "cpu", cpu_name); unity_set_value(uline1[cpu], 0, "pcount", delta[cpu].pcount); unity_set_value(uline1[cpu], 1, "delay", delta[cpu].delay); + if (delta[cpu].delay > curr_max.delay) { + curr_max.delay = delta[cpu].delay; + curr_max.pcount = delta[cpu].pcount; + strcpy(curr_max.cpu_name, cpu_name); + } + if (delta[cpu].delay < curr_min.delay) { + curr_min.delay = delta[cpu].delay; + curr_min.pcount = delta[cpu].pcount; + strcpy(curr_min.cpu_name, cpu_name); + } } } @@ -149,6 +172,7 @@ int full_line(struct unity_line **uline1, struct unity_line *uline2) unity_set_index(uline2, 0, "summary", "avg"); unity_set_value(uline2, 0, "pcount", sum_cnt/nr_cpus); unity_set_value(uline2, 1, "delay", sum_delay/nr_cpus); + tmp = curr; curr = oldp; oldp = tmp; @@ -159,9 +183,9 @@ int full_line(struct unity_line **uline1, struct unity_line *uline2) int call(int t, struct unity_lines* lines) { int i = 0; static double value = 0.0; - struct unity_line* line2; + struct unity_line *line2, *line3, *line4; - unity_alloc_lines(lines, nr_cpus+1); + unity_alloc_lines(lines, nr_cpus+3); for (i = 0; i < nr_cpus; i++) { lines1[i] = unity_get_line(lines, i); unity_set_table(lines1[i], "proc_schedstat"); @@ -169,6 +193,18 @@ int call(int t, struct unity_lines* lines) { line2 = unity_get_line(lines, nr_cpus); unity_set_table(line2, "proc_schedstat"); full_line(lines1, line2); + + line3 = unity_get_line(lines, nr_cpus+1); + unity_set_table(line3, "proc_schedstat"); + unity_set_index(line3, 0, "max", curr_max.cpu_name); + unity_set_value(line3, 0, "pcount", curr_max.delay); + unity_set_value(line3, 1, "delay", curr_max.pcount); + + line4 = unity_get_line(lines, nr_cpus+2); + unity_set_table(line4, "proc_schedstat"); + unity_set_index(line4, 0, "min", curr_min.cpu_name); + unity_set_value(line4, 0, "pcount", curr_min.delay); + unity_set_value(line4, 1, "delay", curr_min.pcount); return 0; } diff --git a/source/tools/monitor/unity/collector/plugin/proto_sender.c b/source/tools/monitor/unity/collector/plugin/proto_sender.c index bdf1397eb270a7dc82fc4d87068787e3585245de..a05211c1ef8953c54f987c9d32621bb14a12b105 100644 --- a/source/tools/monitor/unity/collector/plugin/proto_sender.c +++ b/source/tools/monitor/unity/collector/plugin/proto_sender.c @@ -10,23 +10,19 @@ #define PROTO_QUEUE_SIZE 64 #define gettidv1() syscall(__NR_gettid) -LUALIB_API void luaL_traceback(lua_State *L, lua_State *L1, const char *msg, int level); +extern int lua_reg_errFunc(lua_State *L); +extern int lua_check_ret(int ret); +int lua_load_do_file(lua_State *L, const char* path); -static void report_lua_failed(lua_State *L) { - fprintf(stderr, "\nFATAL ERROR:%s\n\n", lua_tostring(L, -1)); -} - -static int call_init(lua_State *L, struct beeQ* pushQ) { +static int call_init(lua_State *L, int err_func, struct beeQ* pushQ) { int ret; lua_Number lret; lua_getglobal(L, "init"); lua_pushlightuserdata(L, pushQ); lua_pushinteger(L, (int)gettidv1()); - ret = lua_pcall(L, 2, 1, 0); + ret = lua_pcall(L, 2, 1, err_func); if (ret) { - perror("proto_sender lua init func error"); - report_lua_failed(L); goto endCall; } @@ -52,6 +48,7 @@ static int call_init(lua_State *L, struct beeQ* pushQ) { extern int collector_qout(lua_State *L); lua_State * proto_sender_lua(struct beeQ* pushQ) { int ret; + int err_func; /* create a state and load standard library. */ lua_State *L = luaL_newstate(); @@ -61,20 +58,15 @@ lua_State * proto_sender_lua(struct beeQ* pushQ) { } /* opens all standard Lua libraries into the given state. */ luaL_openlibs(L); + err_func = lua_reg_errFunc(L); - ret = luaL_dofile(L, "proto_send.lua"); + ret = lua_load_do_file(L, "proto_send.lua"); if (ret) { - const char *msg = lua_tostring(L, -1); - perror("luaL_dofile error"); - if (msg) { - luaL_traceback(L, L, msg, 0); - fprintf(stderr, "FATAL ERROR:%s\n\n", msg); - } goto endLoad; } lua_register(L, "collector_qout", collector_qout); - ret = call_init(L, pushQ); + ret = call_init(L, err_func, pushQ); if (ret < 0) { goto endCall; } @@ -88,13 +80,13 @@ lua_State * proto_sender_lua(struct beeQ* pushQ) { struct beeQ* proto_que(lua_State *L) { int ret; + int err_func = lua_gettop(L); struct beeQ* que; lua_getglobal(L, "que"); - ret = lua_pcall(L, 0, 1, 0); + ret = lua_pcall(L, 0, 1, err_func); if (ret) { - perror("proto_que lua que func error"); - report_lua_failed(L); + lua_check_ret(ret); goto endCall; } if (!lua_isuserdata(L, -1)) { // check @@ -120,6 +112,7 @@ struct beeQ* proto_que(lua_State *L) { extern volatile int sighup_counter; int proto_send_proc(void* msg, struct beeQ* q) { int ret = 0; + int err_func; struct unity_lines *lines = (struct unity_lines *)msg; int num = lines->num; struct unity_line * pLine = lines->line; @@ -139,14 +132,14 @@ int proto_send_proc(void* msg, struct beeQ* q) { q->qarg = L; counter = sighup_counter; } + err_func = lua_gettop(L); lua_getglobal(L, "send"); lua_pushnumber(L, num); lua_pushlightuserdata(L, pLine); - ret = lua_pcall(L, 2, 1, 0); + ret = lua_pcall(L, 2, 1, err_func); if (ret) { - perror("lua call error"); - report_lua_failed(L); + lua_check_ret(ret); goto endCall; } diff --git a/source/tools/monitor/unity/collector/plugin/threads/sample_threads.c b/source/tools/monitor/unity/collector/plugin/threads/sample_threads.c index ceee07584f0de034498176f082e803bb28cffee1..169e5062fb04c8a5c371ed77a42b60cc7f86e5c0 100644 --- a/source/tools/monitor/unity/collector/plugin/threads/sample_threads.c +++ b/source/tools/monitor/unity/collector/plugin/threads/sample_threads.c @@ -17,6 +17,7 @@ int init(void * arg) { } static int sample_thread_func(struct beeQ* q, void * arg) { + unsigned int ret; while (plugin_is_working()) { static double value = 1.0; struct unity_line* line; @@ -29,7 +30,10 @@ static int sample_thread_func(struct beeQ* q, void * arg) { unity_set_value(line, 1, "value2", 2.0 + value); unity_set_log(line, "log", "hello world."); beeQ_send(q, lines); - sleep(1); + ret = sleep(5); + if (ret > 0) { // interrupt by signal + break; + } } return 0; } diff --git a/source/tools/monitor/unity/collector/plugin/unity_irqoff/Makefile b/source/tools/monitor/unity/collector/plugin/unity_irqoff/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..9e86c3592f7ac7974a5c8a80ebc4267e638f9278 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/unity_irqoff/Makefile @@ -0,0 +1,8 @@ + +newdirs := $(shell find ./ -type d) + +bpfsrcs := unity_irqoff.bpf.c +csrcs := unity_irqoff.c +so := libunity_irqoff.so + +include ../bpfso.mk diff --git a/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.bpf.c b/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.bpf.c new file mode 100644 index 0000000000000000000000000000000000000000..136445aae6cd5bd1c94918cee53c38c98cda2a1b --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.bpf.c @@ -0,0 +1,151 @@ +#include +#include +#include "sched_jit.h" +#include "unity_irqoff.h" + +#define PERF_MAX_STACK_DEPTH 127 +#define MAX_ENTRIES 10240 +#define BPF_F_FAST_STACK_CMP (1ULL << 9) +#define KERN_STACKID_FLAGS (0 | BPF_F_FAST_STACK_CMP) + +BPF_PERF_OUTPUT(perf, 1024); + +struct bpf_map_def SEC("maps") stackmap = { + .type = BPF_MAP_TYPE_STACK_TRACE, + .key_size = sizeof(u32), + .value_size = PERF_MAX_STACK_DEPTH * sizeof(u64), + .max_entries = 10000, +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, u32); + __type(value, struct arg_info); +} arg_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 1); + __type(key, u32); + __type(value, struct tm_info); +} tm_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(u32)); + __uint(value_size, sizeof(u32)); +} events SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, MAX_ENTRIES); + __type(key, u64); + __type(value, struct info); +} info_map SEC(".maps"); + +#define _(P) ({typeof(P) val = 0; bpf_probe_read(&val, sizeof(val), &P); val;}) + +static inline u64 get_thresh(void) +{ + u64 thresh, i = 0; + struct arg_info *argp; + + argp = bpf_map_lookup_elem(&arg_map, &i); + if (argp) + thresh = argp->thresh; + else + thresh = -1; + + return thresh; +} + +SEC("perf_event") +int hw_irqoff_event(struct bpf_perf_event_data *ctx) +{ + int i = 0; + u64 now, delta, thresh, stamp; + struct tm_info *tmifp; + struct event event = {}; + u32 cpu = bpf_get_smp_processor_id(); + + now = bpf_ktime_get_ns(); + tmifp = bpf_map_lookup_elem(&tm_map, &i); + + if (tmifp) { + stamp = tmifp->last_stamp; + thresh = get_thresh(); + if (stamp && (thresh != -1)) { + delta = now - stamp; + if (delta > thresh) { + event.cpu = cpu; + event.stamp = now; + event.delay = delta; + event.pid = bpf_get_current_pid_tgid(); + bpf_get_current_comm(&event.comm, sizeof(event.comm)); + event.ret = bpf_get_stackid(ctx, &stackmap, KERN_STACKID_FLAGS); + bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, + &event, sizeof(event)); + } + } + } + + return 0; +} + +SEC("perf_event") +int sw_irqoff_event1(struct bpf_perf_event_data *ctx) +{ + int ret, i = 0; + struct tm_info *tmifp, tm; + + tmifp = bpf_map_lookup_elem(&tm_map, &i); + if (tmifp) { + tmifp->last_stamp = bpf_ktime_get_ns(); + } else { + __builtin_memset(&tm, 0, sizeof(tm)); + tm.last_stamp = bpf_ktime_get_ns(); + bpf_map_update_elem(&tm_map, &i, &tm, 0); + } + return 0; +} + +SEC("perf_event") +int sw_irqoff_event2(struct bpf_perf_event_data *ctx) +{ + int i = 0; + u64 now, delta, thresh, stamp; + struct tm_info *tmifp, tm; + struct event event = {}; + u32 cpu = bpf_get_smp_processor_id(); + + now = bpf_ktime_get_ns(); + tmifp = bpf_map_lookup_elem(&tm_map, &i); + + if (tmifp) { + stamp = tmifp->last_stamp; + tmifp->last_stamp = now; + thresh = get_thresh(); + if (stamp && (thresh != -1)) { + delta = now - stamp; + if (delta > thresh) { + event.cpu = cpu; + event.delay = delta; + event.stamp = now; + event.pid = bpf_get_current_pid_tgid(); + bpf_get_current_comm(&event.comm, sizeof(event.comm)); + event.ret = bpf_get_stackid(ctx, &stackmap, KERN_STACKID_FLAGS); + bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, + &event, sizeof(event)); + } + } + } else { + __builtin_memset(&tm, 0, sizeof(tm)); + tm.last_stamp = now; + bpf_map_update_elem(&tm_map, &i, &tm, 0); + } + + return 0; +} + +char LICENSE[] SEC("license") = "GPL"; diff --git a/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.c b/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.c new file mode 100644 index 0000000000000000000000000000000000000000..81f89ea9819f12e41c61ee0602ef84573a0602da --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.c @@ -0,0 +1,233 @@ +#include +#include +#include +#include +//#include +//#include +//#include +#include +#include +#include +#include +#include +#include +#include "unity_irqoff.h" +#include "sched_jit.h" +#include "unity_irqoff.skel.h" +#include "../../../../unity/beeQ/beeQ.h" + +struct env { + __u64 sample_period; + __u64 threshold; +} env = { + .threshold = 50*1000*1000, /* 10ms */ +}; + +static int nr_cpus; +struct sched_jit_summary summary, *percpu_summary; +struct bpf_link **sw_mlinks, **hw_mlinks= NULL; + +DEFINE_SEKL_OBJECT(unity_irqoff); + +static int +open_and_attach_perf_event(struct perf_event_attr *attr, + struct bpf_program *prog, + struct bpf_link *links[]) +{ + int i, fd; + + for (i = 0; i < nr_cpus; i++) { + fd = syscall(__NR_perf_event_open, attr, -1, i, -1, 0); + if (fd < 0) { + /* Ignore CPU that is offline */ + if (errno == ENODEV) + continue; + fprintf(stderr, "failed to init perf sampling: %s\n", + strerror(errno)); + return -1; + } + links[i] = bpf_program__attach_perf_event(prog, fd); + if (!links[i]) { + fprintf(stderr, "failed to attach perf event on cpu: %d\n", i); + close(fd); + return -1; + } + } + return 0; +} + +/* surprise! return 0 if failed! */ +static int attach_prog_to_perf(struct unity_irqoff_bpf *obj) +{ + int ret = 0; + + struct perf_event_attr attr_hw = { + .type = PERF_TYPE_HARDWARE, + .freq = 0, + .sample_period = env.sample_period*2, /* refer to watchdog_update_hrtimer_threshold() */ + .config = PERF_COUNT_HW_CPU_CYCLES, + }; + + struct perf_event_attr attr_sw = { + .type = PERF_TYPE_SOFTWARE, + .freq = 0, + .sample_period = env.sample_period, + .config = PERF_COUNT_SW_CPU_CLOCK, + }; + + if (!open_and_attach_perf_event(&attr_hw, obj->progs.hw_irqoff_event, hw_mlinks)) { + ret = 1<progs.sw_irqoff_event1, sw_mlinks)) + ret = ret | 1<progs.sw_irqoff_event2, sw_mlinks)) + ret = 1<num++; + summary->total += e->delay; + + if (e->delay < 10) { + summary->less10ms++; + } else if (e->delay < 50) { + summary->less50ms++; + } else if (e->delay < 100) { + summary->less100ms++; + } else if (e->delay < 500) { + summary->less500ms++; + } else if (e->delay < 1000) { + summary->less1s++; + } else { + summary->plus1s++; + } +} + +void handle_event(void *ctx, int cpu, void *data, __u32 data_sz) +{ + struct event e; + e = *((struct event *)data); + e.delay = e.delay/(1000*1000); + if (e.cpu > nr_cpus - 1) + return; + update_summary(&summary, &e); +} + +static int irqoff_handler(void *arg, struct unity_irqoff_bpf *unity_irqoff) +{ + int arg_key = 0, err = 0; + struct arg_info arg_info = {}; + int arg_fd; + + /*memset(summary, 0, sizeof(struct sched_jit_summary)); */ + struct perf_thread_arguments *perf_args = + calloc(sizeof(struct perf_thread_arguments), 1); + if (!perf_args) { + printf("Failed to malloc perf_thread_arguments\n"); + DESTORY_SKEL_BOJECT(unity_irqoff); + return -ENOMEM; + } + perf_args->mapfd = bpf_map__fd(unity_irqoff->maps.events); + perf_args->sample_cb = handle_event; + perf_args->lost_cb = handle_lost_events; + perf_args->ctx = arg; + perf_thread = beeQ_send_thread(arg, perf_args, thread_worker); + + arg_fd = bpf_map__fd(unity_irqoff->maps.arg_map); + arg_info.thresh = env.threshold; + env.sample_period = env.threshold*2/5; + err = bpf_map_update_elem(arg_fd, &arg_key, &arg_info, 0); + if (err) { + fprintf(stderr, "Failed to update arg_map\n"); + return err; + } + + if (!(err = attach_prog_to_perf(unity_irqoff))) + return err; + return 0; +} + +static void bump_memlock_rlimit1(void) +{ + struct rlimit rlim_new = { + .rlim_cur = RLIM_INFINITY, + .rlim_max = RLIM_INFINITY, + }; + + if (setrlimit(RLIMIT_MEMLOCK, &rlim_new)) { + fprintf(stderr, "Failed to increase RLIMIT_MEMLOCK limit!\n"); + exit(1); + } +} + +int init(void *arg) +{ + int err; + + nr_cpus = libbpf_num_possible_cpus(); + if (nr_cpus < 0) { + fprintf(stderr, "failed to get # of possible cpus: '%s'!\n", + strerror(-nr_cpus)); + return nr_cpus; + } + + bump_memlock_rlimit1(); + + sw_mlinks = calloc(nr_cpus, sizeof(*sw_mlinks)); + if (!sw_mlinks) { + err = errno; + fprintf(stderr, "failed to alloc sw_mlinks or rlinks\n"); + return err; + } + + hw_mlinks = calloc(nr_cpus, sizeof(*hw_mlinks)); + if (!hw_mlinks) { + err = errno; + fprintf(stderr, "failed to alloc hw_mlinks or rlinks\n"); + free(sw_mlinks); + return err; + } + + unity_irqoff = unity_irqoff_bpf__open_and_load(); + if (!unity_irqoff) { + err = errno; + fprintf(stderr, "failed to open and/or load BPF object\n"); + return err; + } + + irqoff_handler(arg, unity_irqoff); + + return 0; +} + +int call(int t, struct unity_lines *lines) +{ + struct unity_line *line; + + unity_alloc_lines(lines, 1); + line = unity_get_line(lines, 0); + unity_set_table(line, "sched_moni_jitter"); + unity_set_index(line, 0, "mod", "irqoff"); + unity_set_value(line, 0, "dltnum", summary.num); + unity_set_value(line, 1, "dlttm", summary.total); + unity_set_value(line, 2, "lt10ms", summary.less10ms); + unity_set_value(line, 3, "lt50ms", summary.less50ms); + unity_set_value(line, 4, "lt100ms", summary.less100ms); + unity_set_value(line, 5, "lt500ms", summary.less500ms); + unity_set_value(line, 6, "lt1s", summary.less1s); + unity_set_value(line, 7, "mts", summary.plus1s); + return 0; +} + +void deinit(void) +{ + free(sw_mlinks); + free(hw_mlinks); + printf("unity_irqoff plugin uninstall.\n"); + DESTORY_SKEL_BOJECT(unity_irqoff); +} diff --git a/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.h b/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.h new file mode 100644 index 0000000000000000000000000000000000000000..1b024f9e2f25857dfb18d0e5fe282add85e9604c --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.h @@ -0,0 +1,48 @@ +#ifndef __IRQOFF_H +#define __IRQOFF_H + +#define TASK_COMM_LEN 16 +#define CPU_ARRY_LEN 4 +#define CONID_LEN 13 + +struct info { + __u64 prev_counter; +}; + +struct tm_info { + __u64 last_stamp; +}; + +struct arg_info { + __u64 thresh; +}; + +#ifndef __VMLINUX_H__ + +#include "../plugin_head.h" + +#define DEFINE_SEKL_OBJECT(skel_name) \ + struct skel_name##_bpf *skel_name = NULL; \ + static pthread_t perf_thread = 0; \ + int thread_worker(struct beeQ *q, void *arg) \ + { \ + perf_thread_worker(arg); \ + return 0; \ + } \ + void handle_lost_events(void *ctx, int cpu, __u64 lost_cnt) \ + { \ + printf("Lost %llu events on CPU #%d!\n", lost_cnt, cpu); \ + } + +#define DESTORY_SKEL_BOJECT(skel_name) \ + if (perf_thread > 0) \ + kill_perf_thread(perf_thread); \ + skel_name##_bpf__destroy(skel_name); + +int init(void *arg); +int call(int t, struct unity_lines *lines); +void deinit(void); + +#endif +#endif /* __IRQOFF_H */ + diff --git a/source/tools/monitor/unity/collector/plugin/unity_nosched/Makefile b/source/tools/monitor/unity/collector/plugin/unity_nosched/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..3f88651ec50565cb95045caa9aeaeff274458803 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/unity_nosched/Makefile @@ -0,0 +1,8 @@ + +newdirs := $(shell find ./ -type d) + +bpfsrcs := unity_nosched.bpf.c +csrcs := unity_nosched.c +so := libunity_nosched.so + +include ../bpfso.mk diff --git a/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.bpf.c b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.bpf.c new file mode 100644 index 0000000000000000000000000000000000000000..850111668bbfb4e91dd34c2831163e259989c2eb --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.bpf.c @@ -0,0 +1,206 @@ +#include +#include +#include "sched_jit.h" +#include "unity_nosched.h" + +BPF_PERF_OUTPUT(perf, 1024); + +#define BPF_F_FAST_STACK_CMP (1ULL << 9) +#define KERN_STACKID_FLAGS (0 | BPF_F_FAST_STACK_CMP) + +#define BIT_WORD(nr) ((nr) / BITS_PER_LONG) +#define BITS_PER_LONG 64 +#define _(P) ({typeof(P) val = 0; bpf_probe_read(&val, sizeof(val), &P); val;}) + +struct bpf_map_def SEC("maps") args_map = { + .type = BPF_MAP_TYPE_HASH, + .key_size = sizeof(int), + .value_size = sizeof(struct args), + .max_entries = 1, +}; + +struct bpf_map_def SEC("maps") stackmap = { + .type = BPF_MAP_TYPE_STACK_TRACE, + .key_size = sizeof(u32), + .value_size = PERF_MAX_STACK_DEPTH * sizeof(u64), + .max_entries = 1000, +}; + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_HASH); + __uint(max_entries, MAX_MONI_NR); + __type(key, u64); + __type(value, struct latinfo); +} info_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(u32)); + __uint(value_size, sizeof(u32)); +} events SEC(".maps"); + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; + +static inline int test_bit(int nr, const volatile unsigned long *addr) +{ + return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); +} + +static inline int test_ti_thread_flag(struct thread_info *ti, int nr) +{ + int result; + unsigned long *addr; + unsigned long tmp = _(ti->flags); + + addr = &tmp; + result = 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); + return result; +} + +static inline int test_tsk_thread_flag_low(struct task_struct *tsk, int flag) +{ + struct thread_info *tfp; + + tfp = (struct thread_info *)(BPF_CORE_READ(tsk, stack)); + return test_ti_thread_flag(tfp, flag); +} + +/* + * Note: This is based on + * 1) ->thread_info is always be the first element of task_struct if CONFIG_THREAD_INFO_IN_TASK=y + * 2) ->state now is the most nearly begin of task_struct except ->thread_info if it has. + * return ture if struct thread_info is in task_struct */ +static bool test_THREAD_INFO_IN_TASK(struct task_struct *p) +{ + volatile long *pstate; + size_t len; + + pstate = &(p->state); + + len = (u64)pstate - (u64)p; + return (len == sizeof(struct thread_info)); +} + +static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag) +{ + struct thread_info *tfp; + + tfp = (struct thread_info *)tsk; + return test_ti_thread_flag(tfp, flag); +} + +static inline int test_tsk_need_resched(struct task_struct *tsk, int flag) +{ + if (test_THREAD_INFO_IN_TASK(tsk)) + return test_tsk_thread_flag(tsk, flag); + else + return test_tsk_thread_flag_low(tsk, flag); +} + +SEC("kprobe/account_process_tick") +int BPF_KPROBE(account_process_tick, struct task_struct *p, int user_tick) +{ + int args_key; + u64 cpuid; + u64 resched_latency, now; + struct latinfo lati, *latp; + struct args args, *argsp; + + __builtin_memset(&args_key, 0, sizeof(int)); + argsp = bpf_map_lookup_elem(&args_map, &args_key); + if (!argsp) + return 0; + + if (_(p->pid) == 0) + return 0; + + if(!test_tsk_need_resched(p, _(argsp->flag))) + return 0; + + now = bpf_ktime_get_ns(); + __builtin_memset(&cpuid, 0, sizeof(u64)); + cpuid = bpf_get_smp_processor_id(); + latp = bpf_map_lookup_elem(&info_map, &cpuid); + if (latp) { + if (!latp->last_seen_need_resched_ns) { + latp->last_seen_need_resched_ns = now; + latp->ticks_without_resched = 0; + latp->last_perf_event = now; + } else { + latp->ticks_without_resched++; + resched_latency = now - latp->last_perf_event; + if (resched_latency > _(argsp->thresh)) { + struct event event = {0}; + event.stamp = latp->last_seen_need_resched_ns; + event.cpu = cpuid; + event.delay = now - latp->last_seen_need_resched_ns; + event.pid = bpf_get_current_pid_tgid(); + bpf_get_current_comm(&event.comm, sizeof(event.comm)); + event.ret = bpf_get_stackid(ctx, &stackmap, KERN_STACKID_FLAGS); + latp->last_perf_event = now; + bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, + &event, sizeof(event)); + } + } + } else { + __builtin_memset(&lati, 0, sizeof(struct latinfo)); + lati.last_seen_need_resched_ns = now; + lati.last_perf_event = now; + bpf_map_update_elem(&info_map, &cpuid, &lati, BPF_ANY); + } + + return 0; +} + +/* +struct trace_event_raw_sched_switch { + struct trace_entry ent; + char prev_comm[16]; + pid_t prev_pid; + int prev_prio; + long int prev_state; + char next_comm[16]; + pid_t next_pid; + int next_prio; + char __data[0]; +}; + */ +SEC("tp/sched/sched_switch") +int handle_switch(struct trace_event_raw_sched_switch *ctx) +{ + int args_key; + u64 cpuid; + struct latinfo lati, *latp; + struct args *argp; + + __builtin_memset(&args_key, 0, sizeof(int)); + argp = bpf_map_lookup_elem(&args_map, &args_key); + if (!argp) + return 0; + + cpuid = bpf_get_smp_processor_id(); + latp = bpf_map_lookup_elem(&info_map, &cpuid); + if (latp) { + u64 now; + struct event event = {0}; + + now = bpf_ktime_get_ns(); + event.enter = latp->last_seen_need_resched_ns; + if (argp->thresh && event.enter && + (now - event.enter > argp->thresh)) { + event.stamp = now; + event.exit = now; + event.cpu = cpuid; + event.delay = now - latp->last_seen_need_resched_ns; + latp->last_perf_event = now; + event.pid = bpf_get_current_pid_tgid(); + bpf_get_current_comm(&event.comm, sizeof(event.comm)); + event.ret = bpf_get_stackid(ctx, &stackmap, KERN_STACKID_FLAGS); + bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, + &event, sizeof(event)); + } + latp->last_seen_need_resched_ns = 0; + } + + return 0; +} diff --git a/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c new file mode 100644 index 0000000000000000000000000000000000000000..2ab26875edb9f49600fba0f7638dcb238def2339 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c @@ -0,0 +1,153 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include "unity_nosched.h" +#include "sched_jit.h" +#include "unity_nosched.skel.h" +#include "../../../../unity/beeQ/beeQ.h" + +#ifdef __x86_64__ +#define TIF_NEED_RESCHED 3 +#elif defined (__aarch64__) +#define TIF_NEED_RESCHED 1 +#endif + +unsigned int nr_cpus; +struct sched_jit_summary summary; + +static void update_summary(struct sched_jit_summary* summary, const struct event *e) +{ + summary->num++; + summary->total += e->delay; + + if (e->delay < 10) { + summary->less10ms++; + } else if (e->delay < 50) { + summary->less50ms++; + } else if (e->delay < 100) { + summary->less100ms++; + } else if (e->delay < 500) { + summary->less500ms++; + } else if (e->delay < 1000) { + summary->less1s++; + } else { + summary->plus1s++; + } +} + +void handle_event(void *ctx, int cpu, void *data, __u32 data_sz) +{ + struct event *e = (struct event *)data; + + e->delay = e->delay/(1000*1000); + if (e->cpu > nr_cpus - 1) + return; + if (e->exit != 0) + update_summary(&summary, e); +} + + +DEFINE_SEKL_OBJECT(unity_nosched); + +static void bump_memlock_rlimit1(void) +{ + struct rlimit rlim_new = { + .rlim_cur = RLIM_INFINITY, + .rlim_max = RLIM_INFINITY, + }; + + if (setrlimit(RLIMIT_MEMLOCK, &rlim_new)) { + fprintf(stderr, "Failed to increase RLIMIT_MEMLOCK limit!\n"); + exit(1); + } +} + +int init(void *arg) +{ + int err, argfd, args_key; + struct args args; + + bump_memlock_rlimit1(); + unity_nosched = unity_nosched_bpf__open(); + if (!unity_nosched) { + err = errno; + printf("failed to open BPF object\n"); + return -err; + } + + err = unity_nosched_bpf__load(unity_nosched); + if (err) { + fprintf(stderr, "Failed to load BPF skeleton\n"); + DESTORY_SKEL_BOJECT(unity_nosched); + return -err; + } + + argfd = bpf_map__fd(unity_nosched->maps.args_map); + args_key = 0; + args.flag = TIF_NEED_RESCHED; + args.thresh = 50*1000*1000; /* 50ms */ + + err = bpf_map_update_elem(argfd, &args_key, &args, 0); + if (err) { + fprintf(stderr, "Failed to update flag map\n"); + DESTORY_SKEL_BOJECT(unity_nosched); + return err; + } + + nr_cpus = libbpf_num_possible_cpus(); + memset(&summary, 0, sizeof(summary)); + { + struct perf_thread_arguments *perf_args = + malloc(sizeof(struct perf_thread_arguments)); + if (!perf_args) { + printf("Failed to malloc perf_thread_arguments\n"); + DESTORY_SKEL_BOJECT(unity_nosched); + return -ENOMEM; + } + memset(perf_args, 0, sizeof(struct perf_thread_arguments)); + perf_args->mapfd = bpf_map__fd(unity_nosched->maps.events); + perf_args->sample_cb = handle_event; + perf_args->lost_cb = handle_lost_events; + perf_args->ctx = arg; + perf_thread = beeQ_send_thread(arg, perf_args, thread_worker); + } + err = unity_nosched_bpf__attach(unity_nosched); + if (err) { + printf("failed to attach BPF programs: %s\n", strerror(err)); + DESTORY_SKEL_BOJECT(unity_nosched); + return err; + } + + printf("unity_nosched plugin install.\n"); + return 0; +} + +int call(int t, struct unity_lines *lines) +{ + struct unity_line *line; + + unity_alloc_lines(lines, 1); + line = unity_get_line(lines, 0); + unity_set_table(line, "sched_moni_jitter"); + unity_set_index(line, 0, "mod", "noschd"); + unity_set_value(line, 0, "dltnum", summary.num); + unity_set_value(line, 1, "dlttm", summary.total); + unity_set_value(line, 2, "lt10ms", summary.less10ms); + unity_set_value(line, 3, "lt50ms", summary.less50ms); + unity_set_value(line, 4, "lt100ms", summary.less100ms); + unity_set_value(line, 5, "lt500ms", summary.less500ms); + unity_set_value(line, 6, "lt1s", summary.less1s); + unity_set_value(line, 7, "mts", summary.plus1s); + return 0; +} + +void deinit(void) +{ + printf("unity_nosched plugin uninstall.\n"); + DESTORY_SKEL_BOJECT(unity_nosched); +} diff --git a/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.h b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.h new file mode 100644 index 0000000000000000000000000000000000000000..f7280f4bf2088230af297d819002d13cb238924b --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.h @@ -0,0 +1,92 @@ + + +#ifndef BPF_SAMPLE_H +#define BPF_SAMPLE_H + +#define MAX_MONI_NR 1024 + +#define PERF_MAX_STACK_DEPTH 32 +struct args { + int flag; + unsigned long long thresh; +}; + +struct latinfo { + unsigned long long last_seen_need_resched_ns; + unsigned long long last_perf_event; + int ticks_without_resched; +}; + +#ifndef __VMLINUX_H__ + +#include "../plugin_head.h" + +#define DEFINE_SEKL_OBJECT(skel_name) \ + struct skel_name##_bpf *skel_name = NULL; \ + static pthread_t perf_thread = 0; \ + int thread_worker(struct beeQ *q, void *arg) \ + { \ + perf_thread_worker(arg); \ + return 0; \ + } \ + void handle_lost_events(void *ctx, int cpu, __u64 lost_cnt) \ + { \ + printf("Lost %llu events on CPU #%d!\n", lost_cnt, cpu); \ + } + +#define LOAD_SKEL_OBJECT(skel_name, perf) \ + ( \ + { \ + __label__ load_bpf_skel_out; \ + int __ret = 0; \ + skel_name = skel_name##_bpf__open(); \ + if (!skel_name) \ + { \ + printf("failed to open BPF object\n"); \ + __ret = -1; \ + goto load_bpf_skel_out; \ + } \ + __ret = skel_name##_bpf__load(skel_name); \ + if (__ret) \ + { \ + printf("failed to load BPF object: %d\n", __ret); \ + DESTORY_SKEL_BOJECT(skel_name); \ + goto load_bpf_skel_out; \ + } \ + __ret = skel_name##_bpf__attach(skel_name); \ + if (__ret) \ + { \ + printf("failed to attach BPF programs: %s\n", strerror(-__ret)); \ + DESTORY_SKEL_BOJECT(skel_name); \ + goto load_bpf_skel_out; \ + } \ + struct perf_thread_arguments *perf_args = malloc(sizeof(struct perf_thread_arguments)); \ + if (!perf_args) \ + { \ + __ret = -ENOMEM; \ + printf("failed to allocate memory: %s\n", strerror(-__ret)); \ + DESTORY_SKEL_BOJECT(skel_name); \ + goto load_bpf_skel_out; \ + } \ + memset(perf_args, 0, sizeof(struct perf_thread_arguments)); \ + perf_args->mapfd = bpf_map__fd(skel_name->maps.perf); \ + perf_args->sample_cb = handle_event; \ + perf_args->lost_cb = handle_lost_events; \ + perf_args->ctx = arg; \ + perf_thread = beeQ_send_thread(arg, perf_args, thread_worker); \ + load_bpf_skel_out: \ + __ret; \ + }) + +#define DESTORY_SKEL_BOJECT(skel_name) \ + if (perf_thread > 0) \ + kill_perf_thread(perf_thread); \ + skel_name##_bpf__destroy(skel_name); + +int init(void *arg); +int call(int t, struct unity_lines *lines); +void deinit(void); + +#endif + +#endif diff --git a/source/tools/monitor/unity/collector/proc_buddyinfo.lua b/source/tools/monitor/unity/collector/proc_buddyinfo.lua index 8d85696eb6ca22865f8683518ede044cefd3e5fa..f2fc8c9bc0485af26e32dae40b4243bfe82d5948 100644 --- a/source/tools/monitor/unity/collector/proc_buddyinfo.lua +++ b/source/tools/monitor/unity/collector/proc_buddyinfo.lua @@ -17,10 +17,10 @@ end function CprocBuddyinfo:proc(elapsed, lines) CvProc.proc(self) - buddyinfo = {} + local buddyinfo = {} for line in io.lines(self.pFile) do if string.find(line,"Normal") then - subline = pystring:split(line,"Normal",1)[2] + local subline = pystring:split(line,"Normal",1)[2] for num in string.gmatch(subline, "%d+") do table.insert(buddyinfo,tonumber(num)) end @@ -31,7 +31,7 @@ function CprocBuddyinfo:proc(elapsed, lines) if not buddyinfo then for line in io.lines(self.pFile) do if string.find(line,"DMA32") then - subline = pystring:split(line,"DMA32",1)[2] + local subline = pystring:split(line,"DMA32",1)[2] for num in string.gmatch(subline, "%d+") do table.insert(buddyinfo,tonumber(num)) end diff --git a/source/tools/monitor/unity/collector/proc_mounts.lua b/source/tools/monitor/unity/collector/proc_mounts.lua index 4de16c327bcba74a277713bff7996ea95b40c3e0..6ad8644a6c1fd70b97f752d1f149df1c5747b1a1 100644 --- a/source/tools/monitor/unity/collector/proc_mounts.lua +++ b/source/tools/monitor/unity/collector/proc_mounts.lua @@ -23,8 +23,10 @@ local function get_lines(fName) local fName = fName or "/proc/mounts" local f = assert(io.open(fName, "r")) + local c = 0 for line in f:lines() do - table.insert(lines, line) + c = c + 1 + lines[c] = line end return lines end diff --git a/source/tools/monitor/unity/collector/proc_netdev.lua b/source/tools/monitor/unity/collector/proc_netdev.lua index 7a2f09543dad999dfd1da4b41e020dd027ef0e57..2682bfdb7b5688cc3b2cb0e737b7f04fd5d26878 100644 --- a/source/tools/monitor/unity/collector/proc_netdev.lua +++ b/source/tools/monitor/unity/collector/proc_netdev.lua @@ -27,7 +27,7 @@ function CprocNetdev:_getNewValue(ifName, data) local now = {} local index = self:_netdevIndex() for i = 1, #index do - table.insert(now, tonumber(data.value[i - 1])) + now[i] = tonumber(data.value[i - 1]) end self._lastData[ifName] = now self._lastIfNames[ifName] = 1 @@ -43,13 +43,13 @@ function CprocNetdev:_calcIf(ifName, data, res, elapsed) } for i, index in ipairs(index) do local nowValue = tonumber(data.value[i -1]) - table.insert(now, nowValue) + now[i] = nowValue local value = (nowValue - res[i]) / elapsed local cell = { name = index, value = value } - table.insert(protoTable.vs, cell) + protoTable.vs[i] = cell end self:appendLine(protoTable) diff --git a/source/tools/monitor/unity/collector/proc_snmp_stat.lua b/source/tools/monitor/unity/collector/proc_snmp_stat.lua index 199bdbea6d5f63dfe9b075f44790253a73849845..917cd7d433b684f40a36071c3dd4714df41a4c1b 100644 --- a/source/tools/monitor/unity/collector/proc_snmp_stat.lua +++ b/source/tools/monitor/unity/collector/proc_snmp_stat.lua @@ -58,12 +58,14 @@ end function CprocSnmpStat:pack(labels, logs) local vs = {} + local c = 0 for k, v in pairs(labels) do local value = { name = k, value = tonumber(v) } - table.insert(vs, value) + c = c + 1 + vs[c] = value end self:appendLine(self:_packProto("pkt_status", nil, vs)) if #logs > 0 then @@ -82,6 +84,7 @@ function CprocSnmpStat:check(now) local labels = self:createLabels() local logs = {} if self._rec then + local c = 0 for k, v in pairs(now) do if self._rec[k] and self._rec[k] < v then -- local delta = v - self._rec[k] @@ -91,7 +94,8 @@ function CprocSnmpStat:check(now) labels[lk] = lv + delta end end - table.insert(logs, string.format("%s: %d", k, tonumber(delta))) + c = c + 1 + logs[c] = string.format("%s: %d", k, tonumber(delta)) end end end diff --git a/source/tools/monitor/unity/collector/proc_sockstat.lua b/source/tools/monitor/unity/collector/proc_sockstat.lua index d906ae97bf744e556c50846b5a3419d18c012d26..bf7eb80def26d6c45ea51efbcc94026dfe4befbd 100644 --- a/source/tools/monitor/unity/collector/proc_sockstat.lua +++ b/source/tools/monitor/unity/collector/proc_sockstat.lua @@ -17,6 +17,7 @@ end function CprocSockStat:proc(elapsed, lines) CvProc.proc(self) local vs = {} + local c = 0 for line in io.lines(self.pFile) do local cells = pystring:split(line, ":", 1) if #cells > 1 then @@ -31,7 +32,8 @@ function CprocSockStat:proc(elapsed, lines) name = title, value = tonumber(bodies[2 * i]) } - table.insert(vs, v) + c = c + 1 + vs[c] = v end end end diff --git a/source/tools/monitor/unity/collector/proc_statm.lua b/source/tools/monitor/unity/collector/proc_statm.lua index d3ebeb6083b24bff86186f1536c45c3e3684b68c..9a9fff79810dff55653c7ddd9dbb1fabb535e0a2 100644 --- a/source/tools/monitor/unity/collector/proc_statm.lua +++ b/source/tools/monitor/unity/collector/proc_statm.lua @@ -16,6 +16,7 @@ end function CprocStatm:proc(elapsed, lines) CvProc.proc(self) local heads = {"size", "resident", "shared", "text", "lib", "data", "dt"} + local c = 0 for line in io.lines("/proc/self/statm") do local vs = {} local data = self._ffi.new("var_long_t") @@ -26,7 +27,8 @@ function CprocStatm:proc(elapsed, lines) name = k, value = tonumber(data.value[i - 1]), } - table.insert(vs, cell) + c = c + 1 + vs[c] = cell end self:appendLine(self:_packProto("self_statm", nil, vs)) end diff --git a/source/tools/monitor/unity/collector/vproc.lua b/source/tools/monitor/unity/collector/vproc.lua index 33c016a5551bd4dff1cecd579e9e2272c80cdb05..30d8fa3d4409e35f2115a5212fc2d8af255501ee 100644 --- a/source/tools/monitor/unity/collector/vproc.lua +++ b/source/tools/monitor/unity/collector/vproc.lua @@ -31,8 +31,10 @@ function CvProc:copyLine(line) end function CvProc:push(lines) + local c = #lines for _, v in ipairs(self._lines["lines"]) do - table.insert(lines["lines"], v) + c = c + 1 + lines["lines"][c] = v end self._lines = nil return lines diff --git a/source/tools/monitor/unity/common/lineParse.lua b/source/tools/monitor/unity/common/lineParse.lua index ee8de957297ffb94db912e101677128c09dfaa2d..33d6486a0e831608f41ce0956c6e87539b1be50e 100644 --- a/source/tools/monitor/unity/common/lineParse.lua +++ b/source/tools/monitor/unity/common/lineParse.lua @@ -81,18 +81,23 @@ function module.pack(title, ls, vs) local line = title if system:keyCount(ls) > 0 then local lss = {} + local c = 0 for k, v in pairs(ls) do - table.insert(lss, k .. "=" .. v) + c = c + 1 + lss[c] = table.concat({k, v}, "=") end line = line .. ',' .. pystring:join(",", lss) end local vss = {} + local c = 0 for k, v in pairs(vs) do local tStr = type(v) if tStr == "number" then - table.insert(vss, k .. '=' .. tostring(v)) + c = c + 1 + vss[c] = table.concat({k, tostring(v)}, "=") elseif tStr == "string" then - table.insert(vss, k .. '=' .. json.encode(v)) + c = c + 1 + vss[c] = table.concat({k, json.encode(v)}, "=") else error("bad value type for " .. tStr) end diff --git a/source/tools/monitor/unity/common/lmd.lua b/source/tools/monitor/unity/common/lmd.lua index 67a0e1bf8693ec77bb142e8b913530e272ade0b6..685c66d1fea6d1692dc3abbce28eb5b8bf4e00f9 100644 --- a/source/tools/monitor/unity/common/lmd.lua +++ b/source/tools/monitor/unity/common/lmd.lua @@ -10,6 +10,7 @@ require("common.class") local pystring = require("common.pystring") local Clmd = class("lmd") +local srcPath = "" function Clmd:_init_() self._escs = '\\`*_{}[]()>#+-.!' @@ -98,9 +99,24 @@ local function pCode(s) end end +local function images(s) + local name, link = unpack(pystring:split(s, "](", 1)) + name = string.sub(name, 3) -- ![]() + link = string.sub(link, 1, -2) + + if string.sub(name, -1, -1) == "\\" then + return s + end + if string.sub(link, -1, -1) == "\\" then + return s + end + local path = srcPath .. link + return string.format('%s', path, name) +end + local function links(s) local name, link = unpack(pystring:split(s, "](", 1)) - name = string.sub(name, 2) + name = string.sub(name, 2) -- []() link = string.sub(link, 1, -2) if string.sub(name, -1, -1) == "\\" then return s @@ -111,6 +127,10 @@ local function links(s) return string.format('%s', link, name) end +local function pImages(s) + return string.gsub(s, "!%[.-%]%(.-%)", function(s) return images(s) end) +end + local function pLink(s) return string.gsub(s, "%[.-%]%(.-%)", function(s) return links(s) end) end @@ -353,6 +373,7 @@ function Clmd:seg(s) s = pItalic(s) s = pDelete(s) s = pCode(s) + s = pImages(s) s = pLink(s) return pEscape(s) end @@ -362,11 +383,12 @@ function Clmd:pSeg(s) return pystring:join("", {"

", pEnter(s), "

"}) end -function Clmd:toHtml(md) +function Clmd:toHtml(md, path) local mds = pystring:split(md, '\n') local res = {} local len = #mds local stop = 0 + srcPath = path or "" for i = 1, len do local line = mds[i] diff --git a/source/tools/monitor/unity/common/pystring.lua b/source/tools/monitor/unity/common/pystring.lua index 25278335fd9d1003e143608efe260766908b6e2b..c38a0d1fc26ae6fb6b2233dac40034adad2140d8 100644 --- a/source/tools/monitor/unity/common/pystring.lua +++ b/source/tools/monitor/unity/common/pystring.lua @@ -55,8 +55,10 @@ end local function setupDelimiter(delimiter) local rt = {} + local i = 0 for c in string.gmatch(delimiter, ".") do - table.insert(rt, checkDelimiter(c)) + i = i + 1 + rt[i] = checkDelimiter(c) end return table.concat(rt) end @@ -73,23 +75,30 @@ end function pystring:split(s, delimiter, n) local result = {} + if not delimiter or delimiter == "" then -- for blank, gsub multi blank to single + s = string.gsub(s, "%s+", " ") + end local delimiter = setupDelimiter(delimiter or " ") local n = n or 2 ^ 63 - 1 local nums = 0 local beg = 1 + local c = 0 while (true) do local iBeg, iEnd = string.find(s, delimiter, beg) if (iBeg) then - table.insert(result, string.sub(s, beg, iBeg - 1)) + c = c + 1 + result[c] = string.sub(s, beg, iBeg - 1) beg = iEnd + 1 nums = nums + 1 if nums >= n then - table.insert(result, string.sub(s, beg, string.len(s))) + c = c + 1 + result[c] = string.sub(s, beg, string.len(s)) break end else - table.insert(result, string.sub(s, beg, string.len(s))) + c = c + 1 + result[c] = string.sub(s, beg, string.len(s)) break end end @@ -112,19 +121,23 @@ function pystring:rsplit(s, delimiter, n) rDel = setupDelimiter(rDel) local nums = 0 local beg = 1 + local c = 0 while (true) do local iBeg, iEnd = string.find(rs, rDel, beg) if (iBeg) then - table.insert(result, string.sub(s, len - (iBeg - 1),len - beg)) + c = c + 1 + result[c] = string.sub(s, len - (iBeg - 1),len - beg) beg = iEnd + 1 nums = nums + 1 if nums >= n then - table.insert(result, string.sub(s, 1, len - beg)) + c = c + 1 + result[c] = string.sub(s, 1, len - beg) break end else - table.insert(result, string.sub(s, 1, len - beg)) + c = c + 1 + result[c] = string.sub(s, 1, len - beg) break end end diff --git a/source/tools/monitor/unity/common/system.lua b/source/tools/monitor/unity/common/system.lua index 1c083430d397f0f951fae47ee0afa152355510eb..68dfa7f5d46d009b8cc28502a75ddb2a53259ddd 100644 --- a/source/tools/monitor/unity/common/system.lua +++ b/source/tools/monitor/unity/common/system.lua @@ -125,4 +125,9 @@ function system:parseYaml(fYaml) return lyaml.load(s) end +function system:posixError(msg, err, errno) + local s = msg .. string.format(": %s, errno: %d", err, errno) + error(s) +end + return system \ No newline at end of file diff --git a/source/tools/monitor/unity/httplib/httpBase.lua b/source/tools/monitor/unity/httplib/httpBase.lua index 7ddbc24e43b3c9aa16f0eff8ef7394103590b2af..3ec1ff4f4c09243a7b1965358e4816db35e307cf 100644 --- a/source/tools/monitor/unity/httplib/httpBase.lua +++ b/source/tools/monitor/unity/httplib/httpBase.lua @@ -5,6 +5,7 @@ --- require("common.class") +local system = require("common.system") local ChttpComm = require("httplib.httpComm") local ChttpBase = class("ChttpBase", ChttpComm) @@ -20,22 +21,44 @@ function ChttpBase:_install(frame) end end +function ChttpBase:_installRe(path, frame) + frame:registerRe(path, self) +end + function ChttpBase:echo(tRet, keep) error("ChttpBase:echo is a virtual function.") end local function checkKeep(tReq) local conn = tReq.header["connection"] - if conn and string.lower(conn) == "keep-alive" then - return true + if tReq.vers == "1.0" then + if conn and string.lower(conn) == "keep-alive" then + return true + else -- for http 1.0, close as default + return false + end + else + if conn and string.lower(conn) == "close" then + return false + else -- for http 1.1 and newer, for keep-alive as default + return true + end end - return false end function ChttpBase:call(tReq) + local keep = checkKeep(tReq) local tRet = self._urlCb[tReq.path](tReq) + local res = self:echo(tRet, keep) + + return res, keep +end + +function ChttpBase:calls(tReq) local keep = checkKeep(tReq) - return self:echo(tRet, keep), keep + local res = self:callRe(tReq, keep) + + return res, keep end return ChttpBase diff --git a/source/tools/monitor/unity/httplib/httpComm.lua b/source/tools/monitor/unity/httplib/httpComm.lua index 12b691f318d6f72b218f1be258387b8b909245bd..3fdff02b5ebb8c3e6c5a8fa3b63737c28c6852b3 100644 --- a/source/tools/monitor/unity/httplib/httpComm.lua +++ b/source/tools/monitor/unity/httplib/httpComm.lua @@ -87,12 +87,15 @@ function ChttpComm:packHeaders(headTable, len) -- just for http out. end local origin = originHeader() + local c = 0 for k, v in pairs(origin) do - table.insert(lines, k .. ": " .. v) + c = c + 1 + lines[c] = table.concat({k, v}, ": ") end for k, v in pairs(headTable) do - table.insert(lines, k .. ": " .. v) + c = c + 1 + lines[c] = table.concat({k, v}, ": ") end return pystring:join("\r\n", lines) .. "\r\n" end diff --git a/source/tools/monitor/unity/httplib/httpHtml.lua b/source/tools/monitor/unity/httplib/httpHtml.lua index 5205af6556a44472e8fee1d3873fc0b8f1d75eaa..34a9a9ea7de65316e94082d4e181c3a32c839e4d 100644 --- a/source/tools/monitor/unity/httplib/httpHtml.lua +++ b/source/tools/monitor/unity/httplib/httpHtml.lua @@ -5,13 +5,35 @@ --- require("common.class") +local unistd = require("posix.unistd") local pystring = require("common.pystring") +local system = require("common.system") local ChttpBase = require("httplib.httpBase") local ChttpHtml = class("ChttpHtml", ChttpBase) function ChttpHtml:_init_(frame) ChttpBase._init_(self) + + self._reCb = { + md = function(s, keep, suffix) return self:renderMd(s, keep, suffix) end, + html = function(s, keep, suffix) return self:renderHtml(s, keep, suffix) end, + jpg = function(s, keep, suffix) return self:renderImage(s, keep, "jpeg") end, + jpeg = function(s, keep, suffix) return self:renderImage(s, keep, suffix) end, + gif = function(s, keep, suffix) return self:renderImage(s, keep, suffix) end, + png = function(s, keep, suffix) return self:renderImage(s, keep, suffix) end, + bmp = function(s, keep, suffix) return self:renderImage(s, keep, suffix) end, + txt = function(s, keep, suffix) return self:renderText(s, keep, suffix) end, + text = function(s, keep, suffix) return self:renderText(s, keep, suffix) end, + log = function(s, keep, suffix) return self:renderText(s, keep, suffix) end, + } +end + +local function loadFile(fPpath) + local f = io.open(fPpath,"r") + local s = f:read("*all") + f:close() + return s end function ChttpHtml:markdown(text) @@ -19,6 +41,43 @@ function ChttpHtml:markdown(text) return md:toHtml(text) end +function ChttpHtml:renderMd(s, keep, suffix) + local tmd = self:markdown(s) + local tRet = { + title = "markdown document render from beaver.", + content = tmd, + cType = "text/html", + } + return self:echo(tRet, keep) +end + +function ChttpHtml:renderHtml(s, keep, suffix) + local cType = "text/html" + return self:pack(cType, keep, s) +end + +function ChttpHtml:renderText(s, keep, suffix) + local cType = "text/plain" + return self:pack(cType, keep, s) +end + +function ChttpHtml:renderImage(s, keep, suffix) + local cType = "image/" .. suffix + return self:pack(cType, keep, s) +end + +function ChttpHtml:reSource(tReq, keep, head, srcPath) + local path = tReq.path + path = srcPath .. pystring:lstrip(path, head) + if unistd.access(path) then + local s = loadFile(path) + local _, suffix = unpack(pystring:rsplit(path, ".", 1)) + if system:keyIsIn(self._reCb, suffix) then + return self._reCb[suffix](s, keep, suffix) + end + end +end + local function htmlPack(title, content) local h1 = [[ @@ -41,16 +100,22 @@ local function htmlPack(title, content) return pystring:join("", bodies) end -function ChttpHtml:echo(tRet, keep) +function ChttpHtml:pack(cType, keep, body) local stat = self:packStat(200) local tHead = { - ["Content-Type"] = "text/html", + ["Content-Type"] = cType, ["Connection"] = (keep and "keep-alive") or "close" } - local body = htmlPack(tRet.title, tRet.content) local headers = self:packHeaders(tHead, #body) local tHttp = {stat, headers, body} return pystring:join("\r\n", tHttp) end +function ChttpHtml:echo(tRet, keep) + local cType = tRet.type or "text/html" + local body = htmlPack(tRet.title, tRet.content) + + return self:pack(cType, keep, body) +end + return ChttpHtml diff --git a/source/tools/monitor/unity/test/beaver/walkDir.lua b/source/tools/monitor/unity/test/beaver/walkDir.lua new file mode 100644 index 0000000000000000000000000000000000000000..27a0fd0d88b0692295af6a704abe56dac87891b7 --- /dev/null +++ b/source/tools/monitor/unity/test/beaver/walkDir.lua @@ -0,0 +1,43 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/2/18 12:13 PM +--- + +package.path = package.path .. ";../../?.lua;" +local dirent = require("posix.dirent") +local sysStat = require("posix.sys.stat") +local system = require("common.system") + +local function join(dir, fName) + local paths = {dir, fName} + return table.concat(paths, "/") +end + +local function walk(orig, dir, tbl) + local ls = dirent.dir(dir) + local len = string.len(orig) + for _, l in ipairs(ls) do + if l == ".." or l == '.' then + goto continue + end + local path = join(dir, l) + local stat, err, errno = sysStat.stat(path) + if stat then + local mode = stat.st_mode + if sysStat.S_ISDIR(mode) > 0 then + walk(orig, path, tbl) + elseif sysStat.S_ISREG(mode) > 0 then + table.insert(tbl, string.sub(path, len + 2)) + end + else + system:posixError(string.format("bad access to file %s", path), err, errno) + end + ::continue:: + end +end + +local tbl = {} +local dir = "../../beaver/guide" +walk(dir, dir, tbl) +print(system:dump(tbl)) diff --git a/source/tools/monitor/unity/test/curl/beaver/beavers.lua b/source/tools/monitor/unity/test/curl/beaver/beavers.lua index 619c4334c48adbd18edb9f9b05638474233f5a4e..af2966739ccb3880ce0d34acdcaaf2c006c2b74c 100644 --- a/source/tools/monitor/unity/test/curl/beaver/beavers.lua +++ b/source/tools/monitor/unity/test/curl/beaver/beavers.lua @@ -86,13 +86,15 @@ function Cbeavers:poll() if err then print("socket select return " .. err) end + local c = 0 for _, read in pairs(reads) do if type(read) == "number" then break elseif read == self._server then local s = read:accept() print("accept " .. s:getfd()) - table.insert(self._ss, s) + c = c + 1 + self._ss[c] = s local co = coroutine.create(function(o, s) self._proc(o, s) end) self._cos[s:getfd()] = co coroutine.resume(co, self, s) diff --git a/source/tools/monitor/unity/test/host/hostIp.lua b/source/tools/monitor/unity/test/host/hostIp.lua index 1b0266a630244aefccb2d605c291163465482e31..84b4d4a0dc9fac64f8bcf7edd770bca1afbf0232 100644 --- a/source/tools/monitor/unity/test/host/hostIp.lua +++ b/source/tools/monitor/unity/test/host/hostIp.lua @@ -9,8 +9,8 @@ local socket = require("socket") local function getAdd(hostName) local _, resolved = socket.dns.toip(hostName) local listTab = {} - for _, v in pairs(resolved.ip) do - table.insert(listTab, v) + for i, v in pairs(resolved.ip) do + listTab[i] = v end return listTab end diff --git a/source/tools/monitor/unity/test/string/py.lua b/source/tools/monitor/unity/test/string/py.lua index 04cb3142b49953f13a015a394ec5b79189a7b16e..8befc8eac37cb50d94c89f7caaa2dc7060223df0 100644 --- a/source/tools/monitor/unity/test/string/py.lua +++ b/source/tools/monitor/unity/test/string/py.lua @@ -14,6 +14,15 @@ assert(#ret == 3) assert(ret[1] == "hello") assert(ret[2] == "lua") assert(ret[3] == "language") +ret = pystring:split("hello lua language lua language") +assert(#ret == 5 ) +assert(ret[1] == "hello") +assert(ret[2] == "lua") +assert(ret[3] == "language") +assert(ret[4] == "lua") +assert(ret[5] == "language") +ret = pystring:split("Node 0, zone DMA 1 0 0 1 2 1 1 0 1 1 3") +assert(#ret == 15) -- 自定符号分割 ret = pystring:split("hello*lua *language", "*") diff --git a/source/tools/monitor/unity/test/unix/pyunix.py b/source/tools/monitor/unity/test/unix/pyunix.py new file mode 100644 index 0000000000000000000000000000000000000000..c215808a01506738909909d95a1c808eb63b8bbf --- /dev/null +++ b/source/tools/monitor/unity/test/unix/pyunix.py @@ -0,0 +1,28 @@ +import os +import time +import socket + +PIPE_PATH = "/tmp/sysom" +MAX_BUFF = 128 * 1024 + + +class CnfPut(object): + def __init__(self): + self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + self._path = PIPE_PATH + if not os.path.exists(self._path): + raise ValueError("pipe path is not exist. please check Netinfo is running.") + + def puts(self, s): + if len(s) > MAX_BUFF: + raise ValueError("message len %d, is too long ,should less than%d" % (len(s), MAX_BUFF)) + return self._sock.sendto(s, self._path) + + +if __name__ == "__main__": + nf = CnfPut() + i = 10 + while True: + nf.puts('io_burst,disk=/dev/vda1 limit=10.0,max=%d,log="io log burst"' % i) + i += 1 + time.sleep(5) \ No newline at end of file diff --git a/source/tools/monitor/unity/tsdb/foxTSDB.lua b/source/tools/monitor/unity/tsdb/foxTSDB.lua index b808489bce6fa04e39e34a25f76575037489dd5a..807e1ea13ed7ba9803864f45b9f6f31b5aa94642 100644 --- a/source/tools/monitor/unity/tsdb/foxTSDB.lua +++ b/source/tools/monitor/unity/tsdb/foxTSDB.lua @@ -240,6 +240,7 @@ function CfoxTSDB:query(start, stop, ms) -- start stop should at the same mday self:curMove(start) -- moveto position + local lenMs = #ms for line in self:loadData(stop) do local time = line.time for _, v in ipairs(line.lines) do @@ -261,34 +262,51 @@ function CfoxTSDB:query(start, stop, ms) -- start stop should at the same mday end tCell.values = values - table.insert(ms, tCell) + lenMs = lenMs + 1 + ms[lenMs] = tCell end end return ms end -function CfoxTSDB:qlast(last, ms) - local now = self:get_us() - local date = self:getDateFrom_us(now) - local beg = now - last * 1e6; - - if self._man then -- has setup - if self.cffi.check_pman_date(self._man, date) == 1 then -- at the same day - return self:query(beg, now, ms) - else - self:_del_() -- destroy old manager - if self:_setupRead(now) ~= 0 then -- try to create new - return ms - else - return self:query(beg, now, ms) - end +function CfoxTSDB:_qlast(date, beg, stop, ms) + if not self._man then -- check _man is already installed. + if self:_setupRead(beg) ~= 0 then -- try to create new + return ms end + end + + if self.cffi.check_pman_date(self._man, date) == 1 then + return self:query(beg, stop, ms) else - if self:_setupRead(now) ~= 0 then -- try to create new + self:_del_() + if self:_setupRead(beg) ~= 0 then -- try to create new return ms - else - return self:query(beg, now, ms) end + return self:query(beg, stop, ms) + end +end + +function CfoxTSDB:qlast(last, ms) + assert(last < 24 * 60 * 60) + + local now = self:get_us() + local beg = now - last * 1e6 + + local dStart = self:getDateFrom_us(beg) + local dStop = self:getDateFrom_us(now) + + if self.cffi.check_foxdate(dStart, dStop) ~= 0 then + self:_qlast(dStart, beg, now, ms) + else + dStop.hour, dStop.min, dStop.sec = 0, 0, 0 + local beg1 = beg + local beg2 = self.cffi.make_stamp(dStop) + local now1 = beg2 - 1 + local now2 = now + + self:_qlast(dStart, beg1, now1, ms) + self:_qlast(dStop, beg2, now2, ms) end end @@ -304,6 +322,7 @@ function CfoxTSDB:qDay(start, stop, ms, tbls, budget) budget = budget or self._qBudget self:curMove(start) local inc = false + local lenMs = #ms for line in self:loadData(stop) do inc = false local time = line.time @@ -336,7 +355,8 @@ function CfoxTSDB:qDay(start, stop, ms, tbls, budget) end tCell.logs = logs - table.insert(ms, tCell) + lenMs = lenMs + 1 + ms[lenMs] = tCell inc = true end end @@ -361,11 +381,13 @@ function CfoxTSDB:qDayTables(start, stop, tbls) end self:curMove(start) + local lenTbls = #tbls for line in self:loadData(stop) do for _, v in ipairs(line.lines) do local title = v.line if not system:valueIsIn(tbls, title) then - table.insert(tbls, title) + lenTbls = lenTbls + 1 + tbls[lenTbls] = title end end end @@ -384,6 +406,7 @@ function CfoxTSDB:qDate(dStart, dStop, tbls) if self.cffi.check_foxdate(dStart, dStop) ~= 0 then self:qDay(beg, now, ms, tbls) else + dStop.hour, dStop.min, dStop.sec = 0, 0, 0 local beg1 = beg local beg2 = self.cffi.make_stamp(dStop) local now1 = beg2 - 1 @@ -412,6 +435,7 @@ function CfoxTSDB:qNow(sec, tbls) if self.cffi.check_foxdate(dStart, dStop) ~= 0 then self:qDay(beg, now, ms, tbls) else + dStop.hour, dStop.min, dStop.sec = 0, 0, 0 local beg1 = beg local beg2 = self.cffi.make_stamp(dStop) local now1 = beg2 - 1 @@ -440,6 +464,7 @@ function CfoxTSDB:qTabelNow(sec) if self.cffi.check_foxdate(dStart, dStop) ~= 0 then self:qDayTables(beg, now, tbls) else + dStop.hour, dStop.min, dStop.sec = 0, 0, 0 local beg1 = beg local beg2 = self.cffi.make_stamp(dStop) local now1 = beg2 - 1