diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index b639f68ef81e47f5d1ca0c3e13497b3d586b058e..0000000000000000000000000000000000000000 Binary files a/.DS_Store and /dev/null differ diff --git a/.gitignore b/.gitignore index 40a844cf97c51e9911797ce9870e063f2b71d5ed..fa19e01d509ccd80cd1f42a00da3993f60245ee7 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ *.ko *.ko.cmd *.idea +**/.idea/ +**/.DS_Store Module.symvers modules.builtin modules.order diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 13566b81b018ad684f3a35fee301741b2734c8f4..0000000000000000000000000000000000000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Editor-based HTTP Client requests -/httpRequests/ -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 53624c9e1f9ab0331fd10b32c26747519ace5794..0000000000000000000000000000000000000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 35eb1ddfbbc029bcab630581847471d7f238ec53..0000000000000000000000000000000000000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/source/.DS_Store b/source/.DS_Store deleted file mode 100644 index fe62bc3d37d7e43ee3c765a76137456f136a5618..0000000000000000000000000000000000000000 Binary files a/source/.DS_Store and /dev/null differ diff --git a/source/tools/.DS_Store b/source/tools/.DS_Store deleted file mode 100644 index 12272494a9145ab77e6821d51a35aa9e656937e1..0000000000000000000000000000000000000000 Binary files a/source/tools/.DS_Store and /dev/null differ diff --git a/source/tools/monitor/.DS_Store b/source/tools/monitor/.DS_Store deleted file mode 100644 index 54403a1caa441bc5047ab36847f73aef327fef34..0000000000000000000000000000000000000000 Binary files a/source/tools/monitor/.DS_Store and /dev/null differ diff --git a/source/tools/monitor/unity/beeQ/pack.sh b/source/tools/monitor/unity/beeQ/pack.sh index 07b479ba32000502b5769d90fc5a59a15a1208c0..dfabfce33cb4029dea2684076f1f7c5059b48574 100755 --- a/source/tools/monitor/unity/beeQ/pack.sh +++ b/source/tools/monitor/unity/beeQ/pack.sh @@ -80,6 +80,12 @@ cp common/*.lua ${APP}/common/ mkdir ${APP}/httplib cp httplib/*.lua ${APP}/httplib/ +mkdir ${APP}/sls_metric +mkdir ${APP}/sls_metric/native +cp sls_metric/native/*.so* ${APP}/sls_metric/native/ +cp sls_metric/native/*.lua ${APP}/sls_metric/native/ +cp sls_metric/*.lua ${APP}/sls_metric/ + mkdir ${APP}/tsdb mkdir ${APP}/tsdb/native cp tsdb/native/*.so* ${APP}/tsdb/native/ diff --git a/source/tools/monitor/unity/beeQ/pushTo.lua b/source/tools/monitor/unity/beeQ/pushTo.lua index bf301ffedc8bbb7ed1e97d450ef577e55268960c..c4d68eff493db881fd85b998e74c2fb2905de1ee 100644 --- a/source/tools/monitor/unity/beeQ/pushTo.lua +++ b/source/tools/monitor/unity/beeQ/pushTo.lua @@ -6,13 +6,26 @@ package.path = package.path .. ";../?.lua;" +local system = require("common.system") local coCli = require("httplib.coCli") local coInflux = require("httplib.coInflux") +local coMetrics = require("httplib.coMetrics") +local coAutoMetrics =require("httplib.coAutoMetrics") function work(fd, fYaml) + local conf = system:parseYaml(fYaml) + local to = conf.pushTo.to + print(to) local frame = coCli.new(fd) - local c = coInflux.new(fYaml) + local _funcs = { + Influx = function(fYaml) return coInflux.new(fYaml) end, + Metrics = function(fYaml) return coMetrics.new(fYaml) end, + AutoMetrics = function(fYaml) return coAutoMetrics.new(fYaml) end + } + local c = _funcs[to](fYaml) + --local c = _funcs[to]("/etc/sysak/base.yaml") frame:poll(c) + print("end push.") return 0 end diff --git a/source/tools/monitor/unity/beeQ/run.sh b/source/tools/monitor/unity/beeQ/run.sh index 1f228ae736c9ee43efa024e219590a697d11f156..97b5be763dc2807fbf432511db096a36bc45a9db 100755 --- a/source/tools/monitor/unity/beeQ/run.sh +++ b/source/tools/monitor/unity/beeQ/run.sh @@ -18,7 +18,7 @@ yaml_path=$1 [ ! $yaml_path ] && yaml_path="/etc/sysak/base.yaml" #download sysak.ko -sysak -oss -d +#sysak -oss -d echo $yaml_yaml_path ./unity-mon $yaml_path diff --git a/source/tools/monitor/unity/common/transPro.lua b/source/tools/monitor/unity/common/transPro.lua new file mode 100644 index 0000000000000000000000000000000000000000..328e99988f53a9cb66ce8284c9f2d01dd90c24b8 --- /dev/null +++ b/source/tools/monitor/unity/common/transPro.lua @@ -0,0 +1,114 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by wrp. +--- DateTime: 2023/7/10 10:46 +--- + +local system = require("common.system") +local pystring = require("common.pystring") +require("common.class") + +local CtransPro = class("CtransPro") + +local function qFormData(from, tData) + local res = {} + local len = #tData + local last = 0 + local c = 0 + for i = len, 1, -1 do + local line = tData[i] + print(line.title) + if from == line.title then + if last == 0 or last == line.time then + c = c + 1 + res[c] = line + last = line.time + else + break + end + end + end + return res +end + +local function packLine_us(title, ls, v, time) + local tLs = {} + for k, v in pairs(ls) do + table.insert(tLs, string.format("%s=\"%s\"", k , v)) + end + local label = "" + if #tLs then + label = pystring:join(",", tLs) + label = "{" .. label .. "}" + end + return string.format("%s%s %.1f %d", title, label, v, time/1000) +end + +local function packLine(title, ls, v, time) + local tLs = {} + local c = 0 + for k, v in pairs(ls) do + c = c + 1 + tLs[c] = string.format("%s=\"%s\"", k , v) + end + local label = "" + if #tLs then + label = pystring:join(",", tLs) + label = "{" .. label .. "}" + end + return string.format("%s%s %.1f", title, label, v) +end + +function CtransPro:_init_(instance, fYaml, help, timestamps) + self._instance = instance + self._help = help or false + local ms = system:parseYaml(fYaml) + self._timestamps = timestamps or false + if self._timestamps == true then + self.pack_line = packLine_us + else + self.pack_line = packLine + end + self._tDescr = ms.metrics + print(self._tDescr) +end + +function CtransPro:export(datas) + local res = {} + local c = 0 + for _, line in ipairs(self._tDescr) do + --local from = line.from -- cpu_total + --local tFroms = qFormData(from, datas) + if #datas then + local title = line.title --sysak_proc_cpu_total + if self._help then + local help = string.format("# HELP %s %s", title, line.help) + c = c + 1 + res[c] = help + local sType = string.format("# TYPE %s %s", title, line.type) + c = c + 1 + res[c] = sType + end + + for _, tFrom in ipairs(datas) do + local labels = system:deepcopy(tFrom.labels) + if not labels then + labels = {} + end + labels.instance = self._instance + for k, v in pairs(tFrom.vs) do + labels[line.head] = v.name + c = c + 1 + res[c] = self.pack_line(title, labels, v.value, 1) + end + end + end + end + --c = c + 1 + --res[c] = "" + local lines = pystring:join("\n", res) + --print(lines) + return lines +end + +return CtransPro \ No newline at end of file diff --git a/source/tools/monitor/unity/httplib/coAutoMetrics.lua b/source/tools/monitor/unity/httplib/coAutoMetrics.lua new file mode 100644 index 0000000000000000000000000000000000000000..b7d87aa6d3d30c526434c176a4997cfdcaae3674 --- /dev/null +++ b/source/tools/monitor/unity/httplib/coAutoMetrics.lua @@ -0,0 +1,27 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by wrp. +--- DateTime: 2023/7/17 15:26 +--- + +require("common.class") + +local CcoMetrics = require("httplib.coMetrics") + +local CcoAutoMetrics = class("coAutoMetrics", CcoMetrics) + +function CcoAutoMetrics:_init_(fYaml) + CcoMetrics._init_(self,fYaml) + + local ts = io.popen('curl 100.100.100.200/latest/meta-data/region-id') + local regionid = ts:read("*all") + + self._project = "sysom-metrics-" .. regionid + self._endpoint = regionid .. "-intranet.log.aliyuncs.com" + self._metricstore = "auto" + self._url = "/prometheus/" ..self._project.."/"..self._metricstore.."/api/v1/write" + self._host = self._project .."." .. self._endpoint + +end + +return CcoAutoMetrics \ No newline at end of file diff --git a/source/tools/monitor/unity/httplib/coHttpCli.lua b/source/tools/monitor/unity/httplib/coHttpCli.lua index 9718385d2e36764b01e00e2b55265005ecf7db89..83c7b1a22da676def801b3ef1d8d1c76e36e8a70 100644 --- a/source/tools/monitor/unity/httplib/coHttpCli.lua +++ b/source/tools/monitor/unity/httplib/coHttpCli.lua @@ -113,18 +113,13 @@ local function setupSocket(host, port) end end -function CcoHttpCli:_init_(fYaml, persistent) - local res = system:parseYaml(fYaml) - local pushTo = res.pushTo +function CcoHttpCli:_init_(pushTo, persistent) + self._host = pushTo.host self._port = pushTo.port or 80 self._url = pushTo.url or "/" self._persistent = persistent - local Cidentity = require("beaver.identity") - local inst = Cidentity.new(fYaml) - self._instance = inst:id() - self.status = enumStat.closed end @@ -151,30 +146,6 @@ function CcoHttpCli:trans(msgs, body, filter) return "" end -function CcoHttpCli:addInstance(line) -- add instance id for line index. - local cells = line.ls - local hasInstance = false - - if cells then - for _, cell in ipairs(cells) do - if cell.name == "instance" then - hasInstance = true - end - end - end - - if not hasInstance then - local cell = { - name = "instance", - index = self._instance - } - if cells then - table.insert(cells, cell) - else - line.ls = {cell} - end - end -end function CcoHttpCli:pack(body) local line = self:packCliHead('GET', self._url) @@ -271,7 +242,7 @@ local function waitChuckSize(fread, s) if string.find(s, "\r\n") then return s end - local add = fread() + local add = fread() --add = nil,后续没有读到数据了 if add then s = s .. add else @@ -283,7 +254,7 @@ end local function readChunks(fread, tReq) local cells = {} local s = tReq.data - local ssize, size + local size local len = 1 local bodies, body @@ -325,8 +296,9 @@ local function waitHttpRest(fread, tReq) if waitDataRest(fread, rest, tReq) < 0 then return -2 end - else -- chunk mode - if #tReq.data > 0 then + elseif tReq.header["Transfer-Encoding"] then + -- chunk mode + if tReq.header["Transfer-Encoding"]=="chunked" and #tReq.data > 0 then if readChunks(fread, tReq) < 0 then return -3 end @@ -520,8 +492,8 @@ function CcoHttpCli:work(cffi, efd) goto failed end self.status = enumStat.receiving - local fread = self:closureRead(fd) - local tReq = self:result(fread) + local fread = self:closureRead(fd) -- fread() = nil + local tReq = self:result(fread) --nil,fread = nil if tReq then self.status = enumStat.connected self:echo(tReq) diff --git a/source/tools/monitor/unity/httplib/coHttpCliInst.lua b/source/tools/monitor/unity/httplib/coHttpCliInst.lua new file mode 100644 index 0000000000000000000000000000000000000000..4481f2d635fc554f254f17c3ab16e5e99305278a --- /dev/null +++ b/source/tools/monitor/unity/httplib/coHttpCliInst.lua @@ -0,0 +1,44 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by wrp. +--- DateTime: 2023/7/17 09:45 +--- + +require("common.class") + +local CcoHttpCli = require("httplib.coHttpCli") + +local CcoHttpCliInst = class("coHttpCliInst", CcoHttpCli) + +function CcoHttpCliInst:_init_(instance, pushTo, persistent) + CcoHttpCli._init_(self, pushTo ,persistent) + + self._instance = instance +end + +function CcoHttpCliInst:addInstance(line) -- add instance id for line index. + local cells = line.ls + local hasInstance = false + + if cells then + for _, cell in ipairs(cells) do + if cell.name == "instance" then + hasInstance = true + end + end + end + + if not hasInstance then + local cell = { + name = "instance", + index = self._instance + } + if cells then + table.insert(cells, cell) + else + line.ls = {cell} + end + end +end + +return CcoHttpCliInst \ No newline at end of file diff --git a/source/tools/monitor/unity/httplib/coInflux.lua b/source/tools/monitor/unity/httplib/coInflux.lua index 6b00a4c930c8b53f66d0f86efb6d90213545ac9f..61f8e8634563983894871a59aee443dfdd1e335d 100644 --- a/source/tools/monitor/unity/httplib/coInflux.lua +++ b/source/tools/monitor/unity/httplib/coInflux.lua @@ -5,14 +5,26 @@ --- require("common.class") -local CcoHttpCli = require("httplib.coHttpCli") +local CcoHttpCliInst = require("httplib.coHttpCliInst") local pystring = require("common.pystring") local lineParse = require("common.lineParse") +local system = require("common.system") -local CcoInflux = class("coInflux", CcoHttpCli) +local CcoInflux = class("coInflux", CcoHttpCliInst) function CcoInflux:_init_(fYaml) - CcoHttpCli._init_(self, fYaml) + local res = system:parseYaml(fYaml) + local pushInflux = { + host = res.pushTo.host, + url = res.pushTo.url, + port = res.pushTo.port + } + + local Cidentity = require("beaver.identity") + local inst = Cidentity.new(fYaml) + local instance = inst:id() + + CcoHttpCliInst._init_(self, instance, pushInflux) end function CcoInflux:echo(tReq) diff --git a/source/tools/monitor/unity/httplib/coMetrics.lua b/source/tools/monitor/unity/httplib/coMetrics.lua new file mode 100644 index 0000000000000000000000000000000000000000..eaf97761fb5389ed270b0d22897507b0a59712da --- /dev/null +++ b/source/tools/monitor/unity/httplib/coMetrics.lua @@ -0,0 +1,111 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by wrp. +--- DateTime: 2023/7/7 17:07 +--- + +require("common.class") + +local CcoHttpCliInst = require("httplib.coHttpCliInst") +local system = require("common.system") +local pystring = require("common.pystring") +local lineParse = require("common.lineParse") +local CtransPro = require("common.transPro") +local base64 = require("base64") + + +local CcoMetrics = class("coMetrics", CcoHttpCliInst) + +function CcoMetrics:_init_(fYaml) + + local res = system:parseYaml(fYaml) + local _metrics = res.metrics + self._mhead = _metrics.head + self._title = _metrics.title + + local Cidentity = require("beaver.identity") + local inst = Cidentity.new(fYaml) + local instance = inst:id() + + self._ak = res.pushTo.ak + self._sk = res.pushTo.sk + self._project = res.pushTo.project + self._endpoint = res.pushTo.endpoint + self._metricstore = res.pushTo.metricstore + self._url = "/prometheus/" ..self._project.."/"..self._metricstore.."/api/v1/write" + self._host = self._project .."." .. self._endpoint + + local pushMetrics = { + host = self._host, + url = self._url, + port = 80 + } + CcoHttpCliInst._init_(self, instance, pushMetrics) + -- go ffi + local ffi = require("sls_metric.native.ffi_lua") + self.ffi = ffi.ffi + self.awesome = ffi.awesome + + --fox ffi + local foxFFI = require("tsdb.native.foxffi") + self.foxffi = foxFFI.ffi + self.foxcffi = foxFFI.cffi + + self._transPro = CtransPro.new(instance, fYaml, false, false) +end + +function CcoMetrics:echo(tReq) + --if tReq.code ~= "204" then + print(tReq.code, tReq.data) + --end +end + +function CcoMetrics:trans(msgs) + local res + local c = 0 + local lines + + lines = msgs.lines + res = self._transPro:export(lines) + local prome = self.ffi.new("GoString") + prome.p = res + prome.n = #res + local prome_ptr = self.ffi.cast("GoString*", prome) + local byte = self.ffi.new("GoSlice") + local byte_ptr = self.ffi.cast("GoSlice*", byte) --{ void *data; GoInt len; GoInt cap; } GoSlice + local data_len = self.awesome.metricSnappy(prome,byte_ptr) + data_len = tonumber(data_len) + local data = self.ffi.cast("GoUint8*", byte_ptr.data) + --local stream = "" + --for i = 0, data_len do + -- --print(data[i]) + -- + -- stream = stream .. string.char(data[i]) + --end + -- + --print(stream) + --return stream + + return self.foxffi.string(data, data_len) + --return "sysak_proc_cpu_total{mode=\"user\",instance=\"i-wz9d3tqjhpb8esj8ps4z\"} 0.8\nsysak_proc_cpu_total{mode=\"total\",instance=\"i-wz9d3tqjhpb8esj8ps4z\"} 3960.0\nsysak_proc_cpu_total{mode=\"user2\",instance=\"i-wz9d3tqjhpb8iesj8ps4z\"} 0.9\n" +end + +function CcoMetrics:pack(body) + local line = self:packCliHead('POST', self._url) + local aksk = self._ak .. ":"..self._sk + local aksk64 = base64.encode(aksk) + print(aksk64) + local head = { + Host = self._host, + ["Content-Encoding"] = "snappy", + ["Content-Type"] = "application/x-protobuf", + --["X-Prometheus-Remote-Write-Version"] = "0.1.0", + ["Content-Length"] = #body, + ["Authorization"] = "Basic " .. aksk64, + } + local heads = self:packCliHeaders(head) + print("pack finish") + return pystring:join("\r\n", {line, heads, body}) +end + +return CcoMetrics \ No newline at end of file diff --git a/source/tools/monitor/unity/httplib/test.lua b/source/tools/monitor/unity/httplib/test.lua new file mode 100644 index 0000000000000000000000000000000000000000..50f6e3609d2b5223381f69853fa81c9d49e4d4ef --- /dev/null +++ b/source/tools/monitor/unity/httplib/test.lua @@ -0,0 +1,9 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by wrp. +--- DateTime: 2023/7/17 15:42 +--- + +local ts = io.popen('curl 100.100.100.200/latest/meta-data/region-id') +local s = ts:read("*all") +print("s "..s) diff --git a/source/tools/monitor/unity/sls_metric/native/README.md b/source/tools/monitor/unity/sls_metric/native/README.md new file mode 100644 index 0000000000000000000000000000000000000000..95e08e3e0ac7bd65c156cfd35a49b1ae706988dc --- /dev/null +++ b/source/tools/monitor/unity/sls_metric/native/README.md @@ -0,0 +1,10 @@ +模块功能: + 将prometheus格式字符串转化为snappy压缩后的字节流 + +run: + sh ./build_shared.sh + luajit ffi_lua.lua + +test: + 修改metricstore信息 + go run test_sls.go \ No newline at end of file diff --git a/source/tools/monitor/unity/sls_metric/native/build_shared.sh b/source/tools/monitor/unity/sls_metric/native/build_shared.sh new file mode 100644 index 0000000000000000000000000000000000000000..743e6b0d96c4ad872263d3088d98eb6a751093b9 --- /dev/null +++ b/source/tools/monitor/unity/sls_metric/native/build_shared.sh @@ -0,0 +1,7 @@ +rm go.mod +rm go.sum +go mod init metricSnappy.go +go mod tidy +go build -o libmetricSnappy.so -buildmode=c-shared metricSnappy.go +cp libmetricSnappy.so ../../beeQ/lib + diff --git a/source/tools/monitor/unity/sls_metric/native/ffi_lua.lua b/source/tools/monitor/unity/sls_metric/native/ffi_lua.lua new file mode 100644 index 0000000000000000000000000000000000000000..2c3649d619edfb765bfa938fed579eb225335af3 --- /dev/null +++ b/source/tools/monitor/unity/sls_metric/native/ffi_lua.lua @@ -0,0 +1,37 @@ +local ffi = require("ffi") +local awesome = ffi.load('metricSnappy') + +ffi.cdef[[ +typedef signed char GoInt8; +typedef unsigned char GoUint8; +typedef long long GoInt64; +typedef GoInt64 GoInt; +typedef double GoFloat64; +typedef struct { const char *p; GoInt n; } GoString; +typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; +extern GoInt metricSnappy(GoString* prome_ptr, GoSlice* ret); +]] + +return {ffi = ffi, awesome = awesome} +-- +---- input prometheus string +--local prome = ffi.new("GoString") +---- an example of prometheus string +--local s = 'sysak_proc_cpu_total{mode="user",instance="i-wz9d3tqjhpb8esj8ps4z"} 0.8\nsysak_proc_cpu_total{mode="total",instance="i-wz9d3tqjhpb8esj8ps4z"} 3960.0' +--prome.p = s; +--prome.n = #s; +--local prome_ptr = ffi.cast("GoString*", prome) +--local byte = ffi.new("GoSlice") +--local byte_ptr = ffi.cast("GoSlice*", byte) +-- +---- execute parse and snappy +--local data_len = awesome.metricSnappy(prome,byte_ptr) +-- +---- read data in lua +--data_len = tonumber(data_len) +--local data = ffi.cast("GoUint8*", byte_ptr.data) +-- +---- show data +--for i = 0, data_len do +-- print(data[i]) +--end diff --git a/source/tools/monitor/unity/sls_metric/native/metricSnappy.go b/source/tools/monitor/unity/sls_metric/native/metricSnappy.go new file mode 100644 index 0000000000000000000000000000000000000000..eeded249fcb4023609753761c2d985de4ec7f12a --- /dev/null +++ b/source/tools/monitor/unity/sls_metric/native/metricSnappy.go @@ -0,0 +1,88 @@ +package main + +import "C" + +import ( + "fmt" + "time" + "strings" + "strconv" + "github.com/gogo/protobuf/proto" + "github.com/prometheus/prometheus/prompb" + "github.com/golang/snappy" + "regexp" +) + +//export metricSnappy +func metricSnappy(prome_ptr *string, ret *[]byte) int { + /* + input: + prome_ptr, *string, prometheus data + ret, *[]byte, receive return byte data + output: + int, length of return data + */ + // initialize + var prome = strings.Split(*prome_ptr, "\n") + timeSeries := []prompb.TimeSeries{}; + timestamp := time.Now().UnixNano() + + for i:=0;i