diff --git a/source/tools/monitor/mservice/master/conf/sysakmon.conf b/source/tools/monitor/mservice/master/conf/sysakmon.conf index 64b7988a1fea221ec2063ea3338dcbbfb6a0f3e1..e1fc4a07a71ec567b53f6e31c9e2801c99a79612 100644 --- a/source/tools/monitor/mservice/master/conf/sysakmon.conf +++ b/source/tools/monitor/mservice/master/conf/sysakmon.conf @@ -19,6 +19,7 @@ mod_squid off mod_nginx off mod_percpu off mod_proc off pidname +mod_cgroup on ####output_interface file,db,nagios output_interface file diff --git a/source/tools/monitor/mservice/master/include/output_http.h b/source/tools/monitor/mservice/master/include/output_http.h index 3663570761a938790ce6e3a8c5270cbfe1d6c426..862d8b024311bad2817c266b58588fb021882250 100644 --- a/source/tools/monitor/mservice/master/include/output_http.h +++ b/source/tools/monitor/mservice/master/include/output_http.h @@ -1,7 +1,6 @@ #ifndef OUTPUT_HTTP_H #define OUTPUT_HTTP_H -extern void output_http(int sk); extern int http_server(void); #endif diff --git a/source/tools/monitor/mservice/master/modules/Makefile b/source/tools/monitor/mservice/master/modules/Makefile index d13a5157675d1d567266e279ef94839e87d8264e..ce80099f32ad599453073094dc31c4ae09ccfeb2 100644 --- a/source/tools/monitor/mservice/master/modules/Makefile +++ b/source/tools/monitor/mservice/master/modules/Makefile @@ -10,12 +10,14 @@ ifeq ($(UNAME_S),Darwin) OBJS = mod_apache.so mod_cpu.so mod_haproxy.so mod_load.so mod_swap.so\ mod_lvs.so mod_mem.so mod_ncpu.so mod_nginx.so \ mod_pcsw.so mod_percpu.so mod_pernic.so \ - mod_proc.so mod_squid.so mod_tcp.so mod_tcpx.so mod_traffic.so mod_udp.so + mod_proc.so mod_squid.so mod_tcp.so mod_tcpx.so mod_traffic.so mod_udp.so \ + mod_cgroup.so else OBJS = mod_apache.so mod_cpu.so mod_haproxy.so mod_io.so mod_load.so mod_swap.so \ mod_lvs.so mod_mem.so mod_ncpu.so mod_nginx.so \ mod_partition.so mod_pcsw.so mod_percpu.so mod_pernic.so \ - mod_proc.so mod_squid.so mod_tcp.so mod_tcpx.so mod_traffic.so mod_udp.so + mod_proc.so mod_squid.so mod_tcp.so mod_tcpx.so mod_traffic.so mod_udp.so \ + mod_cgroup.so endif all: $(OBJS) diff --git a/source/tools/monitor/mservice/master/modules/mod_cgroup.c b/source/tools/monitor/mservice/master/modules/mod_cgroup.c new file mode 100644 index 0000000000000000000000000000000000000000..d40bfd6104c04a849d3c2bb999490a455b541cb5 --- /dev/null +++ b/source/tools/monitor/mservice/master/modules/mod_cgroup.c @@ -0,0 +1,252 @@ +/* + * iostat.c for 2.6.* with file /proc/diskstat + * Linux I/O performance monitoring utility + */ +#include "tsar.h" +#include +#include +#include +#include +#include +#include +#include +#include + +char *cg_usage = " --cg Linux container stats"; + +#define MAX_CGROUPS 64 +/*Todo,user configure ?*/ +#define CGROUP_INFO_AGING_TIME 7200 + +struct cg_load_info { + int load_avg_1; + int load_avg_5; + int load_avg_15; + int nr_running; + int nr_uninterrupt; +}; + +struct cg_cpu_info { + +}; + +struct cg_mem_info { + +}; + +struct cg_blkio_info { + unsigned long long rd_ios; /* Read I/O operations */ + unsigned long long rd_merges; /* Reads merged */ + unsigned long long rd_sectors; /* Sectors read */ + unsigned long long rd_ticks; /* Time in queue + service for read */ + unsigned long long wr_ios; /* Write I/O operations */ + unsigned long long wr_merges; /* Writes merged */ + unsigned long long wr_sectors; /* Sectors written */ + unsigned long long wr_ticks; /* Time in queue + service for write */ + unsigned long long ticks; /* Time of requests in queue */ + unsigned long long aveq; /* Average queue length */ +}; + +struct cgroup_info { + char name[LEN_32]; + int valid; + struct cg_load_info load; +/* struct cg_cpu_info cpu; + struct cg_mem_info mem; + struct cg_blkio_info blkio;*/ +} cgroups[MAX_CGROUPS]; + +unsigned int n_cgs = 0; /* Number of cgroups */ +char buffer[256]; /* Temporary buffer for parsing */ + +static void +set_load_record(double st_array[], U_64 cur_array[]) +{ + int i; + for (i = 0; i < 3; i++) { + st_array[i] = cur_array[i] / 100.0; + } + st_array[3] = cur_array[3]; + st_array[4] = cur_array[4]; +} + +static struct mod_info cg_info[] = { + /* load info */ + {" load1", HIDE_BIT, 0, STATS_NULL}, + {" load5", HIDE_BIT, 0, STATS_NULL}, + {"load15", HIDE_BIT, 0, STATS_NULL}, + {" nrun", HIDE_BIT, 0, STATS_NULL}, + {"nunint", HIDE_BIT, 0, STATS_NULL}, + +/* io info */ +/* + {" rrqms", DETAIL_BIT, MERGE_SUM, STATS_NULL}, + {" wrqms", DETAIL_BIT, MERGE_SUM, STATS_NULL}, + {" %rrqm", DETAIL_BIT, MERGE_AVG, STATS_NULL}, + {" %wrqm", DETAIL_BIT, MERGE_AVG, STATS_NULL}, + {" rs", DETAIL_BIT, MERGE_SUM, STATS_NULL}, + {" ws", DETAIL_BIT, MERGE_SUM, STATS_NULL}, + {" rsecs", DETAIL_BIT, MERGE_SUM, STATS_NULL}, + {" wsecs", DETAIL_BIT, MERGE_SUM, STATS_NULL}, + {"rqsize", DETAIL_BIT, MERGE_AVG, STATS_NULL}, + {"rarqsz", DETAIL_BIT, MERGE_AVG, STATS_NULL}, + {"warqsz", DETAIL_BIT, MERGE_AVG, STATS_NULL}, + {"qusize", DETAIL_BIT, MERGE_AVG, STATS_NULL}, + {" await", DETAIL_BIT, MERGE_AVG, STATS_NULL}, + {"rawait", DETAIL_BIT, MERGE_AVG, STATS_NULL}, + {"wawait", DETAIL_BIT, MERGE_AVG, STATS_NULL}, + {" svctm", DETAIL_BIT, MERGE_AVG, STATS_NULL}, + {" util", SUMMARY_BIT, MERGE_AVG, STATS_NULL} +*/ +}; + +#define NR_CGROUP_INFO sizeof(cg_info)/sizeof(struct mod_info) + +char *get_cgroup_path(const char *name, const char *child, char *path) +{ + FILE *result; + char cmd[LEN_256]; + + snprintf(cmd, LEN_256, "find /sys/fs/cgroup/%s/ -name %s*", child, name); + result = popen(cmd, "r"); + if (!result) + return NULL; + + if (fgets(buffer, sizeof(buffer), result)) { + sscanf(buffer, "%s", path); + pclose(result); + return path; + } + + pclose(result); + return NULL; +} + +static unsigned long cgroup_init_time; +static int need_reinit(void) +{ + if (cgroup_init_time <= (time(NULL) - CGROUP_INFO_AGING_TIME)) + return 1; + + return 0; +} + +static void init_cgroups(void) +{ + int i; + FILE *result; + + if (n_cgs > 0 && !need_reinit()) + return; + + memset(cgroups, 0, sizeof(cgroups)); + n_cgs = 0; + result = popen("docker ps -q", "r"); + for (i = 0; i < MAX_CGROUPS && !feof(result); i++) { + if (feof(result) || !fgets(buffer, sizeof(buffer), result)) + break; + sscanf(buffer, "%31s", cgroups[n_cgs].name); + n_cgs++; + } + pclose(result); + cgroup_init_time = time(NULL); +} + +void get_load_stats(int cg_idx) +{ + char filepath[LEN_1024]; + FILE *file; + int items; + + if (!get_cgroup_path(cgroups[cg_idx].name, "cpuacct", filepath)) + return; + + strcat(filepath, "/cpuacct.proc_stat"); + file = fopen(filepath, "r"); + if (!file) + return; + + while (fgets(buffer, sizeof(buffer), file)) { + items = sscanf(buffer, "load average(1min) %d", &cgroups[cg_idx].load.load_avg_1); + if (items != 0) { + cg_info[0].summary_bit = DETAIL_BIT; + continue; + } + items = sscanf(buffer, "load average(5min) %d", &cgroups[cg_idx].load.load_avg_5); + if (items != 0) { + cg_info[1].summary_bit = DETAIL_BIT; + continue; + } + items = sscanf(buffer, "load average(15min) %d", &cgroups[cg_idx].load.load_avg_15); + if (items != 0) { + cg_info[2].summary_bit = DETAIL_BIT; + continue; + } + items = sscanf(buffer, "nr_running %d", &cgroups[cg_idx].load.nr_running); + if (items != 0) { + cg_info[3].summary_bit = DETAIL_BIT; + continue; + } + items = sscanf(buffer, "nr_uninterruptible %d", &cgroups[cg_idx].load.nr_uninterrupt); + if (items != 0) { + cg_info[4].summary_bit = DETAIL_BIT; + continue; + } + } + + cgroups[cg_idx].valid = 1; + fclose(file); +} + +void get_cgroup_stats(void) +{ + int i; + + for (i = 0; i < n_cgs; i++) { + get_load_stats(i); + } +} + +void +print_cgroup_stats(struct module *mod) +{ + int pos = 0, i; + char buf[LEN_1M]; + + memset(buf, 0, LEN_1M); + + for (i = 0; i < n_cgs; i++) { + if (!cgroups[i].valid) + continue; + pos += snprintf(buf + pos, LEN_1M - pos, "%s=%d,%d,%d,%d,%d" ITEM_SPLIT, + cgroups[i].name, + cgroups[i].load.load_avg_1, + cgroups[i].load.load_avg_5, + cgroups[i].load.load_avg_15, + cgroups[i].load.nr_running, + cgroups[i].load.nr_uninterrupt); + } + + set_mod_record(mod, buf); +} + +void +read_cgroup_stat(struct module *mod, char *parameter) +{ + init_cgroups(); + get_cgroup_stats(); + print_cgroup_stats(mod); +} + +static void +set_cgroup_record(struct module *mod, double st_array[], + U_64 pre_array[], U_64 cur_array[], int inter) +{ + set_load_record(st_array, cur_array); +} + +void +mod_register(struct module *mod) +{ + register_mod_fields(mod, "--cg", cg_usage, cg_info, NR_CGROUP_INFO, read_cgroup_stat, set_cgroup_record); +} diff --git a/source/tools/monitor/mservice/master/src/httpserver.c b/source/tools/monitor/mservice/master/src/httpserver.c index 33175d6896e20014c65c9722fd35bf94de8868e3..f8efd3a3e829e2acca8284ddd21d54b53b6985a7 100644 --- a/source/tools/monitor/mservice/master/src/httpserver.c +++ b/source/tools/monitor/mservice/master/src/httpserver.c @@ -4,22 +4,31 @@ #define ERROR_MSG "HTTP/1.1 404 Not Found\r\n" enum { - REQUEST_METRIC, + REQUEST_METRIC_ROOT, + REQUEST_METRIC_CGROUP, + REQUEST_METRIC_CGROUP_ALL, REQUEST_MAX }; -static int get_request(const char *buf) +static int get_request(const char *buf, char *sub_req, int len) { - char req_str[32]; - - sscanf(buf, "GET /%s HTTP", req_str); - if (strcmp(req_str, "metric") == 0) - return REQUEST_METRIC; + char req_str[LEN_256]; + sscanf(buf, "GET /%255s HTTP", req_str); + if (strcmp(req_str, "metric") == 0 || strcmp(req_str, "metric/") == 0) { + return REQUEST_METRIC_ROOT; + } + else if (strcmp(req_str, "metric/cgroups") == 0 || strcmp(req_str, "metric/cgroups/") == 0) { + return REQUEST_METRIC_CGROUP_ALL; + } + else if (strncmp(req_str, "metric/cgroups/", 15) == 0) { + strncpy(sub_req, req_str + 15, len - 1); + return REQUEST_METRIC_CGROUP; + } return REQUEST_MAX; } -void output_http(int sk) +int output_http(int sk, int req, const char*sub_req) { int i, j, k, n = 0; char detail[LEN_1M] = {0}; @@ -34,25 +43,38 @@ void output_http(int sk) for (i = 0; i < statis.total_mod_num; i++) { mod = mods[i]; - if (mod->enable && strlen(mod->record)) { - precord = mod->record; - j = 0; - for (j = 0; j < mod->n_item; j++) { - if (mod->n_item > 1) { - psub = strstr(precord, "="); - if (!psub) - break; - *psub = 0; - snprintf(opt_line, LEN_64, "%s{%s,", mod->opt_line+2, precord); - precord = strstr(psub + 1, ";"); + if (req != REQUEST_METRIC_ROOT) { + if ((req == REQUEST_METRIC_CGROUP_ALL || req == REQUEST_METRIC_CGROUP) + && strcmp(mod->name, "mod_cgroup")) + continue; + } + + if (mod->enable && strlen(mod->record)) { + precord = mod->record; + j = 0; + for (j = 0; j < mod->n_item; j++) { + psub = strstr(precord, "="); + if (psub) { + int ignore = 0; + *psub = 0; + /*check if we want*/ + if (req == REQUEST_METRIC_CGROUP && strcmp(precord, sub_req)) + ignore = 1; + snprintf(opt_line, LEN_64, "%s{%s,", mod->opt_line+2, precord); + precord = strstr(psub + 1, ";"); if (precord) precord = precord + 1; + if (ignore) + continue; } else { snprintf(opt_line, LEN_64, "%s{", mod->opt_line+2); } st_array = &mod->st_array[j * mod->n_col]; for (k = 0; k < mod->n_col; k++) { + if (HIDE_BIT == mod->info[k].summary_bit) + continue; + n = snprintf(detail, LEN_1M, "%s%s} %6.2f\n", opt_line, trim(mod->info[k].hdr, LEN_128), st_array[k]); if (n >= LEN_1M - 1) { do_debug(LOG_FATAL, "mod %s lenth is overflow %d\n", mod->name, n); @@ -67,27 +89,69 @@ void output_http(int sk) } } + if (strlen(line) == 0) + return -1; + strcat(line, "\n"); sprintf(http_header, "HTTP/1.1 200 OK\r\nContent-Length: %d \r\n\r\n", (int)strlen(line)); write(sk, http_header, strlen(http_header)); write(sk, line, strlen(line)); + return 0; } -static void handle_metric(int sk) +static int prev_collect_time; +static int handle_metric(int sk, int req, const char*sub_req) { + int ret; + pthread_mutex_lock(&module_record_mutex); - init_module_fields(); - /*read twice for metrics which need compute diff*/ - collect_record(); - collect_record_stat(); - usleep(50000); + prev_collect_time = statis.cur_time; + statis.cur_time = time(NULL); + if (statis.cur_time - prev_collect_time > 60 || statis.cur_time <= prev_collect_time) { + /*read twice for metrics which need compute diff*/ + collect_record(); + collect_record_stat(); + conf.print_interval = 1; + sleep(1); + } else { + conf.print_interval = statis.cur_time - prev_collect_time; + } + collect_record(); collect_record_stat(); - output_http(sk); + + ret = output_http(sk, req, sub_req); pthread_mutex_unlock(&module_record_mutex); + + return ret; +} + +static void handle_request(int sk) +{ + char buff[1024] = {0}; + char sub_req[LEN_256] = {0}; + int req; + + if (read(sk, buff, sizeof(buff)) <= 0) + goto error; + + req = get_request(buff, sub_req, LEN_256); + switch (req) { + case REQUEST_METRIC_ROOT: + case REQUEST_METRIC_CGROUP: + case REQUEST_METRIC_CGROUP_ALL: + break; + default: + goto error; + } + + if (handle_metric(sk, req, sub_req)) { +error: + write(sk, ERROR_MSG, strlen(ERROR_MSG)); + } } int http_server(void) @@ -127,15 +191,7 @@ int http_server(void) close(sk); return -1; } - - char buff[1024] = {0}; - int size = read(csk, buff, sizeof(buff)); - - if (size > 0 && get_request(buff) == REQUEST_METRIC) - handle_metric(csk); - else - write(csk, ERROR_MSG, strlen(ERROR_MSG)); - + handle_request(csk); close(csk); } diff --git a/source/tools/monitor/mservice/master/src/tsar.c b/source/tools/monitor/mservice/master/src/tsar.c index 54070f8e7a210874718873f9a3adb4aaf92bbd7a..68621c85160731e56fb072053a73a86332bffcc5 100644 --- a/source/tools/monitor/mservice/master/src/tsar.c +++ b/source/tools/monitor/mservice/master/src/tsar.c @@ -259,9 +259,10 @@ running_cron() void *cron_thread(void *arg) { while (1) { - statis.cur_time = time(NULL); pthread_mutex_lock(&module_record_mutex); + statis.cur_time = time(NULL); running_cron(); + collect_record_stat(); pthread_mutex_unlock(&module_record_mutex); sleep(conf.cron_period); }