diff --git a/0006-add-cpu-online-fault-isolation.patch b/0006-add-cpu-online-fault-isolation.patch new file mode 100644 index 0000000000000000000000000000000000000000..44bae4856357c3152a2cae8b6c8f703c2172faee --- /dev/null +++ b/0006-add-cpu-online-fault-isolation.patch @@ -0,0 +1,983 @@ +From fcdc2d2f99def2745c261a085f7aacc8ce106246 Mon Sep 17 00:00:00 2001 +From: Lostwayzxc +Date: Fri, 21 May 2021 16:28:15 +0800 +Subject: [PATCH 1/2] add cpu online fault isolation + +Add cpu online fault isolation, when CE/UCE occurs, we choose to offline +the error cpu according to threshold algorithm. + +Signed-off-by: Luo Shengwei +--- + .travis.yml | 2 +- + Makefile.am | 6 +- + configure.ac | 11 ++ + misc/rasdaemon.env | 15 ++ + queue.c | 121 ++++++++++++ + queue.h | 42 ++++ + ras-arm-handler.c | 73 +++++++ + ras-cpu-isolation.c | 469 ++++++++++++++++++++++++++++++++++++++++++++ + ras-cpu-isolation.h | 70 +++++++ + ras-events.c | 8 + + ras-record.h | 5 + + 11 files changed, 820 insertions(+), 2 deletions(-) + create mode 100644 queue.c + create mode 100644 queue.h + create mode 100644 ras-cpu-isolation.c + create mode 100644 ras-cpu-isolation.h + +diff --git a/.travis.yml b/.travis.yml +index 79cf4ca..5ab3957 100644 +--- a/.travis.yml ++++ b/.travis.yml +@@ -20,7 +20,7 @@ before_install: + - sudo apt-get install -y sqlite3 + install: + - autoreconf -vfi +-- ./configure --enable-sqlite3 --enable-aer --enable-non-standard --enable-arm --enable-mce --enable-extlog --enable-devlink --enable-diskerror --enable-abrt-report --enable-hisi-ns-decode --enable-memory-ce-pfa ++- ./configure --enable-sqlite3 --enable-aer --enable-non-standard --enable-arm --enable-mce --enable-extlog --enable-devlink --enable-diskerror --enable-abrt-report --enable-hisi-ns-decode --enable-memory-ce-pfa --enable-cpu-fault-isolation + + script: + - make && sudo make install +diff --git a/Makefile.am b/Makefile.am +index f4822b9..6431dd3 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -57,12 +57,16 @@ endif + if WITH_MEMORY_CE_PFA + rasdaemon_SOURCES += rbtree.c ras-page-isolation.c + endif ++if WITH_CPU_FAULT_ISOLATION ++ rasdaemon_SOURCES += ras-cpu-isolation.c queue.c ++endif + rasdaemon_LDADD = -lpthread $(SQLITE3_LIBS) libtrace/libtrace.a + + include_HEADERS = config.h ras-events.h ras-logger.h ras-mc-handler.h \ + ras-aer-handler.h ras-mce-handler.h ras-record.h bitfield.h ras-report.h \ + ras-extlog-handler.h ras-arm-handler.h ras-non-standard-handler.h \ +- ras-devlink-handler.h ras-diskerror-handler.h rbtree.h ras-page-isolation.h ++ ras-devlink-handler.h ras-diskerror-handler.h rbtree.h ras-page-isolation.h \ ++ ras-cpu-isolation.h queue.h + + # This rule can't be called with more than one Makefile job (like make -j8) + # I can't figure out a way to fix that +diff --git a/configure.ac b/configure.ac +index 2d6c59c..a682bb9 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -141,6 +141,16 @@ AS_IF([test "x$enable_memory_ce_pfa" = "xyes" || test "x$enable_all" == "xyes"], + AM_CONDITIONAL([WITH_MEMORY_CE_PFA], [test x$enable_memory_ce_pfa = xyes || test x$enable_all == xyes]) + AM_COND_IF([WITH_MEMORY_CE_PFA], [USE_MEMORY_CE_PFA="yes"], [USE_MEMORY_CE_PFA="no"]) + ++AC_ARG_ENABLE([cpu_fault_isolation], ++ AS_HELP_STRING([--enable-cpu-fault-isolation], [enable cpu online fault isolation])) ++ ++AS_IF([test "x$enable_cpu_fault_isolation" = "xyes" || test "x$enable_all" == "xyes"], [ ++ AC_DEFINE(HAVE_CPU_FAULT_ISOLATION,1,"have cpu online fault isolation") ++ AC_SUBST([WITH_CPU_FAULT_ISOLATION]) ++]) ++AM_CONDITIONAL([WITH_CPU_FAULT_ISOLATION], [test x$enable_cpu_fault_isolation = xyes || test x$enable_all == xyes]) ++AM_COND_IF([WITH_CPU_FAULT_ISOLATION], [USE_CPU_FAULT_ISOLATION="yes"], [USE_CPU_FAULT_ISOLATION="no"]) ++ + test "$sysconfdir" = '${prefix}/etc' && sysconfdir=/etc + + CFLAGS="$CFLAGS -Wall -Wmissing-prototypes -Wstrict-prototypes" +@@ -173,4 +183,5 @@ compile time options summary + DEVLINK : $USE_DEVLINK + Disk I/O errors : $USE_DISKERROR + Memory CE PFA : $USE_MEMORY_CE_PFA ++ CPU fault isolation : $USE_CPU_FAULT_ISOLATION + EOF +diff --git a/misc/rasdaemon.env b/misc/rasdaemon.env +index 12fd766..7498992 100644 +--- a/misc/rasdaemon.env ++++ b/misc/rasdaemon.env +@@ -27,3 +27,18 @@ PAGE_CE_THRESHOLD="50" + # soft-then-hard First try to soft offline, then try hard offlining. + # Note: default offline choice is "soft". + PAGE_CE_ACTION="soft" ++ ++# CPU Online Fault Isolation ++# Specify the threshold of corrected errors. ++# ++# Format: ++# [0-9]+[unit] ++# ++# Supported units: ++# CPU_CE_THRESHOLD: no unit ++# CPU_ISOLATION_CYCLE: D|d (day), H|h (hour), M|m (minute), S|s (second), default is in second ++CPU_CE_THRESHOLD="18" ++CPU_ISOLATION_CYCLE="24h" ++ ++# Prevent excessive isolation from causing an avalanche effect ++CPU_ISOLATION_LIMIT="10" +diff --git a/queue.c b/queue.c +new file mode 100644 +index 0000000..1ef8688 +--- /dev/null ++++ b/queue.c +@@ -0,0 +1,121 @@ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. ++ * ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License as published by ++ * the Free Software Foundation; either version 2 of the License, or ++ * (at your option) any later version. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++*/ ++#include ++#include ++#include "queue.h" ++#include "ras-logger.h" ++ ++ ++int is_empty(struct link_queue *queue) ++{ ++ if (queue) { ++ return queue->size == 0; ++ } ++ ++ return 1; ++} ++ ++struct link_queue* init_queue(void) ++{ ++ struct link_queue* queue; ++ queue = (struct link_queue*) malloc(sizeof(struct link_queue)); ++ ++ if (queue == NULL) { ++ log(TERM, LOG_ERR, "Failed to allocate memory for queue.\n"); ++ return NULL; ++ } ++ ++ queue->size = 0; ++ queue->head = NULL; ++ queue->tail = NULL; ++ ++ return queue; ++} ++ ++void clear_queue(struct link_queue *queue) ++{ ++ if (queue == NULL) { ++ return; ++ } ++ ++ struct queue_node *node = queue->head; ++ struct queue_node *tmp = NULL; ++ ++ while (node != NULL) { ++ tmp = node; ++ node = node->next; ++ free(tmp); ++ } ++ ++ free(queue); ++} ++ ++int push(struct link_queue *queue, struct queue_node *node) ++{ ++ if (!queue || !node) { ++ return -1; ++ } ++ /* there is no element in the queue */ ++ if (queue->head == NULL) { ++ queue->head = node; ++ } ++ else { ++ node->next = queue->tail->next; ++ queue->tail->next = node; ++ } ++ ++ queue->tail = node; ++ (queue->size)++; ++ ++ return 0; ++} ++ ++int pop(struct link_queue *queue) ++{ ++ struct queue_node *tmp = NULL; ++ ++ if (is_empty(queue)) { ++ return -1; ++ } ++ ++ tmp = queue->head; ++ queue->head = queue->head->next; ++ free(tmp); ++ (queue->size)--; ++ ++ return 0; ++} ++ ++struct queue_node* front(struct link_queue *queue) ++{ ++ if (is_empty(queue)) { ++ return NULL; ++ } ++ ++ return queue->head; ++} ++ ++struct queue_node* node_create(time_t time, unsigned value) ++{ ++ struct queue_node *node = NULL; ++ node = (struct queue_node*) malloc(sizeof(struct queue_node)); ++ ++ if (node != NULL) { ++ node->time = time; ++ node->value = value; ++ node->next = NULL; ++ } ++ ++ return node; ++} +diff --git a/queue.h b/queue.h +new file mode 100644 +index 0000000..b60aa81 +--- /dev/null ++++ b/queue.h +@@ -0,0 +1,42 @@ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. ++ * ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License as published by ++ * the Free Software Foundation; either version 2 of the License, or ++ * (at your option) any later version. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++*/ ++ ++#ifndef __RAS_QUEUE_H ++#define __RAS_QUEUE_H ++ ++ ++struct queue_node ++{ ++ time_t time; ++ unsigned value; ++ struct queue_node *next; ++}; ++ ++struct link_queue ++{ ++ struct queue_node *head; ++ struct queue_node *tail; ++ int size; ++}; ++ ++int is_empty(struct link_queue *queue); ++struct link_queue* init_queue(void); ++void clear_queue(struct link_queue *queue); ++int push(struct link_queue *queue, struct queue_node *node); ++int pop(struct link_queue *queue); ++struct queue_node* front(struct link_queue *queue); ++struct queue_node* node_create(time_t time, unsigned value); ++ ++ ++#endif +\ No newline at end of file +diff --git a/ras-arm-handler.c b/ras-arm-handler.c +index 2f170e2..9f9302e 100644 +--- a/ras-arm-handler.c ++++ b/ras-arm-handler.c +@@ -20,6 +20,44 @@ + #include "ras-record.h" + #include "ras-logger.h" + #include "ras-report.h" ++#include "ras-cpu-isolation.h" ++ ++ ++static int is_core_failure(unsigned long value) ++{ ++ /* ++ * core failure: ++ * Bit 0\1\3: (at lease 1) ++ * Bit 2: 0 ++ */ ++ return (value & 0xf) && !(value & (0x1 << 2)); ++} ++ ++static int count_errors(struct event_format *event, const uint8_t* data, int len) ++{ ++ /* ++ * According to UEFI_2_9_2021_03_18 specification chapter N2.4.4, ++ * the length of struct processor error information is 32, the byte ++ * length of the Flags field is 1, and the byte offset is 7 in the struct. ++ */ ++ int pei_err_size = 32; ++ int field_size = 1; ++ int cur_offset = 7; ++ unsigned long value; ++ int num = 0; ++ if (len % pei_err_size != 0) { ++ log(TERM, LOG_ERR, "the event data does not match to the ARM Processor Error Information Structure\n"); ++ return num; ++ } ++ while (cur_offset < len) { ++ value = pevent_read_number(event->pevent, data+cur_offset, field_size); ++ if (is_core_failure(value)) { ++ num++; ++ } ++ cur_offset += pei_err_size; ++ } ++ return num; ++} + + int ras_arm_event_handler(struct trace_seq *s, + struct pevent_record *record, +@@ -78,6 +116,41 @@ int ras_arm_event_handler(struct trace_seq *s, + ev.psci_state = val; + trace_seq_printf(s, "\n psci_state: %d", ev.psci_state); + ++#ifdef HAVE_CPU_FAULT_ISOLATION ++ /* record cpu error */ ++ if (pevent_get_field_val(s, event, "sev", record, &val, 1) < 0) ++ return -1; ++ /* error severity defined from include/acpi/ghes.h */ ++ switch (val) { ++ case GHES_SEV_NO: ++ ev.severity = "Informational"; ++ break; ++ case GHES_SEV_CORRECTED: ++ ev.severity = "Corrected"; ++ break; ++ case GHES_SEV_RECOVERABLE: ++ ev.severity = "Recoverable"; ++ break; ++ default: ++ case GHES_SEV_PANIC: ++ ev.severity = "Fatal"; ++ } ++ ++ if (val == GHES_SEV_CORRECTED || val == GHES_SEV_RECOVERABLE) { ++ int len, nums; ++ ev.error_info = pevent_get_field_raw(s, event, "buf", record, &len, 1); ++ if (!ev.error_info) ++ return -1; ++ ev.length = len; ++ /* relate to enum error_type */ ++ nums = count_errors(event, ev.error_info, len); ++ if (nums > 0) { ++ struct error_info err_info = {nums, now, val}; ++ ras_record_cpu_error(&err_info, ev.mpidr); ++ } ++ } ++#endif ++ + /* Insert data into the SGBD */ + #ifdef HAVE_SQLITE3 + ras_store_arm_record(ras, &ev); +diff --git a/ras-cpu-isolation.c b/ras-cpu-isolation.c +new file mode 100644 +index 0000000..857aa45 +--- /dev/null ++++ b/ras-cpu-isolation.c +@@ -0,0 +1,469 @@ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. ++ * ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License as published by ++ * the Free Software Foundation; either version 2 of the License, or ++ * (at your option) any later version. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++*/ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "ras-logger.h" ++#include "ras-cpu-isolation.h" ++ ++static struct cpu_info* cpu_infos = NULL; ++static unsigned int cpu_nums, cores_per_socket, cores_per_die; ++static const char* cpu_path_format = "/sys/devices/system/cpu/cpu%d/online"; ++static const char* core_siblings_list_path = "/sys/devices/system/cpu/cpu%d/topology/core_siblings_list"; ++static const char* node_path = "/sys/devices/system/node/node%d/cpulist"; ++ ++static const struct param normal_units[] = { ++ { "", 1 }, ++ {} ++}; ++ ++static const struct param cycle_units[] = { ++ { "d", 24 * 60 * 60 }, ++ { "h", 60 * 60 }, ++ { "m", 60 }, ++ { "s", 1 }, ++ {} ++}; ++ ++static struct isolation_param threshold = { ++ .name = "CPU_CE_THRESHOLD", ++ .units = normal_units, ++ .value = 18, ++ .limit = 100 ++}; ++ ++static struct isolation_param cpu_limit = { ++ .name = "CPU_ISOLATION_LIMIT", ++ .units = normal_units, ++ .value = 10, ++ .limit = 30 ++}; ++ ++static struct isolation_param cycle = { ++ .name = "CPU_ISOLATION_CYCLE", ++ .units = cycle_units, ++ .value = 24 * 60 * 60, ++ .limit = 72 * 60 * 60 ++}; ++ ++static const char *cpu_state[] = { ++ [CPU_OFFLINE] = "offline", ++ [CPU_ONLINE] = "online", ++ [CPU_OFFLINE_FAILED] = "offline-failed", ++ [CPU_UNKNOWN] = "unknown" ++}; ++ ++static int open_sys_file(unsigned cpu, int __oflag, const char *format) ++{ ++ int fd; ++ char buf[MAX_PATH_LEN]; ++ snprintf(buf, sizeof(buf), format, cpu); ++ fd = open(buf, __oflag); ++ ++ if (fd == -1) { ++ log(TERM, LOG_ERR, "[%s]:open file: %s failed\n", __func__, buf); ++ return -1; ++ } ++ ++ return fd; ++} ++ ++static void set_socket(void) ++{ ++ int ncores, fd; ++ int begin, end; ++ char buf[20]; ++ ncores = sysconf(_SC_NPROCESSORS_CONF); ++ cores_per_socket = ncores; ++ ++ for (int i = 0; i < ncores; ++i) { ++ fd = open_sys_file(i, O_RDONLY, core_siblings_list_path); ++ if (fd == -1) { ++ continue; ++ } ++ if (read(fd, buf, sizeof(buf)) <= 0) { ++ close(fd); ++ continue; ++ } ++ if (sscanf(buf, "%d-%d", &begin, &end) == 2) { ++ cores_per_socket = end > begin ? end - begin + 1 : ncores; ++ close(fd); ++ return; ++ } ++ } ++} ++ ++static void set_die(void) ++{ ++ int fd, begin, end; ++ char buf[20]; ++ cores_per_die = 0; ++ fd = open_sys_file(0, O_RDONLY, node_path); ++ ++ if (fd == -1) { ++ return; ++ } ++ ++ if (read(fd, buf, sizeof(buf))) { ++ if (sscanf(buf, "%d-%d", &begin, &end) == 2) { ++ cores_per_die = end > begin ? end - begin + 1 : 0; ++ } ++ } ++ ++ close(fd); ++} ++ ++static void init_cpu_info(unsigned int cpus) ++{ ++ cpu_nums = cpus; ++ cpu_infos = calloc(sizeof(*cpu_infos), cpus); ++ ++ if (!cpu_infos) { ++ log(TERM, LOG_ERR, "Failed to allocate memory for cpu infos in %s.\n", __func__); ++ return; ++ } ++ ++ for (unsigned int i = 0; i < cpus; ++i) { ++ cpu_infos[i].state = CPU_ONLINE; ++ cpu_infos[i].ce_queue = init_queue(); ++ } ++ /* set limit of offlined cpu limit according to number of cpu */ ++ cpu_limit.limit = cpus / 3; ++ set_socket(); ++ set_die(); ++} ++ ++static void check_config(struct isolation_param *config) ++{ ++ if (config->value >= config->limit) { ++ log(TERM, LOG_WARNING, "Value: %lu exceed limit: %lu, set to limit\n", ++ config->value, config->limit); ++ config->value = config->limit; ++ } ++ ++ return; ++} ++ ++static int parse_ul_config(struct isolation_param *config, char* env, unsigned long* value) ++{ ++ int env_size, has_unit = 0; ++ ++ if (!env || strlen(env) == 0) { ++ return -1; ++ } ++ ++ env_size = strlen(env); ++ char* unit = NULL; ++ unit = env + env_size - 1; ++ ++ if (isalpha(*unit)) { ++ has_unit = 1; ++ env_size--; ++ } ++ ++ for (int i = 0; i < env_size; ++i) { ++ if (isdigit(env[i])) { ++ if (*value > ULONG_MAX / 10 || (*value == ULONG_MAX / 10 && env[i] - '0' > 5)) { ++ log(TERM, LOG_ERR, "%s is out of range: %lu\n", env, ULONG_MAX); ++ return -1; ++ } ++ *value = 10 * (*value) + (env[i] - '0'); ++ } ++ else { ++ return -1; ++ } ++ } ++ ++ if (has_unit) { ++ for (const struct param *units = config->units; units->name; units++) { ++ /* value character and unit character are both valid */ ++ if (!strcasecmp(unit, units->name)) { ++ if (*value > ULONG_MAX / units->value) { ++ log(TERM, LOG_ERR, "%s is out of range: %lu\n", env, ULONG_MAX); ++ return -1; ++ } ++ *value = (*value) * units->value; ++ return 0; ++ } ++ } ++ log(TERM, LOG_ERR, "Invalid unit %s\n", unit); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++static void init_config(struct isolation_param *config) ++{ ++ char* env = getenv(config->name); ++ unsigned long value = 0; ++ ++ if (parse_ul_config(config, env, &value) < 0) { ++ log(TERM, LOG_ERR, "Invalid %s: %s! Use default value %ld.\n", ++ config->name, env, config->value); ++ return; ++ } ++ ++ config->value = value; ++ check_config(config); ++} ++ ++void ras_error_count_init(unsigned int cpus) ++{ ++ init_cpu_info(cpus); ++ init_config(&threshold); ++ init_config(&cpu_limit); ++ init_config(&cycle); ++} ++ ++void cpu_infos_free(void) ++{ ++ if (cpu_infos) { ++ for (int i = 0; i < cpu_nums; ++i) { ++ clear_queue(cpu_infos[i].ce_queue); ++ } ++ free(cpu_infos); ++ } ++ ++ return; ++} ++ ++static int get_cpu_status(unsigned cpu) ++{ ++ int fd; ++ fd = open_sys_file(cpu, O_RDONLY, cpu_path_format); ++ ++ if (fd == -1) { ++ return CPU_UNKNOWN; ++ } ++ ++ int num = 0; ++ ++ if (read(fd, &num, 1) <= 0) { ++ num = CPU_UNKNOWN; ++ } ++ else { ++ num = num - '0'; ++ } ++ ++ close(fd); ++ ++ return num; ++} ++ ++static int do_cpu_offline(unsigned cpu) ++{ ++ int fd, rc; ++ char buf[2]; ++ cpu_infos[cpu].state = CPU_OFFLINE_FAILED; ++ fd = open_sys_file(cpu, O_RDWR, cpu_path_format); ++ ++ if (fd == -1) { ++ return HANDLE_FAILED; ++ } ++ ++ strcpy(buf, "0"); ++ rc = write(fd, buf, strlen(buf)); ++ close(fd); ++ ++ if (rc < 0) { ++ log(TERM, LOG_ERR, "cpu%d offline failed, errno:%d\n", cpu, errno); ++ return HANDLE_FAILED; ++ } ++ /* check wthether the cpu is isolated successfully */ ++ else { ++ int num = 0; ++ num = get_cpu_status(cpu); ++ if (num == CPU_OFFLINE) { ++ return HANDLE_SUCCEED; ++ } ++ else { ++ return HANDLE_FAILED; ++ } ++ ++ } ++} ++ ++static void do_ce_handler(unsigned cpu, int *ret) ++{ ++ struct link_queue *queue = cpu_infos[cpu].ce_queue; ++ ++ if (queue == NULL) { ++ return; ++ } ++ ++ unsigned tmp; ++ /* ++ * Since we just count all error numbers in setted cycle, we store the time ++ * and error numbers from current event to the queue, then everytime we ++ * calculate the period from beginning time to ending time, if the period ++ * exceeds setted cycle, we pop the beginning time and error until the period ++ * from new beginning time to ending time is less than cycle. ++ */ ++ while (queue && queue->head && queue->tail && queue->tail->time - queue->head->time > cycle.value) { ++ tmp = queue->head->value; ++ if (pop(queue) == 0) { ++ cpu_infos[cpu].ce_nums -= tmp; ++ } ++ } ++ ++ if (cpu_infos[cpu].ce_nums >= threshold.value) { ++ log(TERM, LOG_INFO, "Corrected Errors exceeded threshold %ld, try to offline cpu%d\n", ++ threshold.value, cpu); ++ *ret = do_cpu_offline(cpu); ++ } ++} ++ ++static void do_uce_handler(unsigned cpu, int *ret) ++{ ++ if (cpu_infos[cpu].uce_nums > 0) { ++ log(TERM, LOG_INFO, "Uncorrected Errors occured, try to offline cpu%d\n", cpu); ++ *ret = do_cpu_offline(cpu); ++ } ++} ++ ++static void error_handler(unsigned cpu, struct error_info *err_info, int *ret) ++{ ++ switch (err_info->err_type) ++ { ++ case CE: ++ do_ce_handler(cpu, ret); ++ break; ++ case UCE: ++ do_uce_handler(cpu, ret); ++ break; ++ default: ++ break; ++ } ++} ++ ++static void record_error_info(unsigned cpu, struct error_info *err_info) ++{ ++ switch (err_info->err_type) ++ { ++ case CE: ++ { ++ struct queue_node *node = NULL; ++ node = node_create(err_info->time, err_info->nums); ++ /* if the queue is still NULL, try malloc again */ ++ if (cpu_infos[cpu].ce_queue == NULL) { ++ cpu_infos[cpu].ce_queue = init_queue(); ++ } ++ if (push(cpu_infos[cpu].ce_queue, node) < 0) { ++ /* when the queue is NULL and node is not NULL, free it */ ++ if (node != NULL) { ++ free(node); ++ } ++ log(TERM, LOG_ERR, "Fail to push node to queue\n"); ++ return; ++ } ++ cpu_infos[cpu].ce_nums += err_info->nums; ++ break; ++ } ++ case UCE: ++ cpu_infos[cpu].uce_nums++; ++ break; ++ default: ++ break; ++ } ++} ++ ++static unsigned long get_bit_value(int64_t value, int offset, int size) ++{ ++ if (size <= 0 || offset < 0) { ++ return 0; ++ } ++ ++ value >>= offset; ++ unsigned long res = 0; ++ int i = 0; ++ ++ while (i < size) { ++ res |= (value & (0x1 << (i++))); ++ } ++ ++ return res; ++} ++ ++static unsigned get_cpu_index(int64_t mpidr) ++{ ++ unsigned core_id, socket_id, die_id, cpu; ++ core_id = get_bit_value(mpidr, 8, 8); ++ socket_id = get_bit_value(mpidr, 21, 2); ++ die_id = get_bit_value(mpidr, 19, 2); ++ cpu = core_id + socket_id * cores_per_socket + die_id * cores_per_die; ++ ++ return cpu; ++} ++ ++void ras_record_cpu_error(struct error_info* err_info, int64_t mpidr) ++{ ++ int cur_cpu_status; ++ unsigned cpu; ++ int ret = HANDLE_NOTHING; ++ ++ if (!cpu_infos) { ++ return; ++ } ++ ++ cpu = get_cpu_index(mpidr); ++ ++ if (cpu >= cpu_nums) { ++ return; ++ } ++ ++ log(TERM, LOG_INFO, "Handling error on cpu%d\n", cpu); ++ ++ if (cpu_infos[cpu].state == CPU_OFFLINE) { ++ /* user may online the offlined cpu */ ++ cur_cpu_status = get_cpu_status(cpu); ++ if (cur_cpu_status != CPU_ONLINE) { ++ log(TERM, LOG_INFO, "cpu%d is already offlined, ignore\n", cpu); ++ return; ++ } ++ cpu_infos[cpu].state = CPU_ONLINE; ++ } ++ ++ record_error_info(cpu, err_info); ++ /* Since user may change cpu state, we get current offlined cpu numbers every recording time. */ ++ if (cpu_nums - sysconf(_SC_NPROCESSORS_ONLN) >= cpu_limit.value) { ++ log(TERM, LOG_WARNING, "Offlined cpus have exceeded limit: %lu, choose to do nothing\n", ++ cpu_limit.value); ++ return; ++ } ++ ++ error_handler(cpu, err_info, &ret); ++ /* do nothing */ ++ if (ret == HANDLE_NOTHING) { ++ return; ++ } ++ ++ if (ret == HANDLE_SUCCEED) { ++ cpu_infos[cpu].state = CPU_OFFLINE; ++ struct link_queue *tmp = cpu_infos[cpu].ce_queue; ++ clear_queue(tmp); ++ cpu_infos[cpu].ce_queue = init_queue(); ++ } ++ ++ log(TERM, LOG_INFO, "Result of offlining cpu %d: %s\n", ++ cpu, cpu_state[cpu_infos[cpu].state]); ++ ++ return; ++} +diff --git a/ras-cpu-isolation.h b/ras-cpu-isolation.h +new file mode 100644 +index 0000000..f340585 +--- /dev/null ++++ b/ras-cpu-isolation.h +@@ -0,0 +1,70 @@ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. ++ * ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License as published by ++ * the Free Software Foundation; either version 2 of the License, or ++ * (at your option) any later version. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++*/ ++ ++#ifndef __RAS_CPU_ISOLATION_H ++#define __RAS_CPU_ISOLATION_H ++ ++#include "queue.h" ++ ++#define MAX_PATH_LEN 100 ++ ++struct param { ++ char *name; ++ unsigned long value; ++}; ++ ++struct isolation_param { ++ char *name; ++ const struct param *units; ++ unsigned long value; ++ unsigned long limit; ++}; ++ ++enum cpu_state { ++ CPU_OFFLINE, ++ CPU_ONLINE, ++ CPU_OFFLINE_FAILED, ++ CPU_UNKNOWN, ++}; ++ ++enum error_handle_result { ++ HANDLE_FAILED = -1, ++ HANDLE_SUCCEED, ++ HANDLE_NOTHING, ++}; ++ ++enum error_type { ++ CE = 1, ++ UCE ++}; ++ ++struct cpu_info { ++ unsigned long uce_nums; ++ unsigned long ce_nums; ++ struct link_queue *ce_queue; ++ enum cpu_state state; ++}; ++ ++struct error_info { ++ unsigned long nums; ++ time_t time; ++ enum error_type err_type; ++}; ++ ++ ++void ras_error_count_init(unsigned cpus); ++void ras_record_cpu_error(struct error_info *err_info, int64_t mpidr); ++void cpu_infos_free(void); ++ ++#endif +\ No newline at end of file +diff --git a/ras-events.c b/ras-events.c +index 471d25d..31c4170 100644 +--- a/ras-events.c ++++ b/ras-events.c +@@ -40,6 +40,7 @@ + #include "ras-record.h" + #include "ras-logger.h" + #include "ras-page-isolation.h" ++#include "ras-cpu-isolation.h" + + /* + * Polling time, if read() doesn't block. Currently, trace_pipe_raw never +@@ -874,6 +875,10 @@ int handle_ras_events(int record_events) + + cpus = get_num_cpus(ras); + ++#ifdef HAVE_CPU_FAULT_ISOLATION ++ ras_error_count_init(cpus); ++#endif ++ + #ifdef HAVE_MCE + rc = register_mce_handler(ras, cpus); + if (rc) +@@ -990,6 +995,9 @@ err: + } + free(ras); + } ++#ifdef HAVE_CPU_FAULT_ISOLATION ++ cpu_infos_free(); ++#endif + + return rc; + } +diff --git a/ras-record.h b/ras-record.h +index cc217a9..b453f83 100644 +--- a/ras-record.h ++++ b/ras-record.h +@@ -77,6 +77,11 @@ struct ras_arm_event { + int64_t midr; + int32_t running_state; + int32_t psci_state; ++#ifdef HAVE_CPU_FAULT_ISOLATION ++ const char *severity; ++ const uint8_t *error_info; ++ uint32_t length; ++#endif + }; + + struct devlink_event { +-- +2.27.0 + diff --git a/0007-add-trace-print-and-add-sqlite-store.patch b/0007-add-trace-print-and-add-sqlite-store.patch new file mode 100644 index 0000000000000000000000000000000000000000..3b306fe309e8799f737d8cffa68023bfd35bb030 --- /dev/null +++ b/0007-add-trace-print-and-add-sqlite-store.patch @@ -0,0 +1,77 @@ +From 6192d18c563a0610142488d734a5993fe8dff077 Mon Sep 17 00:00:00 2001 +From: Lostwayzxc +Date: Fri, 21 May 2021 16:29:32 +0800 +Subject: [PATCH 2/2] add trace print of new information and add it to sqilte + +Since we add new information of the event, we add trace print and store it to +Sqlite. + +Signed-off-by: Luo Shengwei +--- + ras-arm-handler.c | 9 +++++++++ + ras-record.c | 8 ++++++++ + 2 files changed, 17 insertions(+) + +diff --git a/ras-arm-handler.c b/ras-arm-handler.c +index 9f9302e..58b33f1 100644 +--- a/ras-arm-handler.c ++++ b/ras-arm-handler.c +@@ -22,6 +22,12 @@ + #include "ras-report.h" + #include "ras-cpu-isolation.h" + ++static void trace_print_hex(struct trace_seq *s, const uint8_t *buf, int buf_len) ++{ ++ for (int i = 0; i < buf_len; ++i) { ++ trace_seq_printf(s, "%2.2x", buf[i]); ++ } ++} + + static int is_core_failure(unsigned long value) + { +@@ -135,6 +141,7 @@ int ras_arm_event_handler(struct trace_seq *s, + case GHES_SEV_PANIC: + ev.severity = "Fatal"; + } ++ trace_seq_printf(s, "\n severity: %s", ev.severity); + + if (val == GHES_SEV_CORRECTED || val == GHES_SEV_RECOVERABLE) { + int len, nums; +@@ -142,6 +149,8 @@ int ras_arm_event_handler(struct trace_seq *s, + if (!ev.error_info) + return -1; + ev.length = len; ++ trace_seq_printf(s, "\n processor_err_info: "); ++ trace_print_hex(s, ev.error_info, len); + /* relate to enum error_type */ + nums = count_errors(event, ev.error_info, len); + if (nums > 0) { +diff --git a/ras-record.c b/ras-record.c +index 549c494..33d4741 100644 +--- a/ras-record.c ++++ b/ras-record.c +@@ -210,6 +210,10 @@ static const struct db_fields arm_event_fields[] = { + { .name="mpidr", .type="INTEGER" }, + { .name="running_state", .type="INTEGER" }, + { .name="psci_state", .type="INTEGER" }, ++#ifdef HAVE_CPU_FAULT_ISOLATION ++ { .name="severity", .type="TEXT" }, ++ { .name="error_info", .type="BLOB" }, ++#endif + }; + + static const struct db_table_descriptor arm_event_tab = { +@@ -233,6 +237,10 @@ int ras_store_arm_record(struct ras_events *ras, struct ras_arm_event *ev) + sqlite3_bind_int64 (priv->stmt_arm_record, 4, ev->mpidr); + sqlite3_bind_int (priv->stmt_arm_record, 5, ev->running_state); + sqlite3_bind_int (priv->stmt_arm_record, 6, ev->psci_state); ++#ifdef HAVE_CPU_FAULT_ISOLATION ++ sqlite3_bind_text (priv->stmt_arm_record, 7, ev->severity, -1, NULL); ++ sqlite3_bind_blob (priv->stmt_arm_record, 8, ev->error_info, ev->length, NULL); ++#endif + + rc = sqlite3_step(priv->stmt_arm_record); + if (rc != SQLITE_OK && rc != SQLITE_DONE) +-- +2.27.0 + diff --git a/rasdaemon.spec b/rasdaemon.spec index 310122306d3d1b968d74efde504df6ef64ba2662..39bc449f9a94c7adf17e1289bceba26cea24d856 100644 --- a/rasdaemon.spec +++ b/rasdaemon.spec @@ -1,6 +1,6 @@ Name: rasdaemon Version: 0.6.6 -Release: 3 +Release: 4 License: GPLv2 Summary: Utility to get Platform Reliability, Availability and Serviceability (RAS) reports via the Kernel tracing events URL: https://github.com/mchehab/rasdaemon.git @@ -24,6 +24,8 @@ Patch2: bugfix-rasdaemon-wait-for-file-access.patch Patch3: bugfix-fix-fd-check.patch Patch4: bugfix-fix-disk-error-log-storm.patch Patch5: backport-rasdaemon-Fix-error-print.patch +Patch6: 0006-add-cpu-online-fault-isolation.patch +Patch7: 0007-add-trace-print-and-add-sqlite-store.patch %description The rasdaemon program is a daemon which monitors the platform @@ -42,7 +44,7 @@ autoheader libtoolize --automake --copy --debug --force automake --add-missing %ifarch %{arm} aarch64 -%configure --enable-mce --enable-aer --enable-sqlite3 --enable-extlog --enable-abrt-report --enable-devlink --enable-diskerror --enable-non-standard --enable-hisi-ns-decode --enable-arm +%configure --enable-mce --enable-aer --enable-sqlite3 --enable-extlog --enable-abrt-report --enable-devlink --enable-diskerror --enable-non-standard --enable-hisi-ns-decode --enable-arm --enable-cpu-fault-isolation %else %configure --enable-mce --enable-aer --enable-sqlite3 --enable-extlog --enable-abrt-report --enable-devlink --enable-diskerror %endif @@ -70,6 +72,9 @@ rm INSTALL %{buildroot}/usr/include/*.h /usr/bin/systemctl enable rasdaemon.service >/dev/null 2>&1 || : %changelog +* Fri May 21 2021 luoshengwei - 0.6.6-4 +- add cpu online fault isolation + * Sat May 15 2021 xujing<17826839720@163.com> - 0.6.6-3 - Type:bugfix - ID:NA