diff --git a/0006-add-cpu-online-fault-isolation.patch b/0006-add-cpu-online-fault-isolation.patch new file mode 100644 index 0000000000000000000000000000000000000000..b796a48845963d1c62298e5f372bc1045b4496a5 --- /dev/null +++ b/0006-add-cpu-online-fault-isolation.patch @@ -0,0 +1,1027 @@ +From 94f9581a6b398f178fcabf0fde2cce7eebb15ea7 Mon Sep 17 00:00:00 2001 +From: Lostwayzxc +Date: Tue, 25 May 2021 20:05:49 +0800 +Subject: [PATCH 1/2] add cpu online fault isolation + +Add cpu online fault isolation, when CE/UCE occurs, we choose to offline +the error cpu according to threshold algorithm. + +Signed-off-by: Luo Shengwei +--- + .travis.yml | 2 +- + Makefile.am | 6 +- + configure.ac | 11 + + misc/rasdaemon.env | 17 ++ + queue.c | 126 +++++++++++ + queue.h | 43 ++++ + ras-arm-handler.c | 73 +++++++ + ras-cpu-isolation.c | 499 ++++++++++++++++++++++++++++++++++++++++++++ + ras-cpu-isolation.h | 76 +++++++ + ras-events.c | 8 + + ras-record.h | 5 + + 11 files changed, 864 insertions(+), 2 deletions(-) + create mode 100644 queue.c + create mode 100644 queue.h + create mode 100644 ras-cpu-isolation.c + create mode 100644 ras-cpu-isolation.h + +diff --git a/.travis.yml b/.travis.yml +index 79cf4ca..5ab3957 100644 +--- a/.travis.yml ++++ b/.travis.yml +@@ -20,7 +20,7 @@ before_install: + - sudo apt-get install -y sqlite3 + install: + - autoreconf -vfi +-- ./configure --enable-sqlite3 --enable-aer --enable-non-standard --enable-arm --enable-mce --enable-extlog --enable-devlink --enable-diskerror --enable-abrt-report --enable-hisi-ns-decode --enable-memory-ce-pfa ++- ./configure --enable-sqlite3 --enable-aer --enable-non-standard --enable-arm --enable-mce --enable-extlog --enable-devlink --enable-diskerror --enable-abrt-report --enable-hisi-ns-decode --enable-memory-ce-pfa --enable-cpu-fault-isolation + + script: + - make && sudo make install +diff --git a/Makefile.am b/Makefile.am +index f4822b9..6431dd3 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -57,12 +57,16 @@ endif + if WITH_MEMORY_CE_PFA + rasdaemon_SOURCES += rbtree.c ras-page-isolation.c + endif ++if WITH_CPU_FAULT_ISOLATION ++ rasdaemon_SOURCES += ras-cpu-isolation.c queue.c ++endif + rasdaemon_LDADD = -lpthread $(SQLITE3_LIBS) libtrace/libtrace.a + + include_HEADERS = config.h ras-events.h ras-logger.h ras-mc-handler.h \ + ras-aer-handler.h ras-mce-handler.h ras-record.h bitfield.h ras-report.h \ + ras-extlog-handler.h ras-arm-handler.h ras-non-standard-handler.h \ +- ras-devlink-handler.h ras-diskerror-handler.h rbtree.h ras-page-isolation.h ++ ras-devlink-handler.h ras-diskerror-handler.h rbtree.h ras-page-isolation.h \ ++ ras-cpu-isolation.h queue.h + + # This rule can't be called with more than one Makefile job (like make -j8) + # I can't figure out a way to fix that +diff --git a/configure.ac b/configure.ac +index 2d6c59c..a682bb9 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -141,6 +141,16 @@ AS_IF([test "x$enable_memory_ce_pfa" = "xyes" || test "x$enable_all" == "xyes"], + AM_CONDITIONAL([WITH_MEMORY_CE_PFA], [test x$enable_memory_ce_pfa = xyes || test x$enable_all == xyes]) + AM_COND_IF([WITH_MEMORY_CE_PFA], [USE_MEMORY_CE_PFA="yes"], [USE_MEMORY_CE_PFA="no"]) + ++AC_ARG_ENABLE([cpu_fault_isolation], ++ AS_HELP_STRING([--enable-cpu-fault-isolation], [enable cpu online fault isolation])) ++ ++AS_IF([test "x$enable_cpu_fault_isolation" = "xyes" || test "x$enable_all" == "xyes"], [ ++ AC_DEFINE(HAVE_CPU_FAULT_ISOLATION,1,"have cpu online fault isolation") ++ AC_SUBST([WITH_CPU_FAULT_ISOLATION]) ++]) ++AM_CONDITIONAL([WITH_CPU_FAULT_ISOLATION], [test x$enable_cpu_fault_isolation = xyes || test x$enable_all == xyes]) ++AM_COND_IF([WITH_CPU_FAULT_ISOLATION], [USE_CPU_FAULT_ISOLATION="yes"], [USE_CPU_FAULT_ISOLATION="no"]) ++ + test "$sysconfdir" = '${prefix}/etc' && sysconfdir=/etc + + CFLAGS="$CFLAGS -Wall -Wmissing-prototypes -Wstrict-prototypes" +@@ -173,4 +183,5 @@ compile time options summary + DEVLINK : $USE_DEVLINK + Disk I/O errors : $USE_DISKERROR + Memory CE PFA : $USE_MEMORY_CE_PFA ++ CPU fault isolation : $USE_CPU_FAULT_ISOLATION + EOF +diff --git a/misc/rasdaemon.env b/misc/rasdaemon.env +index 12fd766..3191d03 100644 +--- a/misc/rasdaemon.env ++++ b/misc/rasdaemon.env +@@ -27,3 +27,20 @@ PAGE_CE_THRESHOLD="50" + # soft-then-hard First try to soft offline, then try hard offlining. + # Note: default offline choice is "soft". + PAGE_CE_ACTION="soft" ++ ++# CPU Online Fault Isolation ++# Whether to enable cpu online fault isolation (yes|no). ++CPU_ISOLATION_ENABLE="no" ++# Specify the threshold of CE numbers. ++# ++# Format: ++# [0-9]+[unit] ++# ++# Supported units: ++# CPU_CE_THRESHOLD: no unit ++# CPU_ISOLATION_CYCLE: D|d (day), H|h (hour), M|m (minute), S|s (second), default is in second ++CPU_CE_THRESHOLD="18" ++CPU_ISOLATION_CYCLE="24h" ++ ++# Prevent excessive isolation from causing an avalanche effect ++CPU_ISOLATION_LIMIT="10" +diff --git a/queue.c b/queue.c +new file mode 100644 +index 0000000..92f3d3c +--- /dev/null ++++ b/queue.c +@@ -0,0 +1,126 @@ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. ++ * ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License as published by ++ * the Free Software Foundation; either version 2 of the License, or ++ * (at your option) any later version. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++*/ ++#include ++#include ++#include "queue.h" ++#include "ras-logger.h" ++ ++ ++int is_empty(struct link_queue *queue) ++{ ++ if (queue) { ++ return queue->size == 0; ++ } ++ ++ return 1; ++} ++ ++struct link_queue* init_queue(void) ++{ ++ struct link_queue* queue; ++ queue = (struct link_queue*) malloc(sizeof(struct link_queue)); ++ ++ if (queue == NULL) { ++ log(TERM, LOG_ERR, "Failed to allocate memory for queue.\n"); ++ return NULL; ++ } ++ ++ queue->size = 0; ++ queue->head = NULL; ++ queue->tail = NULL; ++ ++ return queue; ++} ++ ++void clear_queue(struct link_queue *queue) ++{ ++ if (queue == NULL) { ++ return; ++ } ++ ++ struct queue_node *node = queue->head; ++ struct queue_node *tmp = NULL; ++ ++ while (node != NULL) { ++ tmp = node; ++ node = node->next; ++ free(tmp); ++ } ++ ++ queue->head = NULL; ++ queue->tail = NULL; ++ queue->size = 0; ++} ++ ++void free_queue(struct link_queue *queue) { ++ clear_queue(queue); ++ ++ if (queue) { ++ free(queue); ++ } ++} ++ ++/* It should be guranteed that the param is not NULL */ ++void push(struct link_queue *queue, struct queue_node *node) ++{ ++ /* there is no element in the queue */ ++ if (queue->head == NULL) { ++ queue->head = node; ++ } ++ else { ++ node->next = queue->tail->next; ++ queue->tail->next = node; ++ } ++ ++ queue->tail = node; ++ (queue->size)++; ++} ++ ++int pop(struct link_queue *queue) ++{ ++ if (queue == NULL || is_empty(queue)) { ++ return -1; ++ } ++ ++ struct queue_node *tmp = NULL; ++ tmp = queue->head; ++ queue->head = queue->head->next; ++ free(tmp); ++ (queue->size)--; ++ ++ return 0; ++} ++ ++struct queue_node* front(struct link_queue *queue) ++{ ++ if (queue == NULL) { ++ return NULL; ++ } ++ ++ return queue->head; ++} ++ ++struct queue_node* node_create(time_t time, unsigned value) ++{ ++ struct queue_node *node = NULL; ++ node = (struct queue_node*) malloc(sizeof(struct queue_node)); ++ ++ if (node != NULL) { ++ node->time = time; ++ node->value = value; ++ node->next = NULL; ++ } ++ ++ return node; ++} +diff --git a/queue.h b/queue.h +new file mode 100644 +index 0000000..9684c58 +--- /dev/null ++++ b/queue.h +@@ -0,0 +1,43 @@ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. ++ * ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License as published by ++ * the Free Software Foundation; either version 2 of the License, or ++ * (at your option) any later version. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++*/ ++ ++#ifndef __RAS_QUEUE_H ++#define __RAS_QUEUE_H ++ ++ ++struct queue_node ++{ ++ time_t time; ++ unsigned value; ++ struct queue_node *next; ++}; ++ ++struct link_queue ++{ ++ struct queue_node *head; ++ struct queue_node *tail; ++ int size; ++}; ++ ++int is_empty(struct link_queue *queue); ++struct link_queue* init_queue(void); ++void clear_queue(struct link_queue *queue); ++void free_queue(struct link_queue *queue); ++void push(struct link_queue *queue, struct queue_node *node); ++int pop(struct link_queue *queue); ++struct queue_node* front(struct link_queue *queue); ++struct queue_node* node_create(time_t time, unsigned value); ++ ++ ++#endif +\ No newline at end of file +diff --git a/ras-arm-handler.c b/ras-arm-handler.c +index 2f170e2..10d0099 100644 +--- a/ras-arm-handler.c ++++ b/ras-arm-handler.c +@@ -20,6 +20,44 @@ + #include "ras-record.h" + #include "ras-logger.h" + #include "ras-report.h" ++#include "ras-cpu-isolation.h" ++ ++#ifdef HAVE_CPU_FAULT_ISOLATION ++static int is_core_failure(unsigned long value) ++{ ++ /* ++ * core failure: ++ * Bit 0\1\3: (at lease 1) ++ * Bit 2: 0 ++ */ ++ return (value & 0xf) && !(value & (0x1 << 2)); ++} ++ ++static int count_errors(struct event_format *event, const uint8_t *data, int len) ++{ ++ /* ++ * According to UEFI_2_9_2021_03_18 specification chapter N2.4.4, ++ * the length of struct processor error information is 32, the byte ++ * length of the Flags field is 1, and the byte offset is 7 in the struct. ++ */ ++ int cur_offset = 7; ++ unsigned long value; ++ int num = 0; ++ if (len % PEI_ERR_SIZE != 0) { ++ log(TERM, LOG_ERR, "the event data does not match to the ARM Processor Error Information Structure\n"); ++ return num; ++ } ++ while (cur_offset < len) { ++ value = pevent_read_number(event->pevent, data+cur_offset, FLAGS_SIZE); ++ if (is_core_failure(value)) { ++ num++; ++ log(TERM, LOG_INFO, "Error in cpu core catched\n"); ++ } ++ cur_offset += PEI_ERR_SIZE; ++ } ++ return num; ++} ++#endif + + int ras_arm_event_handler(struct trace_seq *s, + struct pevent_record *record, +@@ -78,6 +116,41 @@ int ras_arm_event_handler(struct trace_seq *s, + ev.psci_state = val; + trace_seq_printf(s, "\n psci_state: %d", ev.psci_state); + ++#ifdef HAVE_CPU_FAULT_ISOLATION ++ /* record cpu error */ ++ if (pevent_get_field_val(s, event, "sev", record, &val, 1) < 0) ++ return -1; ++ /* refer to UEFI_2_9_2021_03_18 specification chapter N2.2 Table N-5 */ ++ switch (val) { ++ case GHES_SEV_NO: ++ ev.severity = "Informational"; ++ break; ++ case GHES_SEV_CORRECTED: ++ ev.severity = "Corrected"; ++ break; ++ case GHES_SEV_RECOVERABLE: ++ ev.severity = "Recoverable"; ++ break; ++ default: ++ case GHES_SEV_PANIC: ++ ev.severity = "Fatal"; ++ } ++ ++ if (val == GHES_SEV_CORRECTED || val == GHES_SEV_RECOVERABLE) { ++ int len, nums; ++ ev.error_info = pevent_get_field_raw(s, event, "buf", record, &len, 1); ++ if (!ev.error_info) ++ return -1; ++ ev.length = len; ++ /* relate to enum error_type */ ++ nums = count_errors(event, ev.error_info, len); ++ if (nums > 0) { ++ struct error_info err_info = {nums, now, val}; ++ ras_record_cpu_error(&err_info, ev.mpidr); ++ } ++ } ++#endif ++ + /* Insert data into the SGBD */ + #ifdef HAVE_SQLITE3 + ras_store_arm_record(ras, &ev); +diff --git a/ras-cpu-isolation.c b/ras-cpu-isolation.c +new file mode 100644 +index 0000000..a809f91 +--- /dev/null ++++ b/ras-cpu-isolation.c +@@ -0,0 +1,499 @@ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. ++ * ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License as published by ++ * the Free Software Foundation; either version 2 of the License, or ++ * (at your option) any later version. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++*/ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "ras-logger.h" ++#include "ras-cpu-isolation.h" ++ ++static struct cpu_info *cpu_infos = NULL; ++static unsigned int ncores, cores_per_socket, cores_per_die; ++static unsigned int sockets, dies = 1; ++static unsigned int enabled = 1; ++static const char *cpu_path_format = "/sys/devices/system/cpu/cpu%d/online"; ++static const char *core_siblings_list_path = "/sys/devices/system/cpu/cpu%d/topology/core_siblings_list"; ++static const char *node_path = "/sys/devices/system/node/possible"; ++ ++static const struct param normal_units[] = { ++ { "", 1 }, ++ {} ++}; ++ ++static const struct param cycle_units[] = { ++ { "d", 24 * 60 * 60 }, ++ { "h", 60 * 60 }, ++ { "m", 60 }, ++ { "s", 1 }, ++ {} ++}; ++ ++static struct isolation_param threshold = { ++ .name = "CPU_CE_THRESHOLD", ++ .units = normal_units, ++ .value = 18, ++ .limit = 10000 ++}; ++ ++static struct isolation_param cpu_limit = { ++ .name = "CPU_ISOLATION_LIMIT", ++ .units = normal_units ++}; ++ ++static struct isolation_param cycle = { ++ .name = "CPU_ISOLATION_CYCLE", ++ .units = cycle_units, ++ .value = 24 * 60 * 60, ++ .limit = 30 * 24 * 60 * 60 ++}; ++ ++static const char *cpu_state[] = { ++ [CPU_OFFLINE] = "offline", ++ [CPU_ONLINE] = "online", ++ [CPU_OFFLINE_FAILED] = "offline-failed", ++ [CPU_UNKNOWN] = "unknown" ++}; ++ ++static int open_sys_file(unsigned cpu, int __oflag, const char *format) ++{ ++ int fd; ++ char buf[MAX_PATH_LEN] = ""; ++ snprintf(buf, sizeof(buf), format, cpu); ++ fd = open(buf, __oflag); ++ ++ if (fd == -1) { ++ log(TERM, LOG_ERR, "[%s]:open file: %s failed\n", __func__, buf); ++ return -1; ++ } ++ ++ return fd; ++} ++ ++static int get_sockets(void) ++{ ++ int fd, j; ++ char buf[MAX_BUF_LEN] = ""; ++ cores_per_socket = ncores; ++ struct cpu_set *cpu_sets = (struct cpu_set *) malloc(sizeof(*cpu_sets) * ncores); ++ ++ if (!cpu_sets) { ++ log(TERM, LOG_ERR, "Failed to allocate memory for cpu sets in %s.\n", __func__); ++ return -1; ++ } ++ ++ for (int i = 0; i < ncores; ++i) { ++ fd = open_sys_file(i, O_RDONLY, core_siblings_list_path); ++ if (fd == -1) { ++ continue; ++ } ++ memset(buf, '\0', strlen(buf)); ++ if (read(fd, buf, sizeof(buf)) <= 0) { ++ close(fd); ++ continue; ++ } ++ for (j = 0; j < sockets; ++j) { ++ if (strcmp(cpu_sets[j].buf, buf) == 0) { ++ break; ++ } ++ } ++ if (j == sockets) { ++ strcpy(cpu_sets[sockets].buf, buf); ++ sockets++; ++ } ++ close(fd); ++ } ++ ++ free(cpu_sets); ++ cores_per_socket = sockets > 0 ? ncores / sockets : ncores; ++ ++ return 0; ++} ++ ++static int get_dies(void) ++{ ++ int fd, begin, end; ++ char buf[20] = ""; ++ cores_per_die = ncores; ++ fd = open(node_path, O_RDONLY); ++ ++ if (fd == -1) { ++ return -1; ++ } ++ ++ if (read(fd, buf, sizeof(buf))) { ++ if (sscanf(buf, "%d-%d", &begin, &end) == 2) { ++ dies = end > begin ? end - begin + 1 : 1; ++ } ++ } ++ ++ close(fd); ++ cores_per_die = ncores / dies; ++ ++ return 0; ++} ++ ++static int get_cpu_status(unsigned cpu) ++{ ++ int fd, num; ++ char buf[2] = ""; ++ fd = open_sys_file(cpu, O_RDONLY, cpu_path_format); ++ ++ if (fd == -1) { ++ return CPU_UNKNOWN; ++ } ++ ++ if (read(fd, buf, 1) <= 0 || sscanf(buf, "%d", &num) != 1) { ++ num = CPU_UNKNOWN; ++ } ++ ++ close(fd); ++ ++ return (num < 0 || num > CPU_UNKNOWN) ? CPU_UNKNOWN : num; ++} ++ ++static int init_cpu_info(unsigned cpus) ++{ ++ ncores = cpus; ++ cpu_infos = (struct cpu_info *) malloc(sizeof(*cpu_infos) * cpus); ++ ++ if (!cpu_infos) { ++ log(TERM, LOG_ERR, "Failed to allocate memory for cpu infos in %s.\n", __func__); ++ return -1; ++ } ++ ++ for (unsigned int i = 0; i < cpus; ++i) { ++ cpu_infos[i].state = get_cpu_status(i); ++ cpu_infos[i].ce_queue = init_queue(); ++ if (cpu_infos[i].ce_queue == NULL) { ++ log(TERM, LOG_ERR, "Failed to allocate memory for cpu ce queue in %s.\n", __func__); ++ return -1; ++ } ++ } ++ /* set limit of offlined cpu limit according to number of cpu */ ++ cpu_limit.limit = cpus - 1; ++ cpu_limit.value = 0; ++ ++ if (get_sockets() < 0 || get_dies() < 0) { ++ log(TERM, LOG_ERR, "Failed to get sockets or nodes of the system\n"); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++static void check_config(struct isolation_param *config) ++{ ++ if (config->value > config->limit) { ++ log(TERM, LOG_WARNING, "Value: %lu exceed limit: %lu, set to limit\n", ++ config->value, config->limit); ++ config->value = config->limit; ++ } ++} ++ ++static int parse_ul_config(struct isolation_param *config, char *env, unsigned long *value) ++{ ++ int env_size, has_unit = 0; ++ ++ if (!env || strlen(env) == 0) { ++ return -1; ++ } ++ ++ env_size = strlen(env); ++ char *unit = NULL; ++ unit = env + env_size - 1; ++ ++ if (isalpha(*unit)) { ++ has_unit = 1; ++ env_size--; ++ if (env_size <= 0) { ++ return -1; ++ } ++ } ++ ++ for (int i = 0; i < env_size; ++i) { ++ if (isdigit(env[i])) { ++ if (*value > ULONG_MAX / 10 || (*value == ULONG_MAX / 10 && env[i] - '0' > 5)) { ++ log(TERM, LOG_ERR, "%s is out of range: %lu\n", env, ULONG_MAX); ++ return -1; ++ } ++ *value = 10 * (*value) + (env[i] - '0'); ++ } ++ else { ++ return -1; ++ } ++ } ++ ++ if (has_unit) { ++ for (const struct param *units = config->units; units->name; units++) { ++ /* value character and unit character are both valid */ ++ if (!strcasecmp(unit, units->name)) { ++ if (*value > (ULONG_MAX / units->value)) { ++ log(TERM, LOG_ERR, "%s is out of range: %lu\n", env, ULONG_MAX); ++ return -1; ++ } ++ *value = (*value) * units->value; ++ return 0; ++ } ++ } ++ log(TERM, LOG_ERR, "Invalid unit %s\n", unit); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++static void init_config(struct isolation_param *config) ++{ ++ char *env = getenv(config->name); ++ unsigned long value = 0; ++ ++ if (parse_ul_config(config, env, &value) < 0) { ++ log(TERM, LOG_ERR, "Invalid %s: %s! Use default value %ld.\n", ++ config->name, env, config->value); ++ return; ++ } ++ ++ config->value = value; ++ check_config(config); ++} ++ ++static int check_config_status(void) ++{ ++ char *env = getenv("CPU_ISOLATION_ENABLE"); ++ ++ if (env == NULL || strcasecmp(env, "yes")) { ++ return -1; ++ } ++ ++ return 0; ++} ++ ++void ras_error_count_init(unsigned cpus) ++{ ++ if (init_cpu_info(cpus) < 0 || check_config_status() < 0) { ++ enabled = 0; ++ log(TERM, LOG_WARNING, "Cpu fault isolation is disabled\n"); ++ return; ++ } ++ ++ log(TERM, LOG_INFO, "Cpu fault isolation is enabled\n"); ++ init_config(&threshold); ++ init_config(&cpu_limit); ++ init_config(&cycle); ++} ++ ++void cpu_infos_free(void) ++{ ++ if (cpu_infos) { ++ for (int i = 0; i < ncores; ++i) { ++ free_queue(cpu_infos[i].ce_queue); ++ } ++ free(cpu_infos); ++ } ++} ++ ++static int do_cpu_offline(unsigned cpu) ++{ ++ int fd, rc; ++ char buf[2] = ""; ++ cpu_infos[cpu].state = CPU_OFFLINE_FAILED; ++ fd = open_sys_file(cpu, O_RDWR, cpu_path_format); ++ ++ if (fd == -1) { ++ return HANDLE_FAILED; ++ } ++ ++ strcpy(buf, "0"); ++ rc = write(fd, buf, strlen(buf)); ++ ++ if (rc < 0) { ++ log(TERM, LOG_ERR, "cpu%d offline failed, errno:%d\n", cpu, errno); ++ close(fd); ++ return HANDLE_FAILED; ++ } ++ ++ close(fd); ++ /* check wthether the cpu is isolated successfully */ ++ cpu_infos[cpu].state = get_cpu_status(cpu); ++ ++ if (cpu_infos[cpu].state == CPU_OFFLINE) { ++ return HANDLE_SUCCEED; ++ } ++ ++ return HANDLE_FAILED; ++} ++ ++static int do_ce_handler(unsigned cpu) ++{ ++ struct link_queue *queue = cpu_infos[cpu].ce_queue; ++ unsigned tmp; ++ /* ++ * Since we just count all error numbers in setted cycle, we store the time ++ * and error numbers from current event to the queue, then everytime we ++ * calculate the period from beginning time to ending time, if the period ++ * exceeds setted cycle, we pop the beginning time and error until the period ++ * from new beginning time to ending time is less than cycle. ++ */ ++ while (queue->head && queue->tail && queue->tail->time - queue->head->time > cycle.value) { ++ tmp = queue->head->value; ++ if (pop(queue) == 0) { ++ cpu_infos[cpu].ce_nums -= tmp; ++ } ++ } ++ ++ if (cpu_infos[cpu].ce_nums >= threshold.value) { ++ log(TERM, LOG_INFO, "Corrected Errors exceeded threshold %ld, try to offline cpu%d\n", ++ threshold.value, cpu); ++ return do_cpu_offline(cpu); ++ } ++ return HANDLE_NOTHING; ++} ++ ++static int do_uce_handler(unsigned cpu) ++{ ++ if (cpu_infos[cpu].uce_nums > 0) { ++ log(TERM, LOG_INFO, "Uncorrected Errors occured, try to offline cpu%d\n", cpu); ++ return do_cpu_offline(cpu); ++ } ++ return HANDLE_NOTHING; ++} ++ ++static int error_handler(unsigned cpu, struct error_info *err_info) ++{ ++ int ret = HANDLE_NOTHING; ++ ++ switch (err_info->err_type) ++ { ++ case CE: ++ ret = do_ce_handler(cpu); ++ break; ++ case UCE: ++ ret = do_uce_handler(cpu); ++ break; ++ default: ++ break; ++ } ++ ++ return ret; ++} ++ ++static void record_error_info(unsigned cpu, struct error_info *err_info) ++{ ++ switch (err_info->err_type) ++ { ++ case CE: ++ { ++ struct queue_node *node = NULL; ++ node = node_create(err_info->time, err_info->nums); ++ if (node == NULL) { ++ log(TERM, LOG_ERR, "Fail to allocate memory for queue node\n"); ++ return; ++ } ++ push(cpu_infos[cpu].ce_queue, node); ++ cpu_infos[cpu].ce_nums += err_info->nums; ++ break; ++ } ++ case UCE: ++ cpu_infos[cpu].uce_nums++; ++ break; ++ default: ++ break; ++ } ++} ++ ++static unsigned long get_bit_value(int64_t value, unsigned offset, unsigned size) ++{ ++ value >>= offset; ++ unsigned long res = 0; ++ int i = 0; ++ ++ while (i < size) { ++ res |= (value & (0x1 << (i++))); ++ } ++ ++ return res; ++} ++ ++static unsigned get_cpu_index(int64_t mpidr) ++{ ++ unsigned core_id, socket_id, die_id, cpu; ++ /* ++ * Adapt to certain BIOS ++ * In the MPIDR: ++ * bit 8:15: core id ++ * bit 19:20: die_id ++ * bit 21:22: socket_id ++ */ ++ core_id = get_bit_value(mpidr, 8, 8); ++ socket_id = get_bit_value(mpidr, 21, 2); ++ die_id = get_bit_value(mpidr, 19, 2); ++ cpu = core_id + socket_id * cores_per_socket + die_id * cores_per_die; ++ ++ return cpu; ++} ++ ++void ras_record_cpu_error(struct error_info *err_info, int64_t mpidr) ++{ ++ unsigned cpu; ++ int ret; ++ ++ if (enabled == 0) { ++ return; ++ } ++ ++ cpu = get_cpu_index(mpidr); ++ ++ if (cpu >= ncores) { ++ log(TERM, LOG_ERR, "The current cpu %d has exceed the total number of cpu:%d\n", cpu, ncores); ++ return; ++ } ++ ++ log(TERM, LOG_INFO, "Handling error on cpu%d\n", cpu); ++ cpu_infos[cpu].state = get_cpu_status(cpu); ++ ++ if (cpu_infos[cpu].state != CPU_ONLINE) { ++ log(TERM, LOG_INFO, "Cpu%d is not online or unknown, ignore\n", cpu); ++ return; ++ } ++ ++ record_error_info(cpu, err_info); ++ /* Since user may change cpu state, we get current offlined cpu numbers every recording time. */ ++ if (ncores - sysconf(_SC_NPROCESSORS_ONLN) >= cpu_limit.value) { ++ log(TERM, LOG_WARNING, "Offlined cpus have exceeded limit: %lu, choose to do nothing\n", ++ cpu_limit.value); ++ return; ++ } ++ ++ ret = error_handler(cpu, err_info); ++ ++ if (ret == HANDLE_NOTHING) { ++ log(TERM, LOG_WARNING, "Doing nothing in the cpu%d\n", cpu); ++ } ++ else if (ret == HANDLE_SUCCEED) { ++ log(TERM, LOG_INFO, "Offline cpu%d succeed, the state is %s\n", ++ cpu, cpu_state[cpu_infos[cpu].state]); ++ clear_queue(cpu_infos[cpu].ce_queue); ++ } ++ else { ++ log(TERM, LOG_INFO, "Offline cpu%d fail, the state is %s\n", ++ cpu, cpu_state[cpu_infos[cpu].state]); ++ } ++ ++ return; ++} +diff --git a/ras-cpu-isolation.h b/ras-cpu-isolation.h +new file mode 100644 +index 0000000..a7d3fdb +--- /dev/null ++++ b/ras-cpu-isolation.h +@@ -0,0 +1,76 @@ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. ++ * ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License as published by ++ * the Free Software Foundation; either version 2 of the License, or ++ * (at your option) any later version. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++*/ ++ ++#ifndef __RAS_CPU_ISOLATION_H ++#define __RAS_CPU_ISOLATION_H ++ ++#include "queue.h" ++ ++#define MAX_PATH_LEN 100 ++#define MAX_BUF_LEN 1024 ++#define PEI_ERR_SIZE 32 ++#define FLAGS_SIZE 1 ++ ++struct param { ++ char *name; ++ unsigned long value; ++}; ++ ++struct isolation_param { ++ char *name; ++ const struct param *units; ++ unsigned long value; ++ unsigned long limit; ++}; ++ ++enum cpu_state { ++ CPU_OFFLINE, ++ CPU_ONLINE, ++ CPU_OFFLINE_FAILED, ++ CPU_UNKNOWN, ++}; ++ ++enum error_handle_result { ++ HANDLE_FAILED = -1, ++ HANDLE_SUCCEED, ++ HANDLE_NOTHING, ++}; ++ ++enum error_type { ++ CE = 1, ++ UCE ++}; ++ ++struct cpu_info { ++ unsigned long uce_nums; ++ unsigned long ce_nums; ++ struct link_queue *ce_queue; ++ enum cpu_state state; ++}; ++ ++struct error_info { ++ unsigned long nums; ++ time_t time; ++ enum error_type err_type; ++}; ++ ++struct cpu_set { ++ char buf[MAX_BUF_LEN]; ++}; ++ ++void ras_error_count_init(unsigned cpus); ++void ras_record_cpu_error(struct error_info *err_info, int64_t mpidr); ++void cpu_infos_free(void); ++ ++#endif +\ No newline at end of file +diff --git a/ras-events.c b/ras-events.c +index 471d25d..31c4170 100644 +--- a/ras-events.c ++++ b/ras-events.c +@@ -40,6 +40,7 @@ + #include "ras-record.h" + #include "ras-logger.h" + #include "ras-page-isolation.h" ++#include "ras-cpu-isolation.h" + + /* + * Polling time, if read() doesn't block. Currently, trace_pipe_raw never +@@ -874,6 +875,10 @@ int handle_ras_events(int record_events) + + cpus = get_num_cpus(ras); + ++#ifdef HAVE_CPU_FAULT_ISOLATION ++ ras_error_count_init(cpus); ++#endif ++ + #ifdef HAVE_MCE + rc = register_mce_handler(ras, cpus); + if (rc) +@@ -990,6 +995,9 @@ err: + } + free(ras); + } ++#ifdef HAVE_CPU_FAULT_ISOLATION ++ cpu_infos_free(); ++#endif + + return rc; + } +diff --git a/ras-record.h b/ras-record.h +index cc217a9..b453f83 100644 +--- a/ras-record.h ++++ b/ras-record.h +@@ -77,6 +77,11 @@ struct ras_arm_event { + int64_t midr; + int32_t running_state; + int32_t psci_state; ++#ifdef HAVE_CPU_FAULT_ISOLATION ++ const char *severity; ++ const uint8_t *error_info; ++ uint32_t length; ++#endif + }; + + struct devlink_event { +-- +2.27.0 + diff --git a/0007-add-trace-print-and-add-sqlite-store.patch b/0007-add-trace-print-and-add-sqlite-store.patch new file mode 100644 index 0000000000000000000000000000000000000000..08361e6cdacc20a9b5bccc5fc251e7014763b7f0 --- /dev/null +++ b/0007-add-trace-print-and-add-sqlite-store.patch @@ -0,0 +1,78 @@ +From 57640072aead2e00037749d66f05fc26e3fe3071 Mon Sep 17 00:00:00 2001 +From: Lostwayzxc +Date: Tue, 25 May 2021 20:07:26 +0800 +Subject: [PATCH 2/2] add trace print of new information and add it to sqilte + +Since we add new information of the event, we add trace print and store it to +Sqlite. + +Signed-off-by: Luo Shengwei +--- + ras-arm-handler.c | 10 ++++++++++ + ras-record.c | 8 ++++++++ + 2 files changed, 18 insertions(+) + +diff --git a/ras-arm-handler.c b/ras-arm-handler.c +index 10d0099..23ad470 100644 +--- a/ras-arm-handler.c ++++ b/ras-arm-handler.c +@@ -23,6 +23,13 @@ + #include "ras-cpu-isolation.h" + + #ifdef HAVE_CPU_FAULT_ISOLATION ++static void trace_print_hex(struct trace_seq *s, const uint8_t *buf, int buf_len) ++{ ++ for (int i = 0; i < buf_len; ++i) { ++ trace_seq_printf(s, "%2.2x", buf[i]); ++ } ++} ++ + static int is_core_failure(unsigned long value) + { + /* +@@ -135,6 +142,7 @@ int ras_arm_event_handler(struct trace_seq *s, + case GHES_SEV_PANIC: + ev.severity = "Fatal"; + } ++ trace_seq_printf(s, "\n severity: %s", ev.severity); + + if (val == GHES_SEV_CORRECTED || val == GHES_SEV_RECOVERABLE) { + int len, nums; +@@ -142,6 +150,8 @@ int ras_arm_event_handler(struct trace_seq *s, + if (!ev.error_info) + return -1; + ev.length = len; ++ trace_seq_printf(s, "\n processor_err_info: "); ++ trace_print_hex(s, ev.error_info, len); + /* relate to enum error_type */ + nums = count_errors(event, ev.error_info, len); + if (nums > 0) { +diff --git a/ras-record.c b/ras-record.c +index 549c494..33d4741 100644 +--- a/ras-record.c ++++ b/ras-record.c +@@ -210,6 +210,10 @@ static const struct db_fields arm_event_fields[] = { + { .name="mpidr", .type="INTEGER" }, + { .name="running_state", .type="INTEGER" }, + { .name="psci_state", .type="INTEGER" }, ++#ifdef HAVE_CPU_FAULT_ISOLATION ++ { .name="severity", .type="TEXT" }, ++ { .name="error_info", .type="BLOB" }, ++#endif + }; + + static const struct db_table_descriptor arm_event_tab = { +@@ -233,6 +237,10 @@ int ras_store_arm_record(struct ras_events *ras, struct ras_arm_event *ev) + sqlite3_bind_int64 (priv->stmt_arm_record, 4, ev->mpidr); + sqlite3_bind_int (priv->stmt_arm_record, 5, ev->running_state); + sqlite3_bind_int (priv->stmt_arm_record, 6, ev->psci_state); ++#ifdef HAVE_CPU_FAULT_ISOLATION ++ sqlite3_bind_text (priv->stmt_arm_record, 7, ev->severity, -1, NULL); ++ sqlite3_bind_blob (priv->stmt_arm_record, 8, ev->error_info, ev->length, NULL); ++#endif + + rc = sqlite3_step(priv->stmt_arm_record); + if (rc != SQLITE_OK && rc != SQLITE_DONE) +-- +2.27.0 + diff --git a/0008-modify-cpu-parse-for-adapting-to-new-bios-version.patch b/0008-modify-cpu-parse-for-adapting-to-new-bios-version.patch new file mode 100644 index 0000000000000000000000000000000000000000..38ef9ac9a09d3e268e744da4570884789ebd48fb --- /dev/null +++ b/0008-modify-cpu-parse-for-adapting-to-new-bios-version.patch @@ -0,0 +1,60 @@ +From 6b767a2fce615384f062ecb392cd332452bf4482 Mon Sep 17 00:00:00 2001 +From: Lostwayzxc +Date: Wed, 1 Sep 2021 21:00:16 +0800 +Subject: [PATCH] modify cpu parse for adapting to new bios version + +--- + ras-cpu-isolation.c | 20 ++++++++++++++++++-- + 1 file changed, 18 insertions(+), 2 deletions(-) + +diff --git a/ras-cpu-isolation.c b/ras-cpu-isolation.c +index 6dcff70..b1643c4 100644 +--- a/ras-cpu-isolation.c ++++ b/ras-cpu-isolation.c +@@ -25,6 +25,7 @@ + + static struct cpu_info *cpu_infos = NULL; + static unsigned int ncores, cores_per_socket, cores_per_die; ++static unsigned int cores_per_cluster = 4; + static unsigned int sockets, dies = 1; + static unsigned int enabled = 1; + static const char *cpu_path_format = "/sys/devices/system/cpu/cpu%d/online"; +@@ -432,18 +433,33 @@ static unsigned long get_bit_value(int64_t value, unsigned offset, unsigned size + + static unsigned get_cpu_index(int64_t mpidr) + { +- unsigned core_id, socket_id, die_id, cpu; ++ unsigned core_id, cluster_id, socket_id, die_id, cpu; + /* + * Adapt to certain BIOS + * In the MPIDR: + * bit 8:15: core id ++ * bit 16:18: cluster id + * bit 19:20: die_id + * bit 21:22: socket_id + */ + core_id = get_bit_value(mpidr, 8, 8); ++ cluster_id = get_bit_value(mpidr, 16, 3); + socket_id = get_bit_value(mpidr, 21, 2); + die_id = get_bit_value(mpidr, 19, 2); +- cpu = core_id + socket_id * cores_per_socket + die_id * cores_per_die; ++ ++ /* When die id parsed from MPIDR is 1, it means TotemA, and when it's 3, ++ * it means TotemB. When cores per die equal to cores per socket, it means ++ * that there is only one die in the socket, in case that the only die is ++ * TotemB in CPU 1620s, we set die id to 0 directly. ++ */ ++ if (cores_per_die == cores_per_socket) { ++ die_id = 0; ++ } ++ else { ++ die_id = (die_id == 1 ? 0:1); ++ } ++ cpu = core_id + socket_id * cores_per_socket + die_id * cores_per_die + ++ cluster_id * cores_per_cluster; + + return cpu; + } +-- +2.27.0 + diff --git a/rasdaemon.spec b/rasdaemon.spec index 7112d825c30083defe23af5283280c717781209c..a4924ec670d91c02c1a58bbf21c24f7e5a359d6f 100644 --- a/rasdaemon.spec +++ b/rasdaemon.spec @@ -1,6 +1,6 @@ Name: rasdaemon Version: 0.6.6 -Release: 5 +Release: 6 License: GPLv2 Summary: Utility to get Platform Reliability, Availability and Serviceability (RAS) reports via the Kernel tracing events URL: https://github.com/mchehab/rasdaemon.git @@ -32,6 +32,9 @@ Patch10: 0001-rasdaemon-Fix-the-issue-of-sprintf-data-type-mismatc.patch Patch11: 0002-rasdaemon-Fix-the-issue-of-command-option-r-for-hip0.patch Patch12: 0003-rasdaemon-Fix-some-print-format-issues-for-hisi-comm.patch Patch13: 0004-rasdaemon-Add-some-modules-supported-by-hisi-common-.patch +Patch14: 0006-add-cpu-online-fault-isolation.patch +Patch15: 0007-add-trace-print-and-add-sqlite-store.patch +Patch16: 0008-modify-cpu-parse-for-adapting-to-new-bios-version.patch %description The rasdaemon program is a daemon which monitors the platform @@ -78,6 +81,12 @@ rm INSTALL %{buildroot}/usr/include/*.h /usr/bin/systemctl enable rasdaemon.service >/dev/null 2>&1 || : %changelog +* Wed Oct 27 2021 luoshengwei - 0.6.6-6 +- Type:feature +- ID:NA +- SUG:NA +- DESC: Sync three patches, add cpu online fault isolation. + * Wed Oct 20 2021 tanxiaofei - 0.6.6-5 - Type:Bugfix - ID:NA