From 18dbd0efc14aa190b2f4c364fa614b0994af5af0 Mon Sep 17 00:00:00 2001 From: Chuan Zheng Date: Wed, 16 Sep 2020 14:21:56 +0800 Subject: [PATCH 01/15] migration/dirtyrate: setup up query-dirtyrate framwork Add get_dirtyrate_thread() functions to setup query-dirtyrate framework. Signed-off-by: Chuan Zheng Signed-off-by: YanYing Zhuang Reviewed-by: Dr. David Alan Gilbert Reviewed-by: David Edmondson Reviewed-by: Li Qiang Message-Id: <1600237327-33618-2-git-send-email-zhengchuan@huawei.com> Signed-off-by: Dr. David Alan Gilbert --- Makefile.target | 1 + migration/dirtyrate.c | 38 ++++++++++++++++++++++++++++++++++++++ migration/dirtyrate.h | 28 ++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+) create mode 100644 migration/dirtyrate.c create mode 100644 migration/dirtyrate.h diff --git a/Makefile.target b/Makefile.target index 933b27453a..5ea840964c 100644 --- a/Makefile.target +++ b/Makefile.target @@ -161,6 +161,7 @@ obj-y += qapi/ obj-y += memory.o obj-y += memory_mapping.o obj-y += migration/ram.o +obj-y += migration/dirtyrate.o LIBS := $(libs_softmmu) $(LIBS) # Hardware support diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c new file mode 100644 index 0000000000..29ef663acb --- /dev/null +++ b/migration/dirtyrate.c @@ -0,0 +1,38 @@ +/* + * Dirtyrate implement code + * + * Copyright (c) 2020 HUAWEI TECHNOLOGIES CO.,LTD. + * + * Authors: + * Chuan Zheng + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#include "qemu/osdep.h" +#include "qapi/error.h" +#include "cpu.h" +#include "qemu/config-file.h" +#include "exec/memory.h" +#include "exec/ram_addr.h" +#include "exec/target_page.h" +#include "qemu/rcu_queue.h" +#include "qapi/qapi-commands-migration.h" +#include "migration.h" +#include "dirtyrate.h" + +static void calculate_dirtyrate(struct DirtyRateConfig config) +{ + /* todo */ + return; +} + +void *get_dirtyrate_thread(void *arg) +{ + struct DirtyRateConfig config = *(struct DirtyRateConfig *)arg; + + calculate_dirtyrate(config); + + return NULL; +} diff --git a/migration/dirtyrate.h b/migration/dirtyrate.h new file mode 100644 index 0000000000..84ab9409ac --- /dev/null +++ b/migration/dirtyrate.h @@ -0,0 +1,28 @@ +/* + * Dirtyrate common functions + * + * Copyright (c) 2020 HUAWEI TECHNOLOGIES CO., LTD. + * + * Authors: + * Chuan Zheng + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#ifndef QEMU_MIGRATION_DIRTYRATE_H +#define QEMU_MIGRATION_DIRTYRATE_H + +/* + * Sample 512 pages per GB as default. + * TODO: Make it configurable. + */ +#define DIRTYRATE_DEFAULT_SAMPLE_PAGES 512 + +struct DirtyRateConfig { + uint64_t sample_pages_per_gigabytes; /* sample pages per GB */ + int64_t sample_period_seconds; /* time duration between two sampling */ +}; + +void *get_dirtyrate_thread(void *arg); +#endif -- Gitee From bce6583a23182e1de4336a2d5e464fea1d274197 Mon Sep 17 00:00:00 2001 From: Chuan Zheng Date: Wed, 16 Sep 2020 14:21:57 +0800 Subject: [PATCH 02/15] migration/dirtyrate: add DirtyRateStatus to denote calculation status add DirtyRateStatus to denote calculating status. Signed-off-by: Chuan Zheng Reviewed-by: Dr. David Alan Gilbert Reviewed-by: Li Qiang Message-Id: <1600237327-33618-3-git-send-email-zhengchuan@huawei.com> Signed-off-by: Dr. David Alan Gilbert atomic name fixup --- migration/dirtyrate.c | 26 ++++++++++++++++++++++++++ qapi/migration.json | 17 +++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c index 29ef663acb..eb4796db1a 100644 --- a/migration/dirtyrate.c +++ b/migration/dirtyrate.c @@ -22,6 +22,19 @@ #include "migration.h" #include "dirtyrate.h" +static int CalculatingState = DIRTY_RATE_STATUS_UNSTARTED; + +static int dirtyrate_set_state(int *state, int old_state, int new_state) +{ + assert(new_state < DIRTY_RATE_STATUS__MAX); + if (qatomic_cmpxchg(state, old_state, new_state) == old_state) { + return 0; + } else { + return -1; + } +} + + static void calculate_dirtyrate(struct DirtyRateConfig config) { /* todo */ @@ -31,8 +44,21 @@ static void calculate_dirtyrate(struct DirtyRateConfig config) void *get_dirtyrate_thread(void *arg) { struct DirtyRateConfig config = *(struct DirtyRateConfig *)arg; + int ret; + + ret = dirtyrate_set_state(&CalculatingState, DIRTY_RATE_STATUS_UNSTARTED, + DIRTY_RATE_STATUS_MEASURING); + if (ret == -1) { + error_report("change dirtyrate state failed."); + return NULL; + } calculate_dirtyrate(config); + ret = dirtyrate_set_state(&CalculatingState, DIRTY_RATE_STATUS_MEASURING, + DIRTY_RATE_STATUS_MEASURED); + if (ret == -1) { + error_report("change dirtyrate state failed."); + } return NULL; } diff --git a/qapi/migration.json b/qapi/migration.json index 9cfbaf8c6c..fdddde0af7 100644 --- a/qapi/migration.json +++ b/qapi/migration.json @@ -1445,3 +1445,20 @@ # Since: 3.0 ## { 'command': 'migrate-pause', 'allow-oob': true } + +## +# @DirtyRateStatus: +# +# An enumeration of dirtyrate status. +# +# @unstarted: the dirtyrate thread has not been started. +# +# @measuring: the dirtyrate thread is measuring. +# +# @measured: the dirtyrate thread has measured and results are available. +# +# Since: 5.2 +# +## +{ 'enum': 'DirtyRateStatus', + 'data': [ 'unstarted', 'measuring', 'measured'] } -- Gitee From 0755134cab463b94bd83e83cfe99ec86f8f4ab85 Mon Sep 17 00:00:00 2001 From: Chuan Zheng Date: Wed, 16 Sep 2020 14:21:58 +0800 Subject: [PATCH 03/15] migration/dirtyrate: Add RamblockDirtyInfo to store sampled page info Add RamblockDirtyInfo to store sampled page info of each ramblock. Signed-off-by: Chuan Zheng Reviewed-by: Dr. David Alan Gilbert Reviewed-by: David Edmondson Reviewed-by: Li Qiang Message-Id: <1600237327-33618-4-git-send-email-zhengchuan@huawei.com> Signed-off-by: Dr. David Alan Gilbert --- migration/dirtyrate.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/migration/dirtyrate.h b/migration/dirtyrate.h index 84ab9409ac..8707df852d 100644 --- a/migration/dirtyrate.h +++ b/migration/dirtyrate.h @@ -19,10 +19,28 @@ */ #define DIRTYRATE_DEFAULT_SAMPLE_PAGES 512 +/* + * Record ramblock idstr + */ +#define RAMBLOCK_INFO_MAX_LEN 256 + struct DirtyRateConfig { uint64_t sample_pages_per_gigabytes; /* sample pages per GB */ int64_t sample_period_seconds; /* time duration between two sampling */ }; +/* + * Store dirtypage info for each ramblock. + */ +struct RamblockDirtyInfo { + char idstr[RAMBLOCK_INFO_MAX_LEN]; /* idstr for each ramblock */ + uint8_t *ramblock_addr; /* base address of ramblock we measure */ + uint64_t ramblock_pages; /* ramblock size in TARGET_PAGE_SIZE */ + uint64_t *sample_page_vfn; /* relative offset address for sampled page */ + uint64_t sample_pages_count; /* count of sampled pages */ + uint64_t sample_dirty_count; /* count of dirty pages we measure */ + uint32_t *hash_result; /* array of hash result for sampled pages */ +}; + void *get_dirtyrate_thread(void *arg); #endif -- Gitee From 91276b2841e2bd4ead912760aad03e5c65cb54a0 Mon Sep 17 00:00:00 2001 From: Chuan Zheng Date: Wed, 16 Sep 2020 14:21:59 +0800 Subject: [PATCH 04/15] migration/dirtyrate: Add dirtyrate statistics series functions Add dirtyrate statistics functions to record/update dirtyrate info. Signed-off-by: Chuan Zheng Reviewed-by: Dr. David Alan Gilbert Reviewed-by: Li Qiang Message-Id: <1600237327-33618-5-git-send-email-zhengchuan@huawei.com> Signed-off-by: Dr. David Alan Gilbert --- migration/dirtyrate.c | 32 ++++++++++++++++++++++++++++++++ migration/dirtyrate.h | 12 ++++++++++++ 2 files changed, 44 insertions(+) diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c index eb4796db1a..82c66134c0 100644 --- a/migration/dirtyrate.c +++ b/migration/dirtyrate.c @@ -23,6 +23,7 @@ #include "dirtyrate.h" static int CalculatingState = DIRTY_RATE_STATUS_UNSTARTED; +static struct DirtyRateStat DirtyStat; static int dirtyrate_set_state(int *state, int old_state, int new_state) { @@ -34,6 +35,37 @@ static int dirtyrate_set_state(int *state, int old_state, int new_state) } } +static void reset_dirtyrate_stat(void) +{ + DirtyStat.total_dirty_samples = 0; + DirtyStat.total_sample_count = 0; + DirtyStat.total_block_mem_MB = 0; + DirtyStat.dirty_rate = -1; + DirtyStat.start_time = 0; + DirtyStat.calc_time = 0; +} + +static void update_dirtyrate_stat(struct RamblockDirtyInfo *info) +{ + DirtyStat.total_dirty_samples += info->sample_dirty_count; + DirtyStat.total_sample_count += info->sample_pages_count; + /* size of total pages in MB */ + DirtyStat.total_block_mem_MB += (info->ramblock_pages * + TARGET_PAGE_SIZE) >> 20; +} + +static void update_dirtyrate(uint64_t msec) +{ + uint64_t dirtyrate; + uint64_t total_dirty_samples = DirtyStat.total_dirty_samples; + uint64_t total_sample_count = DirtyStat.total_sample_count; + uint64_t total_block_mem_MB = DirtyStat.total_block_mem_MB; + + dirtyrate = total_dirty_samples * total_block_mem_MB * + 1000 / (total_sample_count * msec); + + DirtyStat.dirty_rate = dirtyrate; +} static void calculate_dirtyrate(struct DirtyRateConfig config) { diff --git a/migration/dirtyrate.h b/migration/dirtyrate.h index 8707df852d..312debca6f 100644 --- a/migration/dirtyrate.h +++ b/migration/dirtyrate.h @@ -42,5 +42,17 @@ struct RamblockDirtyInfo { uint32_t *hash_result; /* array of hash result for sampled pages */ }; +/* + * Store calculation statistics for each measure. + */ +struct DirtyRateStat { + uint64_t total_dirty_samples; /* total dirty sampled page */ + uint64_t total_sample_count; /* total sampled pages */ + uint64_t total_block_mem_MB; /* size of total sampled pages in MB */ + int64_t dirty_rate; /* dirty rate in MB/s */ + int64_t start_time; /* calculation start time in units of second */ + int64_t calc_time; /* time duration of two sampling in units of second */ +}; + void *get_dirtyrate_thread(void *arg); #endif -- Gitee From 821df1239de26f2a001e786ebb7c16dfd181674f Mon Sep 17 00:00:00 2001 From: Chuan Zheng Date: Wed, 16 Sep 2020 14:22:00 +0800 Subject: [PATCH 05/15] migration/dirtyrate: move RAMBLOCK_FOREACH_MIGRATABLE into ram.h RAMBLOCK_FOREACH_MIGRATABLE is need in dirtyrate measure, move the existing definition up into migration/ram.h Signed-off-by: Chuan Zheng Reviewed-by: Dr. David Alan Gilbert Reviewed-by: David Edmondson Reviewed-by: Li Qiang Message-Id: <1600237327-33618-6-git-send-email-zhengchuan@huawei.com> Signed-off-by: Dr. David Alan Gilbert --- migration/dirtyrate.c | 1 + migration/ram.c | 11 +---------- migration/ram.h | 10 ++++++++++ 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c index 82c66134c0..8781611ce6 100644 --- a/migration/dirtyrate.c +++ b/migration/dirtyrate.c @@ -20,6 +20,7 @@ #include "qemu/rcu_queue.h" #include "qapi/qapi-commands-migration.h" #include "migration.h" +#include "ram.h" #include "dirtyrate.h" static int CalculatingState = DIRTY_RATE_STATUS_UNSTARTED; diff --git a/migration/ram.c b/migration/ram.c index 848059d9fb..1a33c7b3e2 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -159,21 +159,12 @@ out: return ret; } -static bool ramblock_is_ignored(RAMBlock *block) +bool ramblock_is_ignored(RAMBlock *block) { return !qemu_ram_is_migratable(block) || (migrate_ignore_shared() && qemu_ram_is_shared(block)); } -/* Should be holding either ram_list.mutex, or the RCU lock. */ -#define RAMBLOCK_FOREACH_NOT_IGNORED(block) \ - INTERNAL_RAMBLOCK_FOREACH(block) \ - if (ramblock_is_ignored(block)) {} else - -#define RAMBLOCK_FOREACH_MIGRATABLE(block) \ - INTERNAL_RAMBLOCK_FOREACH(block) \ - if (!qemu_ram_is_migratable(block)) {} else - #undef RAMBLOCK_FOREACH int foreach_not_ignored_block(RAMBlockIterFunc func, void *opaque) diff --git a/migration/ram.h b/migration/ram.h index a788ff0e8e..565ec86b1f 100644 --- a/migration/ram.h +++ b/migration/ram.h @@ -37,6 +37,16 @@ extern MigrationStats ram_counters; extern XBZRLECacheStats xbzrle_counters; extern CompressionStats compression_counters; +bool ramblock_is_ignored(RAMBlock *block); +/* Should be holding either ram_list.mutex, or the RCU lock. */ +#define RAMBLOCK_FOREACH_NOT_IGNORED(block) \ + INTERNAL_RAMBLOCK_FOREACH(block) \ + if (ramblock_is_ignored(block)) {} else + +#define RAMBLOCK_FOREACH_MIGRATABLE(block) \ + INTERNAL_RAMBLOCK_FOREACH(block) \ + if (!qemu_ram_is_migratable(block)) {} else + int xbzrle_cache_resize(int64_t new_size, Error **errp); uint64_t ram_bytes_remaining(void); uint64_t ram_bytes_total(void); -- Gitee From bcf9337c7ff15923f2ddf5b57d1c88f0f92a6d5d Mon Sep 17 00:00:00 2001 From: Chuan Zheng Date: Wed, 16 Sep 2020 14:22:01 +0800 Subject: [PATCH 06/15] migration/dirtyrate: Record hash results for each sampled page Record hash results for each sampled page, crc32 is taken to calculate hash results for each sampled length in TARGET_PAGE_SIZE. Signed-off-by: Chuan Zheng Signed-off-by: YanYing Zhuang Reviewed-by: David Edmondson Reviewed-by: Li Qiang Message-Id: <1600237327-33618-7-git-send-email-zhengchuan@huawei.com> Signed-off-by: Dr. David Alan Gilbert --- migration/dirtyrate.c | 109 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c index 8781611ce6..e585119b23 100644 --- a/migration/dirtyrate.c +++ b/migration/dirtyrate.c @@ -10,6 +10,7 @@ * See the COPYING file in the top-level directory. */ +#include #include "qemu/osdep.h" #include "qapi/error.h" #include "cpu.h" @@ -68,6 +69,114 @@ static void update_dirtyrate(uint64_t msec) DirtyStat.dirty_rate = dirtyrate; } +/* + * get hash result for the sampled memory with length of TARGET_PAGE_SIZE + * in ramblock, which starts from ramblock base address. + */ +static uint32_t get_ramblock_vfn_hash(struct RamblockDirtyInfo *info, + uint64_t vfn) +{ + uint32_t crc; + + crc = crc32(0, (info->ramblock_addr + + vfn * TARGET_PAGE_SIZE), TARGET_PAGE_SIZE); + + return crc; +} + +static bool save_ramblock_hash(struct RamblockDirtyInfo *info) +{ + unsigned int sample_pages_count; + int i; + GRand *rand; + + sample_pages_count = info->sample_pages_count; + + /* ramblock size less than one page, return success to skip this ramblock */ + if (unlikely(info->ramblock_pages == 0 || sample_pages_count == 0)) { + return true; + } + + info->hash_result = g_try_malloc0_n(sample_pages_count, + sizeof(uint32_t)); + if (!info->hash_result) { + return false; + } + + info->sample_page_vfn = g_try_malloc0_n(sample_pages_count, + sizeof(uint64_t)); + if (!info->sample_page_vfn) { + g_free(info->hash_result); + return false; + } + + rand = g_rand_new(); + for (i = 0; i < sample_pages_count; i++) { + info->sample_page_vfn[i] = g_rand_int_range(rand, 0, + info->ramblock_pages - 1); + info->hash_result[i] = get_ramblock_vfn_hash(info, + info->sample_page_vfn[i]); + } + g_rand_free(rand); + + return true; +} + +static void get_ramblock_dirty_info(RAMBlock *block, + struct RamblockDirtyInfo *info, + struct DirtyRateConfig *config) +{ + uint64_t sample_pages_per_gigabytes = config->sample_pages_per_gigabytes; + + /* Right shift 30 bits to calc ramblock size in GB */ + info->sample_pages_count = (qemu_ram_get_used_length(block) * + sample_pages_per_gigabytes) >> 30; + /* Right shift TARGET_PAGE_BITS to calc page count */ + info->ramblock_pages = qemu_ram_get_used_length(block) >> + TARGET_PAGE_BITS; + info->ramblock_addr = qemu_ram_get_host_addr(block); + strcpy(info->idstr, qemu_ram_get_idstr(block)); +} + +static bool record_ramblock_hash_info(struct RamblockDirtyInfo **block_dinfo, + struct DirtyRateConfig config, + int *block_count) +{ + struct RamblockDirtyInfo *info = NULL; + struct RamblockDirtyInfo *dinfo = NULL; + RAMBlock *block = NULL; + int total_count = 0; + int index = 0; + bool ret = false; + + RAMBLOCK_FOREACH_MIGRATABLE(block) { + total_count++; + } + + dinfo = g_try_malloc0_n(total_count, sizeof(struct RamblockDirtyInfo)); + if (dinfo == NULL) { + goto out; + } + + RAMBLOCK_FOREACH_MIGRATABLE(block) { + if (index >= total_count) { + break; + } + info = &dinfo[index]; + get_ramblock_dirty_info(block, info, &config); + if (!save_ramblock_hash(info)) { + goto out; + } + index++; + } + ret = true; + +out: + *block_count = index; + *block_dinfo = dinfo; + return ret; +} + static void calculate_dirtyrate(struct DirtyRateConfig config) { /* todo */ -- Gitee From e09099ad105dae1e2e90fb75636993b8d2c9f45b Mon Sep 17 00:00:00 2001 From: Chuan Zheng Date: Wed, 16 Sep 2020 14:22:02 +0800 Subject: [PATCH 07/15] migration/dirtyrate: Compare page hash results for recorded sampled page Compare page hash results for recorded sampled page. Signed-off-by: Chuan Zheng Signed-off-by: YanYing Zhuang Reviewed-by: Dr. David Alan Gilbert Reviewed-by: Li Qiang Message-Id: <1600237327-33618-8-git-send-email-zhengchuan@huawei.com> Signed-off-by: Dr. David Alan Gilbert --- migration/dirtyrate.c | 63 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c index e585119b23..9b0c52295d 100644 --- a/migration/dirtyrate.c +++ b/migration/dirtyrate.c @@ -177,6 +177,69 @@ out: return ret; } +static void calc_page_dirty_rate(struct RamblockDirtyInfo *info) +{ + uint32_t crc; + int i; + + for (i = 0; i < info->sample_pages_count; i++) { + crc = get_ramblock_vfn_hash(info, info->sample_page_vfn[i]); + if (crc != info->hash_result[i]) { + info->sample_dirty_count++; + } + } +} + +static struct RamblockDirtyInfo * +find_block_matched(RAMBlock *block, int count, + struct RamblockDirtyInfo *infos) +{ + int i; + struct RamblockDirtyInfo *matched; + + for (i = 0; i < count; i++) { + if (!strcmp(infos[i].idstr, qemu_ram_get_idstr(block))) { + break; + } + } + + if (i == count) { + return NULL; + } + + if (infos[i].ramblock_addr != qemu_ram_get_host_addr(block) || + infos[i].ramblock_pages != + (qemu_ram_get_used_length(block) >> TARGET_PAGE_BITS)) { + return NULL; + } + + matched = &infos[i]; + + return matched; +} + +static bool compare_page_hash_info(struct RamblockDirtyInfo *info, + int block_count) +{ + struct RamblockDirtyInfo *block_dinfo = NULL; + RAMBlock *block = NULL; + + RAMBLOCK_FOREACH_MIGRATABLE(block) { + block_dinfo = find_block_matched(block, block_count, info); + if (block_dinfo == NULL) { + continue; + } + calc_page_dirty_rate(block_dinfo); + update_dirtyrate_stat(block_dinfo); + } + + if (DirtyStat.total_sample_count == 0) { + return false; + } + + return true; +} + static void calculate_dirtyrate(struct DirtyRateConfig config) { /* todo */ -- Gitee From a070663909dd12d5ef89c81aecde8a2b783ee7b5 Mon Sep 17 00:00:00 2001 From: Chuan Zheng Date: Wed, 16 Sep 2020 14:22:03 +0800 Subject: [PATCH 08/15] migration/dirtyrate: skip sampling ramblock with size below MIN_RAMBLOCK_SIZE In order to sample real RAM, skip ramblock with size below MIN_RAMBLOCK_SIZE which is set as 128M. Signed-off-by: Chuan Zheng Reviewed-by: David Edmondson Reviewed-by: Dr. David Alan Gilbert Reviewed-by: Li Qiang Message-Id: <1600237327-33618-9-git-send-email-zhengchuan@huawei.com> Signed-off-by: Dr. David Alan Gilbert --- migration/dirtyrate.c | 21 +++++++++++++++++++++ migration/dirtyrate.h | 5 +++++ 2 files changed, 26 insertions(+) diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c index 9b0c52295d..870f857ca3 100644 --- a/migration/dirtyrate.c +++ b/migration/dirtyrate.c @@ -138,6 +138,18 @@ static void get_ramblock_dirty_info(RAMBlock *block, strcpy(info->idstr, qemu_ram_get_idstr(block)); } +static bool skip_sample_ramblock(RAMBlock *block) +{ + /* + * Sample only blocks larger than MIN_RAMBLOCK_SIZE. + */ + if (qemu_ram_get_used_length(block) < (MIN_RAMBLOCK_SIZE << 10)) { + return true; + } + + return false; +} + static bool record_ramblock_hash_info(struct RamblockDirtyInfo **block_dinfo, struct DirtyRateConfig config, int *block_count) @@ -150,6 +162,9 @@ static bool record_ramblock_hash_info(struct RamblockDirtyInfo **block_dinfo, bool ret = false; RAMBLOCK_FOREACH_MIGRATABLE(block) { + if (skip_sample_ramblock(block)) { + continue; + } total_count++; } @@ -159,6 +174,9 @@ static bool record_ramblock_hash_info(struct RamblockDirtyInfo **block_dinfo, } RAMBLOCK_FOREACH_MIGRATABLE(block) { + if (skip_sample_ramblock(block)) { + continue; + } if (index >= total_count) { break; } @@ -225,6 +243,9 @@ static bool compare_page_hash_info(struct RamblockDirtyInfo *info, RAMBlock *block = NULL; RAMBLOCK_FOREACH_MIGRATABLE(block) { + if (skip_sample_ramblock(block)) { + continue; + } block_dinfo = find_block_matched(block, block_count, info); if (block_dinfo == NULL) { continue; diff --git a/migration/dirtyrate.h b/migration/dirtyrate.h index 312debca6f..be5b8ec2b1 100644 --- a/migration/dirtyrate.h +++ b/migration/dirtyrate.h @@ -24,6 +24,11 @@ */ #define RAMBLOCK_INFO_MAX_LEN 256 +/* + * Minimum RAMBlock size to sample, in megabytes. + */ +#define MIN_RAMBLOCK_SIZE 128 + struct DirtyRateConfig { uint64_t sample_pages_per_gigabytes; /* sample pages per GB */ int64_t sample_period_seconds; /* time duration between two sampling */ -- Gitee From 73fa074ab54fba42d7c915def2bc0cd55db88fd0 Mon Sep 17 00:00:00 2001 From: Chuan Zheng Date: Wed, 16 Sep 2020 14:22:04 +0800 Subject: [PATCH 09/15] migration/dirtyrate: Implement set_sample_page_period() and is_sample_period_valid() Implement is_sample_period_valid() to check if the sample period is vaild and do set_sample_page_period() to sleep specific time between sample actions. Signed-off-by: Chuan Zheng Reviewed-by: Dr. David Alan Gilbert Reviewed-by: David Edmondson Reviewed-by: Li Qiang Message-Id: <1600237327-33618-10-git-send-email-zhengchuan@huawei.com> Signed-off-by: Dr. David Alan Gilbert --- migration/dirtyrate.c | 24 ++++++++++++++++++++++++ migration/dirtyrate.h | 6 ++++++ 2 files changed, 30 insertions(+) diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c index 870f857ca3..0ecda70678 100644 --- a/migration/dirtyrate.c +++ b/migration/dirtyrate.c @@ -27,6 +27,30 @@ static int CalculatingState = DIRTY_RATE_STATUS_UNSTARTED; static struct DirtyRateStat DirtyStat; +static int64_t set_sample_page_period(int64_t msec, int64_t initial_time) +{ + int64_t current_time; + + current_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); + if ((current_time - initial_time) >= msec) { + msec = current_time - initial_time; + } else { + g_usleep((msec + initial_time - current_time) * 1000); + } + + return msec; +} + +static bool is_sample_period_valid(int64_t sec) +{ + if (sec < MIN_FETCH_DIRTYRATE_TIME_SEC || + sec > MAX_FETCH_DIRTYRATE_TIME_SEC) { + return false; + } + + return true; +} + static int dirtyrate_set_state(int *state, int old_state, int new_state) { assert(new_state < DIRTY_RATE_STATUS__MAX); diff --git a/migration/dirtyrate.h b/migration/dirtyrate.h index be5b8ec2b1..6ec429534d 100644 --- a/migration/dirtyrate.h +++ b/migration/dirtyrate.h @@ -29,6 +29,12 @@ */ #define MIN_RAMBLOCK_SIZE 128 +/* + * Take 1s as minimum time for calculation duration + */ +#define MIN_FETCH_DIRTYRATE_TIME_SEC 1 +#define MAX_FETCH_DIRTYRATE_TIME_SEC 60 + struct DirtyRateConfig { uint64_t sample_pages_per_gigabytes; /* sample pages per GB */ int64_t sample_period_seconds; /* time duration between two sampling */ -- Gitee From 0bf2e570741daf29f0e19b3b866ed43702375a54 Mon Sep 17 00:00:00 2001 From: Chuan Zheng Date: Wed, 16 Sep 2020 14:22:05 +0800 Subject: [PATCH 10/15] migration/dirtyrate: Implement calculate_dirtyrate() function Implement calculate_dirtyrate() function. Signed-off-by: Chuan Zheng Signed-off-by: YanYing Zhuang Reviewed-by: Dr. David Alan Gilbert Reviewed-by: Li Qiang Message-Id: <1600237327-33618-11-git-send-email-zhengchuan@huawei.com> Signed-off-by: Dr. David Alan Gilbert --- migration/dirtyrate.c | 45 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c index 0ecda70678..446078df5a 100644 --- a/migration/dirtyrate.c +++ b/migration/dirtyrate.c @@ -162,6 +162,21 @@ static void get_ramblock_dirty_info(RAMBlock *block, strcpy(info->idstr, qemu_ram_get_idstr(block)); } +static void free_ramblock_dirty_info(struct RamblockDirtyInfo *infos, int count) +{ + int i; + + if (!infos) { + return; + } + + for (i = 0; i < count; i++) { + g_free(infos[i].sample_page_vfn); + g_free(infos[i].hash_result); + } + g_free(infos); +} + static bool skip_sample_ramblock(RAMBlock *block) { /* @@ -287,8 +302,34 @@ static bool compare_page_hash_info(struct RamblockDirtyInfo *info, static void calculate_dirtyrate(struct DirtyRateConfig config) { - /* todo */ - return; + struct RamblockDirtyInfo *block_dinfo = NULL; + int block_count = 0; + int64_t msec = 0; + int64_t initial_time; + + rcu_register_thread(); + reset_dirtyrate_stat(); + rcu_read_lock(); + initial_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); + if (!record_ramblock_hash_info(&block_dinfo, config, &block_count)) { + goto out; + } + rcu_read_unlock(); + + msec = config.sample_period_seconds * 1000; + msec = set_sample_page_period(msec, initial_time); + + rcu_read_lock(); + if (!compare_page_hash_info(block_dinfo, block_count)) { + goto out; + } + + update_dirtyrate(msec); + +out: + rcu_read_unlock(); + free_ramblock_dirty_info(block_dinfo, block_count); + rcu_unregister_thread(); } void *get_dirtyrate_thread(void *arg) -- Gitee From 49f90ba99bded7ff60ae3d249f7bc26945d05fc1 Mon Sep 17 00:00:00 2001 From: Chuan Zheng Date: Wed, 16 Sep 2020 14:22:06 +0800 Subject: [PATCH 11/15] migration/dirtyrate: Implement qmp_cal_dirty_rate()/qmp_get_dirty_rate() function Implement qmp_cal_dirty_rate()/qmp_get_dirty_rate() function which could be called Signed-off-by: Chuan Zheng Message-Id: <1600237327-33618-12-git-send-email-zhengchuan@huawei.com> Reviewed-by: Dr. David Alan Gilbert Signed-off-by: Dr. David Alan Gilbert atomic function fixup Wording fixup in migration.json based on Eric's review --- migration/dirtyrate.c | 62 +++++++++++++++++++++++++++++++++++++++++++ qapi/migration.json | 50 ++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c index 446078df5a..4839352f35 100644 --- a/migration/dirtyrate.c +++ b/migration/dirtyrate.c @@ -61,6 +61,24 @@ static int dirtyrate_set_state(int *state, int old_state, int new_state) } } +static struct DirtyRateInfo *query_dirty_rate_info(void) +{ + int64_t dirty_rate = DirtyStat.dirty_rate; + struct DirtyRateInfo *info = g_malloc0(sizeof(DirtyRateInfo)); + + if (qatomic_read(&CalculatingState) == DIRTY_RATE_STATUS_MEASURED) { + info->dirty_rate = dirty_rate; + } else { + info->dirty_rate = -1; + } + + info->status = CalculatingState; + info->start_time = DirtyStat.start_time; + info->calc_time = DirtyStat.calc_time; + + return info; +} + static void reset_dirtyrate_stat(void) { DirtyStat.total_dirty_samples = 0; @@ -318,6 +336,8 @@ static void calculate_dirtyrate(struct DirtyRateConfig config) msec = config.sample_period_seconds * 1000; msec = set_sample_page_period(msec, initial_time); + DirtyStat.start_time = initial_time / 1000; + DirtyStat.calc_time = msec / 1000; rcu_read_lock(); if (!compare_page_hash_info(block_dinfo, block_count)) { @@ -353,3 +373,45 @@ void *get_dirtyrate_thread(void *arg) } return NULL; } + +void qmp_calc_dirty_rate(int64_t calc_time, Error **errp) +{ + static struct DirtyRateConfig config; + QemuThread thread; + int ret; + + /* + * If the dirty rate is already being measured, don't attempt to start. + */ + if (qatomic_read(&CalculatingState) == DIRTY_RATE_STATUS_MEASURING) { + error_setg(errp, "the dirty rate is already being measured."); + return; + } + + if (!is_sample_period_valid(calc_time)) { + error_setg(errp, "calc-time is out of range[%d, %d].", + MIN_FETCH_DIRTYRATE_TIME_SEC, + MAX_FETCH_DIRTYRATE_TIME_SEC); + return; + } + + /* + * Init calculation state as unstarted. + */ + ret = dirtyrate_set_state(&CalculatingState, CalculatingState, + DIRTY_RATE_STATUS_UNSTARTED); + if (ret == -1) { + error_setg(errp, "init dirty rate calculation state failed."); + return; + } + + config.sample_period_seconds = calc_time; + config.sample_pages_per_gigabytes = DIRTYRATE_DEFAULT_SAMPLE_PAGES; + qemu_thread_create(&thread, "get_dirtyrate", get_dirtyrate_thread, + (void *)&config, QEMU_THREAD_DETACHED); +} + +struct DirtyRateInfo *qmp_query_dirty_rate(Error **errp) +{ + return query_dirty_rate_info(); +} diff --git a/qapi/migration.json b/qapi/migration.json index fdddde0af7..76f5b42493 100644 --- a/qapi/migration.json +++ b/qapi/migration.json @@ -1462,3 +1462,53 @@ ## { 'enum': 'DirtyRateStatus', 'data': [ 'unstarted', 'measuring', 'measured'] } + +## +# @DirtyRateInfo: +# +# Information about current dirty page rate of vm. +# +# @dirty-rate: @dirtyrate describing the dirty page rate of vm +# in units of MB/s. +# If this field returns '-1', it means querying has not +# yet started or completed. +# +# @status: status containing dirtyrate query status includes +# 'unstarted' or 'measuring' or 'measured' +# +# @start-time: start time in units of second for calculation +# +# @calc-time: time in units of second for sample dirty pages +# +# Since: 5.2 +# +## +{ 'struct': 'DirtyRateInfo', + 'data': {'dirty-rate': 'int64', + 'status': 'DirtyRateStatus', + 'start-time': 'int64', + 'calc-time': 'int64'} } + +## +# @calc-dirty-rate: +# +# start calculating dirty page rate for vm +# +# @calc-time: time in units of second for sample dirty pages +# +# Since: 5.2 +# +# Example: +# {"command": "calc-dirty-rate", "data": {"calc-time": 1} } +# +## +{ 'command': 'calc-dirty-rate', 'data': {'calc-time': 'int64'} } + +## +# @query-dirty-rate: +# +# query dirty page rate in units of MB/s for vm +# +# Since: 5.2 +## +{ 'command': 'query-dirty-rate', 'returns': 'DirtyRateInfo' } -- Gitee From df2bc36a41819e01341efc9733c978ad02025020 Mon Sep 17 00:00:00 2001 From: Chuan Zheng Date: Wed, 16 Sep 2020 14:22:07 +0800 Subject: [PATCH 12/15] migration/dirtyrate: Add trace_calls to make it easier to debug Add trace_calls to make it easier to debug Signed-off-by: Chuan Zheng Reviewed-by: Dr. David Alan Gilbert Reviewed-by: David Edmondson Message-Id: <1600237327-33618-13-git-send-email-zhengchuan@huawei.com> Signed-off-by: Dr. David Alan Gilbert --- migration/dirtyrate.c | 9 +++++++++ migration/trace-events | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c index 4839352f35..b87069a798 100644 --- a/migration/dirtyrate.c +++ b/migration/dirtyrate.c @@ -22,6 +22,7 @@ #include "qapi/qapi-commands-migration.h" #include "migration.h" #include "ram.h" +#include "trace.h" #include "dirtyrate.h" static int CalculatingState = DIRTY_RATE_STATUS_UNSTARTED; @@ -54,6 +55,7 @@ static bool is_sample_period_valid(int64_t sec) static int dirtyrate_set_state(int *state, int old_state, int new_state) { assert(new_state < DIRTY_RATE_STATUS__MAX); + trace_dirtyrate_set_state(DirtyRateStatus_str(new_state)); if (qatomic_cmpxchg(state, old_state, new_state) == old_state) { return 0; } else { @@ -76,6 +78,8 @@ static struct DirtyRateInfo *query_dirty_rate_info(void) info->start_time = DirtyStat.start_time; info->calc_time = DirtyStat.calc_time; + trace_query_dirty_rate_info(DirtyRateStatus_str(CalculatingState)); + return info; } @@ -123,6 +127,7 @@ static uint32_t get_ramblock_vfn_hash(struct RamblockDirtyInfo *info, crc = crc32(0, (info->ramblock_addr + vfn * TARGET_PAGE_SIZE), TARGET_PAGE_SIZE); + trace_get_ramblock_vfn_hash(info->idstr, vfn, crc); return crc; } @@ -201,6 +206,8 @@ static bool skip_sample_ramblock(RAMBlock *block) * Sample only blocks larger than MIN_RAMBLOCK_SIZE. */ if (qemu_ram_get_used_length(block) < (MIN_RAMBLOCK_SIZE << 10)) { + trace_skip_sample_ramblock(block->idstr, + qemu_ram_get_used_length(block)); return true; } @@ -260,6 +267,7 @@ static void calc_page_dirty_rate(struct RamblockDirtyInfo *info) for (i = 0; i < info->sample_pages_count; i++) { crc = get_ramblock_vfn_hash(info, info->sample_page_vfn[i]); if (crc != info->hash_result[i]) { + trace_calc_page_dirty_rate(info->idstr, crc, info->hash_result[i]); info->sample_dirty_count++; } } @@ -285,6 +293,7 @@ find_block_matched(RAMBlock *block, int count, if (infos[i].ramblock_addr != qemu_ram_get_host_addr(block) || infos[i].ramblock_pages != (qemu_ram_get_used_length(block) >> TARGET_PAGE_BITS)) { + trace_find_page_matched(block->idstr); return NULL; } diff --git a/migration/trace-events b/migration/trace-events index d8e54c367a..69620c43c2 100644 --- a/migration/trace-events +++ b/migration/trace-events @@ -296,3 +296,11 @@ dirty_bitmap_load_bits_zeroes(void) "" dirty_bitmap_load_header(uint32_t flags) "flags 0x%x" dirty_bitmap_load_enter(void) "" dirty_bitmap_load_success(void) "" + +# dirtyrate.c +dirtyrate_set_state(const char *new_state) "new state %s" +query_dirty_rate_info(const char *new_state) "current state %s" +get_ramblock_vfn_hash(const char *idstr, uint64_t vfn, uint32_t crc) "ramblock name: %s, vfn: %"PRIu64 ", crc: %" PRIu32 +calc_page_dirty_rate(const char *idstr, uint32_t new_crc, uint32_t old_crc) "ramblock name: %s, new crc: %" PRIu32 ", old crc: %" PRIu32 +skip_sample_ramblock(const char *idstr, uint64_t ramblock_size) "ramblock name: %s, ramblock size: %" PRIu64 +find_page_matched(const char *idstr) "ramblock %s addr or size changed" -- Gitee From 58300511c27ca282c9236cb5a26badde206cfca8 Mon Sep 17 00:00:00 2001 From: Chuan Zheng Date: Tue, 29 Sep 2020 11:42:17 +0800 Subject: [PATCH 13/15] migration/dirtyrate: record start_time and calc_time while at the measuring state Querying could include both the start-time and the calc-time while at the measuring state, allowing a caller to determine when they should expect to come back looking for a result. Signed-off-by: Chuan Zheng Message-Id: <1601350938-128320-2-git-send-email-zhengchuan@huawei.com> Reviewed-by: David Edmondson Signed-off-by: Dr. David Alan Gilbert --- migration/dirtyrate.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c index b87069a798..c72d678e33 100644 --- a/migration/dirtyrate.c +++ b/migration/dirtyrate.c @@ -83,14 +83,14 @@ static struct DirtyRateInfo *query_dirty_rate_info(void) return info; } -static void reset_dirtyrate_stat(void) +static void init_dirtyrate_stat(int64_t start_time, int64_t calc_time) { DirtyStat.total_dirty_samples = 0; DirtyStat.total_sample_count = 0; DirtyStat.total_block_mem_MB = 0; DirtyStat.dirty_rate = -1; - DirtyStat.start_time = 0; - DirtyStat.calc_time = 0; + DirtyStat.start_time = start_time; + DirtyStat.calc_time = calc_time; } static void update_dirtyrate_stat(struct RamblockDirtyInfo *info) @@ -335,7 +335,6 @@ static void calculate_dirtyrate(struct DirtyRateConfig config) int64_t initial_time; rcu_register_thread(); - reset_dirtyrate_stat(); rcu_read_lock(); initial_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); if (!record_ramblock_hash_info(&block_dinfo, config, &block_count)) { @@ -365,6 +364,8 @@ void *get_dirtyrate_thread(void *arg) { struct DirtyRateConfig config = *(struct DirtyRateConfig *)arg; int ret; + int64_t start_time; + int64_t calc_time; ret = dirtyrate_set_state(&CalculatingState, DIRTY_RATE_STATUS_UNSTARTED, DIRTY_RATE_STATUS_MEASURING); @@ -373,6 +374,10 @@ void *get_dirtyrate_thread(void *arg) return NULL; } + start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) / 1000; + calc_time = config.sample_period_seconds; + init_dirtyrate_stat(start_time, calc_time); + calculate_dirtyrate(config); ret = dirtyrate_set_state(&CalculatingState, DIRTY_RATE_STATUS_MEASURING, -- Gitee From d5af915cfd8f7b43f70aba223a0d7c76b3a56600 Mon Sep 17 00:00:00 2001 From: Chuan Zheng Date: Tue, 29 Sep 2020 11:42:18 +0800 Subject: [PATCH 14/15] migration/dirtyrate: present dirty rate only when querying the rate has completed Make dirty_rate field optional, present dirty rate only when querying the rate has completed. The qmp results is shown as follow: @unstarted: {"return":{"status":"unstarted","start-time":0,"calc-time":0},"id":"libvirt-12"} @measuring: {"return":{"status":"measuring","start-time":102931,"calc-time":1},"id":"libvirt-85"} @measured: {"return":{"status":"measured","dirty-rate":4,"start-time":150146,"calc-time":1},"id":"libvirt-15"} Signed-off-by: Chuan Zheng Reviewed-by: David Edmondson Message-Id: <1601350938-128320-3-git-send-email-zhengchuan@huawei.com> Signed-off-by: Dr. David Alan Gilbert --- migration/dirtyrate.c | 3 +-- qapi/migration.json | 8 +++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c index c72d678e33..afb1b0501d 100644 --- a/migration/dirtyrate.c +++ b/migration/dirtyrate.c @@ -69,9 +69,8 @@ static struct DirtyRateInfo *query_dirty_rate_info(void) struct DirtyRateInfo *info = g_malloc0(sizeof(DirtyRateInfo)); if (qatomic_read(&CalculatingState) == DIRTY_RATE_STATUS_MEASURED) { + info->has_dirty_rate = true; info->dirty_rate = dirty_rate; - } else { - info->dirty_rate = -1; } info->status = CalculatingState; diff --git a/qapi/migration.json b/qapi/migration.json index 76f5b42493..6844ddfab3 100644 --- a/qapi/migration.json +++ b/qapi/migration.json @@ -1468,10 +1468,8 @@ # # Information about current dirty page rate of vm. # -# @dirty-rate: @dirtyrate describing the dirty page rate of vm -# in units of MB/s. -# If this field returns '-1', it means querying has not -# yet started or completed. +# @dirty-rate: an estimate of the dirty page rate of the VM in units of +# MB/s, present only when estimating the rate has completed. # # @status: status containing dirtyrate query status includes # 'unstarted' or 'measuring' or 'measured' @@ -1484,7 +1482,7 @@ # ## { 'struct': 'DirtyRateInfo', - 'data': {'dirty-rate': 'int64', + 'data': {'*dirty-rate': 'int64', 'status': 'DirtyRateStatus', 'start-time': 'int64', 'calc-time': 'int64'} } -- Gitee From 70232a287afcc19a11f196b74869514dd2ac3503 Mon Sep 17 00:00:00 2001 From: Chuan Zheng Date: Fri, 30 Oct 2020 11:58:01 +0800 Subject: [PATCH 15/15] migration/dirtyrate: simplify includes in dirtyrate.c Remove redundant blank line which is left by Commit 662770af7c6e8c, also take this opportunity to remove redundant includes in dirtyrate.c. Signed-off-by: Chuan Zheng Message-Id: <1604030281-112946-1-git-send-email-zhengchuan@huawei.com> Reviewed-by: Dr. David Alan Gilbert Signed-off-by: Dr. David Alan Gilbert --- migration/dirtyrate.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c index afb1b0501d..f65ea85620 100644 --- a/migration/dirtyrate.c +++ b/migration/dirtyrate.c @@ -10,17 +10,15 @@ * See the COPYING file in the top-level directory. */ -#include #include "qemu/osdep.h" +#include #include "qapi/error.h" #include "cpu.h" -#include "qemu/config-file.h" #include "exec/memory.h" #include "exec/ram_addr.h" #include "exec/target_page.h" #include "qemu/rcu_queue.h" #include "qapi/qapi-commands-migration.h" -#include "migration.h" #include "ram.h" #include "trace.h" #include "dirtyrate.h" -- Gitee