From f0dec67048e57ae9b1d43e44ce5703f736f81303 Mon Sep 17 00:00:00 2001 From: zhengchuan Date: Sat, 17 Dec 2022 11:03:35 +0800 Subject: [PATCH] add function of set migration thread affinity during migration Signed-off-by:zhengchuan --- libvirt.spec | 11 +- ...ion-pin-add-domainMigrationPid-for-q.patch | 252 ++++++++++++++++++ ...ion-pin-add-migrationpin-for-migrati.patch | 180 +++++++++++++ ...ion-pin-add-qemu-monitor-callback-fu.patch | 114 ++++++++ ...ion-pin-add-some-migration-multiFd-p.patch | 144 ++++++++++ ...d-pin-add-qemu-monitor-callback-func.patch | 115 ++++++++ ...d-pin-support-migration-multifd-thre.patch | 195 ++++++++++++++ 7 files changed, 1010 insertions(+), 1 deletion(-) create mode 100644 migration-migration-pin-add-domainMigrationPid-for-q.patch create mode 100644 migration-migration-pin-add-migrationpin-for-migrati.patch create mode 100644 migration-migration-pin-add-qemu-monitor-callback-fu.patch create mode 100644 migration-migration-pin-add-some-migration-multiFd-p.patch create mode 100644 migration-multifd-pin-add-qemu-monitor-callback-func.patch create mode 100644 migration-multifd-pin-support-migration-multifd-thre.patch diff --git a/libvirt.spec b/libvirt.spec index db11c2c..c934d07 100644 --- a/libvirt.spec +++ b/libvirt.spec @@ -101,7 +101,7 @@ Summary: Library providing a simple virtualization API Name: libvirt Version: 6.2.0 -Release: 48 +Release: 49 License: LGPLv2+ URL: https://libvirt.org/ @@ -426,6 +426,12 @@ Patch0313: qemu_capabilities-Introduce-QEMU_CAPS_DIRTYRATE_MODE.patch Patch0314: include-Introduce-virDomainDirtyRateCalcFlags.patch Patch0315: qemu_driver-Add-mode-parameter-to-qemuDomainStartDir.patch Patch0316: update-the-Chinese-translation-of-nwfilter.patch +Patch0317: migration-migration-pin-add-some-migration-multiFd-p.patch +Patch0318: migration-migration-pin-add-qemu-monitor-callback-fu.patch +Patch0319: migration-migration-pin-add-migrationpin-for-migrati.patch +Patch0320: migration-migration-pin-add-domainMigrationPid-for-q.patch +Patch0321: migration-multifd-pin-add-qemu-monitor-callback-func.patch +Patch0322: migration-multifd-pin-support-migration-multifd-thre.patch Requires: libvirt-daemon = %{version}-%{release} Requires: libvirt-daemon-config-network = %{version}-%{release} @@ -2160,6 +2166,9 @@ exit 0 %changelog +* Sat Dec 17 2022 zhengchuan - 6.2.0-49 +- add function of set migration thread affinity during migration + * Sat Dec 10 2022 yezengruan - 6.2.0-48 - update the Chinese translation of nwfilter diff --git a/migration-migration-pin-add-domainMigrationPid-for-q.patch b/migration-migration-pin-add-domainMigrationPid-for-q.patch new file mode 100644 index 0000000..bf958f8 --- /dev/null +++ b/migration-migration-pin-add-domainMigrationPid-for-q.patch @@ -0,0 +1,252 @@ +From 18b3736c7a0dd1915401981750a5d1a3b3560031 Mon Sep 17 00:00:00 2001 +From: zhengchuan +Date: Wed, 30 Nov 2022 15:01:12 +0800 +Subject: [PATCH 4/6] migration/migration-pin: add domainMigrationPid for + qemuMonitorCallbacks + +add domainMigrationPid for qemuMonitorCallbacks + +Signed-off-by:zhengchuan +--- + src/qemu/qemu_process.c | 189 ++++++++++++++++++++++++++++++++++++++++ + src/qemu/qemu_process.h | 13 +++ + 2 files changed, 202 insertions(+) + +diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c +index 9cbf34a046..ce1c597da1 100644 +--- a/src/qemu/qemu_process.c ++++ b/src/qemu/qemu_process.c +@@ -1914,6 +1914,7 @@ static qemuMonitorCallbacks monitorCallbacks = { + .domainPRManagerStatusChanged = qemuProcessHandlePRManagerStatusChanged, + .domainRdmaGidStatusChanged = qemuProcessHandleRdmaGidStatusChanged, + .domainGuestCrashloaded = qemuProcessHandleGuestCrashloaded, ++ .domainMigrationPid = qemuProcessHandleMigrationPid, + }; + + static void +@@ -2782,6 +2783,194 @@ qemuProcessResctrlCreate(virQEMUDriverPtr driver, + } + + ++int ++qemuProcessSetupMigration(virDomainObjPtr vm, ++ virDomainMigrationIDDefPtr migration) ++{ ++ return qemuProcessSetupPid(vm, migration->thread_id, ++ VIR_CGROUP_THREAD_MIGRATION_THREAD, ++ 0, ++ vm->def->cputune.emulatorpin, ++ vm->def->cputune.emulator_period, ++ vm->def->cputune.emulator_quota, ++ &migration->sched); ++} ++ ++ ++unsigned char * ++virParseCPUList(int *cpumaplen, const char *cpulist, int maxcpu) ++{ ++ unsigned char *cpumap = NULL; ++ virBitmapPtr map = NULL; ++ ++ if (cpulist[0] == 'r') { ++ map = virBitmapNew(maxcpu); ++ if (!map) ++ return NULL; ++ virBitmapSetAll(map); ++ } else { ++ if (virBitmapParse(cpulist, &map, 1024) < 0 || ++ virBitmapIsAllClear(map)) { ++ goto cleanup; ++ } ++ ++ int lastcpu = virBitmapLastSetBit(map); ++ if (lastcpu >= maxcpu) ++ goto cleanup; ++ } ++ ++ if (virBitmapToData(map, &cpumap, cpumaplen) < 0) ++ VIR_ERROR(_("Bitmap to data failure")); ++ ++ cleanup: ++ virBitmapFree(map); ++ return cpumap; ++} ++ ++ ++/* ++ * If priv->pcpumap is NULL, it means migrationpin command is not called, ++ * otherwise we set the affinity of migration thread by migrationpin. ++ */ ++static virBitmapPtr ++qemuProcessGetPcpumap(qemuDomainObjPrivatePtr priv) ++{ ++ int cpumaplen = 0; ++ int maxcpu = 0; ++ g_autofree unsigned char *cpumap = NULL; ++ virBitmapPtr pcpumap = NULL; ++ ++ if(priv->pcpumap) ++ return priv->pcpumap; ++ ++ if (!(priv->migrationThreadPinList) || STREQ(priv->migrationThreadPinList, "")) { ++ VIR_ERROR(_("didn't set the migratin thread pin")); ++ return NULL; ++ } ++ ++ /* judge whether migration.pin is default value or not */ ++ if (STREQ(priv->migrationThreadPinList, "none")) ++ return NULL; ++ ++ maxcpu = virHostCPUGetCount(); ++ if (maxcpu < 0) { ++ VIR_ERROR(_("get the cpu count of host failure")); ++ return NULL; ++ } ++ ++ cpumap = virParseCPUList(&cpumaplen, priv->migrationThreadPinList, maxcpu); ++ if (!cpumap) { ++ VIR_ERROR(_("parse migration.pin params failure : migration.pin = %s"), ++ priv->migrationThreadPinList); ++ return NULL; ++ } ++ ++ if (!(pcpumap = virBitmapNewData(cpumap, cpumaplen))) { ++ VIR_ERROR(_("Bitmap data failure")); ++ return pcpumap; ++ } ++ ++ return pcpumap; ++} ++ ++ ++/* ++ * In order to set migration thread affinity when vm is migrating, ++ * we should create the cgroup for migration thread. ++ */ ++static void ++qemuProcessSetMigthreadAffinity(qemuDomainObjPrivatePtr priv, ++ virBitmapPtr pcpumap, ++ int mpid) ++{ ++ int migration_id = 0; ++ virCgroupPtr cgroup_migthread = NULL; ++ ++ if (!pcpumap) ++ return; ++ ++ if (virCgroupHasController(priv->cgroup, ++ VIR_CGROUP_CONTROLLER_CPUSET)) { ++ if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_MIGRATION_THREAD, ++ migration_id, false, &cgroup_migthread) < 0) ++ goto cleanup; ++ ++ if (qemuSetupCgroupCpusetCpus(cgroup_migthread, pcpumap) < 0) { ++ virReportError(VIR_ERR_OPERATION_INVALID, ++ _("failed to set cpuset.cpus in cgroup" ++ " for migration%d thread"), migration_id); ++ goto cleanup; ++ } ++ } ++ ++ if (virProcessSetAffinity(mpid, pcpumap) < 0) ++ VIR_WARN("failed to set affinity in migration"); ++ ++ cleanup: ++ if (cgroup_migthread) ++ virCgroupFree(&cgroup_migthread); ++ return; ++} ++ ++ ++int ++qemuProcessHandleMigrationPid(qemuMonitorPtr mon ATTRIBUTE_UNUSED, ++ virDomainObjPtr vm, ++ int mpid, ++ void *opaque G_GNUC_UNUSED) ++{ ++ qemuDomainObjPrivatePtr priv; ++ char *mpidStr = NULL; ++ virDomainMigrationIDDefPtr migration = NULL; ++ virBitmapPtr pcpumap = NULL; ++ virObjectLock(vm); ++ ++ VIR_INFO("Migrating domain %p %s, migration pid %d", ++ vm, vm->def->name, mpid); ++ ++ priv = vm->privateData; ++ if (priv->job.asyncJob == QEMU_ASYNC_JOB_NONE) { ++ VIR_DEBUG("got MIGRATION_PID event without a migration job"); ++ goto cleanup; ++ } ++ ++ if (VIR_ALLOC(migration) < 0) { ++ VIR_ERROR(_("alloc migrationIDDefPtr failure")); ++ goto cleanup; ++ } ++ migration->thread_id = mpid; ++ ++ if (qemuProcessSetupMigration(vm, migration) < 0) { ++ VIR_ERROR(_("fail to setup migration cgroup")); ++ goto cleanup; ++ } ++ ++ mpidStr = g_strdup_printf("%d", mpid); ++ ++ VIR_FREE(priv->migrationPids); ++ priv->migrationPids = mpidStr; ++ ++ pcpumap = qemuProcessGetPcpumap(priv); ++ ++ if (!pcpumap) ++ goto cleanup; ++ ++ qemuProcessSetMigthreadAffinity(priv, pcpumap, mpid); ++ ++ cleanup: ++ /* ++ * If the value of pcpumap is setted by priv->migrationThreadPinList, ++ * we need to free pcpumap. ++ */ ++ if (pcpumap != priv->pcpumap) ++ virBitmapFree(pcpumap); ++ virDomainMigrationIDDefFree(migration); ++ virObjectUnlock(vm); ++ ++ return 0; ++} ++ ++ + static char * + qemuProcessBuildPRHelperPidfilePath(virDomainObjPtr vm) + { +diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h +index 15e67b9762..be062b9f95 100644 +--- a/src/qemu/qemu_process.h ++++ b/src/qemu/qemu_process.h +@@ -25,6 +25,7 @@ + #include "qemu_domain.h" + #include "virstoragefile.h" + #include "vireventthread.h" ++#include "domain_conf.h" + + int qemuProcessPrepareMonitorChr(virDomainChrSourceDefPtr monConfig, + const char *domainDir); +@@ -236,3 +237,15 @@ qemuProcessQMPPtr qemuProcessQMPNew(const char *binary, + void qemuProcessQMPFree(qemuProcessQMPPtr proc); + + int qemuProcessQMPStart(qemuProcessQMPPtr proc); ++ ++int qemuProcessSetupMigration(virDomainObjPtr vm, ++ virDomainMigrationIDDefPtr migration); ++ ++unsigned char * virParseCPUList(int *cpumaplen, ++ const char *cpulist, ++ int maxcpu); ++ ++int qemuProcessHandleMigrationPid(qemuMonitorPtr mon ATTRIBUTE_UNUSED, ++ virDomainObjPtr vm, ++ int mpid, ++ void *opaque); +-- +2.25.1 + diff --git a/migration-migration-pin-add-migrationpin-for-migrati.patch b/migration-migration-pin-add-migrationpin-for-migrati.patch new file mode 100644 index 0000000..f0d3cf1 --- /dev/null +++ b/migration-migration-pin-add-migrationpin-for-migrati.patch @@ -0,0 +1,180 @@ +From 4483bab5e3d6b031c851e6b10323a37818f147e4 Mon Sep 17 00:00:00 2001 +From: zhengchuan +Date: Wed, 30 Nov 2022 15:01:12 +0800 +Subject: [PATCH 3/6] migration/migration-pin: add migrationpin for migration + parameters + +Add a migrationpin to the migration parameters of live migration to bind cores +to the migration thread during VM migration. + +Signed-off-by:zhengchuan +--- + include/libvirt/libvirt-domain.h | 8 ++++++++ + src/qemu/qemu_domain.c | 1 + + src/qemu/qemu_domain.h | 1 + + src/qemu/qemu_migration.c | 3 +++ + src/qemu/qemu_migration.h | 1 + + src/qemu/qemu_migration_params.c | 20 ++++++++++++++++++++ + src/qemu/qemu_migration_params.h | 4 ++++ + 7 files changed, 38 insertions(+) + +diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h +index c7956c81d6..8fe723198d 100644 +--- a/include/libvirt/libvirt-domain.h ++++ b/include/libvirt/libvirt-domain.h +@@ -1016,6 +1016,14 @@ typedef enum { + */ + # define VIR_MIGRATE_PARAM_COMPRESSION_MT_DTHREADS "compression.mt.dthreads" + ++/** ++ * VIR_MIGRATE_PARAM_MIGRATIONPIN: ++ * ++ * virDomainMigrate* params field: the pin of migration threads for ++ * migration as VIR_TYPED_PARAM_STRING. ++ */ ++# define VIR_MIGRATE_PARAM_MIGRATIONPIN "migration.pin" ++ + /** + * VIR_MIGRATE_PARAM_COMPRESSION_XBZRLE_CACHE: + * +diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c +index 71d034fa42..b7fb4eb9f9 100644 +--- a/src/qemu/qemu_domain.c ++++ b/src/qemu/qemu_domain.c +@@ -2323,6 +2323,7 @@ qemuDomainObjPrivateFree(void *data) + VIR_FREE(priv->lockState); + VIR_FREE(priv->origname); + VIR_FREE(priv->migrationPids); ++ VIR_FREE(priv->migrationThreadPinList); + virBitmapFree(priv->pcpumap); + + VIR_FREE(priv->migrationMultiFdPids); +diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h +index 38a842ebf6..7603724ccd 100644 +--- a/src/qemu/qemu_domain.h ++++ b/src/qemu/qemu_domain.h +@@ -333,6 +333,7 @@ struct _qemuDomainObjPrivate { + char *origname; + int nbdPort; /* Port used for migration with NBD */ + char *migrationPids; ++ char *migrationThreadPinList; + unsigned short migrationPort; + int preMigrationState; + char *migrationMultiFdPids; +diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c +index faf4e223a8..b47e69c642 100644 +--- a/src/qemu/qemu_migration.c ++++ b/src/qemu/qemu_migration.c +@@ -2533,6 +2533,7 @@ qemuMigrationDstPrepareAny(virQEMUDriverPtr driver, + + priv = vm->privateData; + priv->origname = g_strdup(origname); ++ VIR_FREE(priv->migrationPids); + + if (taint_hook) { + /* Domain XML has been altered by a hook script. */ +@@ -3601,6 +3602,8 @@ qemuMigrationSrcRun(virQEMUDriverPtr driver, + migrate_speed * 1024 * 1024) < 0) + goto error; + ++ qemuMigrationMigrationParamsToVM(migParams, vm); ++ + if (qemuMigrationParamsApply(driver, vm, QEMU_ASYNC_JOB_MIGRATION_OUT, + migParams) < 0) + goto error; +diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h +index b6f88d3fd9..316a1a10f4 100644 +--- a/src/qemu/qemu_migration.h ++++ b/src/qemu/qemu_migration.h +@@ -84,6 +84,7 @@ + VIR_MIGRATE_PARAM_BANDWIDTH_POSTCOPY, VIR_TYPED_PARAM_ULLONG, \ + VIR_MIGRATE_PARAM_PARALLEL_CONNECTIONS, VIR_TYPED_PARAM_INT, \ + VIR_MIGRATE_PARAM_TLS_DESTINATION, VIR_TYPED_PARAM_STRING, \ ++ VIR_MIGRATE_PARAM_MIGRATIONPIN, VIR_TYPED_PARAM_STRING, \ + NULL + + +diff --git a/src/qemu/qemu_migration_params.c b/src/qemu/qemu_migration_params.c +index df9d5d205a..4001fb2b34 100644 +--- a/src/qemu/qemu_migration_params.c ++++ b/src/qemu/qemu_migration_params.c +@@ -108,6 +108,7 @@ VIR_ENUM_IMPL(qemuMigrationParam, + "xbzrle-cache-size", + "max-postcopy-bandwidth", + "multifd-channels", ++ "migrationpin", + ); + + typedef struct _qemuMigrationParamsAlwaysOnItem qemuMigrationParamsAlwaysOnItem; +@@ -201,6 +202,10 @@ static const qemuMigrationParamsTPMapItem qemuMigrationParamsTPMap[] = { + {.typedParam = VIR_MIGRATE_PARAM_TLS_DESTINATION, + .param = QEMU_MIGRATION_PARAM_TLS_HOSTNAME, + .party = QEMU_MIGRATION_SOURCE}, ++ ++ {.typedParam = VIR_MIGRATE_PARAM_MIGRATIONPIN, ++ .param = QEMU_MIGRATION_PARAM_MIGRATIONPIN, ++ .party = QEMU_MIGRATION_SOURCE}, + }; + + static const qemuMigrationParamType qemuMigrationParamTypes[] = { +@@ -217,6 +222,7 @@ static const qemuMigrationParamType qemuMigrationParamTypes[] = { + [QEMU_MIGRATION_PARAM_XBZRLE_CACHE_SIZE] = QEMU_MIGRATION_PARAM_TYPE_ULL, + [QEMU_MIGRATION_PARAM_MAX_POSTCOPY_BANDWIDTH] = QEMU_MIGRATION_PARAM_TYPE_ULL, + [QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS] = QEMU_MIGRATION_PARAM_TYPE_INT, ++ [QEMU_MIGRATION_PARAM_MIGRATIONPIN] = QEMU_MIGRATION_PARAM_TYPE_STRING, + }; + G_STATIC_ASSERT(G_N_ELEMENTS(qemuMigrationParamTypes) == QEMU_MIGRATION_PARAM_LAST); + +@@ -534,6 +540,16 @@ qemuMigrationParamsSetCompression(virTypedParameterPtr params, + } + + ++void ++qemuMigrationMigrationParamsToVM(const qemuMigrationParamsPtr migParams, const virDomainObjPtr vm) ++{ ++ if (migParams && migParams->params[QEMU_MIGRATION_PARAM_MIGRATIONPIN].set) { ++ qemuDomainObjPrivatePtr priv = vm->privateData; ++ priv->migrationThreadPinList = g_strdup(migParams->params[QEMU_MIGRATION_PARAM_MIGRATIONPIN].value.s); ++ } ++} ++ ++ + qemuMigrationParamsPtr + qemuMigrationParamsFromFlags(virTypedParameterPtr params, + int nparams, +@@ -741,6 +757,10 @@ qemuMigrationParamsToJSON(qemuMigrationParamsPtr migParams) + if (!pv->set) + continue; + ++ if (i == QEMU_MIGRATION_PARAM_MIGRATIONPIN) { ++ continue; ++ } ++ + rc = 0; + switch (qemuMigrationParamTypes[i]) { + case QEMU_MIGRATION_PARAM_TYPE_INT: +diff --git a/src/qemu/qemu_migration_params.h b/src/qemu/qemu_migration_params.h +index 9aea24725f..f3deffb4cc 100644 +--- a/src/qemu/qemu_migration_params.h ++++ b/src/qemu/qemu_migration_params.h +@@ -58,6 +58,7 @@ typedef enum { + QEMU_MIGRATION_PARAM_XBZRLE_CACHE_SIZE, + QEMU_MIGRATION_PARAM_MAX_POSTCOPY_BANDWIDTH, + QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS, ++ QEMU_MIGRATION_PARAM_MIGRATIONPIN, + + QEMU_MIGRATION_PARAM_LAST + } qemuMigrationParam; +@@ -74,6 +75,9 @@ typedef enum { + virBitmapPtr + qemuMigrationParamsGetAlwaysOnCaps(qemuMigrationParty party); + ++void ++qemuMigrationMigrationParamsToVM(const qemuMigrationParamsPtr migParams, const virDomainObjPtr vm); ++ + qemuMigrationParamsPtr + qemuMigrationParamsFromFlags(virTypedParameterPtr params, + int nparams, +-- +2.25.1 + diff --git a/migration-migration-pin-add-qemu-monitor-callback-fu.patch b/migration-migration-pin-add-qemu-monitor-callback-fu.patch new file mode 100644 index 0000000..68cf739 --- /dev/null +++ b/migration-migration-pin-add-qemu-monitor-callback-fu.patch @@ -0,0 +1,114 @@ +From b0035c45ecfed66f0406a6064b33f296fd65afd9 Mon Sep 17 00:00:00 2001 +From: zhengchuan +Date: Wed, 30 Nov 2022 11:30:12 +0800 +Subject: [PATCH 2/6] migration/migration-pin: add qemu monitor callback + functions + +add qemu monitor callback functions + +Signed-off-by:zhengchuan +--- + src/qemu/qemu_monitor.c | 12 ++++++++++++ + src/qemu/qemu_monitor.h | 8 ++++++++ + src/qemu/qemu_monitor_json.c | 15 +++++++++++++++ + 3 files changed, 35 insertions(+) + +diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c +index 04592fdcd5..fcda2c0771 100644 +--- a/src/qemu/qemu_monitor.c ++++ b/src/qemu/qemu_monitor.c +@@ -1547,6 +1547,18 @@ qemuMonitorEmitPRManagerStatusChanged(qemuMonitorPtr mon, + } + + ++int ++qemuMonitorEmitMigrationPid(qemuMonitorPtr mon, ++ int mpid) ++{ ++ int ret = -1; ++ VIR_DEBUG("mon=%p, pass=%d", mon, mpid); ++ QEMU_MONITOR_CALLBACK(mon, ret, domainMigrationPid, mon->vm, mpid); ++ ++ return ret; ++} ++ ++ + int + qemuMonitorEmitRdmaGidStatusChanged(qemuMonitorPtr mon, + const char *netdev, +diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h +index 4f9b70d820..4b34ea3b4a 100644 +--- a/src/qemu/qemu_monitor.h ++++ b/src/qemu/qemu_monitor.h +@@ -340,6 +340,11 @@ typedef int (*qemuMonitorDomainGuestCrashloadedCallback)(qemuMonitorPtr mon, + virDomainObjPtr vm, + void *opaque); + ++typedef int (*qemuMonitorDomainMigrationPidCallback)(qemuMonitorPtr mon, ++ virDomainObjPtr vm, ++ int mcpid, ++ void *opaque); ++ + typedef struct _qemuMonitorCallbacks qemuMonitorCallbacks; + typedef qemuMonitorCallbacks *qemuMonitorCallbacksPtr; + struct _qemuMonitorCallbacks { +@@ -376,6 +381,7 @@ struct _qemuMonitorCallbacks { + qemuMonitorDomainPRManagerStatusChangedCallback domainPRManagerStatusChanged; + qemuMonitorDomainRdmaGidStatusChangedCallback domainRdmaGidStatusChanged; + qemuMonitorDomainGuestCrashloadedCallback domainGuestCrashloaded; ++ qemuMonitorDomainMigrationPidCallback domainMigrationPid; + }; + + qemuMonitorPtr qemuMonitorOpen(virDomainObjPtr vm, +@@ -477,6 +483,8 @@ int qemuMonitorEmitMigrationStatus(qemuMonitorPtr mon, + int qemuMonitorEmitMigrationPass(qemuMonitorPtr mon, + int pass); + ++int qemuMonitorEmitMigrationPid(qemuMonitorPtr mon, int mpid); ++ + int qemuMonitorEmitAcpiOstInfo(qemuMonitorPtr mon, + const char *alias, + const char *slotType, +diff --git a/src/qemu/qemu_monitor_json.c b/src/qemu/qemu_monitor_json.c +index a1617cdfed..7d2fbec5f4 100644 +--- a/src/qemu/qemu_monitor_json.c ++++ b/src/qemu/qemu_monitor_json.c +@@ -112,6 +112,7 @@ static void qemuMonitorJSONHandleBlockThreshold(qemuMonitorPtr mon, virJSONValue + static void qemuMonitorJSONHandleDumpCompleted(qemuMonitorPtr mon, virJSONValuePtr data); + static void qemuMonitorJSONHandlePRManagerStatusChanged(qemuMonitorPtr mon, virJSONValuePtr data); + static void qemuMonitorJSONHandleRdmaGidStatusChanged(qemuMonitorPtr mon, virJSONValuePtr data); ++static void qemuMonitorJSONHandleMigrationPid(qemuMonitorPtr mon, virJSONValuePtr data); + + typedef struct { + const char *type; +@@ -134,6 +135,7 @@ static qemuEventHandler eventHandlers[] = { + { "JOB_STATUS_CHANGE", qemuMonitorJSONHandleJobStatusChange, }, + { "MIGRATION", qemuMonitorJSONHandleMigrationStatus, }, + { "MIGRATION_PASS", qemuMonitorJSONHandleMigrationPass, }, ++ { "MIGRATION_PID", qemuMonitorJSONHandleMigrationPid, }, + { "NIC_RX_FILTER_CHANGED", qemuMonitorJSONHandleNicRxFilterChanged, }, + { "POWERDOWN", qemuMonitorJSONHandlePowerdown, }, + { "PR_MANAGER_STATUS_CHANGED", qemuMonitorJSONHandlePRManagerStatusChanged, }, +@@ -158,6 +160,19 @@ static qemuEventHandler eventHandlers[] = { + /* We use bsearch, so keep this list sorted. */ + }; + ++static void qemuMonitorJSONHandleMigrationPid(qemuMonitorPtr mon, ++ virJSONValuePtr data) ++{ ++ int mpid; ++ ++ if (virJSONValueObjectGetNumberInt(data, "pid", &mpid) < 0) { ++ VIR_WARN("missing migration pid in migration-pid event"); ++ return; ++ } ++ ++ qemuMonitorEmitMigrationPid(mon, mpid); ++} ++ + static int + qemuMonitorEventCompare(const void *key, const void *elt) + { +-- +2.25.1 + diff --git a/migration-migration-pin-add-some-migration-multiFd-p.patch b/migration-migration-pin-add-some-migration-multiFd-p.patch new file mode 100644 index 0000000..2b22a7c --- /dev/null +++ b/migration-migration-pin-add-some-migration-multiFd-p.patch @@ -0,0 +1,144 @@ +From 36295c0cc955fee9afc7b0fb70822c8e1e030b87 Mon Sep 17 00:00:00 2001 +From: zhengchuan +Date: Wed, 14 Dec 2022 11:07:58 +0800 +Subject: [PATCH 1/6] migration/migration-pin:add some migration/multiFd params + +add migration/multiFd params. + +Signed-off-by:zhengchuan +--- + src/conf/domain_conf.c | 9 +++++++++ + src/conf/domain_conf.h | 12 ++++++++++++ + src/libvirt_private.syms | 1 + + src/qemu/qemu_domain.c | 6 ++++++ + src/qemu/qemu_domain.h | 4 ++++ + src/util/vircgroup.c | 3 +++ + src/util/vircgroup.h | 1 + + 7 files changed, 36 insertions(+) + +diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c +index 8b1ca76d39..1689d92c51 100644 +--- a/src/conf/domain_conf.c ++++ b/src/conf/domain_conf.c +@@ -32617,3 +32617,12 @@ virHostdevIsVFIODevice(const virDomainHostdevDef *hostdev) + hostdev->source.subsys.type == VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI && + hostdev->source.subsys.u.pci.backend == VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VFIO; + } ++ ++void ++virDomainMigrationIDDefFree(virDomainMigrationIDDefPtr def) ++{ ++ if (!def) ++ return; ++ virBitmapFree(def->cpumask); ++ VIR_FREE(def); ++} +diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h +index c0a323d465..98fa323679 100644 +--- a/src/conf/domain_conf.h ++++ b/src/conf/domain_conf.h +@@ -27,6 +27,7 @@ + #include + + #include "internal.h" ++#include "viralloc.h" + #include "virconftypes.h" + #include "capabilities.h" + #include "virstorageencryption.h" +@@ -3775,3 +3776,14 @@ virHostdevIsMdevDevice(const virDomainHostdevDef *hostdev) + bool + virHostdevIsVFIODevice(const virDomainHostdevDef *hostdev) + ATTRIBUTE_NONNULL(1); ++ ++typedef struct _virDomainMigrationIDDef virDomainMigrationIDDef; ++typedef virDomainMigrationIDDef *virDomainMigrationIDDefPtr; ++struct _virDomainMigrationIDDef { ++ bool autofill; ++ int thread_id; ++ virBitmapPtr cpumask; ++ virDomainThreadSchedParam sched; ++}; ++ ++void virDomainMigrationIDDefFree(virDomainMigrationIDDefPtr def); +diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms +index 14ad7ecde3..f30eb7ffb5 100644 +--- a/src/libvirt_private.syms ++++ b/src/libvirt_private.syms +@@ -492,6 +492,7 @@ virDomainMemoryModelTypeToString; + virDomainMemoryRemove; + virDomainMemorySourceTypeFromString; + virDomainMemorySourceTypeToString; ++virDomainMigrationIDDefFree; + virDomainNetAllocateActualDevice; + virDomainNetAppendIPAddress; + virDomainNetARPInterfaces; +diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c +index b2ac1d37cc..71d034fa42 100644 +--- a/src/qemu/qemu_domain.c ++++ b/src/qemu/qemu_domain.c +@@ -2322,6 +2322,12 @@ qemuDomainObjPrivateFree(void *data) + qemuDomainObjFreeJob(priv); + VIR_FREE(priv->lockState); + VIR_FREE(priv->origname); ++ VIR_FREE(priv->migrationPids); ++ virBitmapFree(priv->pcpumap); ++ ++ VIR_FREE(priv->migrationMultiFdPids); ++ priv->migrationMultiFdPids = NULL; ++ priv->migrationMultiFdCount = 0; + + virChrdevFree(priv->devs); + +diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h +index 164fb36f0c..38a842ebf6 100644 +--- a/src/qemu/qemu_domain.h ++++ b/src/qemu/qemu_domain.h +@@ -332,8 +332,11 @@ struct _qemuDomainObjPrivate { + unsigned long migMaxBandwidth; + char *origname; + int nbdPort; /* Port used for migration with NBD */ ++ char *migrationPids; + unsigned short migrationPort; + int preMigrationState; ++ char *migrationMultiFdPids; ++ unsigned int migrationMultiFdCount; + + virChrdevsPtr devs; + +@@ -354,6 +357,7 @@ struct _qemuDomainObjPrivate { + /* Bitmaps below hold data from the auto NUMA feature */ + virBitmapPtr autoNodeset; + virBitmapPtr autoCpuset; ++ virBitmapPtr pcpumap; + + bool signalIOError; /* true if the domain condition should be signalled on + I/O error */ +diff --git a/src/util/vircgroup.c b/src/util/vircgroup.c +index f52a8cc843..b075dd1ecb 100644 +--- a/src/util/vircgroup.c ++++ b/src/util/vircgroup.c +@@ -969,6 +969,9 @@ virCgroupNewThread(virCgroupPtr domain, + case VIR_CGROUP_THREAD_IOTHREAD: + name = g_strdup_printf("iothread%d", id); + break; ++ case VIR_CGROUP_THREAD_MIGRATION_THREAD: ++ name = g_strdup_printf("migthread%d", id); ++ break; + case VIR_CGROUP_THREAD_LAST: + virReportError(VIR_ERR_INTERNAL_ERROR, + _("unexpected name value %d"), nameval); +diff --git a/src/util/vircgroup.h b/src/util/vircgroup.h +index 1dcd0688f1..5639f36c1f 100644 +--- a/src/util/vircgroup.h ++++ b/src/util/vircgroup.h +@@ -54,6 +54,7 @@ typedef enum { + VIR_CGROUP_THREAD_VCPU = 0, + VIR_CGROUP_THREAD_EMULATOR, + VIR_CGROUP_THREAD_IOTHREAD, ++ VIR_CGROUP_THREAD_MIGRATION_THREAD, + + VIR_CGROUP_THREAD_LAST + } virCgroupThreadName; +-- +2.25.1 + diff --git a/migration-multifd-pin-add-qemu-monitor-callback-func.patch b/migration-multifd-pin-add-qemu-monitor-callback-func.patch new file mode 100644 index 0000000..bf1a89a --- /dev/null +++ b/migration-multifd-pin-add-qemu-monitor-callback-func.patch @@ -0,0 +1,115 @@ +From d0c3db4eb01ac0c31003c333173aa2b8dceb8032 Mon Sep 17 00:00:00 2001 +From: zhengchuan +Date: Wed, 30 Nov 2022 15:59:27 +0800 +Subject: [PATCH 5/6] migration/multifd-pin: add qemu monitor callback + functions + +add qemu monitor callback functions + +Signed-off-by:zhengchuan +--- + src/qemu/qemu_monitor.c | 12 ++++++++++++ + src/qemu/qemu_monitor.h | 9 +++++++++ + src/qemu/qemu_monitor_json.c | 15 +++++++++++++++ + 3 files changed, 36 insertions(+) + +diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c +index fcda2c0771..ffd1d348e5 100644 +--- a/src/qemu/qemu_monitor.c ++++ b/src/qemu/qemu_monitor.c +@@ -1559,6 +1559,18 @@ qemuMonitorEmitMigrationPid(qemuMonitorPtr mon, + } + + ++int ++qemuMonitorEmitMigrationMultiFdPids(qemuMonitorPtr mon, ++ int mpid) ++{ ++ int ret = -1; ++ VIR_DEBUG("mon=%p, pass=%d", mon, mpid); ++ QEMU_MONITOR_CALLBACK(mon, ret, domainMigrationMultiFdPids, mon->vm, mpid); ++ ++ return ret; ++} ++ ++ + int + qemuMonitorEmitRdmaGidStatusChanged(qemuMonitorPtr mon, + const char *netdev, +diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h +index 4b34ea3b4a..76d0bbb753 100644 +--- a/src/qemu/qemu_monitor.h ++++ b/src/qemu/qemu_monitor.h +@@ -345,6 +345,11 @@ typedef int (*qemuMonitorDomainMigrationPidCallback)(qemuMonitorPtr mon, + int mcpid, + void *opaque); + ++typedef int (*qemuMonitorDomainMigrationMultiFdPidsCallback)(qemuMonitorPtr mon, ++ virDomainObjPtr vm, ++ int mcpid, ++ void *opaque); ++ + typedef struct _qemuMonitorCallbacks qemuMonitorCallbacks; + typedef qemuMonitorCallbacks *qemuMonitorCallbacksPtr; + struct _qemuMonitorCallbacks { +@@ -382,6 +387,7 @@ struct _qemuMonitorCallbacks { + qemuMonitorDomainRdmaGidStatusChangedCallback domainRdmaGidStatusChanged; + qemuMonitorDomainGuestCrashloadedCallback domainGuestCrashloaded; + qemuMonitorDomainMigrationPidCallback domainMigrationPid; ++ qemuMonitorDomainMigrationMultiFdPidsCallback domainMigrationMultiFdPids; + }; + + qemuMonitorPtr qemuMonitorOpen(virDomainObjPtr vm, +@@ -485,6 +491,9 @@ int qemuMonitorEmitMigrationPass(qemuMonitorPtr mon, + + int qemuMonitorEmitMigrationPid(qemuMonitorPtr mon, int mpid); + ++int qemuMonitorEmitMigrationMultiFdPids(qemuMonitorPtr mon, ++ int mpid); ++ + int qemuMonitorEmitAcpiOstInfo(qemuMonitorPtr mon, + const char *alias, + const char *slotType, +diff --git a/src/qemu/qemu_monitor_json.c b/src/qemu/qemu_monitor_json.c +index 7d2fbec5f4..a02d0e2780 100644 +--- a/src/qemu/qemu_monitor_json.c ++++ b/src/qemu/qemu_monitor_json.c +@@ -113,6 +113,7 @@ static void qemuMonitorJSONHandleDumpCompleted(qemuMonitorPtr mon, virJSONValueP + static void qemuMonitorJSONHandlePRManagerStatusChanged(qemuMonitorPtr mon, virJSONValuePtr data); + static void qemuMonitorJSONHandleRdmaGidStatusChanged(qemuMonitorPtr mon, virJSONValuePtr data); + static void qemuMonitorJSONHandleMigrationPid(qemuMonitorPtr mon, virJSONValuePtr data); ++static void qemuMonitorJSONHandleMigrationMultiFdPids(qemuMonitorPtr mon, virJSONValuePtr data); + + typedef struct { + const char *type; +@@ -134,6 +135,7 @@ static qemuEventHandler eventHandlers[] = { + { "GUEST_PANICKED", qemuMonitorJSONHandleGuestPanic, }, + { "JOB_STATUS_CHANGE", qemuMonitorJSONHandleJobStatusChange, }, + { "MIGRATION", qemuMonitorJSONHandleMigrationStatus, }, ++ { "MIGRATION_MULTIFD_PID", qemuMonitorJSONHandleMigrationMultiFdPids, }, + { "MIGRATION_PASS", qemuMonitorJSONHandleMigrationPass, }, + { "MIGRATION_PID", qemuMonitorJSONHandleMigrationPid, }, + { "NIC_RX_FILTER_CHANGED", qemuMonitorJSONHandleNicRxFilterChanged, }, +@@ -173,6 +175,19 @@ static void qemuMonitorJSONHandleMigrationPid(qemuMonitorPtr mon, + qemuMonitorEmitMigrationPid(mon, mpid); + } + ++static void qemuMonitorJSONHandleMigrationMultiFdPids(qemuMonitorPtr mon, ++ virJSONValuePtr data) ++{ ++ int mpid; ++ ++ if (virJSONValueObjectGetNumberInt(data, "pid", &mpid) < 0) { ++ VIR_WARN("missing multifd pid in migration-multifd-pid event"); ++ return; ++ } ++ ++ qemuMonitorEmitMigrationMultiFdPids(mon, mpid); ++} ++ + static int + qemuMonitorEventCompare(const void *key, const void *elt) + { +-- +2.25.1 + diff --git a/migration-multifd-pin-support-migration-multifd-thre.patch b/migration-multifd-pin-support-migration-multifd-thre.patch new file mode 100644 index 0000000..1c292e5 --- /dev/null +++ b/migration-multifd-pin-support-migration-multifd-thre.patch @@ -0,0 +1,195 @@ +From 8c07702d9280ef5235bb2bdb1ee472452cffca02 Mon Sep 17 00:00:00 2001 +From: zhengchuan +Date: Wed, 30 Nov 2022 16:47:30 +0800 +Subject: [PATCH 6/6] migration/multifd-pin: support migration multifd thread + pin + +support migration multifd thread pin by configuration. + +Signed-off-by:zhengchuan +--- + src/qemu/qemu_migration.c | 2 + + src/qemu/qemu_process.c | 119 ++++++++++++++++++++++++++++++++++++++ + src/qemu/qemu_process.h | 5 ++ + 3 files changed, 126 insertions(+) + +diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c +index b47e69c642..c2a694eff1 100644 +--- a/src/qemu/qemu_migration.c ++++ b/src/qemu/qemu_migration.c +@@ -2534,6 +2534,8 @@ qemuMigrationDstPrepareAny(virQEMUDriverPtr driver, + priv = vm->privateData; + priv->origname = g_strdup(origname); + VIR_FREE(priv->migrationPids); ++ VIR_FREE(priv->migrationMultiFdPids); ++ priv->migrationMultiFdCount = 0; + + if (taint_hook) { + /* Domain XML has been altered by a hook script. */ +diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c +index ce1c597da1..0f3e651630 100644 +--- a/src/qemu/qemu_process.c ++++ b/src/qemu/qemu_process.c +@@ -1629,6 +1629,57 @@ qemuProcessHandleSpiceMigrated(qemuMonitorPtr mon G_GNUC_UNUSED, + } + + ++static void ++qemuProcessHandleMigrationPinStatus(qemuDomainObjPrivatePtr priv, int status) ++{ ++ if (priv->job.asyncJob != QEMU_ASYNC_JOB_MIGRATION_OUT) ++ return; ++ ++ switch (status) { ++ case QEMU_MONITOR_MIGRATION_STATUS_INACTIVE: ++ case QEMU_MONITOR_MIGRATION_STATUS_SETUP: ++ case QEMU_MONITOR_MIGRATION_STATUS_ACTIVE: ++ case QEMU_MONITOR_MIGRATION_STATUS_PRE_SWITCHOVER: ++ case QEMU_MONITOR_MIGRATION_STATUS_DEVICE: ++ case QEMU_MONITOR_MIGRATION_STATUS_POSTCOPY: ++ case QEMU_MONITOR_MIGRATION_STATUS_CANCELLING: ++ case QEMU_MONITOR_MIGRATION_STATUS_COMPLETED: ++ case QEMU_MONITOR_MIGRATION_STATUS_WAIT_UNPLUG: ++ break; ++ case QEMU_MONITOR_MIGRATION_STATUS_ERROR: ++ /* ++ * migration thread is still running, ++ * so we can't delete migration Cgroup. ++ */ ++ VIR_FREE(priv->migrationPids); ++ VIR_FREE(priv->migrationMultiFdPids); ++ VIR_FREE(priv->migrationThreadPinList); ++ priv->migrationMultiFdCount = 0; ++ virBitmapFree(priv->pcpumap); ++ priv->pcpumap = NULL; ++ break; ++ case QEMU_MONITOR_MIGRATION_STATUS_CANCELLED: ++ VIR_FREE(priv->migrationPids); ++ VIR_FREE(priv->migrationMultiFdPids); ++ VIR_FREE(priv->migrationThreadPinList); ++ priv->migrationMultiFdCount = 0; ++ virBitmapFree(priv->pcpumap); ++ priv->pcpumap = NULL; ++ if (virCgroupDelThread(priv->cgroup, ++ VIR_CGROUP_THREAD_MIGRATION_THREAD, 0) < 0) ++ VIR_WARN("Failed to delete migration thread Cgroup!"); ++ VIR_INFO("success to free pcpumap and migrationPids"); ++ break; ++ default: ++ VIR_WARN("got unknown migration status'%s'", ++ qemuMonitorMigrationStatusTypeToString(status)); ++ break; ++ } ++ ++ return; ++} ++ ++ + static int + qemuProcessHandleMigrationStatus(qemuMonitorPtr mon G_GNUC_UNUSED, + virDomainObjPtr vm, +@@ -1675,6 +1726,8 @@ qemuProcessHandleMigrationStatus(qemuMonitorPtr mon G_GNUC_UNUSED, + } + } + ++ qemuProcessHandleMigrationPinStatus(priv, status); ++ + cleanup: + virObjectUnlock(vm); + virObjectEventStateQueue(driver->domainEventState, event); +@@ -1915,6 +1968,7 @@ static qemuMonitorCallbacks monitorCallbacks = { + .domainRdmaGidStatusChanged = qemuProcessHandleRdmaGidStatusChanged, + .domainGuestCrashloaded = qemuProcessHandleGuestCrashloaded, + .domainMigrationPid = qemuProcessHandleMigrationPid, ++ .domainMigrationMultiFdPids = qemuProcessHandleMigrationMultiFdPids, + }; + + static void +@@ -2971,6 +3025,71 @@ qemuProcessHandleMigrationPid(qemuMonitorPtr mon ATTRIBUTE_UNUSED, + } + + ++int ++qemuProcessHandleMigrationMultiFdPids(qemuMonitorPtr mon ATTRIBUTE_UNUSED, ++ virDomainObjPtr vm, ++ int mpid, ++ void *opaque G_GNUC_UNUSED) ++{ ++ qemuDomainObjPrivatePtr priv; ++ char *mpidOldStr = NULL; ++ char *mpidStr = NULL; ++ virDomainMigrationIDDefPtr migration = NULL; ++ virBitmapPtr pcpumap = NULL; ++ virObjectLock(vm); ++ ++ VIR_INFO("Migrating domain %p %s, migration-multifd pid %d", ++ vm, vm->def->name, mpid); ++ ++ priv = vm->privateData; ++ if (priv->job.asyncJob == QEMU_ASYNC_JOB_NONE) { ++ VIR_DEBUG("got MIGRATION_MULTIFD_PID event without a migration job"); ++ goto cleanup; ++ } ++ ++ if (VIR_ALLOC(migration) < 0) { ++ VIR_ERROR(_("alloc migrationIDDefPtr failure")); ++ goto cleanup; ++ } ++ migration->thread_id = mpid; ++ ++ if (qemuProcessSetupMigration(vm, migration) < 0) { ++ VIR_ERROR(_("fail to setup migration multiFd cgroup")); ++ goto cleanup; ++ } ++ ++ mpidOldStr = priv->migrationMultiFdPids; ++ if (!mpidOldStr) { ++ mpidStr = g_strdup_printf("%d", mpid); ++ } else { ++ mpidStr = g_strdup_printf("%s/%d", mpidOldStr, mpid); ++ } ++ ++ VIR_FREE(priv->migrationMultiFdPids); ++ priv->migrationMultiFdPids = mpidStr; ++ priv->migrationMultiFdCount++; ++ ++ pcpumap = qemuProcessGetPcpumap(priv); ++ ++ if (!pcpumap) ++ goto cleanup; ++ ++ qemuProcessSetMigthreadAffinity(priv, pcpumap, mpid); ++ ++ cleanup: ++ /* ++ * If the value of pcpumap is setted by priv->migrationThreadPinList, ++ * we need to free pcpumap. ++ */ ++ if (pcpumap != priv->pcpumap) ++ virBitmapFree(pcpumap); ++ virDomainMigrationIDDefFree(migration); ++ virObjectUnlock(vm); ++ ++ return 0; ++} ++ ++ + static char * + qemuProcessBuildPRHelperPidfilePath(virDomainObjPtr vm) + { +diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h +index be062b9f95..561795edeb 100644 +--- a/src/qemu/qemu_process.h ++++ b/src/qemu/qemu_process.h +@@ -249,3 +249,8 @@ int qemuProcessHandleMigrationPid(qemuMonitorPtr mon ATTRIBUTE_UNUSED, + virDomainObjPtr vm, + int mpid, + void *opaque); ++ ++int qemuProcessHandleMigrationMultiFdPids(qemuMonitorPtr mon ATTRIBUTE_UNUSED, ++ virDomainObjPtr vm, ++ int mpid, ++ void *opaque); +-- +2.25.1 + -- Gitee