diff --git a/add-virtcca-support.patch b/add-virtcca-support.patch new file mode 100644 index 0000000000000000000000000000000000000000..007d437d8f71221645aca102725fade1ca3cd72c --- /dev/null +++ b/add-virtcca-support.patch @@ -0,0 +1,1984 @@ +diff --git a/nova/api/openstack/compute/migrate_server.py b/nova/api/openstack/compute/migrate_server.py +index 0d209e0..1b73773 100644 +--- a/nova/api/openstack/compute/migrate_server.py ++++ b/nova/api/openstack/compute/migrate_server.py +@@ -141,6 +141,7 @@ class MigrateServerController(wsgi.Controller): + else: + raise exc.HTTPBadRequest(explanation=ex.format_message()) + except ( ++ exception.OperationNotSupportedForVirtCCA, + exception.OperationNotSupportedForSEV, + exception.OperationNotSupportedForVTPM, + exception.OperationNotSupportedForVDPAInterface, +diff --git a/nova/api/openstack/compute/servers.py b/nova/api/openstack/compute/servers.py +index db52da8..2490327 100644 +--- a/nova/api/openstack/compute/servers.py ++++ b/nova/api/openstack/compute/servers.py +@@ -1008,7 +1008,10 @@ class ServersController(wsgi.Controller): + + try: + self.compute_api.reboot(context, instance, reboot_type) +- except exception.InstanceIsLocked as e: ++ except ( ++ exception.OperationNotSupportedForVirtCCA, ++ exception.InstanceIsLocked ++ )as e: + raise exc.HTTPConflict(explanation=e.format_message()) + except exception.InstanceInvalidState as state_error: + common.raise_http_conflict_for_instance_invalid_state(state_error, +@@ -1321,6 +1324,10 @@ class ServersController(wsgi.Controller): + raise exc.HTTPBadRequest(explanation=err.format_message()) + except exception.OverQuota as e: + raise exc.HTTPForbidden(explanation=e.format_message()) ++ except ( ++ exception.OperationNotSupportedForVirtCCA ++ ) as e: ++ raise exc.HTTPConflict(explanation=e.format_message()) + + # Starting with microversion 2.45 we return a response body containing + # the snapshot image id without the Location header. +diff --git a/nova/api/openstack/compute/shelve.py b/nova/api/openstack/compute/shelve.py +index 281437f..4cb4977 100644 +--- a/nova/api/openstack/compute/shelve.py ++++ b/nova/api/openstack/compute/shelve.py +@@ -50,6 +50,7 @@ class ShelveController(wsgi.Controller): + try: + self.compute_api.shelve(context, instance) + except ( ++ exception.OperationNotSupportedForVirtCCA, + exception.InstanceIsLocked, + exception.OperationNotSupportedForVTPM, + exception.OperationNotSupportedForVDPAInterface, +diff --git a/nova/api/openstack/compute/suspend_server.py b/nova/api/openstack/compute/suspend_server.py +index db5e8ff..b1a69b6 100644 +--- a/nova/api/openstack/compute/suspend_server.py ++++ b/nova/api/openstack/compute/suspend_server.py +@@ -39,6 +39,7 @@ class SuspendServerController(wsgi.Controller): + 'project_id': server.project_id}) + self.compute_api.suspend(context, server) + except ( ++ exception.OperationNotSupportedForVirtCCA, + exception.OperationNotSupportedForSEV, + exception.OperationNotSupportedForVDPAInterface, + exception.InstanceIsLocked, +diff --git a/nova/cmd/manage.py b/nova/cmd/manage.py +index 3d33602..10856ab 100644 +--- a/nova/cmd/manage.py ++++ b/nova/cmd/manage.py +@@ -2441,7 +2441,15 @@ class PlacementCommands(object): + NOVA_RCS = [orc.VCPU, orc.MEMORY_MB, orc.DISK_GB, orc.VGPU, + orc.NET_BW_EGR_KILOBIT_PER_SEC, + orc.NET_BW_IGR_KILOBIT_PER_SEC, +- orc.PCPU, orc.MEM_ENCRYPTION_CONTEXT] ++ orc.PCPU, orc.MEM_ENCRYPTION_CONTEXT, ++ orc.SECURE_MEM_CONTEXT, orc.VIRTCCA_KAE_VF, ++ orc.VIRTCCA_HUGEPAGE, orc.SECURE_NUMA_0, orc.SECURE_NUMA_1, ++ orc.SECURE_NUMA_2, orc.SECURE_NUMA_3, orc.SECURE_NUMA_4, ++ orc.SECURE_NUMA_5, orc.SECURE_NUMA_6, orc.SECURE_NUMA_7, ++ orc.VIRTCCA_VM, orc.VIRTCCA_VCPU, ++ orc.SECURE_NUMA_1G_PAGE_0, orc.SECURE_NUMA_1G_PAGE_1, ++ orc.SECURE_NUMA_1G_PAGE_2, orc.SECURE_NUMA_1G_PAGE_3, orc.SECURE_NUMA_1G_PAGE_4, ++ orc.SECURE_NUMA_1G_PAGE_5, orc.SECURE_NUMA_1G_PAGE_6, orc.SECURE_NUMA_1G_PAGE_7] + + # Since the RP can be a child RP, we need to get the root RP as it's + # the compute node UUID +diff --git a/nova/compute/api.py b/nova/compute/api.py +index 774eb53..07de2f8 100644 +--- a/nova/compute/api.py ++++ b/nova/compute/api.py +@@ -246,6 +246,24 @@ def reject_sev_instances(operation): + return outer + + ++def reject_virtcca_instances(operation): ++ """Reject requests to decorated function if instance has virtCCA enabled. ++ ++ Raise OperationNotSupportedForVirtCCA if instance has virtCCA enabled. ++ """ ++ ++ def outer(f): ++ @functools.wraps(f) ++ def inner(self, context, instance, *args, **kw): ++ if hardware.get_secure_mem_constraint(instance.flavor): ++ raise exception.OperationNotSupportedForVirtCCA( ++ instance_uuid=instance.uuid, ++ operation=operation) ++ return f(self, context, instance, *args, **kw) ++ return inner ++ return outer ++ ++ + def reject_vtpm_instances(operation): + """Reject requests to decorated function if instance has vTPM enabled. + +@@ -3185,6 +3203,7 @@ class API(base.Base): + + # NOTE(melwitt): We don't check instance lock for snapshot because lock is + # intended to prevent accidental change/delete of instances ++ @reject_virtcca_instances(instance_actions.SNAPSHOT) + @check_instance_state(vm_state=[vm_states.ACTIVE, vm_states.STOPPED, + vm_states.PAUSED, vm_states.SUSPENDED]) + def snapshot(self, context, instance, name, extra_properties=None): +@@ -3242,6 +3261,7 @@ class API(base.Base): + + # NOTE(melwitt): We don't check instance lock for snapshot because lock is + # intended to prevent accidental change/delete of instances ++ @reject_virtcca_instances(instance_actions.SNAPSHOT_VOLUME_BACKED) + @check_instance_state(vm_state=[vm_states.ACTIVE, vm_states.STOPPED, + vm_states.PAUSED, vm_states.SUSPENDED]) + def snapshot_volume_backed(self, context, instance, name, +@@ -3383,6 +3403,7 @@ class API(base.Base): + + return self.image_api.create(context, image_meta) + ++ @reject_virtcca_instances(instance_actions.REBOOT) + @check_instance_lock + def reboot(self, context, instance, reboot_type): + """Reboot the given instance.""" +@@ -4192,6 +4213,7 @@ class API(base.Base): + + # FIXME(sean-k-mooney): Shelve works but unshelve does not due to bug + # #1851545, so block it for now ++ @reject_virtcca_instances(instance_actions.SHELVE) + @reject_vdpa_instances(instance_actions.SHELVE) + @reject_vtpm_instances(instance_actions.SHELVE) + @block_accelerators(until_service=54) +@@ -4382,6 +4404,7 @@ class API(base.Base): + # FIXME(sean-k-mooney): Suspend does not work because we do not unplug + # the vDPA devices before calling managed save as we do with SR-IOV + # devices ++ @reject_virtcca_instances(instance_actions.SUSPEND) + @reject_vdpa_instances(instance_actions.SUSPEND) + @block_accelerators() + @reject_sev_instances(instance_actions.SUSPEND) +@@ -5141,6 +5164,7 @@ class API(base.Base): + + return _metadata + ++ @reject_virtcca_instances(instance_actions.LIVE_MIGRATION) + @reject_vdpa_instances(instance_actions.LIVE_MIGRATION) + @block_accelerators() + @reject_vtpm_instances(instance_actions.LIVE_MIGRATION) +diff --git a/nova/compute/claims.py b/nova/compute/claims.py +index bcc6f30..7df11b1 100644 +--- a/nova/compute/claims.py ++++ b/nova/compute/claims.py +@@ -135,12 +135,24 @@ class Claim(NopClaim): + if pci_requests.requests: + pci_stats = self.tracker.pci_tracker.stats + +- instance_topology = ( +- hardware.numa_fit_instance_to_host( +- host_topology, requested_topology, +- limits=limit, ++ if hardware.get_secure_mem_constraint(self.instance.flavor): ++ host_secure_numa_topology = self.tracker.driver.get_host_secure_numa_topology() ++ host_secure_numa_page = self.tracker.driver.get_secure_numa_page() ++ instance_topology = ( ++ hardware.virtcca_numa_fit_instance_to_host( ++ host_topology, host_secure_numa_topology, ++ requested_topology, self.instance.flavor, ++ host_secure_numa_page, limits=limit, + pci_requests=pci_requests.requests, +- pci_stats=pci_stats)) ++ pci_stats=pci_stats, ++ imageMeta=self.instance.image_meta)) ++ else: ++ instance_topology = ( ++ hardware.numa_fit_instance_to_host( ++ host_topology, requested_topology, ++ limits=limit, ++ pci_requests=pci_requests.requests, ++ pci_stats=pci_stats)) + + if requested_topology and not instance_topology: + if pci_requests.requests: +diff --git a/nova/compute/instance_actions.py b/nova/compute/instance_actions.py +index 1089975..8075e83 100644 +--- a/nova/compute/instance_actions.py ++++ b/nova/compute/instance_actions.py +@@ -71,3 +71,5 @@ UNLOCK = 'unlock' + BACKUP = 'createBackup' + CREATE_IMAGE = 'createImage' + RESET_STATE = 'resetState' ++SNAPSHOT = 'snapshot' ++SNAPSHOT_VOLUME_BACKED = 'snapshot_volume_backed' +diff --git a/nova/compute/resource_tracker.py b/nova/compute/resource_tracker.py +index 0febdd7..d4688be 100644 +--- a/nova/compute/resource_tracker.py ++++ b/nova/compute/resource_tracker.py +@@ -1488,8 +1488,12 @@ class ResourceTracker(object): + instance, + sign=sign) + # new instance, update compute node resource usage: +- self._update_usage(self._get_usage_dict(instance, instance), +- nodename, sign=sign) ++ if hardware.get_secure_mem_constraint(instance.flavor): ++ self._virtcca_update_usage(self._get_usage_dict(instance, instance), ++ nodename, instance.flavor, sign=sign) ++ else: ++ self._update_usage(self._get_usage_dict(instance, instance), ++ nodename, sign=sign) + + # Stop tracking removed instances in the is_bfv cache. This needs to + # happen *after* calling _get_usage_dict() since that relies on the +@@ -1898,6 +1902,47 @@ class ResourceTracker(object): + """Resets the failed_builds stats for the given node.""" + self.stats[nodename].build_succeeded() + ++ def _virtcca_update_usage(self, usage, nodename, flavor, sign=1): ++ # TODO(stephenfin): We don't use the CPU, RAM and disk fields for much ++ # except 'Aggregate(Core|Ram|Disk)Filter', the 'os-hypervisors' API, ++ # and perhaps some out-of-tree filters. Once the in-tree stuff is ++ # removed or updated to use information from placement, we can think ++ # about dropping the fields from the 'ComputeNode' object entirely ++ mem_usage = usage['memory_mb'] ++ disk_usage = usage.get('root_gb', 0) ++ vcpus_usage = usage.get('vcpus', 0) ++ ++ cn = self.compute_nodes[nodename] ++ cn.memory_mb_used += sign * mem_usage ++ cn.local_gb_used += sign * disk_usage ++ cn.local_gb_used += sign * usage.get('ephemeral_gb', 0) ++ cn.local_gb_used += sign * usage.get('swap', 0) / 1024 ++ cn.vcpus_used += sign * vcpus_usage ++ ++ # free ram and disk may be negative, depending on policy: ++ cn.free_ram_mb = cn.memory_mb - cn.memory_mb_used ++ cn.free_disk_gb = cn.local_gb - cn.local_gb_used ++ ++ stats = self.stats[nodename] ++ cn.running_vms = stats.num_instances ++ ++ # calculate the NUMA usage, assuming the instance is actually using ++ # NUMA, of course ++ if cn.numa_topology and usage.get('numa_topology'): ++ instance_numa_topology = usage.get('numa_topology') ++ # the ComputeNode.numa_topology field is a StringField, so ++ # deserialize ++ host_numa_topology = objects.NUMATopology.obj_from_db_obj( ++ cn.numa_topology) ++ ++ free = sign == -1 ++ ++ secure_numa_topology = self.driver.get_host_secure_numa_topology() ++ # ...and reserialize once we save it back ++ host_numa_topology, _ = hardware.virtcca_numa_usage_from_instance_numa( ++ host_numa_topology, secure_numa_topology, instance_numa_topology, flavor, free) ++ cn.numa_topology = host_numa_topology._to_json() ++ + @utils.synchronized(COMPUTE_RESOURCE_SEMAPHORE, fair=True) + def claim_pci_devices(self, context, pci_requests, instance_numa_topology): + """Claim instance PCI resources +diff --git a/nova/exception.py b/nova/exception.py +index ad322b5..b52e303 100644 +--- a/nova/exception.py ++++ b/nova/exception.py +@@ -531,6 +531,12 @@ class OperationNotSupportedForSEV(NovaException): + code = 409 + + ++class OperationNotSupportedForVirtCCA(NovaException): ++ msg_fmt = _("Operation '%(operation)s' not supported for virtCCA-enabled " ++ "instance (%(instance_uuid)s).") ++ code = 409 ++ ++ + class OperationNotSupportedForVTPM(NovaException): + msg_fmt = _("Operation '%(operation)s' not supported for vTPM-enabled " + "instance (%(instance_uuid)s).") +@@ -2380,3 +2386,20 @@ class ProviderConfigException(NovaException): + """ + msg_fmt = _("An error occurred while processing " + "a provider config file: %(error)s") ++ ++ ++class InvalidKaeVirtFunctionNumException(NovaException): ++ msg_fmt = _("Invalid KAE VF nums %(kae_vf)s " ++ "KAE VF nums should > 0 and <= %(kae_vf_max)s") ++ ++class InvalidKaeVirtFunctionValException(NovaException): ++ msg_fmt = _(" KAE VF value should be a number") ++ ++class InvalidVirtccaSecureMemoryException(NovaException): ++ msg_fmt = _("Invalid sercure memory nums: %(memory_mb)s, " ++ "Secure memory should be greater than %(min_memory_mb)s M" ++ "and be %(aligned)s M aligend") ++ ++class InvalidVirtccaSwiotlbException(NovaException): ++ msg_fmt = _("Invalid swiotlb memory nums: %(memory_mb)s, " ++ "swiotlb memory should be %(aligned)s M aligned") +\ No newline at end of file +diff --git a/nova/scheduler/filter_scheduler.py b/nova/scheduler/filter_scheduler.py +index 692ea50..2e842fb 100644 +--- a/nova/scheduler/filter_scheduler.py ++++ b/nova/scheduler/filter_scheduler.py +@@ -19,9 +19,11 @@ You can customize this scheduler by specifying your own Host Filters and + Weighing Functions. + """ + ++import math + import random + + from oslo_log import log as logging ++import os_resource_classes as orc + + from nova.compute import utils as compute_utils + import nova.conf +@@ -33,6 +35,7 @@ from nova import rpc + from nova.scheduler.client import report + from nova.scheduler import driver + from nova.scheduler import utils ++from nova.virt import hardware + + CONF = nova.conf.CONF + LOG = logging.getLogger(__name__) +@@ -235,6 +238,24 @@ class FilterScheduler(driver.Scheduler): + # information in the provider summaries, we'll just try to + # claim resources using the first allocation_request + alloc_req = alloc_reqs[0] ++ if hardware.get_secure_mem_constraint(spec_obj.flavor): ++ page_1g_total = 0 ++ for pages in host.secure_numa_page.values(): ++ page_1g_total += pages ++ ++ page_1g_need = 0 ++ if spec_obj.numa_topology: ++ for cell in spec_obj.numa_topology.cells: ++ page_1g_need += cell.memory // hardware.VIRTCCA_1G_SIZE ++ ++ page_1g_need = max(page_1g_need - 1, 0) ++ else: ++ page_1g_need = (spec_obj.flavor.memory_mb - hardware.VIRTCCA_1G_SIZE) // hardware.VIRTCCA_1G_SIZE ++ ++ if page_1g_need > page_1g_total: ++ LOG.info("Schedule CVM on host '%s' failed, no available buddy pages ", host.uuid) ++ continue ++ + if utils.claim_resources(elevated, self.placement_client, + spec_obj, instance_uuid, alloc_req, + allocation_request_version=allocation_request_version): +diff --git a/nova/scheduler/filters/numa_topology_filter.py b/nova/scheduler/filters/numa_topology_filter.py +index 74d6012..59215a3 100644 +--- a/nova/scheduler/filters/numa_topology_filter.py ++++ b/nova/scheduler/filters/numa_topology_filter.py +@@ -77,6 +77,8 @@ class NUMATopologyFilter(filters.BaseHostFilter): + requested_topology = spec_obj.numa_topology + host_topology = host_state.numa_topology + pci_requests = spec_obj.pci_requests ++ host_secure_topology = host_state.secure_numa_topology ++ host_secure_numa_page = host_state.secure_numa_page + + network_metadata = None + if 'network_metadata' in spec_obj: +@@ -97,11 +99,19 @@ class NUMATopologyFilter(filters.BaseHostFilter): + if network_metadata: + limits.network_metadata = network_metadata + +- instance_topology = (hardware.numa_fit_instance_to_host( +- host_topology, requested_topology, +- limits=limits, ++ if hardware.get_secure_mem_constraint(spec_obj.flavor): ++ instance_topology = (hardware.virtcca_numa_fit_instance_to_host( ++ host_topology, host_secure_topology, requested_topology, ++ spec_obj.flavor, host_secure_numa_page, limits=limits, + pci_requests=pci_requests, + pci_stats=host_state.pci_stats)) ++ else: ++ instance_topology = (hardware.numa_fit_instance_to_host( ++ host_topology, requested_topology, ++ limits=limits, ++ pci_requests=pci_requests, ++ pci_stats=host_state.pci_stats)) ++ + if not instance_topology: + LOG.debug("%(host)s, %(node)s fails NUMA topology " + "requirements. The instance does not fit on this " +diff --git a/nova/scheduler/host_manager.py b/nova/scheduler/host_manager.py +index 1118678..fb4f9fa 100644 +--- a/nova/scheduler/host_manager.py ++++ b/nova/scheduler/host_manager.py +@@ -29,6 +29,7 @@ except ImportError: + import iso8601 + from oslo_log import log as logging + from oslo_utils import timeutils ++import os_traits as ot + + import nova.conf + from nova import context as context_module +@@ -37,6 +38,7 @@ from nova import objects + from nova.pci import stats as pci_stats + from nova.scheduler import filters + from nova.scheduler import weights ++from nova.scheduler.client import report + from nova import utils + from nova.virt import hardware + +@@ -157,6 +159,12 @@ class HostState(object): + + self.updated = None + ++ # numa topology for virtcca ++ self.secure_numa_topology = None ++ self.secure_numa_page = None ++ ++ self.reportclient = report.SchedulerReportClient() ++ + def update(self, compute=None, service=None, aggregates=None, + inst_dict=None): + """Update all information about a host.""" +@@ -296,14 +304,26 @@ class HostState(object): + pci_requests = None + + # Calculate the NUMA usage... +- if self.numa_topology and spec_obj.numa_topology: +- spec_obj.numa_topology = hardware.numa_fit_instance_to_host( +- self.numa_topology, spec_obj.numa_topology, +- limits=self.limits.get('numa_topology'), +- pci_requests=pci_requests, pci_stats=self.pci_stats) ++ if hardware.get_secure_mem_constraint(spec_obj.flavor): ++ if self.numa_topology and spec_obj.numa_topology: ++ spec_obj.numa_topology = hardware.virtcca_numa_fit_instance_to_host( ++ self.numa_topology, self.secure_numa_topology, spec_obj.numa_topology, ++ spec_obj.flavor, self.secure_numa_page, limits=self.limits.get('numa_topology'), ++ pci_requests=pci_requests, pci_stats=self.pci_stats, ++ imageMeta=spec_obj.image) ++ ++ self.numa_topology, self.secure_numa_topology = hardware.virtcca_numa_usage_from_instance_numa( ++ self.numa_topology, self.secure_numa_topology, spec_obj.numa_topology, spec_obj.flavor) ++ ++ else: ++ if self.numa_topology and spec_obj.numa_topology: ++ spec_obj.numa_topology = hardware.numa_fit_instance_to_host( ++ self.numa_topology, spec_obj.numa_topology, ++ limits=self.limits.get('numa_topology'), ++ pci_requests=pci_requests, pci_stats=self.pci_stats) + +- self.numa_topology = hardware.numa_usage_from_instance_numa( +- self.numa_topology, spec_obj.numa_topology) ++ self.numa_topology = hardware.numa_usage_from_instance_numa( ++ self.numa_topology, spec_obj.numa_topology) + + # ...and the PCI usage + if pci_requests: +@@ -326,6 +346,53 @@ class HostState(object): + 'num_io_ops': self.num_io_ops, + 'num_instances': self.num_instances}) + ++ def update_secure_numa(self, context, compute): ++ def _get_secure_numa_inventory(provider_inventory, orc_secure_numa): ++ if orc_secure_numa not in provider_inventory: ++ LOG.warning("Resource %s not found in provider inventory", orc_secure_numa) ++ return None ++ return provider_inventory[orc_secure_numa] ++ ++ def _generate_orc_names(prefix, cells): ++ return [f"{prefix}{cell.id}" for cell in cells] ++ ++ @utils.synchronized(self._lock_name) ++ def _locked_update_resource_provider_secure_numa(self, context): ++ if not hasattr(self, 'uuid') or not self.uuid: ++ raise exception.HostNotFound(host=self.uuid) ++ ++ trait_info = self.reportclient.get_provider_traits(context, self.uuid) ++ if ot.HW_CPU_AARCH64_HISI_VIRTCCA not in trait_info.traits: ++ LOG.warning("Host %s lacks trait: %s", self.host, ot.HW_CPU_AARCH64_HISI_VIRTCCA) ++ return ++ ++ provider_tree = self.reportclient.get_provider_tree_and_ensure_root(context, self.uuid) ++ provider_inventory = provider_tree.data(self.uuid).inventory ++ ++ self.secure_numa_topology = self.numa_topology.obj_clone() ++ secure_numa_names = _generate_orc_names( ++ hardware.VIRTCCA_SECURE_NUMA_PREFIX, self.secure_numa_topology.cells) ++ ++ for cell, orc_secure_numa in zip(self.secure_numa_topology.cells, secure_numa_names): ++ orc_secure_numa_inventory = _get_secure_numa_inventory(provider_inventory, orc_secure_numa) ++ if not orc_secure_numa_inventory: ++ continue ++ cell.memory = orc_secure_numa_inventory['total'] ++ cell.memory_usage = orc_secure_numa_inventory['reserved'] ++ ++ self.secure_numa_page = {} ++ secure_numa_page_names = _generate_orc_names( ++ hardware.VIRTCCA_SECURE_NUMA_1G_PAGE_PREFIX, self.secure_numa_topology.cells) ++ ++ for orc_secure_numa in secure_numa_page_names: ++ orc_secure_numa_inventory = _get_secure_numa_inventory(provider_inventory, orc_secure_numa) ++ if not orc_secure_numa_inventory: ++ continue ++ self.secure_numa_page[orc_secure_numa] = \ ++ orc_secure_numa_inventory['total'] - orc_secure_numa_inventory['reserved'] ++ ++ ++ return _locked_update_resource_provider_secure_numa(self, context) + + class HostManager(object): + """Base HostManager class.""" +@@ -809,6 +876,7 @@ class HostManager(object): + dict(service), + self._get_aggregates_info(host), + self._get_instance_info(context, compute)) ++ host_state.update_secure_numa(context, compute) + + seen_nodes.add(state_key) + +diff --git a/nova/scheduler/utils.py b/nova/scheduler/utils.py +index b71c209..21f38fd 100644 +--- a/nova/scheduler/utils.py ++++ b/nova/scheduler/utils.py +@@ -15,6 +15,7 @@ + """Utility methods for scheduling.""" + + import collections ++import math + import re + import sys + import typing as ty +@@ -24,6 +25,7 @@ import os_resource_classes as orc + import os_traits + from oslo_log import log as logging + from oslo_serialization import jsonutils ++from oslo_utils import units + + from nova.compute import flavors + from nova.compute import utils as compute_utils +@@ -184,6 +186,12 @@ class ResourceRequest(object): + + res_req._translate_memory_encryption(request_spec.flavor, image) + ++ res_req._translate_secure_memory(request_spec.flavor, image) ++ ++ res_req._translate_virtcca_kae(request_spec.flavor) ++ ++ res_req._translate_virtcca_hugepage(request_spec.flavor, image) ++ + res_req._translate_vpmems_request(request_spec.flavor) + + res_req._translate_vtpm_request(request_spec.flavor, image) +@@ -291,6 +299,65 @@ class ResourceRequest(object): + LOG.debug("Added %s=1 to requested resources", + orc.MEM_ENCRYPTION_CONTEXT) + ++ def _translate_secure_memory(self, flavor, image): ++ """When the 'hw:mem_secure' extra spec is requested, this triggers: ++ 1. Original RAM allocation is converted to 'SECURE_MEM_CONTEXT' resource class. ++ 2. Base RAM request is converted to 'sw:swiotlb', ++ and the default value is 'VIRTCCA_BOUNCE_BUFFER_MB' if it is not set. ++ """ ++ if not hardware.get_secure_mem_constraint(flavor): ++ return ++ ++ if flavor.memory_mb < hardware.VIRTCCA_SECURE_MEMORY_MIN or \ ++ flavor.memory_mb % hardware.VIRTCCA_MEM_ALIGNED != 0: ++ raise exception.InvalidVirtccaSecureMemoryException( ++ memory_mb=flavor.memory_mb, ++ min_memory_mb=hardware.VIRTCCA_SECURE_MEMORY_MIN, ++ aligned=hardware.VIRTCCA_MEM_ALIGNED) ++ ++ swiotlb = hardware.get_swiotlb_buffer_constraint(flavor, hardware.VIRTCCA_SWIOTLB_BUFFER_DEFAULT) ++ if swiotlb % hardware.VIRTCCA_SWIOTLB_ALIGNED != 0: ++ raise exception.InvalidVirtccaSwiotlbException( ++ memory_mb=swiotlb, aligned=hardware.VIRTCCA_SWIOTLB_ALIGNED) ++ ++ hugepage = hardware._get_numa_pagesize_constraint(flavor, image) ++ if hugepage and hugepage == hardware.VIRTCCA_HUGEPAGE_SIZE / units.Ki: ++ swiotlb = 0 ++ ++ self._add_resource(orc.SECURE_MEM_CONTEXT, flavor.memory_mb) ++ self._add_resource(orc.MEMORY_MB, swiotlb) ++ self._add_resource(orc.VIRTCCA_VM, 1) ++ self._add_resource(orc.VIRTCCA_VCPU, flavor.vcpus) ++ ++ LOG.debug("Added %s=%d, %s=1, %s=%d, modified %s=%d to requested resources", ++ orc.SECURE_MEM_CONTEXT, flavor.memory_mb, ++ orc.VIRTCCA_VM, orc.VIRTCCA_VCPU, flavor.vcpus, ++ orc.MEMORY_MB, swiotlb) ++ ++ def _translate_virtcca_kae(self, flavor): ++ if not hardware.get_secure_mem_constraint(flavor): ++ return ++ ++ kae_vf = hardware.get_kae_vf_constraint(flavor) ++ if kae_vf <= 0: ++ return ++ ++ self._add_resource(orc.VIRTCCA_KAE_VF, kae_vf) ++ ++ def _translate_virtcca_hugepage(self, flavor, image): ++ hugepage = hardware._get_numa_pagesize_constraint(flavor, image) ++ LOG.info("hugepage %s units.Ki %s", hugepage, units.Ki) ++ if not hugepage or hugepage != hardware.VIRTCCA_HUGEPAGE_SIZE / units.Ki: ++ return ++ ++ hugepage_gb = flavor.memory_mb / units.Ki ++ if hardware.get_secure_mem_constraint(flavor): ++ hugepage_gb = min(hugepage_gb, hardware.VIRTCCA_MAX_HUGEPAGE / units.Gi) ++ ++ self._add_resource(orc.VIRTCCA_HUGEPAGE, hugepage_gb) ++ LOG.debug("modified %s=%d to requested resources", ++ orc.VIRTCCA_HUGEPAGE, hugepage_gb) ++ + def _translate_vpmems_request(self, flavor): + """When the hw:pmem extra spec is present, require hosts which can + provide enough vpmem resources. +diff --git a/nova/virt/driver.py b/nova/virt/driver.py +index df3b1e5..1d64508 100644 +--- a/nova/virt/driver.py ++++ b/nova/virt/driver.py +@@ -1789,6 +1789,15 @@ class ComputeDriver(object): + """ + return True + ++ def get_host_secure_numa_topology(self): ++ """Get secure numa topology of virtcca. ++ """ ++ return None ++ ++ def get_host_secure_numa_page(self): ++ """Get secure numa topology of virtcca. ++ """ ++ return None + + def load_compute_driver(virtapi, compute_driver=None): + """Load a compute driver module. +diff --git a/nova/virt/hardware.py b/nova/virt/hardware.py +index feeef57..79de856 100644 +--- a/nova/virt/hardware.py ++++ b/nova/virt/hardware.py +@@ -38,6 +38,22 @@ MEMPAGES_SMALL = -1 + MEMPAGES_LARGE = -2 + MEMPAGES_ANY = -3 + ++VIRTCCA_SWIOTLB_BUFFER_DEFAULT = 128 ++VIRTCCA_SWIOTLB_ALIGNED = 64 ++VIRTCCA_MEM_ALIGNED = 2 ++VIRTCCA_MAX_HUGEPAGE = 3 * units.Gi ++VIRTCCA_HUGEPAGE_SIZE = units.Gi ++VIRTCCA_MAX_NUMA_NUMS = 8 ++VIRTCCA_SECURE_NUMA_PREFIX = 'SECURE_NUMA_' ++VIRTCCA_SECURE_NUMA_1G_PAGE_PREFIX = 'SECURE_NUMA_1G_PAGE_' ++ ++VIRTCCA_SECURE_MEMORY_MIN = 1024 ++VIRTCCA_1G_SIZE = 1024 ++ ++# Single VM can support up to 11 VFs, ++# and a physical machine can support up to 48 VFs ++VIRTCCA_KAE_VF_MAX = 11 ++KAE_VF_MAX = 48 + + class VTPMConfig(ty.NamedTuple): + version: str +@@ -1096,6 +1112,20 @@ def _get_flavor_image_meta( + return flavor_value, image_value + + ++def _get_flavor_meta( ++ key: str, ++ flavor: 'objects.Flavor', ++ default: ty.Any = None, ++ prefix: str = 'hw', ++) -> ty.Any: ++ """Extract flavor base variants of metadata.""" ++ flavor_key = ':'.join([prefix, key]) ++ ++ flavor_value = flavor.get('extra_specs', {}).get(flavor_key, default) ++ ++ return flavor_value ++ ++ + def _get_unique_flavor_image_meta( + key: str, + flavor: 'objects.Flavor', +@@ -1268,6 +1298,70 @@ def _check_mem_encryption_machine_type(image_meta, machine_type=None): + image_id=image_meta.id, image_name=image_meta.name, + reason=_("q35 type is required for SEV to work")) + ++def get_secure_mem_constraint( ++ flavor: 'objects.Flavor', ++) -> bool: ++ flavor_mem_secure_str = _get_flavor_meta( ++ 'mem_secure', flavor) ++ ++ flavor_mem_secure = None ++ if flavor_mem_secure_str is not None: ++ flavor_mem_secure = strutils.bool_from_string(flavor_mem_secure_str) ++ ++ if not flavor_mem_secure: ++ return False ++ ++ requesters = [] ++ if flavor_mem_secure: ++ requesters.append("hw:mem_secure extra spec in %s flavor" % ++ flavor.name) ++ ++ LOG.debug("Memory secure requested by %s", " and ".join(requesters)) ++ return True ++ ++def get_swiotlb_buffer_constraint( ++ flavor: 'objects.Flavor', ++ default: ty.Any = None, ++) -> int: ++ swiotlb_str = _get_flavor_meta( ++ 'swiotlb', flavor, default, prefix='sw') ++ ++ return int(swiotlb_str) ++ ++def get_qemu_cmdline_constraint( ++ flavor: 'objects.Flavor', ++) -> str: ++ """ Return the requested qemu commandline parameters ++ ++ param flavor:a Flavor object to read extra specs from ++ ++ returns: a series of qemu parameters ++ """ ++ flavor_qemu_commandline = _get_flavor_meta( ++ 'qemu_cmdline', flavor, None, 'sw') ++ ++ return flavor_qemu_commandline ++ ++def get_kae_vf_constraint( ++ flavor: 'objects.Flavor', ++) -> int: ++ flavor_qemu_commandline = get_qemu_cmdline_constraint(flavor) ++ ++ kv_pairs = [kv for kv in flavor_qemu_commandline.split(",") if '=' in kv] ++ params = dict(kv.split("=") for kv in kv_pairs) ++ ++ if 'kae' not in params: ++ return 0 ++ ++ kae_vf = params.get('kae') ++ if not kae_vf.isdigit(): ++ raise exception.InvalidKaeVirtFunctionValException() ++ ++ if int(kae_vf) <= 0 or int(kae_vf) > VIRTCCA_KAE_VF_MAX: ++ raise exception.InvalidKaeVirtFunctionNumException( ++ kae_vf=int(kae_vf), kae_vf_max=VIRTCCA_KAE_VF_MAX) ++ ++ return int(kae_vf) + + def _get_numa_pagesize_constraint( + flavor: 'objects.Flavor', +@@ -2411,3 +2505,384 @@ def check_hw_rescue_props(image_meta): + """ + hw_rescue_props = ['hw_rescue_device', 'hw_rescue_bus'] + return any(key in image_meta.properties for key in hw_rescue_props) ++ ++ ++def _virtcca_numa_fit_instance_cell( ++ host_cell: 'objects.NUMACell', ++ host_secure_cell: 'objects.NUMACell', ++ instance_cell: 'objects.InstanceNUMACell', ++ flavor: 'objects.Flavor', ++ secure_numa_page: dict, ++ limits: ty.Optional['objects.NUMATopologyLimit'] = None, ++ cpuset_reserved: int = 0, ++ imageMeta: 'objects.ImageMeta' = None, ++ cell_index: int = 0, ++) -> ty.Optional['objects.InstanceNUMACell']: ++ ++ LOG.debug('Attempting to fit instance cell %(cell)s on host_secure_cell ' ++ '%(host_secure_cell)s', {'cell': instance_cell, 'host_secure_cell': host_secure_cell}) ++ ++ origin_secure_mem = instance_cell.memory ++ ++ instance_cell.memory = 0 ++ if instance_cell.pagesize == None and cell_index == 0: ++ instance_cell.memory = get_swiotlb_buffer_constraint(flavor, VIRTCCA_SWIOTLB_BUFFER_DEFAULT) ++ ++ if ( ++ 'pagesize' in instance_cell and ++ instance_cell.pagesize and ++ instance_cell.pagesize != _get_smallest_pagesize(host_cell) ++ ): ++ # The instance has requested a page size. Verify that the requested ++ # size is valid and that there are available pages of that size on the ++ # host. ++ if (instance_cell.pagesize != VIRTCCA_HUGEPAGE_SIZE / units.Ki): ++ raise exception.MemoryPageSizeNotSupported(pagesize=instance_cell.pagesize) ++ ++ pagesize = instance_cell.pagesize ++ if not pagesize: ++ LOG.debug('Host does not support requested memory pagesize, ' ++ 'or not enough free pages of the requested size. ' ++ 'Requested: %d kB', instance_cell.pagesize) ++ return None ++ LOG.debug('Selected memory pagesize: %(selected_mem_pagesize)d kB. ' ++ 'Requested memory pagesize: %(requested_mem_pagesize)d ' ++ '(small = -1, large = -2, any = -3)', ++ {'selected_mem_pagesize': pagesize, ++ 'requested_mem_pagesize': instance_cell.pagesize}) ++ instance_cell.pagesize = pagesize ++ else: ++ # The instance provides a NUMA topology but does not define any ++ # particular page size for its memory. ++ if host_cell.mempages: ++ # The host supports explicit page sizes. Use a pagesize-aware ++ # memory check using the smallest available page size. ++ pagesize = _get_smallest_pagesize(host_cell) ++ LOG.debug('No specific pagesize requested for instance, ' ++ 'selected pagesize: %d', pagesize) ++ # we want to allow overcommit in this case as we're not using ++ # hugepages ++ if not host_cell.can_fit_pagesize(pagesize, ++ instance_cell.memory * units.Ki, ++ use_free=False): ++ LOG.debug('Not enough available memory to schedule instance ' ++ 'with pagesize %(pagesize)d. Required: ' ++ '%(required)s, available: %(available)s, total: ' ++ '%(total)s.', ++ {'required': instance_cell.memory, ++ 'available': host_cell.avail_memory, ++ 'total': host_cell.memory, ++ 'pagesize': pagesize}) ++ return None ++ else: ++ # The host does not support explicit page sizes. Ignore pagesizes ++ # completely. ++ # NOTE(stephenfin): Do not allow an instance to overcommit against ++ # itself on any NUMA cell, i.e. with 'ram_allocation_ratio = 2.0' ++ # on a host with 1GB RAM, we should allow two 1GB instances but not ++ # one 2GB instance. ++ if instance_cell.memory > host_cell.memory: ++ LOG.debug('Not enough host cell memory to fit instance cell. ' ++ 'Required: %(required)d, actual: %(actual)d', ++ {'required': instance_cell.memory, ++ 'actual': host_cell.memory}) ++ return None ++ ++ # NOTE(stephenfin): As with memory, do not allow an instance to overcommit ++ # against itself on any NUMA cell ++ if instance_cell.cpu_policy in ( ++ fields.CPUAllocationPolicy.DEDICATED, ++ fields.CPUAllocationPolicy.MIXED, ++ ): ++ required_cpus = len(instance_cell.pcpuset) + cpuset_reserved ++ if required_cpus > len(host_cell.pcpuset): ++ LOG.debug('Not enough host cell CPUs to fit instance cell; ' ++ 'required: %(required)d + %(cpuset_reserved)d as ' ++ 'overhead, actual: %(actual)d', { ++ 'required': len(instance_cell.pcpuset), ++ 'actual': len(host_cell.pcpuset), ++ 'cpuset_reserved': cpuset_reserved ++ }) ++ return None ++ ++ if instance_cell.cpu_policy in ( ++ fields.CPUAllocationPolicy.SHARED, ++ fields.CPUAllocationPolicy.MIXED, ++ None, ++ ): ++ required_cpus = len(instance_cell.cpuset) ++ if required_cpus > len(host_cell.cpuset): ++ LOG.debug('Not enough host cell CPUs to fit instance cell; ' ++ 'required: %(required)d, actual: %(actual)d', { ++ 'required': len(instance_cell.cpuset), ++ 'actual': len(host_cell.cpuset), ++ }) ++ return None ++ ++ if instance_cell.cpu_policy in ( ++ fields.CPUAllocationPolicy.DEDICATED, ++ fields.CPUAllocationPolicy.MIXED, ++ ): ++ LOG.debug('Instance has requested pinned CPUs') ++ required_cpus = len(instance_cell.pcpuset) + cpuset_reserved ++ if required_cpus > host_cell.avail_pcpus: ++ LOG.debug('Not enough available CPUs to schedule instance. ' ++ 'Oversubscription is not possible with pinned ' ++ 'instances. Required: %(required)d (%(vcpus)d + ' ++ '%(num_cpu_reserved)d), actual: %(actual)d', ++ {'required': required_cpus, ++ 'vcpus': len(instance_cell.pcpuset), ++ 'actual': host_cell.avail_pcpus, ++ 'num_cpu_reserved': cpuset_reserved}) ++ return None ++ ++ if instance_cell.memory > host_cell.avail_memory: ++ LOG.debug('Not enough available memory to schedule instance. ' ++ 'Oversubscription is not possible with pinned ' ++ 'instances. Required: %(required)s, available: ' ++ '%(available)s, total: %(total)s. ', ++ {'required': instance_cell.memory, ++ 'available': host_cell.avail_memory, ++ 'total': host_cell.memory}) ++ return None ++ ++ # Try to pack the instance cell onto cores ++ instance_cell = _pack_instance_onto_cores( ++ host_cell, instance_cell, num_cpu_reserved=cpuset_reserved, ++ ) ++ if not instance_cell: ++ LOG.debug('Failed to map instance cell CPUs to host cell CPUs') ++ return None ++ ++ if instance_cell.cpu_policy in ( ++ fields.CPUAllocationPolicy.SHARED, ++ fields.CPUAllocationPolicy.MIXED, ++ None, ++ ) and limits: ++ LOG.debug( ++ 'Instance has requested shared CPUs; considering limitations ' ++ 'on usable CPU and memory.') ++ cpu_usage = host_cell.cpu_usage + len(instance_cell.cpuset) ++ cpu_limit = len(host_cell.cpuset) * limits.cpu_allocation_ratio ++ if cpu_usage > cpu_limit: ++ LOG.debug('Host cell has limitations on usable CPUs. There are ' ++ 'not enough free CPUs to schedule this instance. ' ++ 'Usage: %(usage)d, limit: %(limit)d', ++ {'usage': cpu_usage, 'limit': cpu_limit}) ++ return None ++ ++ ram_usage = host_cell.memory_usage + instance_cell.memory ++ ram_limit = host_cell.memory * limits.ram_allocation_ratio ++ if ram_usage > ram_limit: ++ LOG.debug('Host cell has limitations on usable memory. There is ' ++ 'not enough free memory to schedule this instance. ' ++ 'Usage: %(usage)d, limit: %(limit)d', ++ {'usage': ram_usage, 'limit': ram_limit}) ++ return None ++ ++ if origin_secure_mem > host_secure_cell.avail_memory: ++ LOG.debug('Not enough available secure memory to schedule instance. ' ++ 'Oversubscription is not possible with pinned ' ++ 'instances. Required: %(required)s, available: ' ++ '%(available)s, total: %(total)s. ', ++ {'required': origin_secure_mem, ++ 'available': host_secure_cell.avail_memory, ++ 'total': host_secure_cell.memory}) ++ instance_cell.memory = origin_secure_mem ++ return None ++ ++ orc_numa_name = f"{VIRTCCA_SECURE_NUMA_1G_PAGE_PREFIX}{host_secure_cell.id}" ++ numa_entry_1g = origin_secure_mem // VIRTCCA_1G_SIZE ++ if cell_index == 0 and numa_entry_1g > 0: ++ numa_entry_1g = numa_entry_1g -1 ++ ++ if numa_entry_1g > secure_numa_page.get(orc_numa_name, 0): ++ instance_cell.memory = origin_secure_mem ++ return None ++ ++ instance_cell.memory = origin_secure_mem ++ instance_cell.id = host_cell.id ++ return instance_cell ++ ++def virtcca_numa_fit_instance_to_host( ++ host_topology: 'objects.NUMATopology', ++ host_secure_topology: 'objects.NUMATopology', ++ instance_topology: 'objects.InstanceNUMATopology', ++ flavor: 'objects.Flavor', ++ secure_numa_page: dict, ++ limits: ty.Optional['objects.NUMATopologyLimit'] = None, ++ pci_requests: ty.Optional['objects.InstancePCIRequests'] = None, ++ pci_stats: ty.Optional[stats.PciDeviceStats] = None, ++ imageMeta: 'objects.ImageMeta' = None, ++): ++ if not (host_topology and instance_topology and host_secure_topology): ++ LOG.info("Require host NUMA, host secure NUMA and " ++ "instance NUMA topology to fit instance on host.") ++ return ++ elif len(host_secure_topology) < len(instance_topology): ++ LOG.info("There are not enough secure NUMA nodes on the system to schedule " ++ "the instance correctly. Required: %(required)s, actual: " ++ "%(actual)s", ++ {'required': len(instance_topology), ++ 'actual': len(host_secure_topology)}) ++ return ++ ++ emulator_threads_policy = None ++ if 'emulator_threads_policy' in instance_topology: ++ emulator_threads_policy = instance_topology.emulator_threads_policy ++ ++ network_metadata = None ++ if limits and 'network_metadata' in limits: ++ network_metadata = limits.network_metadata ++ ++ host_cells = host_topology.cells ++ ++ # If PCI device(s) are not required, prefer host cells that don't have ++ # devices attached. Presence of a given numa_node in a PCI pool is ++ # indicative of a PCI device being associated with that node ++ if not pci_requests and pci_stats: ++ # TODO(stephenfin): pci_stats can't be None here but mypy can't figure ++ # that out for some reason ++ host_cells = sorted(host_cells, key=lambda cell: cell.id in [ ++ pool['numa_node'] for pool in pci_stats.pools]) # type: ignore ++ ++ for host_cell_perm in itertools.permutations( ++ host_cells, len(instance_topology)): ++ chosen_instance_cells: ty.List['objects.InstanceNUMACell'] = [] ++ chosen_host_cells: ty.List['objects.NUMACell'] = [] ++ cell_index = 0 ++ ++ for host_cell, instance_cell in zip( ++ host_cell_perm, instance_topology.cells): ++ try: ++ cpuset_reserved = 0 ++ if (instance_topology.emulator_threads_isolated and ++ len(chosen_instance_cells) == 0): ++ # For the case of isolate emulator threads, to ++ # make predictable where that CPU overhead is ++ # located we always configure it to be on host ++ # NUMA node associated to the guest NUMA node ++ # 0. ++ cpuset_reserved = 1 ++ for host_secure_cell in host_secure_topology.cells: ++ if host_secure_cell.id == host_cell.id: ++ got_cell = _virtcca_numa_fit_instance_cell( ++ host_cell, host_secure_cell, instance_cell, flavor, ++ secure_numa_page, limits, cpuset_reserved, imageMeta, cell_index) ++ except exception.MemoryPageSizeNotSupported: ++ # This exception will been raised if instance cell's ++ # custom pagesize is not supported with host cell in ++ # _numa_cell_supports_pagesize_request function. ++ break ++ if got_cell is None: ++ break ++ cell_index = cell_index + 1 ++ chosen_host_cells.append(host_cell) ++ chosen_instance_cells.append(got_cell) ++ ++ if len(chosen_instance_cells) != len(host_cell_perm): ++ continue ++ ++ if pci_requests and pci_stats and not pci_stats.support_requests( ++ pci_requests, chosen_instance_cells): ++ continue ++ ++ if network_metadata and not _numa_cells_support_network_metadata( ++ host_topology, chosen_host_cells, network_metadata): ++ continue ++ ++ return objects.InstanceNUMATopology( ++ cells=chosen_instance_cells, ++ emulator_threads_policy=emulator_threads_policy) ++ ++ ++def virtcca_numa_usage_from_instance_numa(host_topology, host_secure_topology, ++ instance_topology, flavor, free=False): ++ """Update the host topology and secure memory topology usage. ++ ++ Update the host NUMA topology based on usage by the provided instance NUMA ++ topology. ++ ++ :param host_topology: objects.NUMATopology to update usage information ++ :param instance_topology: objects.InstanceNUMATopology from which to ++ retrieve usage information. ++ :param free: If true, decrease, rather than increase, host usage based on ++ instance usage. ++ ++ :returns: Updated objects.NUMATopology for host ++ """ ++ if not host_topology or not instance_topology or not host_secure_topology: ++ return (host_topology, host_secure_topology) ++ ++ cells = [] ++ secure_cells = [] ++ sign = -1 if free else 1 ++ ++ for host_cell, host_secure_cell in zip(host_topology.cells, host_secure_topology.cells): ++ memory_usage = host_cell.memory_usage ++ shared_cpus_usage = host_cell.cpu_usage ++ secure_memory_usage = host_secure_cell.memory_usage ++ ++ new_cell = objects.NUMACell( ++ id=host_cell.id, ++ cpuset=host_cell.cpuset, ++ pcpuset=host_cell.pcpuset, ++ memory=host_cell.memory, ++ cpu_usage=0, ++ memory_usage=0, ++ mempages=host_cell.mempages, ++ pinned_cpus=host_cell.pinned_cpus, ++ siblings=host_cell.siblings) ++ ++ if 'network_metadata' in host_cell: ++ new_cell.network_metadata = host_cell.network_metadata ++ ++ for cellid, instance_cell in enumerate(instance_topology.cells): ++ if instance_cell.id != host_cell.id: ++ continue ++ ++ if instance_cell.pagesize == None and cellid == 0: ++ memory_usage = memory_usage + sign * get_swiotlb_buffer_constraint( ++ flavor, VIRTCCA_SWIOTLB_BUFFER_DEFAULT) ++ ++ # new_cell.mempages = _numa_pagesize_usage_from_cell( ++ # new_cell, instance_cell, sign) ++ ++ secure_memory_usage = secure_memory_usage + sign * instance_cell.memory ++ ++ shared_cpus_usage += sign * len(instance_cell.cpuset) ++ ++ if instance_cell.cpu_policy in ( ++ None, fields.CPUAllocationPolicy.SHARED, ++ ): ++ continue ++ ++ pinned_cpus = set(instance_cell.cpu_pinning.values()) ++ if instance_cell.cpuset_reserved: ++ pinned_cpus |= instance_cell.cpuset_reserved ++ ++ if free: ++ if (instance_cell.cpu_thread_policy == ++ fields.CPUThreadAllocationPolicy.ISOLATE): ++ new_cell.unpin_cpus_with_siblings(pinned_cpus) ++ else: ++ new_cell.unpin_cpus(pinned_cpus) ++ else: ++ if (instance_cell.cpu_thread_policy == ++ fields.CPUThreadAllocationPolicy.ISOLATE): ++ new_cell.pin_cpus_with_siblings(pinned_cpus) ++ else: ++ new_cell.pin_cpus(pinned_cpus) ++ ++ # NOTE(stephenfin): We don't need to set 'pinned_cpus' here since that ++ # was done in the above '(un)pin_cpus(_with_siblings)' functions ++ new_cell.memory_usage = max(0, memory_usage) ++ new_cell.cpu_usage = max(0, shared_cpus_usage) ++ cells.append(new_cell) ++ ++ new_secure_cell = new_cell.obj_clone() ++ new_secure_cell.memory_usage = max(0, secure_memory_usage) ++ new_secure_cell.memory = host_secure_cell.memory ++ secure_cells.append(new_secure_cell) ++ ++ return (objects.NUMATopology(cells=cells), objects.NUMATopology(cells=secure_cells)) +\ No newline at end of file +diff --git a/nova/virt/libvirt/config.py b/nova/virt/libvirt/config.py +index 7a99cce..3a3555c 100644 +--- a/nova/virt/libvirt/config.py ++++ b/nova/virt/libvirt/config.py +@@ -39,6 +39,8 @@ from nova.virt import hardware + # Namespace to use for Nova specific metadata items in XML + NOVA_NS = "http://openstack.org/xmlns/libvirt/nova/1.1" + ++QEMU_NS = "http://libvirt.org/schemas/domain/qemu/1.0" ++ + + class LibvirtConfigObject(object): + +@@ -264,6 +266,8 @@ class LibvirtConfigDomainCapsFeatures(LibvirtConfigObject): + feature = None + if c.tag == "sev": + feature = LibvirtConfigDomainCapsFeatureSev() ++ if c.tag == "virtcca": ++ feature = LibvirtConfigDomainCapsFeatureVirtcca() + if feature: + feature.parse_dom(c) + self.features.append(feature) +@@ -303,6 +307,20 @@ class LibvirtConfigDomainCapsFeatureSev(LibvirtConfigObject): + self.cbitpos = int(c.text) + + ++class LibvirtConfigDomainCapsFeatureVirtcca(LibvirtConfigObject): ++ ++ def __init__(self, **kwargs): ++ super(LibvirtConfigDomainCapsFeatureVirtcca, self).__init__( ++ root_name='virtcca', **kwargs) ++ self.supported = False ++ ++ def parse_dom(self, xmldoc): ++ super(LibvirtConfigDomainCapsFeatureVirtcca, self).parse_dom(xmldoc) ++ ++ if xmldoc.get('supported') == 'yes': ++ self.supported = True ++ ++ + class LibvirtConfigDomainCapsOS(LibvirtConfigObject): + + def __init__(self, **kwargs): +@@ -2751,6 +2769,31 @@ class LibvirtConfigGuestFeatureHyperV(LibvirtConfigGuestFeature): + return root + + ++class LibvirtConfigGuestQEMUCommandline(LibvirtConfigObject): ++ ++ def __init__(self, **kwargs): ++ super(LibvirtConfigGuestQEMUCommandline, self).__init__( ++ root_name='commandline', ns_prefix="qemu", ns_uri=QEMU_NS) ++ ++ self.arg1 = None ++ self.arg2 = None ++ ++ def format_dom(self): ++ root = super(LibvirtConfigGuestQEMUCommandline, self).format_dom() ++ ++ if self.arg1 is not None: ++ arg1_node = self._new_node("arg") ++ arg1_node.set("value", self.arg1) ++ root.append(arg1_node) ++ ++ if self.arg2 is not None: ++ arg2_node = self._new_node("arg") ++ arg2_node.set("value", self.arg2) ++ root.append(arg2_node) ++ ++ return root ++ ++ + class LibvirtConfigGuestSEVLaunchSecurity(LibvirtConfigObject): + + def __init__(self, **kwargs): +@@ -2823,6 +2866,8 @@ class LibvirtConfigGuest(LibvirtConfigObject): + self.idmaps = [] + self.perf_events = [] + self.launch_security = None ++ self.launch_security_virtcca = None ++ self.qemu_cmdline = None + + def _format_basic_props(self, root): + root.append(self._text_node("uuid", self.uuid)) +@@ -2946,6 +2991,16 @@ class LibvirtConfigGuest(LibvirtConfigObject): + if self.launch_security is not None: + root.append(self.launch_security.format_dom()) + ++ def _format_virtcca(self, root): ++ if self.launch_security_virtcca is not None: ++ launch_security = self._new_node("launchSecurity") ++ launch_security.set("type", self.launch_security_virtcca) ++ root.append(launch_security) ++ ++ def _format_qemu_commandline(self, root): ++ if self.qemu_cmdline is not None: ++ root.append(self.qemu_cmdline.format_dom()) ++ + def format_dom(self): + root = super(LibvirtConfigGuest, self).format_dom() + +@@ -2976,6 +3031,10 @@ class LibvirtConfigGuest(LibvirtConfigObject): + + self._format_sev(root) + ++ self._format_virtcca(root) ++ ++ self._format_qemu_commandline(root) ++ + return root + + def _parse_basic_props(self, xmldoc): +diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py +index 8719d67..706669d 100644 +--- a/nova/virt/libvirt/driver.py ++++ b/nova/virt/libvirt/driver.py +@@ -145,6 +145,10 @@ VALID_DISK_CACHEMODES = [ + "default", "none", "writethrough", "writeback", "directsync", "unsafe", + ] + ++VIRTCCA_LAUNCH_SECURITY_TYPE = 'cvm' ++ ++VIRTCCA_QEMU_CMDLINE_ARG = "-object" ++ + # The libvirt driver will prefix any disable reason codes with this string. + DISABLE_PREFIX = 'AUTO: ' + # Disable reason for the service which was enabled or disabled without reason +@@ -5166,6 +5170,10 @@ class LibvirtDriver(driver.ComputeDriver): + inst_type['extra_specs'], + disk_unit=disk_unit, + boot_order=boot_order) ++ ++ if self._virtcca_enabled(inst_type): ++ designer.set_driver_iommu_for_device(conf) ++ + return conf + + def _get_guest_fs_config(self, instance, name, image_type=None): +@@ -5804,7 +5812,7 @@ class LibvirtDriver(driver.ComputeDriver): + if image_meta.properties.get("os_command_line"): + guest.os_cmdline = image_meta.properties.os_command_line + +- def _set_clock(self, guest, os_type, image_meta): ++ def _set_clock(self, guest, os_type, image_meta, virtcca_enabled): + # NOTE(mikal): Microsoft Windows expects the clock to be in + # "localtime". If the clock is set to UTC, then you can use a + # registry key to let windows know, but Microsoft says this is +@@ -5818,7 +5826,10 @@ class LibvirtDriver(driver.ComputeDriver): + guest.set_clock(clk) + + if CONF.libvirt.virt_type == "kvm": +- self._set_kvm_timers(clk, os_type, image_meta) ++ if not virtcca_enabled: ++ self._set_kvm_timers(clk, os_type, image_meta) ++ else: ++ LOG.info('Set kvm timers is disabled.It is not supported by virtCCA-enabled') + + def _set_kvm_timers(self, clk, os_type, image_meta): + # TODO(berrange) One day this should be per-guest +@@ -6197,6 +6208,7 @@ class LibvirtDriver(driver.ComputeDriver): + instance: 'objects.Instance', + image_meta: 'objects.ImageMeta', + flavor: 'objects.Flavor', ++ virtcca_enabled: 'bool', + ) -> None: + if CONF.libvirt.virt_type in ("kvm", "qemu"): + arch = libvirt_utils.get_arch(image_meta) +@@ -6251,6 +6263,13 @@ class LibvirtDriver(driver.ComputeDriver): + else: + guest.os_loader_secure = False + ++ if ( ++ guest.os_loader_secure and ++ virtcca_enabled ++ ): ++ guest.os_loader_secure = False ++ LOG.info('Secure boot os disabled. It is not supported by virtCCA-enabled') ++ + try: + loader, nvram_template, requires_smm = ( + self._host.get_loader( +@@ -6264,8 +6283,11 @@ class LibvirtDriver(driver.ComputeDriver): + raise + + guest.os_loader = loader +- guest.os_loader_type = 'pflash' +- guest.os_nvram_template = nvram_template ++ if virtcca_enabled: ++ guest.os_loader_type = 'rom' ++ else: ++ guest.os_loader_type = 'pflash' ++ guest.os_nvram_template = nvram_template + + # if the feature set says we need SMM then enable it + if requires_smm: +@@ -6613,10 +6635,13 @@ class LibvirtDriver(driver.ComputeDriver): + guest.cputune = guest_numa_config.cputune + guest.numatune = guest_numa_config.numatune + +- guest.membacking = self._get_guest_memory_backing_config( +- instance.numa_topology, +- guest_numa_config.numatune, +- flavor, image_meta) ++ virtcca_enabled = self._virtcca_enabled(flavor) ++ ++ if not virtcca_enabled: ++ guest.membacking = self._get_guest_memory_backing_config( ++ instance.numa_topology, ++ guest_numa_config.numatune, ++ flavor, image_meta) + + guest.metadata.append(self._get_guest_config_meta( + instance, network_info)) +@@ -6625,7 +6650,10 @@ class LibvirtDriver(driver.ComputeDriver): + for event in self._supported_perf_events: + guest.add_perf_event(event) + +- self._update_guest_cputune(guest, flavor) ++ if not virtcca_enabled: ++ self._update_guest_cputune(guest, flavor) ++ else: ++ LOG.info('CPU Qos is disabled. It is not supported by virtCCA-enabled') + + guest.cpu = self._get_guest_cpu_config( + flavor, image_meta, guest_numa_config.numaconfig, +@@ -6649,14 +6677,16 @@ class LibvirtDriver(driver.ComputeDriver): + + sev_enabled = self._sev_enabled(flavor, image_meta) + +- self._configure_guest_by_virt_type(guest, instance, image_meta, flavor) ++ ++ ++ self._configure_guest_by_virt_type(guest, instance, image_meta, flavor, virtcca_enabled) + if CONF.libvirt.virt_type != 'lxc': + self._conf_non_lxc( + guest, root_device_name, rescue, instance, inst_path, + image_meta, disk_info) + + self._set_features(guest, instance.os_type, image_meta, flavor) +- self._set_clock(guest, instance.os_type, image_meta) ++ self._set_clock(guest, instance.os_type, image_meta, virtcca_enabled) + + storage_configs = self._get_guest_storage_config(context, + instance, image_meta, disk_info, rescue, block_device_info, +@@ -6668,17 +6698,20 @@ class LibvirtDriver(driver.ComputeDriver): + config = self.vif_driver.get_config( + instance, vif, image_meta, flavor, CONF.libvirt.virt_type, + ) ++ if virtcca_enabled: ++ designer.set_driver_iommu_for_device(config) + guest.add_device(config) + + self._create_consoles(guest, instance, flavor, image_meta) + + self._guest_add_spice_channel(guest) + +- if self._guest_add_video_device(guest): +- self._add_video_driver(guest, image_meta, flavor) ++ if not virtcca_enabled: ++ if self._guest_add_video_device(guest): ++ self._add_video_driver(guest, image_meta, flavor) + +- self._guest_add_pointer_device(guest, image_meta) +- self._guest_add_keyboard_device(guest, image_meta) ++ self._guest_add_pointer_device(guest, image_meta) ++ self._guest_add_keyboard_device(guest, image_meta) + + # Some features are only supported 'qemu' and 'kvm' hypervisor + if CONF.libvirt.virt_type in ('qemu', 'kvm'): +@@ -6689,7 +6722,8 @@ class LibvirtDriver(driver.ComputeDriver): + if self._guest_needs_pcie(guest): + self._guest_add_pcie_root_ports(guest) + +- self._guest_add_usb_root_controller(guest, image_meta) ++ if not virtcca_enabled: ++ self._guest_add_usb_root_controller(guest, image_meta) + + self._guest_add_pci_devices(guest, instance) + +@@ -6716,7 +6750,10 @@ class LibvirtDriver(driver.ComputeDriver): + + self._guest_add_watchdog_action(guest, flavor, image_meta) + +- self._guest_add_memory_balloon(guest) ++ if not virtcca_enabled: ++ self._guest_add_memory_balloon(guest) ++ else: ++ LOG.info('Memory balloon is disabled. It is not supported by virtCCA-enabled') + + if mdevs: + self._guest_add_mdevs(guest, mdevs) +@@ -6726,11 +6763,24 @@ class LibvirtDriver(driver.ComputeDriver): + self._guest_configure_sev(guest, caps.host.cpu.arch, + guest.os_mach_type) + ++ if virtcca_enabled: ++ self._guest_configure_virtcca(guest, flavor) ++ + if vpmems: + self._guest_add_vpmems(guest, vpmems) + + return guest + ++ def _guest_configure_virtcca(self, guest, flavor): ++ guest.launch_security_virtcca = VIRTCCA_LAUNCH_SECURITY_TYPE ++ self._guest_add_qemu(guest, flavor) ++ ++ def _guest_add_qemu(self, guest, flavor): ++ qemu_cmdline = vconfig.LibvirtConfigGuestQEMUCommandline() ++ qemu_cmdline.arg1 = VIRTCCA_QEMU_CMDLINE_ARG ++ qemu_cmdline.arg2 = hardware.get_qemu_cmdline_constraint(flavor) ++ guest.qemu_cmdline = qemu_cmdline ++ + def _get_ordered_vpmems(self, instance, flavor): + resources = self._get_resources(instance) + ordered_vpmem_resources = self._get_ordered_vpmem_resources( +@@ -6842,6 +6892,12 @@ class LibvirtDriver(driver.ComputeDriver): + + return None + ++ def _virtcca_enabled(self, flavor): ++ if not self._host.supports_hisi_virtcca: ++ return False ++ ++ return hardware.get_secure_mem_constraint(flavor) ++ + def _guest_add_mdevs(self, guest, chosen_mdevs): + for chosen_mdev in chosen_mdevs: + mdev = vconfig.LibvirtConfigGuestHostdevMDEV() +@@ -8129,6 +8185,34 @@ class LibvirtDriver(driver.ComputeDriver): + + return objects.NUMATopology(cells=cells) + ++ def _get_host_hugepage(self): ++ if not self._has_numa_support(): ++ return ++ ++ caps = self._host.get_capabilities() ++ topology = caps.host.topology ++ ++ if topology is None or not topology.cells: ++ return ++ ++ def _get_reserved_memory_for_cell(self, cell_id, page_size): ++ cell = self._reserved_hugepages.get(cell_id, {}) ++ return cell.get(page_size, 0) ++ ++ hugepage_total = 0 ++ hugepage_reserved = 0 ++ ++ for cell in topology.cells: ++ for pages in cell.mempages: ++ if pages.size == hardware.VIRTCCA_HUGEPAGE_SIZE / units.Ki: ++ hugepage_total = hugepage_total + pages.total ++ hugepage_reserved = hugepage_reserved + _get_reserved_memory_for_cell(self, cell.id, pages.size) ++ ++ return ( ++ hugepage_total, ++ hugepage_reserved ++ ) ++ + def get_all_volume_usage(self, context, compute_host_bdms): + """Return usage info for volumes attached to vms on + a given host. +@@ -8236,6 +8320,13 @@ class LibvirtDriver(driver.ComputeDriver): + pcpus = len(self._get_pcpu_available()) + memory_enc_slots = self._get_memory_encrypted_slots() + ++ secure_memory_total = self._get_secure_memory_mb_total() ++ virtcca_kae_vf_total = self._host.get_virtcca_kae_vf_total() ++ secure_numa_memory = self._get_secure_memory_numa() ++ secure_numa_page = self.get_secure_numa_page() ++ secure_memory_slab = self.get_secure_memory_slab() ++ hugepage_total, hugepage_reserved = self._get_host_hugepage() ++ + # NOTE(yikun): If the inv record does not exists, the allocation_ratio + # will use the CONF.xxx_allocation_ratio value if xxx_allocation_ratio + # is set, and fallback to use the initial_xxx_allocation_ratio +@@ -8289,6 +8380,73 @@ class LibvirtDriver(driver.ComputeDriver): + 'reserved': 0, + } + ++ if secure_memory_total: ++ result[orc.SECURE_MEM_CONTEXT] = { ++ 'total': secure_memory_total, ++ 'min_unit': 1, ++ 'max_unit': secure_memory_total, ++ 'step_size': 1, ++ 'allocation_ratio': 1.0, ++ 'reserved': 0, ++ } ++ ++ if virtcca_kae_vf_total > 0: ++ result[orc.VIRTCCA_KAE_VF] = { ++ 'total': hardware.KAE_VF_MAX, ++ 'min_unit': 1, ++ 'max_unit': hardware.VIRTCCA_KAE_VF_MAX, ++ 'step_size': 1, ++ 'allocation_ratio': 1.0, ++ 'reserved': 0, ++ } ++ ++ if secure_numa_memory: ++ for rc, numa_memory in secure_numa_memory.items(): ++ result[rc] = { ++ 'total': numa_memory.get('size'), ++ 'min_unit': 1, ++ 'max_unit': numa_memory.get('size'), ++ 'step_size': 1, ++ 'allocation_ratio': 1.0, ++ 'reserved': numa_memory.get('size') - numa_memory.get('free'), ++ } ++ ++ if secure_numa_page: ++ for rc, numa_page in secure_numa_page.items(): ++ reserved_numa_page = 0 ++ if numa_page == 0: ++ numa_page = 1 ++ reserved_numa_page = 1 ++ result[rc] = { ++ 'total': numa_page, ++ 'min_unit': 1, ++ 'max_unit': numa_page, ++ 'step_size': 1, ++ 'allocation_ratio': 1.0, ++ 'reserved': reserved_numa_page, ++ } ++ ++ if secure_memory_slab: ++ for rc, slab in secure_memory_slab.items(): ++ result[rc] = { ++ 'total': slab, ++ 'min_unit': 1, ++ 'max_unit': slab, ++ 'step_size': 1, ++ 'allocation_ratio': 1.0, ++ 'reserved': 0, ++ } ++ ++ if hugepage_total: ++ result[orc.VIRTCCA_HUGEPAGE] = { ++ 'total': hugepage_total, ++ 'min_unit': 1, ++ 'max_unit': hugepage_total, ++ 'step_size': 1, ++ 'allocation_ratio': 1.0, ++ 'reserved': hugepage_reserved, ++ } ++ + # If a sharing DISK_GB provider exists in the provider tree, then our + # storage is shared, and we should not report the DISK_GB inventory in + # the compute node provider. +@@ -8390,6 +8548,97 @@ class LibvirtDriver(driver.ComputeDriver): + else: + return db_const.MAX_INT + ++ def _get_secure_memory_mb_total(self): ++ if not self._host.supports_hisi_virtcca: ++ return 0 ++ ++ secure_memory_total = 0 ++ secure_memory_info = self._host.get_secure_memory_info() ++ ++ if not secure_memory_info: ++ return secure_memory_total ++ ++ for node in secure_memory_info.values(): ++ if isinstance(node, dict) and 'size' in node: ++ secure_memory_total += node['size'] ++ ++ return secure_memory_total ++ ++ def _convert_numa_info(self, numa_info, prefix): ++ """Collect the secure numa info and convert key to orc.prefix_x ++ Returns: ++ dict: {orc.prefix_x: {numa_info}} ++ """ ++ numas_info = {} ++ for node_id in range(hardware.VIRTCCA_MAX_NUMA_NUMS): ++ orc_numa_name = f"{prefix}{node_id}" ++ if hasattr(orc, orc_numa_name) and (node_id in numa_info): ++ numas_info[getattr(orc, orc_numa_name)] = numa_info[node_id] ++ ++ return numas_info ++ ++ def _get_secure_memory_numa(self): ++ """Collect the memory of secure numa in orc ++ Returns: ++ dict: {orc.SECURE_NUMA_x: {memory}} ++ """ ++ if not self._host.supports_hisi_virtcca: ++ return {} ++ ++ numa_nodes_info = self._host.get_secure_memory_info() ++ return self._convert_numa_info(numa_nodes_info, hardware.VIRTCCA_SECURE_NUMA_PREFIX) ++ ++ def get_secure_numa_page(self): ++ """Collect the memory of secure numa in orc ++ Returns: ++ dict: {orc.SECURE_NUMA_1G_PAGE_x: page_numa} ++ """ ++ if not self._host.supports_hisi_virtcca: ++ return {} ++ ++ numa_pages_info = self._host.get_virtcca_buddy_info() ++ orc_numa_pages_info = self._convert_numa_info(numa_pages_info, ++ hardware.VIRTCCA_SECURE_NUMA_1G_PAGE_PREFIX) ++ orc_numa_pages_1g_info = {} ++ for rc, numa_pages in orc_numa_pages_info.items(): ++ orc_numa_pages_1g_info[rc] = numa_pages[9] ++ return orc_numa_pages_1g_info ++ ++ def get_secure_memory_slab(self): ++ """Collect the memory of secure slab info in orc ++ Returns: ++ dict: {orc.VIRTCCA_VM: {vm}, orc.VIRTCCA_VCPU: {vcpu}} ++ """ ++ if not self._host.supports_hisi_virtcca: ++ return {} ++ ++ if not (hasattr(orc, 'VIRTCCA_VM') or hasattr(orc, 'VIRTCCA_VCPU')): ++ return {} ++ ++ slab_info = {} ++ total_td = 0 ++ total_tec = 0 ++ numa_nodes_info = self._host.get_virtcca_slab_info() ++ ++ for node in numa_nodes_info.values(): ++ total_td += node['td']['total'] ++ total_tec += node['tec']['total'] ++ ++ slab_info[getattr(orc, 'VIRTCCA_VM')] = total_td ++ slab_info[getattr(orc, 'VIRTCCA_VCPU')] = total_tec ++ ++ return slab_info ++ ++ def get_host_secure_numa_topology(self): ++ topology = self._get_host_numa_topology().obj_clone() ++ numa_nodes = self._host.get_secure_memory_info() ++ ++ for cell in topology.cells: ++ cell.memory = numa_nodes[cell.id].get('size') ++ cell.memory_usage = numa_nodes[cell.id].get('size') - numa_nodes[cell.id].get('free') ++ ++ return topology ++ + @property + def static_traits(self) -> ty.Dict[str, bool]: + if self._static_traits is not None: +@@ -11729,6 +11978,8 @@ class LibvirtDriver(driver.ComputeDriver): + """ + traits = self._get_cpu_feature_traits() + traits[ot.HW_CPU_X86_AMD_SEV] = self._host.supports_amd_sev ++ traits[ot.HW_CPU_AARCH64_HISI_VIRTCCA] = self._host.supports_hisi_virtcca ++ traits[ot.HW_CPU_AARCH64_HISI_KAE] = self._host.supports_hisi_virtcca_kae + traits[ot.HW_CPU_HYPERTHREADING] = self._host.has_hyperthreading + + return traits +diff --git a/nova/virt/libvirt/host.py b/nova/virt/libvirt/host.py +index 14adf9c..7872a2d 100644 +--- a/nova/virt/libvirt/host.py ++++ b/nova/virt/libvirt/host.py +@@ -34,6 +34,7 @@ import inspect + import operator + import os + import queue ++import re + import socket + import threading + import typing as ty +@@ -64,6 +65,7 @@ from nova.virt.libvirt import event as libvirtevent + from nova.virt.libvirt import guest as libvirt_guest + from nova.virt.libvirt import migration as libvirt_migrate + from nova.virt.libvirt import utils as libvirt_utils ++from nova.virt import hardware + + if ty.TYPE_CHECKING: + import libvirt +@@ -84,6 +86,11 @@ CONF = nova.conf.CONF + HV_DRIVER_QEMU = "QEMU" + + SEV_KERNEL_PARAM_FILE = '/sys/module/kvm_amd/parameters/sev' ++VIRTCCA_ENABLE_SYSFS = '/sys/kernel/tmm/virtcca_enabled' ++TMM_MEMORY_INFO_SYSFS = '/sys/kernel/tmm/memory_info' ++VIRTCCA_KAE_SYSFS = '/sys/kernel/tmm/kae_vf_nums' ++VIRTCCA_BUDDY_INFO_SYSFS = '/sys/kernel/tmm/buddy_info' ++VIRTCCA_SLAB_INFO_SYSFS = '/sys/kernel/tmm/slab_info' + + # These are taken from the spec + # https://github.com/qemu/qemu/blob/v5.2.0/docs/interop/firmware.json +@@ -158,6 +165,8 @@ class Host(object): + # kernel, QEMU, and/or libvirt. These are determined on demand and + # memoized by various properties below + self._supports_amd_sev: ty.Optional[bool] = None ++ self._supports_hisi_virtcca: ty.Optional[bool] = None ++ self._supports_hisi_virtcca_kae: ty.Optional[bool] = None + self._supports_uefi: ty.Optional[bool] = None + self._supports_secure_boot: ty.Optional[bool] = None + +@@ -1598,6 +1607,234 @@ class Host(object): + LOG.debug("No AMD SEV support detected for any (arch, machine_type)") + return self._supports_amd_sev + ++ def _sysfs_supports_hisi_virtcca(self) -> bool: ++ if not os.path.exists(VIRTCCA_ENABLE_SYSFS): ++ LOG.info("%s does not exist ", VIRTCCA_ENABLE_SYSFS) ++ return False ++ ++ with open(VIRTCCA_ENABLE_SYSFS, 'r') as f: ++ contents = f.read() ++ LOG.debug("%s containes [%s]", VIRTCCA_ENABLE_SYSFS, contents) ++ return contents =="1\n" ++ ++ @property ++ def supports_hisi_virtcca(self) -> bool: ++ if self._supports_hisi_virtcca is not None: ++ return self._supports_hisi_virtcca ++ ++ self._supports_hisi_virtcca = False ++ ++ caps = self.get_capabilities() ++ if caps.host.cpu.arch != fields.Architecture.AARCH64: ++ return False ++ ++ if not self._sysfs_supports_hisi_virtcca(): ++ LOG.info("kernel doesn't support HISI VIRTCCA") ++ return False ++ ++ domain_caps = self.get_domain_capabilities() ++ for arch in domain_caps: ++ for machine_type in domain_caps[arch]: ++ LOG.debug("Checking VIRTCCA support for arch %s " ++ "and machine type %s", arch, machine_type) ++ for feature in domain_caps[arch][machine_type].features: ++ feature_is_virtcca = isinstance( ++ feature, vconfig.LibvirtConfigDomainCapsFeatureVirtcca) ++ if feature_is_virtcca and feature.supported: ++ LOG.info("HISI VIRTCCA support detected") ++ self._supports_hisi_virtcca = True ++ return self._supports_hisi_virtcca ++ ++ LOG.debug("No HISI VIRTCCA support detected for any (arch, machine_type)") ++ return False ++ ++ def parse_memory_value(self, line): ++ """Extract memory value in Mi units from the given line string. ++ """ ++ match = re.search(r'(\d+)Mi', line) ++ if match: ++ return int(match.group(1)) ++ else: ++ return 0 ++ ++ def _parse_numa_info(self, lines): ++ if not lines: ++ LOG.warning("Empty content in the systemc file") ++ return {} ++ ++ first_line = lines[0].strip() ++ if not first_line.startswith('available:'): ++ LOG.error("Invalid file format: first line not starting with 'available:'") ++ return {} ++ ++ try: ++ numa_available = int(first_line.split()[1]) ++ except (IndexError, ValueError): ++ LOG.error("Failed to parse available NUMA nodes from %s ", first_line) ++ return {} ++ ++ numa_info = {} ++ current_line = 1 ++ ++ for node in range(numa_available): ++ if current_line + 3 >= len(lines): ++ LOG.debug("Not enough data to read node {node}") ++ break ++ ++ node_line = lines[current_line] ++ if not node_line.startswith('numa node'): ++ LOG.error("The expected node line %d is incorrect ", current_line) ++ break ++ ++ try: ++ node_number = int(node_line.split()[2]) ++ except (IndexError, ValueError): ++ LOG.error("Failed to parse conent in line %d", current_line) ++ ++ try: ++ size = self.parse_memory_value(lines[current_line]) ++ free = self.parse_memory_value(lines[current_line + 1]) ++ cvm_used = self.parse_memory_value(lines[current_line + 2]) ++ meta_used = self.parse_memory_value(lines[current_line + 3]) ++ except IndexError: ++ LOG.error("Failed to reading node %d, file length exceeded", node_number) ++ break ++ ++ numa_info[node_number] = { ++ 'size': size, ++ 'free': free, ++ 'cvm_used': cvm_used, ++ 'meta_used': meta_used, ++ } ++ ++ current_line += 4 ++ ++ return numa_info ++ ++ def get_secure_memory_info(self): ++ """Read and parse secure NUMA memory information from system files, ++ Returns: ++ dict: A dictionary containing secure memory information for each NUMA node, ++ formatted as {node_number: {attributes}}. ++ """ ++ if not os.path.exists(TMM_MEMORY_INFO_SYSFS): ++ LOG.info("%s does not exist", TMM_MEMORY_INFO_SYSFS) ++ return {} ++ ++ with open(TMM_MEMORY_INFO_SYSFS, 'r') as f: ++ lines = f.readlines() ++ ++ return self._parse_numa_info(lines) ++ ++ def _sysfs_hisi_virtcca_kae_vf(self) -> bool: ++ if not os.path.exists(VIRTCCA_KAE_SYSFS): ++ LOG.debug("%s does not exist ", VIRTCCA_KAE_SYSFS) ++ return False ++ ++ with open(VIRTCCA_KAE_SYSFS, 'r') as f: ++ contents = f.readline().strip() ++ if contents.isdigit(): ++ return int(contents) >= 0 and int(contents) <= hardware.KAE_VF_MAX ++ ++ return False ++ ++ @property ++ def supports_hisi_virtcca_kae(self) -> bool: ++ if self._supports_hisi_virtcca_kae is not None: ++ return self._supports_hisi_virtcca_kae ++ ++ self._supports_hisi_virtcca_kae = False ++ # virtcca-kae is based on the virtcca capability ++ if not self._supports_hisi_virtcca: ++ return False ++ ++ if not self._sysfs_hisi_virtcca_kae_vf(): ++ return False ++ ++ self._supports_hisi_virtcca_kae = True ++ LOG.info("HISI VIRTCCA KAE detected") ++ return True ++ ++ def get_virtcca_kae_vf_total(self) -> int: ++ if not os.path.exists(VIRTCCA_KAE_SYSFS): ++ return -1 ++ ++ with open(VIRTCCA_KAE_SYSFS, 'r') as f: ++ contents = f.readline().strip() ++ if not contents.isdigit(): ++ return -1 ++ if int(contents) < 0 or int(contents) > hardware.KAE_VF_MAX: ++ return -1 ++ ++ return int(contents) ++ ++ def get_virtcca_buddy_info(self) -> dict: ++ """Read and parse secure buddy info from sysfs, ++ Returns: ++ dict: A dictionary containing page nums for each numa node, ++ formatted as {numa_node: [order_nums]} ++ """ ++ if not os.path.exists(VIRTCCA_BUDDY_INFO_SYSFS): ++ return {} ++ ++ with open(VIRTCCA_BUDDY_INFO_SYSFS, 'r') as f: ++ lines = f.readlines() ++ nodes = list(map(int, lines[1].split()[1:])) ++ buddy_info = {node: [] for node in nodes} ++ ++ for line in lines[2:]: ++ parts = line.split(":") ++ values = parts[1].strip().split() ++ value_list = list(map(int, values)) ++ ++ for node, val in zip(nodes, value_list): ++ buddy_info[node].append(val) ++ ++ return buddy_info ++ ++ def get_virtcca_slab_info(self) -> dict: ++ """Read and parse secure buddy info from sysfs, ++ Returns: ++ dict: A dictionary containing page nums for each numa node, ++ formatted as {numa_node: [order_nums]} ++ """ ++ if not os.path.exists(VIRTCCA_SLAB_INFO_SYSFS): ++ return {} ++ ++ with open(VIRTCCA_SLAB_INFO_SYSFS, 'r') as f: ++ slab_data = {} ++ ++ for line in f: ++ line = line.strip() ++ if not line.startswith('numa node'): ++ continue ++ ++ node_match = re.search(r'node (\d+)', line) ++ type_match = re.search(r'(td|tec|ttt) meta_data', line) ++ value_match = re.search(r'(\d+)$', line) ++ ++ if not all([node_match, type_match, value_match]): ++ continue ++ ++ node = node_match.group(1) ++ meta_type = type_match.group(1) ++ value = value_match.group(1) ++ is_free = 'free' in line.lower() ++ ++ if node not in slab_data: ++ slab_data[node] = { ++ 'td': {'total': 0, 'free': 0}, ++ 'tec': {'total': 0, 'free': 0}, ++ 'ttt': {'total': 0, 'free': 0}, ++ } ++ ++ if is_free: ++ slab_data[node][meta_type]['free'] = int(value) ++ else: ++ slab_data[node][meta_type]['total'] = int(value) ++ ++ return slab_data ++ + @property + def loaders(self) -> ty.List[dict]: + """Retrieve details of loader configuration for the host. diff --git a/openstack-nova.spec b/openstack-nova.spec index 446d2b89b9ef3ff8c96ce1caa0ef58a22ed1a2a3..ab8bbd8c2b7f7ade63c2b18d0175f14dc742c41a 100644 --- a/openstack-nova.spec +++ b/openstack-nova.spec @@ -17,7 +17,7 @@ Name: openstack-nova # Liberty semver reset # https://review.openstack.org/#/q/I6a35fa0dda798fad93b804d00a46af80f08d475c,n,z Version: 23.2.2 -Release: 3 +Release: 4 Summary: OpenStack Compute (nova) License: ASL 2.0 @@ -52,6 +52,7 @@ Source41: nova_migration-rootwrap_cold_migration Patch01: add-loongarch64-support.patch Patch02: add-loongarch64-libvirt-support.patch +Patch03: add-virtcca-support.patch BuildArch: noarch @@ -727,6 +728,9 @@ exit 0 %endif %changelog +* Thu Jun 5 2025 liuhao - 23.2.2-4 +- Add virtcca support + * Tue Jun 11 2024 zhaixiaojuan - 23.2.2-3 - Add loongarch64 libvirt support