diff --git a/hw/vfio/container.c b/hw/vfio/container.c
index 95b8cff33bb1adc0dd63df7b4082ef8a6fad8c53..56c55d00f19749a82e3d8b71690054208d76c42f 100644
--- a/hw/vfio/container.c
+++ b/hw/vfio/container.c
@@ -736,14 +736,14 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
     group->container = container;
     QLIST_INSERT_HEAD(&container->group_list, group, container_next);
 
+    bcontainer->listener = vfio_memory_listener;
     if (kvm_csv3_enabled()) {
         shared_memory_listener_register(&bcontainer->listener,
                                         bcontainer->space->as);
+    } else {
+        memory_listener_register(&bcontainer->listener, bcontainer->space->as);
     }
 
-    bcontainer->listener = vfio_memory_listener;
-    memory_listener_register(&bcontainer->listener, bcontainer->space->as);
-
     if (bcontainer->error) {
         ret = -1;
         error_propagate_prepend(errp, bcontainer->error,