diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c index b06d76216599e73dfc094738c2258b1d43cbeb3a..4f06a54c2e431f7c8c4157d30ae39137ed3969c0 100644 --- a/drivers/vfio/pci/vfio_pci.c +++ b/drivers/vfio/pci/vfio_pci.c @@ -108,6 +108,61 @@ static inline bool vfio_pci_is_vga(struct pci_dev *pdev) return (pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA; } +static struct vfio_pci { + struct mutex vendor_drivers_lock; + struct list_head vendor_drivers_list; +} vfio_pci; + +struct pci_dev *vfio_pci_pdev(void *device_data) +{ + struct vfio_pci_device *vdev = device_data; + + return vdev->pdev; +} +EXPORT_SYMBOL_GPL(vfio_pci_pdev); + +int vfio_pci_num_regions(void *device_data) +{ + struct vfio_pci_device *vdev = device_data; + + return vdev->num_regions; +} +EXPORT_SYMBOL_GPL(vfio_pci_num_regions); + +int vfio_pci_irq_type(void *device_data) +{ + struct vfio_pci_device *vdev = device_data; + + return vdev->irq_type; +} +EXPORT_SYMBOL_GPL(vfio_pci_irq_type); + +void *vfio_pci_vendor_data(void *device_data) +{ + struct vfio_pci_device *vdev = device_data; + + return vdev->vendor_data; +} +EXPORT_SYMBOL_GPL(vfio_pci_vendor_data); + +int vfio_pci_set_vendor_regions(void *device_data, int num_vendor_regions) +{ + struct vfio_pci_device *vdev = device_data; + + vdev->num_vendor_regions = num_vendor_regions; + return 0; +} +EXPORT_SYMBOL_GPL(vfio_pci_set_vendor_regions); + +int vfio_pci_set_vendor_irqs(void *device_data, int num_vendor_irqs) +{ + struct vfio_pci_device *vdev = device_data; + + vdev->num_vendor_irqs = num_vendor_irqs; + return 0; +} +EXPORT_SYMBOL_GPL(vfio_pci_set_vendor_irqs); + static void vfio_pci_probe_mmaps(struct vfio_pci_device *vdev) { struct resource *res; @@ -463,7 +518,7 @@ static void vfio_pci_disable(struct vfio_pci_device *vdev) vfio_pci_set_power_state(vdev, PCI_D3hot); } -static void vfio_pci_release(void *device_data) +void vfio_pci_release(void *device_data) { struct vfio_pci_device *vdev = device_data; @@ -491,8 +546,9 @@ static void vfio_pci_release(void *device_data) module_put(THIS_MODULE); } +EXPORT_SYMBOL_GPL(vfio_pci_release); -static int vfio_pci_open(void *device_data) +int vfio_pci_open(void *device_data) { struct vfio_pci_device *vdev = device_data; int ret = 0; @@ -516,6 +572,7 @@ static int vfio_pci_open(void *device_data) module_put(THIS_MODULE); return ret; } +EXPORT_SYMBOL_GPL(vfio_pci_open); static int vfio_pci_get_irq_count(struct vfio_pci_device *vdev, int irq_type) { @@ -707,7 +764,7 @@ struct vfio_devices { int max_index; }; -static long vfio_pci_ioctl(void *device_data, +long vfio_pci_ioctl(void *device_data, unsigned int cmd, unsigned long arg) { struct vfio_pci_device *vdev = device_data; @@ -729,8 +786,9 @@ static long vfio_pci_ioctl(void *device_data, if (vdev->reset_works) info.flags |= VFIO_DEVICE_FLAGS_RESET; - info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions; - info.num_irqs = VFIO_PCI_NUM_IRQS; + info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions + + vdev->num_vendor_regions; + info.num_irqs = VFIO_PCI_NUM_IRQS + vdev->num_vendor_irqs; return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; @@ -1217,6 +1275,7 @@ static long vfio_pci_ioctl(void *device_data, return -ENOTTY; } +EXPORT_SYMBOL_GPL(vfio_pci_ioctl); static ssize_t vfio_pci_rw(void *device_data, char __user *buf, size_t count, loff_t *ppos, bool iswrite) @@ -1250,7 +1309,7 @@ static ssize_t vfio_pci_rw(void *device_data, char __user *buf, return -EINVAL; } -static ssize_t vfio_pci_read(void *device_data, char __user *buf, +ssize_t vfio_pci_read(void *device_data, char __user *buf, size_t count, loff_t *ppos) { if (!count) @@ -1258,8 +1317,9 @@ static ssize_t vfio_pci_read(void *device_data, char __user *buf, return vfio_pci_rw(device_data, buf, count, ppos, false); } +EXPORT_SYMBOL_GPL(vfio_pci_read); -static ssize_t vfio_pci_write(void *device_data, const char __user *buf, +ssize_t vfio_pci_write(void *device_data, const char __user *buf, size_t count, loff_t *ppos) { if (!count) @@ -1267,6 +1327,7 @@ static ssize_t vfio_pci_write(void *device_data, const char __user *buf, return vfio_pci_rw(device_data, (char __user *)buf, count, ppos, true); } +EXPORT_SYMBOL_GPL(vfio_pci_write); /* Return 1 on zap and vma_lock acquired, 0 on contention (only with @try) */ static int vfio_pci_zap_and_vma_lock(struct vfio_pci_device *vdev, bool try) @@ -1477,7 +1538,7 @@ static const struct vm_operations_struct vfio_pci_mmap_ops = { .fault = vfio_pci_mmap_fault, }; -static int vfio_pci_mmap(void *device_data, struct vm_area_struct *vma) +int vfio_pci_mmap(void *device_data, struct vm_area_struct *vma) { struct vfio_pci_device *vdev = device_data; struct pci_dev *pdev = vdev->pdev; @@ -1546,8 +1607,9 @@ static int vfio_pci_mmap(void *device_data, struct vm_area_struct *vma) return 0; } +EXPORT_SYMBOL_GPL(vfio_pci_mmap); -static void vfio_pci_request(void *device_data, unsigned int count) +void vfio_pci_request(void *device_data, unsigned int count) { struct vfio_pci_device *vdev = device_data; struct pci_dev *pdev = vdev->pdev; @@ -1567,6 +1629,7 @@ static void vfio_pci_request(void *device_data, unsigned int count) mutex_unlock(&vdev->igate); } +EXPORT_SYMBOL_GPL(vfio_pci_request); static const struct vfio_device_ops vfio_pci_ops = { .name = "vfio-pci", @@ -1582,6 +1645,39 @@ static const struct vfio_device_ops vfio_pci_ops = { static int vfio_pci_reflck_attach(struct vfio_pci_device *vdev); static void vfio_pci_reflck_put(struct vfio_pci_reflck *reflck); +static int probe_vendor_drivers(struct vfio_pci_device *vdev) +{ + struct vfio_pci_vendor_driver *driver; + int ret = -ENODEV; + + request_module("vfio-pci:%x-%x", vdev->pdev->vendor, + vdev->pdev->device); + + mutex_lock(&vfio_pci.vendor_drivers_lock); + list_for_each_entry(driver, &vfio_pci.vendor_drivers_list, next) { + void *data; + + if (vdev->pdev->vendor != driver->ops->vendor || + vdev->pdev->device != driver->ops->device) + continue; + + if (!try_module_get(driver->ops->owner)) + continue; + + data = driver->ops->probe(vdev->pdev); + if (IS_ERR(data)) { + module_put(driver->ops->owner); + continue; + } + vdev->vendor_driver = driver; + vdev->vendor_data = data; + ret = 0; + break; + } + mutex_unlock(&vfio_pci.vendor_drivers_lock); + return ret; +} + static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { struct vfio_pci_device *vdev; @@ -1625,7 +1721,11 @@ static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) INIT_LIST_HEAD(&vdev->vma_list); init_rwsem(&vdev->memory_lock); - ret = vfio_add_group_dev(&pdev->dev, &vfio_pci_ops, vdev); + if (probe_vendor_drivers(vdev)) + ret = vfio_add_group_dev(&pdev->dev, &vfio_pci_ops, vdev); + else + ret = vfio_add_group_dev(&pdev->dev, + vdev->vendor_driver->ops->device_ops, vdev); if (ret) { vfio_iommu_group_put(group, &pdev->dev); kfree(vdev); @@ -1679,6 +1779,11 @@ static void vfio_pci_remove(struct pci_dev *pdev) kfree(vdev->region); mutex_destroy(&vdev->ioeventfds_lock); + if (vdev->vendor_driver) { + vdev->vendor_driver->ops->remove(vdev->vendor_data); + module_put(vdev->vendor_driver->ops->owner); + } + if (!disable_idle_d3) vfio_pci_set_power_state(vdev, PCI_D0); @@ -2007,6 +2112,9 @@ static int __init vfio_pci_init(void) if (ret) return ret; + mutex_init(&vfio_pci.vendor_drivers_lock); + INIT_LIST_HEAD(&vfio_pci.vendor_drivers_list); + /* Register and scan for devices */ ret = pci_register_driver(&vfio_pci_driver); if (ret) @@ -2021,6 +2129,60 @@ static int __init vfio_pci_init(void) return ret; } +int __vfio_pci_register_vendor_driver(struct vfio_pci_vendor_driver_ops *ops) +{ + struct vfio_pci_vendor_driver *driver, *tmp; + + if (!ops || !ops->device_ops) + return -EINVAL; + + driver = kzalloc(sizeof(*driver), GFP_KERNEL); + if (!driver) + return -ENOMEM; + + driver->ops = ops; + + mutex_lock(&vfio_pci.vendor_drivers_lock); + + /* Check for duplicates */ + list_for_each_entry(tmp, &vfio_pci.vendor_drivers_list, next) { + if (tmp->ops->device_ops == ops->device_ops) { + mutex_unlock(&vfio_pci.vendor_drivers_lock); + kfree(driver); + return -EINVAL; + } + } + + list_add(&driver->next, &vfio_pci.vendor_drivers_list); + + mutex_unlock(&vfio_pci.vendor_drivers_lock); + + if (!try_module_get(THIS_MODULE)) + return -ENODEV; + + return 0; +} +EXPORT_SYMBOL_GPL(__vfio_pci_register_vendor_driver); + +void vfio_pci_unregister_vendor_driver(struct vfio_device_ops *device_ops) +{ + struct vfio_pci_vendor_driver *driver, *tmp; + + mutex_lock(&vfio_pci.vendor_drivers_lock); + list_for_each_entry_safe(driver, tmp, + &vfio_pci.vendor_drivers_list, next) { + if (driver->ops->device_ops == device_ops) { + list_del(&driver->next); + mutex_unlock(&vfio_pci.vendor_drivers_lock); + kfree(driver); + module_put(THIS_MODULE); + return; + } + } + mutex_unlock(&vfio_pci.vendor_drivers_lock); +} +EXPORT_SYMBOL_GPL(vfio_pci_unregister_vendor_driver); + module_init(vfio_pci_init); module_exit(vfio_pci_cleanup); diff --git a/drivers/vfio/pci/vfio_pci_private.h b/drivers/vfio/pci/vfio_pci_private.h index 987b4d311fde9981c8f1a5a512aac4106e317eda..033716ea58e8395c876b18f539717e15c7e97119 100644 --- a/drivers/vfio/pci/vfio_pci_private.h +++ b/drivers/vfio/pci/vfio_pci_private.h @@ -89,6 +89,11 @@ struct vfio_pci_mmap_vma { struct list_head vma_next; }; +struct vfio_pci_vendor_driver { + const struct vfio_pci_vendor_driver_ops *ops; + struct list_head next; +}; + struct vfio_pci_device { struct pci_dev *pdev; void __iomem *barmap[PCI_STD_RESOURCE_END + 1]; @@ -102,6 +107,8 @@ struct vfio_pci_device { int num_ctx; int irq_type; int num_regions; + int num_vendor_regions; + int num_vendor_irqs; struct vfio_pci_region *region; u8 msi_qmax; u8 msix_bar; @@ -130,6 +137,8 @@ struct vfio_pci_device { struct mutex vma_lock; struct list_head vma_list; struct rw_semaphore memory_lock; + void *vendor_data; + struct vfio_pci_vendor_driver *vendor_driver; }; #define is_intx(vdev) (vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX) diff --git a/drivers/vfio/pci/vfio_pci_rdwr.c b/drivers/vfio/pci/vfio_pci_rdwr.c index 83f81d24df78eae51fc17221ff3bcd828a487694..ba2672e89f071f7cde2c5566874e983d0ffa0a29 100644 --- a/drivers/vfio/pci/vfio_pci_rdwr.c +++ b/drivers/vfio/pci/vfio_pci_rdwr.c @@ -153,6 +153,16 @@ static int vfio_pci_setup_barmap(struct vfio_pci_device *vdev, int bar) return 0; } +void __iomem *vfio_pci_get_barmap(void *device_data, int bar) +{ + int ret; + struct vfio_pci_device *vdev = device_data; + + ret = vfio_pci_setup_barmap(vdev, bar); + return ret ? ERR_PTR(ret) : vdev->barmap[bar]; +} +EXPORT_SYMBOL_GPL(vfio_pci_get_barmap); + ssize_t vfio_pci_bar_rw(struct vfio_pci_device *vdev, char __user *buf, size_t count, loff_t *ppos, bool iswrite) { diff --git a/include/linux/vfio.h b/include/linux/vfio.h index da29802d62767f4f46c5b67ef27124f837486c08..bb000f772ad66e807dee5f12e1f09f57680e025b 100644 --- a/include/linux/vfio.h +++ b/include/linux/vfio.h @@ -197,4 +197,59 @@ extern int vfio_virqfd_enable(void *opaque, void *data, struct virqfd **pvirqfd, int fd); extern void vfio_virqfd_disable(struct virqfd **pvirqfd); +extern int vfio_pci_num_regions(void *device_data); +extern struct pci_dev *vfio_pci_pdev(void *device_data); +extern long vfio_pci_ioctl(void *device_data, + unsigned int cmd, unsigned long arg); +extern ssize_t vfio_pci_read(void *device_data, char __user *buf, + size_t count, loff_t *ppos); +extern ssize_t vfio_pci_write(void *device_data, const char __user *buf, + size_t count, loff_t *ppos); +extern int vfio_pci_mmap(void *device_data, struct vm_area_struct *vma); +extern void vfio_pci_request(void *device_data, unsigned int count); +extern int vfio_pci_open(void *device_data); +extern void vfio_pci_release(void *device_data); +extern void *vfio_pci_vendor_data(void *device_data); +extern int vfio_pci_set_vendor_regions(void *device_data, + int num_vendor_regions); + +struct vfio_pci_vendor_driver_ops { + char *name; + struct module *owner; + /* Used to match device */ + unsigned short vendor; + unsigned short device; + void *(*probe)(struct pci_dev *pdev); + void (*remove)(void *vendor_data); + struct vfio_device_ops *device_ops; +}; +int __vfio_pci_register_vendor_driver(struct vfio_pci_vendor_driver_ops *ops); +void vfio_pci_unregister_vendor_driver(struct vfio_device_ops *device_ops); + +#define vfio_pci_register_vendor_driver(__name, __probe, __remove, \ + __device_ops) \ +static struct vfio_pci_vendor_driver_ops __ops ## _node = { \ + .owner = THIS_MODULE, \ + .name = __name, \ + .probe = __probe, \ + .remove = __remove, \ + .device_ops = __device_ops, \ +}; \ +__vfio_pci_register_vendor_driver(&__ops ## _node) + +#define module_vfio_pci_register_vendor_handler(name, probe, remove, \ + device_ops) \ +static int __init device_ops ## _module_init(void) \ +{ \ + vfio_pci_register_vendor_driver(name, probe, remove, \ + device_ops); \ + return 0; \ +}; \ +static void __exit device_ops ## _module_exit(void) \ +{ \ + vfio_pci_unregister_vendor_driver(device_ops); \ +}; \ +module_init(device_ops ## _module_init); \ +module_exit(device_ops ## _module_exit) + #endif /* VFIO_H */