From 9577933a1d13d376e547a44e935c25f8ccb709cb Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 30 Oct 2020 08:49:06 -0700 Subject: [PATCH 01/85] dmaengine: idxd: Update calculation of group offset to be more readable ANBZ: #22 commit 5a71270197f39ff3e526cf130bc5d6e84db8f2d7 upstream. Create helper macros to make group offset calculation more readable. Intel-SIG: commit 5a71270197f3 dmaengine: idxd: Update calculation of group offset to be more readable. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/160407294683.839093.10740868559754142070.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 12 +++++------- drivers/dma/idxd/registers.h | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 97cc1653b5ff..d255bb016c4d 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -619,24 +619,22 @@ static void idxd_group_config_write(struct idxd_group *group) dev_dbg(dev, "Writing group %d cfg registers\n", group->id); /* setup GRPWQCFG */ - for (i = 0; i < 4; i++) { - grpcfg_offset = idxd->grpcfg_offset + - group->id * 64 + i * sizeof(u64); - iowrite64(group->grpcfg.wqs[i], - idxd->reg_base + grpcfg_offset); + for (i = 0; i < GRPWQCFG_STRIDES; i++) { + grpcfg_offset = GRPWQCFG_OFFSET(idxd, group->id, i); + iowrite64(group->grpcfg.wqs[i], idxd->reg_base + grpcfg_offset); dev_dbg(dev, "GRPCFG wq[%d:%d: %#x]: %#llx\n", group->id, i, grpcfg_offset, ioread64(idxd->reg_base + grpcfg_offset)); } /* setup GRPENGCFG */ - grpcfg_offset = idxd->grpcfg_offset + group->id * 64 + 32; + grpcfg_offset = GRPENGCFG_OFFSET(idxd, group->id); iowrite64(group->grpcfg.engines, idxd->reg_base + grpcfg_offset); dev_dbg(dev, "GRPCFG engs[%d: %#x]: %#llx\n", group->id, grpcfg_offset, ioread64(idxd->reg_base + grpcfg_offset)); /* setup GRPFLAGS */ - grpcfg_offset = idxd->grpcfg_offset + group->id * 64 + 40; + grpcfg_offset = GRPFLGCFG_OFFSET(idxd, group->id); iowrite32(group->grpcfg.flags.bits, idxd->reg_base + grpcfg_offset); dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#x\n", group->id, grpcfg_offset, diff --git a/drivers/dma/idxd/registers.h b/drivers/dma/idxd/registers.h index 20628b958509..751ecb4f9f81 100644 --- a/drivers/dma/idxd/registers.h +++ b/drivers/dma/idxd/registers.h @@ -360,4 +360,22 @@ union wqcfg { #define WQCFG_STRIDES(_idxd_dev) ((_idxd_dev)->wqcfg_size / sizeof(u32)) +#define GRPCFG_SIZE 64 +#define GRPWQCFG_STRIDES 4 + +/* + * This macro calculates the offset into the GRPCFG register + * idxd - struct idxd * + * n - wq id + * ofs - the index of the 32b dword for the config register + * + * The WQCFG register block is divided into groups per each wq. The n index + * allows us to move to the register group that's for that particular wq. + * Each register is 32bits. The ofs gives us the number of register to access. + */ +#define GRPWQCFG_OFFSET(idxd_dev, n, ofs) ((idxd_dev)->grpcfg_offset +\ + (n) * GRPCFG_SIZE + sizeof(u64) * (ofs)) +#define GRPENGCFG_OFFSET(idxd_dev, n) ((idxd_dev)->grpcfg_offset + (n) * GRPCFG_SIZE + 32) +#define GRPFLGCFG_OFFSET(idxd_dev, n) ((idxd_dev)->grpcfg_offset + (n) * GRPCFG_SIZE + 40) + #endif -- Gitee From e11e5dd39c29af1b2f45fc532052ff112af1a19d Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Apr 2021 16:37:15 -0700 Subject: [PATCH 02/85] dmaengine: idxd: cleanup pci interrupt vector allocation management ANBZ: #22 commit 5fc8e85ff12ce0530ac658686902a0ee64600f56 upstream. The devm managed lifetime is incompatible with 'struct device' objects that resides in idxd context. This is one of the series that clean up the idxd driver 'struct device' lifetime. Remove devm managed pci interrupt vectors and replace with unmanged allocators. Intel-SIG: commit 5fc8e85ff12c dmaengine: idxd: cleanup pci interrupt vector allocation management. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reported-by: Jason Gunthorpe Fixes: bfe1d56091c1 ("dmaengine: idxd: Init and probe for Intel data accelerators") Signed-off-by: Dave Jiang Reviewed-by: Dan Williams Link: https://lore.kernel.org/r/161852983563.2203940.8116028229124776669.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 4 +-- drivers/dma/idxd/idxd.h | 2 +- drivers/dma/idxd/init.c | 64 +++++++++++++++++---------------------- 3 files changed, 30 insertions(+), 40 deletions(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index d255bb016c4d..3f696abd74ac 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -19,7 +19,7 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand, /* Interrupt control bits */ void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id) { - struct irq_data *data = irq_get_irq_data(idxd->msix_entries[vec_id].vector); + struct irq_data *data = irq_get_irq_data(idxd->irq_entries[vec_id].vector); pci_msi_mask_irq(data); } @@ -36,7 +36,7 @@ void idxd_mask_msix_vectors(struct idxd_device *idxd) void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id) { - struct irq_data *data = irq_get_irq_data(idxd->msix_entries[vec_id].vector); + struct irq_data *data = irq_get_irq_data(idxd->irq_entries[vec_id].vector); pci_msi_unmask_irq(data); } diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index a0371035abd7..786597812cd8 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -36,6 +36,7 @@ struct idxd_device_driver { struct idxd_irq_entry { struct idxd_device *idxd; int id; + int vector; struct llist_head pending_llist; struct list_head work_list; /* @@ -220,7 +221,6 @@ struct idxd_device { union sw_err_reg sw_err; wait_queue_head_t cmd_waitq; - struct msix_entry *msix_entries; int num_wq_irqs; struct idxd_irq_entry *irq_entries; diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 40ea8a190340..c776f0ece919 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -61,7 +61,6 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) { struct pci_dev *pdev = idxd->pdev; struct device *dev = &pdev->dev; - struct msix_entry *msix; struct idxd_irq_entry *irq_entry; int i, msixcnt; int rc = 0; @@ -69,23 +68,13 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) msixcnt = pci_msix_vec_count(pdev); if (msixcnt < 0) { dev_err(dev, "Not MSI-X interrupt capable.\n"); - goto err_no_irq; + return -ENOSPC; } - idxd->msix_entries = devm_kzalloc(dev, sizeof(struct msix_entry) * - msixcnt, GFP_KERNEL); - if (!idxd->msix_entries) { - rc = -ENOMEM; - goto err_no_irq; - } - - for (i = 0; i < msixcnt; i++) - idxd->msix_entries[i].entry = i; - - rc = pci_enable_msix_exact(pdev, idxd->msix_entries, msixcnt); - if (rc) { - dev_err(dev, "Failed enabling %d MSIX entries.\n", msixcnt); - goto err_no_irq; + rc = pci_alloc_irq_vectors(pdev, msixcnt, msixcnt, PCI_IRQ_MSIX); + if (rc != msixcnt) { + dev_err(dev, "Failed enabling %d MSIX entries: %d\n", msixcnt, rc); + return -ENOSPC; } dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt); @@ -98,58 +87,57 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) GFP_KERNEL); if (!idxd->irq_entries) { rc = -ENOMEM; - goto err_no_irq; + goto err_irq_entries; } for (i = 0; i < msixcnt; i++) { idxd->irq_entries[i].id = i; idxd->irq_entries[i].idxd = idxd; + idxd->irq_entries[i].vector = pci_irq_vector(pdev, i); spin_lock_init(&idxd->irq_entries[i].list_lock); } - msix = &idxd->msix_entries[0]; irq_entry = &idxd->irq_entries[0]; - rc = devm_request_threaded_irq(dev, msix->vector, idxd_irq_handler, - idxd_misc_thread, 0, "idxd-misc", - irq_entry); + rc = request_threaded_irq(irq_entry->vector, idxd_irq_handler, idxd_misc_thread, + 0, "idxd-misc", irq_entry); if (rc < 0) { dev_err(dev, "Failed to allocate misc interrupt.\n"); - goto err_no_irq; + goto err_misc_irq; } - dev_dbg(dev, "Allocated idxd-misc handler on msix vector %d\n", - msix->vector); + dev_dbg(dev, "Allocated idxd-misc handler on msix vector %d\n", irq_entry->vector); /* first MSI-X entry is not for wq interrupts */ idxd->num_wq_irqs = msixcnt - 1; for (i = 1; i < msixcnt; i++) { - msix = &idxd->msix_entries[i]; irq_entry = &idxd->irq_entries[i]; init_llist_head(&idxd->irq_entries[i].pending_llist); INIT_LIST_HEAD(&idxd->irq_entries[i].work_list); - rc = devm_request_threaded_irq(dev, msix->vector, - idxd_irq_handler, - idxd_wq_thread, 0, - "idxd-portal", irq_entry); + rc = request_threaded_irq(irq_entry->vector, idxd_irq_handler, + idxd_wq_thread, 0, "idxd-portal", irq_entry); if (rc < 0) { - dev_err(dev, "Failed to allocate irq %d.\n", - msix->vector); - goto err_no_irq; + dev_err(dev, "Failed to allocate irq %d.\n", irq_entry->vector); + goto err_wq_irqs; } - dev_dbg(dev, "Allocated idxd-msix %d for vector %d\n", - i, msix->vector); + dev_dbg(dev, "Allocated idxd-msix %d for vector %d\n", i, irq_entry->vector); } idxd_unmask_error_interrupts(idxd); idxd_msix_perm_setup(idxd); return 0; - err_no_irq: + err_wq_irqs: + while (--i >= 0) { + irq_entry = &idxd->irq_entries[i]; + free_irq(irq_entry->vector, irq_entry); + } + err_misc_irq: /* Disable error interrupt generation */ idxd_mask_error_interrupts(idxd); - pci_disable_msix(pdev); + err_irq_entries: + pci_free_irq_vectors(pdev); dev_err(dev, "No usable interrupts\n"); return rc; } @@ -495,7 +483,8 @@ static void idxd_shutdown(struct pci_dev *pdev) for (i = 0; i < msixcnt; i++) { irq_entry = &idxd->irq_entries[i]; - synchronize_irq(idxd->msix_entries[i].vector); + synchronize_irq(irq_entry->vector); + free_irq(irq_entry->vector, irq_entry); if (i == 0) continue; idxd_flush_pending_llist(irq_entry); @@ -503,6 +492,7 @@ static void idxd_shutdown(struct pci_dev *pdev) } idxd_msix_perm_clear(idxd); + pci_free_irq_vectors(pdev); destroy_workqueue(idxd->wq); } -- Gitee From c389bcc49236c1f346081b63b2970304c72e17d2 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Apr 2021 16:37:21 -0700 Subject: [PATCH 03/85] dmaengine: idxd: removal of pcim managed mmio mapping ANBZ: #22 commit a39c7cd0438ee2f0b859ee1eb86cdc52217d2223 upstream. The devm managed lifetime is incompatible with 'struct device' objects that resides in idxd context. This is one of the series that clean up the idxd driver 'struct device' lifetime. Remove pcim_* management of the PCI device and the ioremap of MMIO BAR and replace with unmanaged versions. This is for consistency of removing all the pcim/devm based calls. Intel-SIG: commit a39c7cd0438e dmaengine: idxd: removal of pcim managed mmio mapping. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reported-by: Jason Gunthorpe Fixes: bfe1d56091c1 ("dmaengine: idxd: Init and probe for Intel data accelerators") Signed-off-by: Dave Jiang Reviewed-by: Dan Williams Link: https://lore.kernel.org/r/161852984150.2203940.8043988289748519056.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/init.c | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index c776f0ece919..053529d428a3 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -384,32 +384,36 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) struct idxd_device *idxd; int rc; - rc = pcim_enable_device(pdev); + rc = pci_enable_device(pdev); if (rc) return rc; dev_dbg(dev, "Alloc IDXD context\n"); idxd = idxd_alloc(pdev); - if (!idxd) - return -ENOMEM; + if (!idxd) { + rc = -ENOMEM; + goto err_idxd_alloc; + } dev_dbg(dev, "Mapping BARs\n"); - idxd->reg_base = pcim_iomap(pdev, IDXD_MMIO_BAR, 0); - if (!idxd->reg_base) - return -ENOMEM; + idxd->reg_base = pci_iomap(pdev, IDXD_MMIO_BAR, 0); + if (!idxd->reg_base) { + rc = -ENOMEM; + goto err_iomap; + } dev_dbg(dev, "Set DMA masks\n"); rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); if (rc) rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); if (rc) - return rc; + goto err; rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); if (rc) rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); if (rc) - return rc; + goto err; idxd_set_type(idxd); @@ -423,13 +427,13 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) rc = idxd_probe(idxd); if (rc) { dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n"); - return -ENODEV; + goto err; } rc = idxd_setup_sysfs(idxd); if (rc) { dev_err(dev, "IDXD sysfs setup failed\n"); - return -ENODEV; + goto err; } idxd->state = IDXD_DEV_CONF_READY; @@ -438,6 +442,13 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) idxd->hw.version); return 0; + + err: + pci_iounmap(pdev, idxd->reg_base); + err_iomap: + err_idxd_alloc: + pci_disable_device(pdev); + return rc; } static void idxd_flush_pending_llist(struct idxd_irq_entry *ie) @@ -493,6 +504,8 @@ static void idxd_shutdown(struct pci_dev *pdev) idxd_msix_perm_clear(idxd); pci_free_irq_vectors(pdev); + pci_iounmap(pdev, idxd->reg_base); + pci_disable_device(pdev); destroy_workqueue(idxd->wq); } -- Gitee From c072e7a5dfe78c740a7694a752887c57bcc4aa13 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Apr 2021 16:37:27 -0700 Subject: [PATCH 04/85] dmaengine: idxd: use ida for device instance enumeration ANBZ: #22 commit f7f7739847bd68b3c3103fd1b50d943038bd14c7 upstream. The idr is only used for an device id, never to lookup context from that id. Switch to plain ida. Intel-SIG: commit f7f7739847bd dmaengine: idxd: use ida for device instance enumeration. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Fixes: bfe1d56091c1 ("dmaengine: idxd: Init and probe for Intel data accelerators") Reported-by: Jason Gunthorpe Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/161852984730.2203940.15032482460902003819.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/init.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 053529d428a3..904561169dfa 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -34,8 +34,7 @@ MODULE_PARM_DESC(sva, "Toggle SVA support on/off"); bool support_enqcmd; -static struct idr idxd_idrs[IDXD_TYPE_MAX]; -static DEFINE_MUTEX(idxd_idr_lock); +static struct ida idxd_idas[IDXD_TYPE_MAX]; static struct pci_device_id idxd_pci_tbl[] = { /* DSA ver 1.0 platforms */ @@ -348,12 +347,10 @@ static int idxd_probe(struct idxd_device *idxd) dev_dbg(dev, "IDXD interrupt setup complete.\n"); - mutex_lock(&idxd_idr_lock); - idxd->id = idr_alloc(&idxd_idrs[idxd->type], idxd, 0, 0, GFP_KERNEL); - mutex_unlock(&idxd_idr_lock); + idxd->id = ida_alloc(&idxd_idas[idxd->type], GFP_KERNEL); if (idxd->id < 0) { rc = -ENOMEM; - goto err_idr_fail; + goto err_ida_fail; } idxd->major = idxd_cdev_get_major(idxd); @@ -361,7 +358,7 @@ static int idxd_probe(struct idxd_device *idxd) dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id); return 0; - err_idr_fail: + err_ida_fail: idxd_mask_error_interrupts(idxd); idxd_mask_msix_vectors(idxd); err_setup: @@ -518,9 +515,7 @@ static void idxd_remove(struct pci_dev *pdev) idxd_shutdown(pdev); if (device_pasid_enabled(idxd)) idxd_disable_system_pasid(idxd); - mutex_lock(&idxd_idr_lock); - idr_remove(&idxd_idrs[idxd->type], idxd->id); - mutex_unlock(&idxd_idr_lock); + ida_free(&idxd_idas[idxd->type], idxd->id); } static struct pci_driver idxd_pci_driver = { @@ -550,7 +545,7 @@ static int __init idxd_init_module(void) support_enqcmd = true; for (i = 0; i < IDXD_TYPE_MAX; i++) - idr_init(&idxd_idrs[i]); + ida_init(&idxd_idas[i]); err = idxd_register_bus_type(); if (err < 0) -- Gitee From aea27b47b4fa8a6544fb88c9e9dfd522c05850b6 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Apr 2021 16:37:33 -0700 Subject: [PATCH 05/85] dmaengine: idxd: fix idxd conf_dev 'struct device' lifetime ANBZ: #22 commit 47c16ac27d4cb664cee53ee0b9b7e2f907923fb3 upstream. The devm managed lifetime is incompatible with 'struct device' objects that resides in idxd context. This is one of the series that clean up the idxd driver 'struct device' lifetime. Fix idxd->conf_dev 'struct device' lifetime. Address issues flagged by CONFIG_DEBUG_KOBJECT_RELEASE. Add release functions in order to free the allocated memory at the appropriate time. Intel-SIG: commit 47c16ac27d4c dmaengine: idxd: fix idxd conf_dev 'struct device' lifetime. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reported-by: Jason Gunthorpe Fixes: bfe1d56091c1 ("dmaengine: idxd: Init and probe for Intel data accelerators") Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/161852985319.2203940.4650791514462735368.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/idxd.h | 36 ++++++++++------ drivers/dma/idxd/init.c | 56 ++++++++++++++++-------- drivers/dma/idxd/sysfs.c | 92 ++++++++++++++-------------------------- 3 files changed, 94 insertions(+), 90 deletions(-) diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 786597812cd8..849b8a59aef2 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -8,6 +8,7 @@ #include #include #include +#include #include "registers.h" #define IDXD_DRIVER_VERSION "1.00" @@ -256,6 +257,23 @@ extern struct bus_type dsa_bus_type; extern struct bus_type iax_bus_type; extern bool support_enqcmd; +extern struct device_type dsa_device_type; +extern struct device_type iax_device_type; + +static inline bool is_dsa_dev(struct device *dev) +{ + return dev->type == &dsa_device_type; +} + +static inline bool is_iax_dev(struct device *dev) +{ + return dev->type == &iax_device_type; +} + +static inline bool is_idxd_dev(struct device *dev) +{ + return is_dsa_dev(dev) || is_iax_dev(dev); +} static inline bool wq_dedicated(struct idxd_wq *wq) { @@ -293,18 +311,6 @@ static inline int idxd_get_wq_portal_full_offset(int wq_id, return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot); } -static inline void idxd_set_type(struct idxd_device *idxd) -{ - struct pci_dev *pdev = idxd->pdev; - - if (pdev->device == PCI_DEVICE_ID_INTEL_DSA_SPR0) - idxd->type = IDXD_TYPE_DSA; - else if (pdev->device == PCI_DEVICE_ID_INTEL_IAX_SPR0) - idxd->type = IDXD_TYPE_IAX; - else - idxd->type = IDXD_TYPE_UNKNOWN; -} - static inline void idxd_wq_get(struct idxd_wq *wq) { wq->client_count++; @@ -320,14 +326,16 @@ static inline int idxd_wq_refcount(struct idxd_wq *wq) return wq->client_count; }; +struct ida *idxd_ida(struct idxd_device *idxd); const char *idxd_get_dev_name(struct idxd_device *idxd); int idxd_register_bus_type(void); void idxd_unregister_bus_type(void); -int idxd_setup_sysfs(struct idxd_device *idxd); -void idxd_cleanup_sysfs(struct idxd_device *idxd); +int idxd_register_devices(struct idxd_device *idxd); +void idxd_unregister_devices(struct idxd_device *idxd); int idxd_register_driver(void); void idxd_unregister_driver(void); struct bus_type *idxd_get_bus_type(struct idxd_device *idxd); +struct device_type *idxd_get_device_type(struct idxd_device *idxd); /* device interrupt control */ void idxd_msix_perm_setup(struct idxd_device *idxd); diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 904561169dfa..fbdf3f22573c 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -51,6 +51,11 @@ static char *idxd_name[] = { "iax" }; +struct ida *idxd_ida(struct idxd_device *idxd) +{ + return &idxd_idas[idxd->type]; +} + const char *idxd_get_dev_name(struct idxd_device *idxd) { return idxd_name[idxd->type]; @@ -81,9 +86,8 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) * We implement 1 completion list per MSI-X entry except for * entry 0, which is for errors and others. */ - idxd->irq_entries = devm_kcalloc(dev, msixcnt, - sizeof(struct idxd_irq_entry), - GFP_KERNEL); + idxd->irq_entries = kcalloc_node(msixcnt, sizeof(struct idxd_irq_entry), + GFP_KERNEL, dev_to_node(dev)); if (!idxd->irq_entries) { rc = -ENOMEM; goto err_irq_entries; @@ -262,16 +266,44 @@ static void idxd_read_caps(struct idxd_device *idxd) } } +static inline void idxd_set_type(struct idxd_device *idxd) +{ + struct pci_dev *pdev = idxd->pdev; + + if (pdev->device == PCI_DEVICE_ID_INTEL_DSA_SPR0) + idxd->type = IDXD_TYPE_DSA; + else if (pdev->device == PCI_DEVICE_ID_INTEL_IAX_SPR0) + idxd->type = IDXD_TYPE_IAX; + else + idxd->type = IDXD_TYPE_UNKNOWN; +} + static struct idxd_device *idxd_alloc(struct pci_dev *pdev) { struct device *dev = &pdev->dev; struct idxd_device *idxd; + int rc; - idxd = devm_kzalloc(dev, sizeof(struct idxd_device), GFP_KERNEL); + idxd = kzalloc_node(sizeof(*idxd), GFP_KERNEL, dev_to_node(dev)); if (!idxd) return NULL; idxd->pdev = pdev; + idxd_set_type(idxd); + idxd->id = ida_alloc(idxd_ida(idxd), GFP_KERNEL); + if (idxd->id < 0) + return NULL; + + device_initialize(&idxd->conf_dev); + idxd->conf_dev.parent = dev; + idxd->conf_dev.bus = idxd_get_bus_type(idxd); + idxd->conf_dev.type = idxd_get_device_type(idxd); + rc = dev_set_name(&idxd->conf_dev, "%s%d", idxd_get_dev_name(idxd), idxd->id); + if (rc < 0) { + put_device(&idxd->conf_dev); + return NULL; + } + spin_lock_init(&idxd->dev_lock); return idxd; @@ -347,20 +379,11 @@ static int idxd_probe(struct idxd_device *idxd) dev_dbg(dev, "IDXD interrupt setup complete.\n"); - idxd->id = ida_alloc(&idxd_idas[idxd->type], GFP_KERNEL); - if (idxd->id < 0) { - rc = -ENOMEM; - goto err_ida_fail; - } - idxd->major = idxd_cdev_get_major(idxd); dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id); return 0; - err_ida_fail: - idxd_mask_error_interrupts(idxd); - idxd_mask_msix_vectors(idxd); err_setup: if (device_pasid_enabled(idxd)) idxd_disable_system_pasid(idxd); @@ -412,7 +435,6 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (rc) goto err; - idxd_set_type(idxd); idxd_type_init(idxd); @@ -427,7 +449,7 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err; } - rc = idxd_setup_sysfs(idxd); + rc = idxd_register_devices(idxd); if (rc) { dev_err(dev, "IDXD sysfs setup failed\n"); goto err; @@ -443,6 +465,7 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) err: pci_iounmap(pdev, idxd->reg_base); err_iomap: + put_device(&idxd->conf_dev); err_idxd_alloc: pci_disable_device(pdev); return rc; @@ -511,11 +534,10 @@ static void idxd_remove(struct pci_dev *pdev) struct idxd_device *idxd = pci_get_drvdata(pdev); dev_dbg(&pdev->dev, "%s called\n", __func__); - idxd_cleanup_sysfs(idxd); idxd_shutdown(pdev); if (device_pasid_enabled(idxd)) idxd_disable_system_pasid(idxd); - ida_free(&idxd_idas[idxd->type], idxd->id); + idxd_unregister_devices(idxd); } static struct pci_driver idxd_pci_driver = { diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 7a08f0d63b43..ed2429a59bd7 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -16,51 +16,26 @@ static char *idxd_wq_type_names[] = { [IDXD_WQT_USER] = "user", }; -static void idxd_conf_device_release(struct device *dev) +static void idxd_conf_sub_device_release(struct device *dev) { dev_dbg(dev, "%s for %s\n", __func__, dev_name(dev)); } static struct device_type idxd_group_device_type = { .name = "group", - .release = idxd_conf_device_release, + .release = idxd_conf_sub_device_release, }; static struct device_type idxd_wq_device_type = { .name = "wq", - .release = idxd_conf_device_release, + .release = idxd_conf_sub_device_release, }; static struct device_type idxd_engine_device_type = { .name = "engine", - .release = idxd_conf_device_release, + .release = idxd_conf_sub_device_release, }; -static struct device_type dsa_device_type = { - .name = "dsa", - .release = idxd_conf_device_release, -}; - -static struct device_type iax_device_type = { - .name = "iax", - .release = idxd_conf_device_release, -}; - -static inline bool is_dsa_dev(struct device *dev) -{ - return dev ? dev->type == &dsa_device_type : false; -} - -static inline bool is_iax_dev(struct device *dev) -{ - return dev ? dev->type == &iax_device_type : false; -} - -static inline bool is_idxd_dev(struct device *dev) -{ - return is_dsa_dev(dev) || is_iax_dev(dev); -} - static inline bool is_idxd_wq_dev(struct device *dev) { return dev ? dev->type == &idxd_wq_device_type : false; @@ -405,7 +380,7 @@ struct bus_type *idxd_get_bus_type(struct idxd_device *idxd) return idxd_bus_types[idxd->type]; } -static struct device_type *idxd_get_device_type(struct idxd_device *idxd) +struct device_type *idxd_get_device_type(struct idxd_device *idxd) { if (idxd->type == IDXD_TYPE_DSA) return &dsa_device_type; @@ -1652,6 +1627,30 @@ static const struct attribute_group *idxd_attribute_groups[] = { NULL, }; +static void idxd_conf_device_release(struct device *dev) +{ + struct idxd_device *idxd = container_of(dev, struct idxd_device, conf_dev); + + kfree(idxd->groups); + kfree(idxd->wqs); + kfree(idxd->engines); + kfree(idxd->irq_entries); + ida_free(idxd_ida(idxd), idxd->id); + kfree(idxd); +} + +struct device_type dsa_device_type = { + .name = "dsa", + .release = idxd_conf_device_release, + .groups = idxd_attribute_groups, +}; + +struct device_type iax_device_type = { + .name = "iax", + .release = idxd_conf_device_release, + .groups = idxd_attribute_groups, +}; + static int idxd_setup_engine_sysfs(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; @@ -1753,39 +1752,14 @@ static int idxd_setup_wq_sysfs(struct idxd_device *idxd) return rc; } -static int idxd_setup_device_sysfs(struct idxd_device *idxd) +int idxd_register_devices(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; int rc; - char devname[IDXD_NAME_SIZE]; - sprintf(devname, "%s%d", idxd_get_dev_name(idxd), idxd->id); - idxd->conf_dev.parent = dev; - dev_set_name(&idxd->conf_dev, "%s", devname); - idxd->conf_dev.bus = idxd_get_bus_type(idxd); - idxd->conf_dev.groups = idxd_attribute_groups; - idxd->conf_dev.type = idxd_get_device_type(idxd); - - dev_dbg(dev, "IDXD device register: %s\n", dev_name(&idxd->conf_dev)); - rc = device_register(&idxd->conf_dev); - if (rc < 0) { - put_device(&idxd->conf_dev); - return rc; - } - - return 0; -} - -int idxd_setup_sysfs(struct idxd_device *idxd) -{ - struct device *dev = &idxd->pdev->dev; - int rc; - - rc = idxd_setup_device_sysfs(idxd); - if (rc < 0) { - dev_dbg(dev, "Device sysfs registering failed: %d\n", rc); + rc = device_add(&idxd->conf_dev); + if (rc < 0) return rc; - } rc = idxd_setup_wq_sysfs(idxd); if (rc < 0) { @@ -1811,7 +1785,7 @@ int idxd_setup_sysfs(struct idxd_device *idxd) return 0; } -void idxd_cleanup_sysfs(struct idxd_device *idxd) +void idxd_unregister_devices(struct idxd_device *idxd) { int i; -- Gitee From d6cf21afedf547902801ea11f58ad2a9b4d10649 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Apr 2021 16:37:39 -0700 Subject: [PATCH 06/85] dmaengine: idxd: fix wq conf_dev 'struct device' lifetime ANBZ: #22 commit 7c5dd23e57c14cf7177b8a5e0fd08916e0c60005 upstream. Remove devm_* allocation and fix wq->conf_dev 'struct device' lifetime. Address issues flagged by CONFIG_DEBUG_KOBJECT_RELEASE. Add release functions in order to free the allocated memory for the wq context at device destruction time. Intel-SIG: commit 7c5dd23e57c1 dmaengine: idxd: fix wq conf_dev 'struct device' lifetime. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reported-by: Jason Gunthorpe Fixes: bfe1d56091c1 ("dmaengine: idxd: Init and probe for Intel data accelerators") Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/161852985907.2203940.6840120734115043753.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 6 +-- drivers/dma/idxd/idxd.h | 20 +++++++- drivers/dma/idxd/init.c | 105 ++++++++++++++++++++++++++++---------- drivers/dma/idxd/irq.c | 6 +-- drivers/dma/idxd/sysfs.c | 100 ++++++++++++++++-------------------- 5 files changed, 146 insertions(+), 91 deletions(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 3f696abd74ac..c4183294a704 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -520,7 +520,7 @@ void idxd_device_wqs_clear_state(struct idxd_device *idxd) lockdep_assert_held(&idxd->dev_lock); for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = &idxd->wqs[i]; + struct idxd_wq *wq = idxd->wqs[i]; if (wq->state == IDXD_WQ_ENABLED) { idxd_wq_disable_cleanup(wq); @@ -738,7 +738,7 @@ static int idxd_wqs_config_write(struct idxd_device *idxd) int i, rc; for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = &idxd->wqs[i]; + struct idxd_wq *wq = idxd->wqs[i]; rc = idxd_wq_config_write(wq); if (rc < 0) @@ -816,7 +816,7 @@ static int idxd_wqs_setup(struct idxd_device *idxd) } for (i = 0; i < idxd->max_wqs; i++) { - wq = &idxd->wqs[i]; + wq = idxd->wqs[i]; group = wq->group; if (!wq->group) diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 849b8a59aef2..7b5bbde1db9f 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -195,7 +195,7 @@ struct idxd_device { spinlock_t dev_lock; /* spinlock for device */ struct completion *cmd_done; struct idxd_group *groups; - struct idxd_wq *wqs; + struct idxd_wq **wqs; struct idxd_engine *engines; struct iommu_sva *sva; @@ -259,6 +259,7 @@ extern struct bus_type iax_bus_type; extern bool support_enqcmd; extern struct device_type dsa_device_type; extern struct device_type iax_device_type; +extern struct device_type idxd_wq_device_type; static inline bool is_dsa_dev(struct device *dev) { @@ -275,6 +276,23 @@ static inline bool is_idxd_dev(struct device *dev) return is_dsa_dev(dev) || is_iax_dev(dev); } +static inline bool is_idxd_wq_dev(struct device *dev) +{ + return dev->type == &idxd_wq_device_type; +} + +static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq) +{ + if (wq->type == IDXD_WQT_KERNEL && strcmp(wq->name, "dmaengine") == 0) + return true; + return false; +} + +static inline bool is_idxd_wq_cdev(struct idxd_wq *wq) +{ + return wq->type == IDXD_WQT_USER; +} + static inline bool wq_dedicated(struct idxd_wq *wq) { return test_bit(WQ_FLAG_DEDICATED, &wq->flags); diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index fbdf3f22573c..996cc50079f0 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -145,16 +145,74 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) return rc; } +static int idxd_setup_wqs(struct idxd_device *idxd) +{ + struct device *dev = &idxd->pdev->dev; + struct idxd_wq *wq; + int i, rc; + + idxd->wqs = kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *), + GFP_KERNEL, dev_to_node(dev)); + if (!idxd->wqs) + return -ENOMEM; + + for (i = 0; i < idxd->max_wqs; i++) { + wq = kzalloc_node(sizeof(*wq), GFP_KERNEL, dev_to_node(dev)); + if (!wq) { + rc = -ENOMEM; + goto err; + } + + wq->id = i; + wq->idxd = idxd; + device_initialize(&wq->conf_dev); + wq->conf_dev.parent = &idxd->conf_dev; + wq->conf_dev.bus = idxd_get_bus_type(idxd); + wq->conf_dev.type = &idxd_wq_device_type; + rc = dev_set_name(&wq->conf_dev, "wq%d.%d", idxd->id, wq->id); + if (rc < 0) { + put_device(&wq->conf_dev); + goto err; + } + + mutex_init(&wq->wq_lock); + init_waitqueue_head(&wq->err_queue); + wq->max_xfer_bytes = idxd->max_xfer_bytes; + wq->max_batch_size = idxd->max_batch_size; + wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev)); + if (!wq->wqcfg) { + put_device(&wq->conf_dev); + rc = -ENOMEM; + goto err; + } + idxd->wqs[i] = wq; + } + + return 0; + + err: + while (--i >= 0) + put_device(&idxd->wqs[i]->conf_dev); + return rc; +} + static int idxd_setup_internals(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; - int i; + int i, rc; init_waitqueue_head(&idxd->cmd_waitq); + + rc = idxd_setup_wqs(idxd); + if (rc < 0) + return rc; + idxd->groups = devm_kcalloc(dev, idxd->max_groups, sizeof(struct idxd_group), GFP_KERNEL); - if (!idxd->groups) - return -ENOMEM; + if (!idxd->groups) { + rc = -ENOMEM; + goto err; + } for (i = 0; i < idxd->max_groups; i++) { idxd->groups[i].idxd = idxd; @@ -163,40 +221,31 @@ static int idxd_setup_internals(struct idxd_device *idxd) idxd->groups[i].tc_b = -1; } - idxd->wqs = devm_kcalloc(dev, idxd->max_wqs, sizeof(struct idxd_wq), - GFP_KERNEL); - if (!idxd->wqs) - return -ENOMEM; - idxd->engines = devm_kcalloc(dev, idxd->max_engines, sizeof(struct idxd_engine), GFP_KERNEL); - if (!idxd->engines) - return -ENOMEM; - - for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = &idxd->wqs[i]; - - wq->id = i; - wq->idxd = idxd; - mutex_init(&wq->wq_lock); - init_waitqueue_head(&wq->err_queue); - wq->max_xfer_bytes = idxd->max_xfer_bytes; - wq->max_batch_size = idxd->max_batch_size; - wq->wqcfg = devm_kzalloc(dev, idxd->wqcfg_size, GFP_KERNEL); - if (!wq->wqcfg) - return -ENOMEM; + if (!idxd->engines) { + rc = -ENOMEM; + goto err; } + for (i = 0; i < idxd->max_engines; i++) { idxd->engines[i].idxd = idxd; idxd->engines[i].id = i; } idxd->wq = create_workqueue(dev_name(dev)); - if (!idxd->wq) - return -ENOMEM; + if (!idxd->wq) { + rc = -ENOMEM; + goto err; + } return 0; + + err: + for (i = 0; i < idxd->max_wqs; i++) + put_device(&idxd->wqs[i]->conf_dev); + return rc; } static void idxd_read_table_offsets(struct idxd_device *idxd) @@ -371,11 +420,11 @@ static int idxd_probe(struct idxd_device *idxd) rc = idxd_setup_internals(idxd); if (rc) - goto err_setup; + goto err; rc = idxd_setup_interrupts(idxd); if (rc) - goto err_setup; + goto err; dev_dbg(dev, "IDXD interrupt setup complete.\n"); @@ -384,7 +433,7 @@ static int idxd_probe(struct idxd_device *idxd) dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id); return 0; - err_setup: + err: if (device_pasid_enabled(idxd)) idxd_disable_system_pasid(idxd); return rc; diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c index f287233fff65..fc0781e3f36d 100644 --- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -45,7 +45,7 @@ static void idxd_device_reinit(struct work_struct *work) goto out; for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = &idxd->wqs[i]; + struct idxd_wq *wq = idxd->wqs[i]; if (wq->state == IDXD_WQ_ENABLED) { rc = idxd_wq_enable(wq); @@ -130,7 +130,7 @@ static int process_misc_interrupts(struct idxd_device *idxd, u32 cause) if (idxd->sw_err.valid && idxd->sw_err.wq_idx_valid) { int id = idxd->sw_err.wq_idx; - struct idxd_wq *wq = &idxd->wqs[id]; + struct idxd_wq *wq = idxd->wqs[id]; if (wq->type == IDXD_WQT_USER) wake_up_interruptible(&wq->err_queue); @@ -138,7 +138,7 @@ static int process_misc_interrupts(struct idxd_device *idxd, u32 cause) int i; for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = &idxd->wqs[i]; + struct idxd_wq *wq = idxd->wqs[i]; if (wq->type == IDXD_WQT_USER) wake_up_interruptible(&wq->err_queue); diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index ed2429a59bd7..ba142b4d3cef 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -26,34 +26,11 @@ static struct device_type idxd_group_device_type = { .release = idxd_conf_sub_device_release, }; -static struct device_type idxd_wq_device_type = { - .name = "wq", - .release = idxd_conf_sub_device_release, -}; - static struct device_type idxd_engine_device_type = { .name = "engine", .release = idxd_conf_sub_device_release, }; -static inline bool is_idxd_wq_dev(struct device *dev) -{ - return dev ? dev->type == &idxd_wq_device_type : false; -} - -static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq) -{ - if (wq->type == IDXD_WQT_KERNEL && - strcmp(wq->name, "dmaengine") == 0) - return true; - return false; -} - -static inline bool is_idxd_wq_cdev(struct idxd_wq *wq) -{ - return wq->type == IDXD_WQT_USER; -} - static int idxd_config_bus_match(struct device *dev, struct device_driver *drv) { @@ -297,7 +274,7 @@ static int idxd_config_bus_remove(struct device *dev) dev_dbg(dev, "%s removing dev %s\n", __func__, dev_name(&idxd->conf_dev)); for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = &idxd->wqs[i]; + struct idxd_wq *wq = idxd->wqs[i]; if (wq->state == IDXD_WQ_DISABLED) continue; @@ -309,7 +286,7 @@ static int idxd_config_bus_remove(struct device *dev) idxd_unregister_dma_device(idxd); rc = idxd_device_disable(idxd); for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = &idxd->wqs[i]; + struct idxd_wq *wq = idxd->wqs[i]; mutex_lock(&wq->wq_lock); idxd_wq_disable_cleanup(wq); @@ -678,7 +655,7 @@ static ssize_t group_work_queues_show(struct device *dev, struct idxd_device *idxd = group->idxd; for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = &idxd->wqs[i]; + struct idxd_wq *wq = idxd->wqs[i]; if (!wq->group) continue; @@ -935,7 +912,7 @@ static int total_claimed_wq_size(struct idxd_device *idxd) int wq_size = 0; for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = &idxd->wqs[i]; + struct idxd_wq *wq = idxd->wqs[i]; wq_size += wq->size; } @@ -1339,6 +1316,20 @@ static const struct attribute_group *idxd_wq_attribute_groups[] = { NULL, }; +static void idxd_conf_wq_release(struct device *dev) +{ + struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + + kfree(wq->wqcfg); + kfree(wq); +} + +struct device_type idxd_wq_device_type = { + .name = "wq", + .release = idxd_conf_wq_release, + .groups = idxd_wq_attribute_groups, +}; + /* IDXD device attribs */ static ssize_t version_show(struct device *dev, struct device_attribute *attr, char *buf) @@ -1469,7 +1460,7 @@ static ssize_t clients_show(struct device *dev, spin_lock_irqsave(&idxd->dev_lock, flags); for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = &idxd->wqs[i]; + struct idxd_wq *wq = idxd->wqs[i]; count += wq->client_count; } @@ -1719,70 +1710,67 @@ static int idxd_setup_group_sysfs(struct idxd_device *idxd) return rc; } -static int idxd_setup_wq_sysfs(struct idxd_device *idxd) +static int idxd_register_wq_devices(struct idxd_device *idxd) { - struct device *dev = &idxd->pdev->dev; - int i, rc; + int i, rc, j; for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = &idxd->wqs[i]; - - wq->conf_dev.parent = &idxd->conf_dev; - dev_set_name(&wq->conf_dev, "wq%d.%d", idxd->id, wq->id); - wq->conf_dev.bus = idxd_get_bus_type(idxd); - wq->conf_dev.groups = idxd_wq_attribute_groups; - wq->conf_dev.type = &idxd_wq_device_type; - dev_dbg(dev, "WQ device register: %s\n", - dev_name(&wq->conf_dev)); - rc = device_register(&wq->conf_dev); - if (rc < 0) { - put_device(&wq->conf_dev); + struct idxd_wq *wq = idxd->wqs[i]; + + rc = device_add(&wq->conf_dev); + if (rc < 0) goto cleanup; - } } return 0; cleanup: - while (i--) { - struct idxd_wq *wq = &idxd->wqs[i]; + j = i - 1; + for (; i < idxd->max_wqs; i++) + put_device(&idxd->wqs[i]->conf_dev); - device_unregister(&wq->conf_dev); - } + while (j--) + device_unregister(&idxd->wqs[j]->conf_dev); return rc; } int idxd_register_devices(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; - int rc; + int rc, i; rc = device_add(&idxd->conf_dev); if (rc < 0) return rc; - rc = idxd_setup_wq_sysfs(idxd); + rc = idxd_register_wq_devices(idxd); if (rc < 0) { - /* unregister conf dev */ - dev_dbg(dev, "Work Queue sysfs registering failed: %d\n", rc); - return rc; + dev_dbg(dev, "WQ devices registering failed: %d\n", rc); + goto err_wq; } rc = idxd_setup_group_sysfs(idxd); if (rc < 0) { /* unregister conf dev */ dev_dbg(dev, "Group sysfs registering failed: %d\n", rc); - return rc; + goto err; } rc = idxd_setup_engine_sysfs(idxd); if (rc < 0) { /* unregister conf dev */ dev_dbg(dev, "Engine sysfs registering failed: %d\n", rc); - return rc; + goto err; } return 0; + + err: + for (i = 0; i < idxd->max_wqs; i++) + device_unregister(&idxd->wqs[i]->conf_dev); + err_wq: + device_del(&idxd->conf_dev); + return rc; } void idxd_unregister_devices(struct idxd_device *idxd) @@ -1790,7 +1778,7 @@ void idxd_unregister_devices(struct idxd_device *idxd) int i; for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = &idxd->wqs[i]; + struct idxd_wq *wq = idxd->wqs[i]; device_unregister(&wq->conf_dev); } -- Gitee From f07b4027fa250c1e3d58a182a1b7f6d20b78fd0a Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Apr 2021 16:37:44 -0700 Subject: [PATCH 07/85] dmaengine: idxd: fix engine conf_dev lifetime ANBZ: #22 commit 75b911309060f42ba94bbbf46f5f497d35d5cd02 upstream. Remove devm_* allocation and fix engine->conf_dev 'struct device' lifetime. Address issues flagged by CONFIG_DEBUG_KOBJECT_RELEASE. Add release functions in order to free the allocated memory at the engine conf_dev destruction time. Intel-SIG: commit 75b911309060 dmaengine: idxd: fix engine conf_dev lifetime. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reported-by: Jason Gunthorpe Fixes: bfe1d56091c1 ("dmaengine: idxd: Init and probe for Intel data accelerators") Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/161852986460.2203940.16603218225412118431.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 2 +- drivers/dma/idxd/idxd.h | 3 +- drivers/dma/idxd/init.c | 60 +++++++++++++++++++++++++------- drivers/dma/idxd/sysfs.c | 72 +++++++++++++++++++-------------------- 4 files changed, 86 insertions(+), 51 deletions(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index c4183294a704..be1dcddfe3c4 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -786,7 +786,7 @@ static int idxd_engines_setup(struct idxd_device *idxd) } for (i = 0; i < idxd->max_engines; i++) { - eng = &idxd->engines[i]; + eng = idxd->engines[i]; group = eng->group; if (!group) diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 7b5bbde1db9f..67ebcaa8aafe 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -196,7 +196,7 @@ struct idxd_device { struct completion *cmd_done; struct idxd_group *groups; struct idxd_wq **wqs; - struct idxd_engine *engines; + struct idxd_engine **engines; struct iommu_sva *sva; unsigned int pasid; @@ -260,6 +260,7 @@ extern bool support_enqcmd; extern struct device_type dsa_device_type; extern struct device_type iax_device_type; extern struct device_type idxd_wq_device_type; +extern struct device_type idxd_engine_device_type; static inline bool is_dsa_dev(struct device *dev) { diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 996cc50079f0..0d8fd01d5a48 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -196,6 +196,46 @@ static int idxd_setup_wqs(struct idxd_device *idxd) return rc; } +static int idxd_setup_engines(struct idxd_device *idxd) +{ + struct idxd_engine *engine; + struct device *dev = &idxd->pdev->dev; + int i, rc; + + idxd->engines = kcalloc_node(idxd->max_engines, sizeof(struct idxd_engine *), + GFP_KERNEL, dev_to_node(dev)); + if (!idxd->engines) + return -ENOMEM; + + for (i = 0; i < idxd->max_engines; i++) { + engine = kzalloc_node(sizeof(*engine), GFP_KERNEL, dev_to_node(dev)); + if (!engine) { + rc = -ENOMEM; + goto err; + } + + engine->id = i; + engine->idxd = idxd; + device_initialize(&engine->conf_dev); + engine->conf_dev.parent = &idxd->conf_dev; + engine->conf_dev.type = &idxd_engine_device_type; + rc = dev_set_name(&engine->conf_dev, "engine%d.%d", idxd->id, engine->id); + if (rc < 0) { + put_device(&engine->conf_dev); + goto err; + } + + idxd->engines[i] = engine; + } + + return 0; + + err: + while (--i >= 0) + put_device(&idxd->engines[i]->conf_dev); + return rc; +} + static int idxd_setup_internals(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; @@ -207,6 +247,10 @@ static int idxd_setup_internals(struct idxd_device *idxd) if (rc < 0) return rc; + rc = idxd_setup_engines(idxd); + if (rc < 0) + goto err_engine; + idxd->groups = devm_kcalloc(dev, idxd->max_groups, sizeof(struct idxd_group), GFP_KERNEL); if (!idxd->groups) { @@ -221,19 +265,6 @@ static int idxd_setup_internals(struct idxd_device *idxd) idxd->groups[i].tc_b = -1; } - idxd->engines = devm_kcalloc(dev, idxd->max_engines, - sizeof(struct idxd_engine), GFP_KERNEL); - if (!idxd->engines) { - rc = -ENOMEM; - goto err; - } - - - for (i = 0; i < idxd->max_engines; i++) { - idxd->engines[i].idxd = idxd; - idxd->engines[i].id = i; - } - idxd->wq = create_workqueue(dev_name(dev)); if (!idxd->wq) { rc = -ENOMEM; @@ -243,6 +274,9 @@ static int idxd_setup_internals(struct idxd_device *idxd) return 0; err: + for (i = 0; i < idxd->max_engines; i++) + put_device(&idxd->engines[i]->conf_dev); + err_engine: for (i = 0; i < idxd->max_wqs; i++) put_device(&idxd->wqs[i]->conf_dev); return rc; diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index ba142b4d3cef..531bc88b92eb 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -26,11 +26,6 @@ static struct device_type idxd_group_device_type = { .release = idxd_conf_sub_device_release, }; -static struct device_type idxd_engine_device_type = { - .name = "engine", - .release = idxd_conf_sub_device_release, -}; - static int idxd_config_bus_match(struct device *dev, struct device_driver *drv) { @@ -464,6 +459,19 @@ static const struct attribute_group *idxd_engine_attribute_groups[] = { NULL, }; +static void idxd_conf_engine_release(struct device *dev) +{ + struct idxd_engine *engine = container_of(dev, struct idxd_engine, conf_dev); + + kfree(engine); +} + +struct device_type idxd_engine_device_type = { + .name = "engine", + .release = idxd_conf_engine_release, + .groups = idxd_engine_attribute_groups, +}; + /* Group attributes */ static void idxd_set_free_tokens(struct idxd_device *idxd) @@ -626,7 +634,7 @@ static ssize_t group_engines_show(struct device *dev, struct idxd_device *idxd = group->idxd; for (i = 0; i < idxd->max_engines; i++) { - struct idxd_engine *engine = &idxd->engines[i]; + struct idxd_engine *engine = idxd->engines[i]; if (!engine->group) continue; @@ -1642,37 +1650,27 @@ struct device_type iax_device_type = { .groups = idxd_attribute_groups, }; -static int idxd_setup_engine_sysfs(struct idxd_device *idxd) +static int idxd_register_engine_devices(struct idxd_device *idxd) { - struct device *dev = &idxd->pdev->dev; - int i, rc; + int i, j, rc; for (i = 0; i < idxd->max_engines; i++) { - struct idxd_engine *engine = &idxd->engines[i]; - - engine->conf_dev.parent = &idxd->conf_dev; - dev_set_name(&engine->conf_dev, "engine%d.%d", - idxd->id, engine->id); - engine->conf_dev.bus = idxd_get_bus_type(idxd); - engine->conf_dev.groups = idxd_engine_attribute_groups; - engine->conf_dev.type = &idxd_engine_device_type; - dev_dbg(dev, "Engine device register: %s\n", - dev_name(&engine->conf_dev)); - rc = device_register(&engine->conf_dev); - if (rc < 0) { - put_device(&engine->conf_dev); + struct idxd_engine *engine = idxd->engines[i]; + + rc = device_add(&engine->conf_dev); + if (rc < 0) goto cleanup; - } } return 0; cleanup: - while (i--) { - struct idxd_engine *engine = &idxd->engines[i]; + j = i - 1; + for (; i < idxd->max_engines; i++) + put_device(&idxd->engines[i]->conf_dev); - device_unregister(&engine->conf_dev); - } + while (j--) + device_unregister(&idxd->engines[j]->conf_dev); return rc; } @@ -1749,23 +1747,25 @@ int idxd_register_devices(struct idxd_device *idxd) goto err_wq; } - rc = idxd_setup_group_sysfs(idxd); + rc = idxd_register_engine_devices(idxd); if (rc < 0) { - /* unregister conf dev */ - dev_dbg(dev, "Group sysfs registering failed: %d\n", rc); - goto err; + dev_dbg(dev, "Engine devices registering failed: %d\n", rc); + goto err_engine; } - rc = idxd_setup_engine_sysfs(idxd); + rc = idxd_setup_group_sysfs(idxd); if (rc < 0) { /* unregister conf dev */ - dev_dbg(dev, "Engine sysfs registering failed: %d\n", rc); - goto err; + dev_dbg(dev, "Group sysfs registering failed: %d\n", rc); + goto err_group; } return 0; - err: + err_group: + for (i = 0; i < idxd->max_engines; i++) + device_unregister(&idxd->engines[i]->conf_dev); + err_engine: for (i = 0; i < idxd->max_wqs; i++) device_unregister(&idxd->wqs[i]->conf_dev); err_wq: @@ -1784,7 +1784,7 @@ void idxd_unregister_devices(struct idxd_device *idxd) } for (i = 0; i < idxd->max_engines; i++) { - struct idxd_engine *engine = &idxd->engines[i]; + struct idxd_engine *engine = idxd->engines[i]; device_unregister(&engine->conf_dev); } -- Gitee From aef91b801723232fbba6af4fa242674b6896ca93 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Apr 2021 16:37:51 -0700 Subject: [PATCH 08/85] dmaengine: idxd: fix group conf_dev lifetime ANBZ: #22 commit defe49f96012ca91e8e673cb95b5c30b4a3735e8 upstream. Remove devm_* allocation and fix group->conf_dev 'struct device' lifetime. Address issues flagged by CONFIG_DEBUG_KOBJECT_RELEASE. Add release functions in order to free the allocated memory at the group->conf_dev destruction time. Intel-SIG: commit defe49f96012 dmaengine: idxd: fix group conf_dev lifetime. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reported-by: Jason Gunthorpe Fixes: bfe1d56091c1 ("dmaengine: idxd: Init and probe for Intel data accelerators") Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/161852987144.2203940.8830315575880047.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 8 ++--- drivers/dma/idxd/idxd.h | 3 +- drivers/dma/idxd/init.c | 68 ++++++++++++++++++++++++++++++--------- drivers/dma/idxd/sysfs.c | 68 +++++++++++++++++---------------------- 4 files changed, 88 insertions(+), 59 deletions(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index be1dcddfe3c4..4fef57717049 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -659,7 +659,7 @@ static int idxd_groups_config_write(struct idxd_device *idxd) ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET)); for (i = 0; i < idxd->max_groups; i++) { - struct idxd_group *group = &idxd->groups[i]; + struct idxd_group *group = idxd->groups[i]; idxd_group_config_write(group); } @@ -754,7 +754,7 @@ static void idxd_group_flags_setup(struct idxd_device *idxd) /* TC-A 0 and TC-B 1 should be defaults */ for (i = 0; i < idxd->max_groups; i++) { - struct idxd_group *group = &idxd->groups[i]; + struct idxd_group *group = idxd->groups[i]; if (group->tc_a == -1) group->tc_a = group->grpcfg.flags.tc_a = 0; @@ -781,7 +781,7 @@ static int idxd_engines_setup(struct idxd_device *idxd) struct idxd_group *group; for (i = 0; i < idxd->max_groups; i++) { - group = &idxd->groups[i]; + group = idxd->groups[i]; group->grpcfg.engines = 0; } @@ -810,7 +810,7 @@ static int idxd_wqs_setup(struct idxd_device *idxd) struct device *dev = &idxd->pdev->dev; for (i = 0; i < idxd->max_groups; i++) { - group = &idxd->groups[i]; + group = idxd->groups[i]; for (j = 0; j < 4; j++) group->grpcfg.wqs[j] = 0; } diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 67ebcaa8aafe..89daf746d121 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -194,7 +194,7 @@ struct idxd_device { spinlock_t dev_lock; /* spinlock for device */ struct completion *cmd_done; - struct idxd_group *groups; + struct idxd_group **groups; struct idxd_wq **wqs; struct idxd_engine **engines; @@ -261,6 +261,7 @@ extern struct device_type dsa_device_type; extern struct device_type iax_device_type; extern struct device_type idxd_wq_device_type; extern struct device_type idxd_engine_device_type; +extern struct device_type idxd_group_device_type; static inline bool is_dsa_dev(struct device *dev) { diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 0d8fd01d5a48..a00dce9a2468 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -236,11 +236,54 @@ static int idxd_setup_engines(struct idxd_device *idxd) return rc; } -static int idxd_setup_internals(struct idxd_device *idxd) +static int idxd_setup_groups(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; + struct idxd_group *group; int i, rc; + idxd->groups = kcalloc_node(idxd->max_groups, sizeof(struct idxd_group *), + GFP_KERNEL, dev_to_node(dev)); + if (!idxd->groups) + return -ENOMEM; + + for (i = 0; i < idxd->max_groups; i++) { + group = kzalloc_node(sizeof(*group), GFP_KERNEL, dev_to_node(dev)); + if (!group) { + rc = -ENOMEM; + goto err; + } + + group->id = i; + group->idxd = idxd; + device_initialize(&group->conf_dev); + group->conf_dev.parent = &idxd->conf_dev; + group->conf_dev.bus = idxd_get_bus_type(idxd); + group->conf_dev.type = &idxd_group_device_type; + rc = dev_set_name(&group->conf_dev, "group%d.%d", idxd->id, group->id); + if (rc < 0) { + put_device(&group->conf_dev); + goto err; + } + + idxd->groups[i] = group; + group->tc_a = -1; + group->tc_b = -1; + } + + return 0; + + err: + while (--i >= 0) + put_device(&idxd->groups[i]->conf_dev); + return rc; +} + +static int idxd_setup_internals(struct idxd_device *idxd) +{ + struct device *dev = &idxd->pdev->dev; + int rc, i; + init_waitqueue_head(&idxd->cmd_waitq); rc = idxd_setup_wqs(idxd); @@ -251,29 +294,22 @@ static int idxd_setup_internals(struct idxd_device *idxd) if (rc < 0) goto err_engine; - idxd->groups = devm_kcalloc(dev, idxd->max_groups, - sizeof(struct idxd_group), GFP_KERNEL); - if (!idxd->groups) { - rc = -ENOMEM; - goto err; - } - - for (i = 0; i < idxd->max_groups; i++) { - idxd->groups[i].idxd = idxd; - idxd->groups[i].id = i; - idxd->groups[i].tc_a = -1; - idxd->groups[i].tc_b = -1; - } + rc = idxd_setup_groups(idxd); + if (rc < 0) + goto err_group; idxd->wq = create_workqueue(dev_name(dev)); if (!idxd->wq) { rc = -ENOMEM; - goto err; + goto err_wkq_create; } return 0; - err: + err_wkq_create: + for (i = 0; i < idxd->max_groups; i++) + put_device(&idxd->groups[i]->conf_dev); + err_group: for (i = 0; i < idxd->max_engines; i++) put_device(&idxd->engines[i]->conf_dev); err_engine: diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 531bc88b92eb..9586b55abce5 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -16,16 +16,6 @@ static char *idxd_wq_type_names[] = { [IDXD_WQT_USER] = "user", }; -static void idxd_conf_sub_device_release(struct device *dev) -{ - dev_dbg(dev, "%s for %s\n", __func__, dev_name(dev)); -} - -static struct device_type idxd_group_device_type = { - .name = "group", - .release = idxd_conf_sub_device_release, -}; - static int idxd_config_bus_match(struct device *dev, struct device_driver *drv) { @@ -435,7 +425,7 @@ static ssize_t engine_group_id_store(struct device *dev, if (prevg) prevg->num_engines--; - engine->group = &idxd->groups[id]; + engine->group = idxd->groups[id]; engine->group->num_engines++; return count; @@ -479,7 +469,7 @@ static void idxd_set_free_tokens(struct idxd_device *idxd) int i, tokens; for (i = 0, tokens = 0; i < idxd->max_groups; i++) { - struct idxd_group *g = &idxd->groups[i]; + struct idxd_group *g = idxd->groups[i]; tokens += g->tokens_reserved; } @@ -784,6 +774,19 @@ static const struct attribute_group *idxd_group_attribute_groups[] = { NULL, }; +static void idxd_conf_group_release(struct device *dev) +{ + struct idxd_group *group = container_of(dev, struct idxd_group, conf_dev); + + kfree(group); +} + +struct device_type idxd_group_device_type = { + .name = "group", + .release = idxd_conf_group_release, + .groups = idxd_group_attribute_groups, +}; + /* IDXD work queue attribs */ static ssize_t wq_clients_show(struct device *dev, struct device_attribute *attr, char *buf) @@ -856,7 +859,7 @@ static ssize_t wq_group_id_store(struct device *dev, return count; } - group = &idxd->groups[id]; + group = idxd->groups[id]; prevg = wq->group; if (prevg) @@ -1674,37 +1677,27 @@ static int idxd_register_engine_devices(struct idxd_device *idxd) return rc; } -static int idxd_setup_group_sysfs(struct idxd_device *idxd) +static int idxd_register_group_devices(struct idxd_device *idxd) { - struct device *dev = &idxd->pdev->dev; - int i, rc; + int i, j, rc; for (i = 0; i < idxd->max_groups; i++) { - struct idxd_group *group = &idxd->groups[i]; - - group->conf_dev.parent = &idxd->conf_dev; - dev_set_name(&group->conf_dev, "group%d.%d", - idxd->id, group->id); - group->conf_dev.bus = idxd_get_bus_type(idxd); - group->conf_dev.groups = idxd_group_attribute_groups; - group->conf_dev.type = &idxd_group_device_type; - dev_dbg(dev, "Group device register: %s\n", - dev_name(&group->conf_dev)); - rc = device_register(&group->conf_dev); - if (rc < 0) { - put_device(&group->conf_dev); + struct idxd_group *group = idxd->groups[i]; + + rc = device_add(&group->conf_dev); + if (rc < 0) goto cleanup; - } } return 0; cleanup: - while (i--) { - struct idxd_group *group = &idxd->groups[i]; + j = i - 1; + for (; i < idxd->max_groups; i++) + put_device(&idxd->groups[i]->conf_dev); - device_unregister(&group->conf_dev); - } + while (j--) + device_unregister(&idxd->groups[j]->conf_dev); return rc; } @@ -1753,10 +1746,9 @@ int idxd_register_devices(struct idxd_device *idxd) goto err_engine; } - rc = idxd_setup_group_sysfs(idxd); + rc = idxd_register_group_devices(idxd); if (rc < 0) { - /* unregister conf dev */ - dev_dbg(dev, "Group sysfs registering failed: %d\n", rc); + dev_dbg(dev, "Group device registering failed: %d\n", rc); goto err_group; } @@ -1790,7 +1782,7 @@ void idxd_unregister_devices(struct idxd_device *idxd) } for (i = 0; i < idxd->max_groups; i++) { - struct idxd_group *group = &idxd->groups[i]; + struct idxd_group *group = idxd->groups[i]; device_unregister(&group->conf_dev); } -- Gitee From 28fbbd3bf39a2a8b646fa99ae32092bf9b4a5ecc Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Apr 2021 16:38:03 -0700 Subject: [PATCH 09/85] dmaengine: idxd: iax bus removal ANBZ: #22 commit 4b73e4ebd43ce48101a4c09bf13d439a954d61c5 upstream. There is no need to have an additional bus for the IAX device. The removal of IAX will change user ABI as /sys/bus/iax will no longer exist. The iax device will be moved to the dsa bus. The device id for dsa and iax will now be combined rather than unique for each device type in order to accommodate the iax devices. This is in preparation for fixing the sub-driver code for idxd. There's no hardware deployment for Sapphire Rapids platform yet, which means that users have no reason to have developed scripts against this ABI. There is some exposure to released versions of accel-config, but those are being fixed up and an accel-config upgrade is reasonable to get IAX support. As far as accel-config is concerned IAX support starts when these devices appear under /sys/bus/dsa, and old accel-config just assumes that an empty / missing /sys/bus/iax just means a lack of platform support. Intel-SIG: commit 4b73e4ebd43c dmaengine: idxd: iax bus removal. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Fixes: f25b463883a8 ("dmaengine: idxd: add IAX configuration support in the IDXD driver") Suggested-by: Dan Williams Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/161852988298.2203940.4529909758034944428.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/cdev.c | 2 +- drivers/dma/idxd/idxd.h | 3 +- drivers/dma/idxd/init.c | 21 ++++-------- drivers/dma/idxd/sysfs.c | 74 +++------------------------------------- 4 files changed, 13 insertions(+), 87 deletions(-) diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index 1d8a3876b745..2d976905b2a3 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -268,7 +268,7 @@ int idxd_wq_add_cdev(struct idxd_wq *wq) device_initialize(dev); dev->parent = &wq->conf_dev; - dev->bus = idxd_get_bus_type(idxd); + dev->bus = &dsa_bus_type; dev->type = &idxd_cdev_device_type; dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor); diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 89daf746d121..b17415aa42bd 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -257,6 +257,7 @@ extern struct bus_type dsa_bus_type; extern struct bus_type iax_bus_type; extern bool support_enqcmd; +extern struct ida idxd_ida; extern struct device_type dsa_device_type; extern struct device_type iax_device_type; extern struct device_type idxd_wq_device_type; @@ -346,7 +347,6 @@ static inline int idxd_wq_refcount(struct idxd_wq *wq) return wq->client_count; }; -struct ida *idxd_ida(struct idxd_device *idxd); const char *idxd_get_dev_name(struct idxd_device *idxd); int idxd_register_bus_type(void); void idxd_unregister_bus_type(void); @@ -354,7 +354,6 @@ int idxd_register_devices(struct idxd_device *idxd); void idxd_unregister_devices(struct idxd_device *idxd); int idxd_register_driver(void); void idxd_unregister_driver(void); -struct bus_type *idxd_get_bus_type(struct idxd_device *idxd); struct device_type *idxd_get_device_type(struct idxd_device *idxd); /* device interrupt control */ diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index a00dce9a2468..8667c86a7d7c 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -33,8 +33,7 @@ MODULE_PARM_DESC(sva, "Toggle SVA support on/off"); #define DRV_NAME "idxd" bool support_enqcmd; - -static struct ida idxd_idas[IDXD_TYPE_MAX]; +DEFINE_IDA(idxd_ida); static struct pci_device_id idxd_pci_tbl[] = { /* DSA ver 1.0 platforms */ @@ -51,11 +50,6 @@ static char *idxd_name[] = { "iax" }; -struct ida *idxd_ida(struct idxd_device *idxd) -{ - return &idxd_idas[idxd->type]; -} - const char *idxd_get_dev_name(struct idxd_device *idxd) { return idxd_name[idxd->type]; @@ -167,7 +161,7 @@ static int idxd_setup_wqs(struct idxd_device *idxd) wq->idxd = idxd; device_initialize(&wq->conf_dev); wq->conf_dev.parent = &idxd->conf_dev; - wq->conf_dev.bus = idxd_get_bus_type(idxd); + wq->conf_dev.bus = &dsa_bus_type; wq->conf_dev.type = &idxd_wq_device_type; rc = dev_set_name(&wq->conf_dev, "wq%d.%d", idxd->id, wq->id); if (rc < 0) { @@ -258,7 +252,7 @@ static int idxd_setup_groups(struct idxd_device *idxd) group->idxd = idxd; device_initialize(&group->conf_dev); group->conf_dev.parent = &idxd->conf_dev; - group->conf_dev.bus = idxd_get_bus_type(idxd); + group->conf_dev.bus = &dsa_bus_type; group->conf_dev.type = &idxd_group_device_type; rc = dev_set_name(&group->conf_dev, "group%d.%d", idxd->id, group->id); if (rc < 0) { @@ -409,13 +403,13 @@ static struct idxd_device *idxd_alloc(struct pci_dev *pdev) idxd->pdev = pdev; idxd_set_type(idxd); - idxd->id = ida_alloc(idxd_ida(idxd), GFP_KERNEL); + idxd->id = ida_alloc(&idxd_ida, GFP_KERNEL); if (idxd->id < 0) return NULL; device_initialize(&idxd->conf_dev); idxd->conf_dev.parent = dev; - idxd->conf_dev.bus = idxd_get_bus_type(idxd); + idxd->conf_dev.bus = &dsa_bus_type; idxd->conf_dev.type = idxd_get_device_type(idxd); rc = dev_set_name(&idxd->conf_dev, "%s%d", idxd_get_dev_name(idxd), idxd->id); if (rc < 0) { @@ -669,7 +663,7 @@ static struct pci_driver idxd_pci_driver = { static int __init idxd_init_module(void) { - int err, i; + int err; /* * If the CPU does not support MOVDIR64B or ENQCMDS, there's no point in @@ -685,9 +679,6 @@ static int __init idxd_init_module(void) else support_enqcmd = true; - for (i = 0; i < IDXD_TYPE_MAX; i++) - ida_init(&idxd_idas[i]); - err = idxd_register_bus_type(); if (err < 0) return err; diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 9586b55abce5..b97a0a817dfb 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -301,19 +301,6 @@ struct bus_type dsa_bus_type = { .shutdown = idxd_config_bus_shutdown, }; -struct bus_type iax_bus_type = { - .name = "iax", - .match = idxd_config_bus_match, - .probe = idxd_config_bus_probe, - .remove = idxd_config_bus_remove, - .shutdown = idxd_config_bus_shutdown, -}; - -static struct bus_type *idxd_bus_types[] = { - &dsa_bus_type, - &iax_bus_type -}; - static struct idxd_device_driver dsa_drv = { .drv = { .name = "dsa", @@ -323,25 +310,6 @@ static struct idxd_device_driver dsa_drv = { }, }; -static struct idxd_device_driver iax_drv = { - .drv = { - .name = "iax", - .bus = &iax_bus_type, - .owner = THIS_MODULE, - .mod_name = KBUILD_MODNAME, - }, -}; - -static struct idxd_device_driver *idxd_drvs[] = { - &dsa_drv, - &iax_drv -}; - -struct bus_type *idxd_get_bus_type(struct idxd_device *idxd) -{ - return idxd_bus_types[idxd->type]; -} - struct device_type *idxd_get_device_type(struct idxd_device *idxd) { if (idxd->type == IDXD_TYPE_DSA) @@ -355,28 +323,12 @@ struct device_type *idxd_get_device_type(struct idxd_device *idxd) /* IDXD generic driver setup */ int idxd_register_driver(void) { - int i, rc; - - for (i = 0; i < IDXD_TYPE_MAX; i++) { - rc = driver_register(&idxd_drvs[i]->drv); - if (rc < 0) - goto drv_fail; - } - - return 0; - -drv_fail: - while (--i >= 0) - driver_unregister(&idxd_drvs[i]->drv); - return rc; + return driver_register(&dsa_drv.drv); } void idxd_unregister_driver(void) { - int i; - - for (i = 0; i < IDXD_TYPE_MAX; i++) - driver_unregister(&idxd_drvs[i]->drv); + driver_unregister(&dsa_drv.drv); } /* IDXD engine attributes */ @@ -1637,7 +1589,7 @@ static void idxd_conf_device_release(struct device *dev) kfree(idxd->wqs); kfree(idxd->engines); kfree(idxd->irq_entries); - ida_free(idxd_ida(idxd), idxd->id); + ida_free(&idxd_ida, idxd->id); kfree(idxd); } @@ -1792,26 +1744,10 @@ void idxd_unregister_devices(struct idxd_device *idxd) int idxd_register_bus_type(void) { - int i, rc; - - for (i = 0; i < IDXD_TYPE_MAX; i++) { - rc = bus_register(idxd_bus_types[i]); - if (rc < 0) - goto bus_err; - } - - return 0; - -bus_err: - while (--i >= 0) - bus_unregister(idxd_bus_types[i]); - return rc; + return bus_register(&dsa_bus_type); } void idxd_unregister_bus_type(void) { - int i; - - for (i = 0; i < IDXD_TYPE_MAX; i++) - bus_unregister(idxd_bus_types[i]); + bus_unregister(&dsa_bus_type); } -- Gitee From 69571b199a57e7dc3228ce1d22f07ece4b77c171 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Apr 2021 16:38:09 -0700 Subject: [PATCH 10/85] dmaengine: idxd: remove detection of device type ANBZ: #22 commit 435b512dbc0dac42b34348393049b386bb1a19bd upstream. Move all static data type for per device type to an idxd_driver_data data structure. The data can be attached to the pci_device_id and provided by the pci probe function. This removes a lot of unnecessary type detection and setup code. Intel-SIG: commit 435b512dbc0d dmaengine: idxd: remove detection of device type. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Suggested-by: Dan Williams Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/161852988924.2203940.2787590808682466398.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/cdev.c | 11 +++---- drivers/dma/idxd/device.c | 16 +++------- drivers/dma/idxd/idxd.h | 13 +++++--- drivers/dma/idxd/init.c | 65 +++++++++++++++------------------------ drivers/dma/idxd/submit.c | 2 +- drivers/dma/idxd/sysfs.c | 16 ++-------- 6 files changed, 48 insertions(+), 75 deletions(-) diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index 2d976905b2a3..302cba5ff779 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -45,7 +45,7 @@ static void idxd_cdev_dev_release(struct device *dev) struct idxd_cdev_context *cdev_ctx; struct idxd_wq *wq = idxd_cdev->wq; - cdev_ctx = &ictx[wq->idxd->type]; + cdev_ctx = &ictx[wq->idxd->data->type]; ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor); kfree(idxd_cdev); } @@ -239,7 +239,7 @@ static const struct file_operations idxd_cdev_fops = { int idxd_cdev_get_major(struct idxd_device *idxd) { - return MAJOR(ictx[idxd->type].devt); + return MAJOR(ictx[idxd->data->type].devt); } int idxd_wq_add_cdev(struct idxd_wq *wq) @@ -258,7 +258,7 @@ int idxd_wq_add_cdev(struct idxd_wq *wq) idxd_cdev->wq = wq; cdev = &idxd_cdev->cdev; dev = &idxd_cdev->dev; - cdev_ctx = &ictx[wq->idxd->type]; + cdev_ctx = &ictx[wq->idxd->data->type]; minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL); if (minor < 0) { kfree(idxd_cdev); @@ -272,8 +272,7 @@ int idxd_wq_add_cdev(struct idxd_wq *wq) dev->type = &idxd_cdev_device_type; dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor); - rc = dev_set_name(dev, "%s/wq%u.%u", idxd_get_dev_name(idxd), - idxd->id, wq->id); + rc = dev_set_name(dev, "%s/wq%u.%u", idxd->data->name_prefix, idxd->id, wq->id); if (rc < 0) goto err; @@ -298,7 +297,7 @@ void idxd_wq_del_cdev(struct idxd_wq *wq) struct idxd_cdev *idxd_cdev; struct idxd_cdev_context *cdev_ctx; - cdev_ctx = &ictx[wq->idxd->type]; + cdev_ctx = &ictx[wq->idxd->data->type]; idxd_cdev = wq->idxd_cdev; wq->idxd_cdev = NULL; cdev_device_del(&idxd_cdev->cdev, &idxd_cdev->dev); diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 4fef57717049..016df87cf5c5 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -144,14 +144,8 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq) if (rc < 0) return rc; - if (idxd->type == IDXD_TYPE_DSA) - align = 32; - else if (idxd->type == IDXD_TYPE_IAX) - align = 64; - else - return -ENODEV; - - wq->compls_size = num_descs * idxd->compl_size + align; + align = idxd->data->align; + wq->compls_size = num_descs * idxd->data->compl_size + align; wq->compls_raw = dma_alloc_coherent(dev, wq->compls_size, &wq->compls_addr_raw, GFP_KERNEL); if (!wq->compls_raw) { @@ -178,11 +172,11 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq) struct idxd_desc *desc = wq->descs[i]; desc->hw = wq->hw_descs[i]; - if (idxd->type == IDXD_TYPE_DSA) + if (idxd->data->type == IDXD_TYPE_DSA) desc->completion = &wq->compls[i]; - else if (idxd->type == IDXD_TYPE_IAX) + else if (idxd->data->type == IDXD_TYPE_IAX) desc->iax_completion = &wq->iax_compls[i]; - desc->compl_dma = wq->compls_addr + idxd->compl_size * i; + desc->compl_dma = wq->compls_addr + idxd->data->compl_size * i; desc->id = i; desc->wq = wq; desc->cpu = -1; diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index b17415aa42bd..8055e872953c 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -178,9 +178,17 @@ struct idxd_dma_dev { struct dma_device dma; }; -struct idxd_device { +struct idxd_driver_data { + const char *name_prefix; enum idxd_type type; + struct device_type *dev_type; + int compl_size; + int align; +}; + +struct idxd_device { struct device conf_dev; + struct idxd_driver_data *data; struct list_head list; struct idxd_hw hw; enum idxd_device_state state; @@ -218,7 +226,6 @@ struct idxd_device { int token_limit; int nr_tokens; /* non-reserved tokens */ unsigned int wqcfg_size; - int compl_size; union sw_err_reg sw_err; wait_queue_head_t cmd_waitq; @@ -347,14 +354,12 @@ static inline int idxd_wq_refcount(struct idxd_wq *wq) return wq->client_count; }; -const char *idxd_get_dev_name(struct idxd_device *idxd); int idxd_register_bus_type(void); void idxd_unregister_bus_type(void); int idxd_register_devices(struct idxd_device *idxd); void idxd_unregister_devices(struct idxd_device *idxd); int idxd_register_driver(void); void idxd_unregister_driver(void); -struct device_type *idxd_get_device_type(struct idxd_device *idxd); /* device interrupt control */ void idxd_msix_perm_setup(struct idxd_device *idxd); diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 8667c86a7d7c..b791df8b45b2 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -35,26 +35,33 @@ MODULE_PARM_DESC(sva, "Toggle SVA support on/off"); bool support_enqcmd; DEFINE_IDA(idxd_ida); +static struct idxd_driver_data idxd_driver_data[] = { + [IDXD_TYPE_DSA] = { + .name_prefix = "dsa", + .type = IDXD_TYPE_DSA, + .compl_size = sizeof(struct dsa_completion_record), + .align = 32, + .dev_type = &dsa_device_type, + }, + [IDXD_TYPE_IAX] = { + .name_prefix = "iax", + .type = IDXD_TYPE_IAX, + .compl_size = sizeof(struct iax_completion_record), + .align = 64, + .dev_type = &iax_device_type, + }, +}; + static struct pci_device_id idxd_pci_tbl[] = { /* DSA ver 1.0 platforms */ - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_DSA_SPR0) }, + { PCI_DEVICE_DATA(INTEL, DSA_SPR0, &idxd_driver_data[IDXD_TYPE_DSA]) }, /* IAX ver 1.0 platforms */ - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IAX_SPR0) }, + { PCI_DEVICE_DATA(INTEL, IAX_SPR0, &idxd_driver_data[IDXD_TYPE_IAX]) }, { 0, } }; MODULE_DEVICE_TABLE(pci, idxd_pci_tbl); -static char *idxd_name[] = { - "dsa", - "iax" -}; - -const char *idxd_get_dev_name(struct idxd_device *idxd) -{ - return idxd_name[idxd->type]; -} - static int idxd_setup_interrupts(struct idxd_device *idxd) { struct pci_dev *pdev = idxd->pdev; @@ -379,19 +386,7 @@ static void idxd_read_caps(struct idxd_device *idxd) } } -static inline void idxd_set_type(struct idxd_device *idxd) -{ - struct pci_dev *pdev = idxd->pdev; - - if (pdev->device == PCI_DEVICE_ID_INTEL_DSA_SPR0) - idxd->type = IDXD_TYPE_DSA; - else if (pdev->device == PCI_DEVICE_ID_INTEL_IAX_SPR0) - idxd->type = IDXD_TYPE_IAX; - else - idxd->type = IDXD_TYPE_UNKNOWN; -} - -static struct idxd_device *idxd_alloc(struct pci_dev *pdev) +static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_data *data) { struct device *dev = &pdev->dev; struct idxd_device *idxd; @@ -402,7 +397,7 @@ static struct idxd_device *idxd_alloc(struct pci_dev *pdev) return NULL; idxd->pdev = pdev; - idxd_set_type(idxd); + idxd->data = data; idxd->id = ida_alloc(&idxd_ida, GFP_KERNEL); if (idxd->id < 0) return NULL; @@ -410,8 +405,8 @@ static struct idxd_device *idxd_alloc(struct pci_dev *pdev) device_initialize(&idxd->conf_dev); idxd->conf_dev.parent = dev; idxd->conf_dev.bus = &dsa_bus_type; - idxd->conf_dev.type = idxd_get_device_type(idxd); - rc = dev_set_name(&idxd->conf_dev, "%s%d", idxd_get_dev_name(idxd), idxd->id); + idxd->conf_dev.type = idxd->data->dev_type; + rc = dev_set_name(&idxd->conf_dev, "%s%d", idxd->data->name_prefix, idxd->id); if (rc < 0) { put_device(&idxd->conf_dev); return NULL; @@ -503,18 +498,11 @@ static int idxd_probe(struct idxd_device *idxd) return rc; } -static void idxd_type_init(struct idxd_device *idxd) -{ - if (idxd->type == IDXD_TYPE_DSA) - idxd->compl_size = sizeof(struct dsa_completion_record); - else if (idxd->type == IDXD_TYPE_IAX) - idxd->compl_size = sizeof(struct iax_completion_record); -} - static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { struct device *dev = &pdev->dev; struct idxd_device *idxd; + struct idxd_driver_data *data = (struct idxd_driver_data *)id->driver_data; int rc; rc = pci_enable_device(pdev); @@ -522,7 +510,7 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) return rc; dev_dbg(dev, "Alloc IDXD context\n"); - idxd = idxd_alloc(pdev); + idxd = idxd_alloc(pdev, data); if (!idxd) { rc = -ENOMEM; goto err_idxd_alloc; @@ -548,9 +536,6 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (rc) goto err; - - idxd_type_init(idxd); - dev_dbg(dev, "Set PCI master\n"); pci_set_master(pdev); pci_set_drvdata(pdev, idxd); diff --git a/drivers/dma/idxd/submit.c b/drivers/dma/idxd/submit.c index a7a61bcc17d5..dfc8900d5de3 100644 --- a/drivers/dma/idxd/submit.c +++ b/drivers/dma/idxd/submit.c @@ -15,7 +15,7 @@ static struct idxd_desc *__get_desc(struct idxd_wq *wq, int idx, int cpu) desc = wq->descs[idx]; memset(desc->hw, 0, sizeof(struct dsa_hw_desc)); - memset(desc->completion, 0, idxd->compl_size); + memset(desc->completion, 0, idxd->data->compl_size); desc->cpu = cpu; if (device_pasid_enabled(idxd)) diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index b97a0a817dfb..581ce56ae4f5 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -310,16 +310,6 @@ static struct idxd_device_driver dsa_drv = { }, }; -struct device_type *idxd_get_device_type(struct idxd_device *idxd) -{ - if (idxd->type == IDXD_TYPE_DSA) - return &dsa_device_type; - else if (idxd->type == IDXD_TYPE_IAX) - return &iax_device_type; - else - return NULL; -} - /* IDXD generic driver setup */ int idxd_register_driver(void) { @@ -453,7 +443,7 @@ static ssize_t group_tokens_reserved_store(struct device *dev, if (rc < 0) return -EINVAL; - if (idxd->type == IDXD_TYPE_IAX) + if (idxd->data->type == IDXD_TYPE_IAX) return -EOPNOTSUPP; if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) @@ -501,7 +491,7 @@ static ssize_t group_tokens_allowed_store(struct device *dev, if (rc < 0) return -EINVAL; - if (idxd->type == IDXD_TYPE_IAX) + if (idxd->data->type == IDXD_TYPE_IAX) return -EOPNOTSUPP; if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) @@ -546,7 +536,7 @@ static ssize_t group_use_token_limit_store(struct device *dev, if (rc < 0) return -EINVAL; - if (idxd->type == IDXD_TYPE_IAX) + if (idxd->data->type == IDXD_TYPE_IAX) return -EOPNOTSUPP; if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) -- Gitee From fadc0a985d01e555651a5ba800b54bed43238ef1 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 20 Apr 2021 11:46:22 -0700 Subject: [PATCH 11/85] dmaengine: idxd: add percpu_ref to descriptor submission path ANBZ: #22 commit 93a40a6d7428921897bb7fed5ffb4ce83df05432 upstream. Current submission path has no way to restrict the submitter from stop submiting on shutdown path or wq disable path. This provides a way to quiesce the submission path. Modeling after 'struct reqeust_queue' usage of percpu_ref. One of the abilities of per_cpu reference counting is the ability to stop new references from being taken while awaiting outstanding references to be dropped. On wq shutdown, we want to block any new submissions to the kernel workqueue and quiesce before disabling. The percpu_ref allows us to block any new submissions and wait for any current submission calls to finish submitting to the workqueue. A percpu_ref is embedded in each idxd_wq context to allow control for individual wq. The wq->wq_active counter is elevated before calling movdir64b() or enqcmds() to submit a descriptor to the wq and dropped once the submission call completes. The function is gated by percpu_ref_tryget_live(). On shutdown with percpu_ref_kill() called, any new submission would be blocked from acquiring a ref and failed. Once all references are dropped for the wq, shutdown can continue. Intel-SIG: commit 93a40a6d7428 dmaengine: idxd: add percpu_ref to descriptor submission path. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/161894438293.3202472.14894701611500822232.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 26 +++++ drivers/dma/idxd/idxd.h | 4 + drivers/dma/idxd/init.c | 1 + drivers/dma/idxd/submit.c | 5 + drivers/dma/idxd/sysfs.c | 233 ++++++++++++++++++++------------------ 5 files changed, 161 insertions(+), 108 deletions(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 016df87cf5c5..6d674fdedb4d 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -384,6 +384,32 @@ void idxd_wq_disable_cleanup(struct idxd_wq *wq) memset(wq->name, 0, WQ_NAME_SIZE); } +static void idxd_wq_ref_release(struct percpu_ref *ref) +{ + struct idxd_wq *wq = container_of(ref, struct idxd_wq, wq_active); + + complete(&wq->wq_dead); +} + +int idxd_wq_init_percpu_ref(struct idxd_wq *wq) +{ + int rc; + + memset(&wq->wq_active, 0, sizeof(wq->wq_active)); + rc = percpu_ref_init(&wq->wq_active, idxd_wq_ref_release, 0, GFP_KERNEL); + if (rc < 0) + return rc; + reinit_completion(&wq->wq_dead); + return 0; +} + +void idxd_wq_quiesce(struct idxd_wq *wq) +{ + percpu_ref_kill(&wq->wq_active); + wait_for_completion(&wq->wq_dead); + percpu_ref_exit(&wq->wq_active); +} + /* Device control bits */ static inline bool idxd_is_enabled(struct idxd_device *idxd) { diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 8055e872953c..1b539cbf3c14 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -108,6 +108,8 @@ struct idxd_dma_chan { struct idxd_wq { void __iomem *portal; + struct percpu_ref wq_active; + struct completion wq_dead; struct device conf_dev; struct idxd_cdev *idxd_cdev; struct wait_queue_head err_queue; @@ -395,6 +397,8 @@ void idxd_wq_unmap_portal(struct idxd_wq *wq); void idxd_wq_disable_cleanup(struct idxd_wq *wq); int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid); int idxd_wq_disable_pasid(struct idxd_wq *wq); +void idxd_wq_quiesce(struct idxd_wq *wq); +int idxd_wq_init_percpu_ref(struct idxd_wq *wq); /* submission */ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc); diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index b791df8b45b2..bf505ba3a119 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -178,6 +178,7 @@ static int idxd_setup_wqs(struct idxd_device *idxd) mutex_init(&wq->wq_lock); init_waitqueue_head(&wq->err_queue); + init_completion(&wq->wq_dead); wq->max_xfer_bytes = idxd->max_xfer_bytes; wq->max_batch_size = idxd->max_batch_size; wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev)); diff --git a/drivers/dma/idxd/submit.c b/drivers/dma/idxd/submit.c index dfc8900d5de3..02f9f51e29a6 100644 --- a/drivers/dma/idxd/submit.c +++ b/drivers/dma/idxd/submit.c @@ -86,6 +86,9 @@ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc) if (idxd->state != IDXD_DEV_ENABLED) return -EIO; + if (!percpu_ref_tryget_live(&wq->wq_active)) + return -ENXIO; + portal = wq->portal; /* @@ -108,6 +111,8 @@ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc) return rc; } + percpu_ref_put(&wq->wq_active); + /* * Pending the descriptor to the lockless list for the irq_entry * that we designated the descriptor to. diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 581ce56ae4f5..dad5c0be9ae8 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -47,6 +47,127 @@ static int idxd_config_bus_match(struct device *dev, return matched; } +static int enable_wq(struct idxd_wq *wq) +{ + struct idxd_device *idxd = wq->idxd; + struct device *dev = &idxd->pdev->dev; + unsigned long flags; + int rc; + + mutex_lock(&wq->wq_lock); + + if (idxd->state != IDXD_DEV_ENABLED) { + mutex_unlock(&wq->wq_lock); + dev_warn(dev, "Enabling while device not enabled.\n"); + return -EPERM; + } + + if (wq->state != IDXD_WQ_DISABLED) { + mutex_unlock(&wq->wq_lock); + dev_warn(dev, "WQ %d already enabled.\n", wq->id); + return -EBUSY; + } + + if (!wq->group) { + mutex_unlock(&wq->wq_lock); + dev_warn(dev, "WQ not attached to group.\n"); + return -EINVAL; + } + + if (strlen(wq->name) == 0) { + mutex_unlock(&wq->wq_lock); + dev_warn(dev, "WQ name not set.\n"); + return -EINVAL; + } + + /* Shared WQ checks */ + if (wq_shared(wq)) { + if (!device_swq_supported(idxd)) { + dev_warn(dev, "PASID not enabled and shared WQ.\n"); + mutex_unlock(&wq->wq_lock); + return -ENXIO; + } + /* + * Shared wq with the threshold set to 0 means the user + * did not set the threshold or transitioned from a + * dedicated wq but did not set threshold. A value + * of 0 would effectively disable the shared wq. The + * driver does not allow a value of 0 to be set for + * threshold via sysfs. + */ + if (wq->threshold == 0) { + dev_warn(dev, "Shared WQ and threshold 0.\n"); + mutex_unlock(&wq->wq_lock); + return -EINVAL; + } + } + + rc = idxd_wq_alloc_resources(wq); + if (rc < 0) { + mutex_unlock(&wq->wq_lock); + dev_warn(dev, "WQ resource alloc failed\n"); + return rc; + } + + spin_lock_irqsave(&idxd->dev_lock, flags); + rc = idxd_device_config(idxd); + spin_unlock_irqrestore(&idxd->dev_lock, flags); + if (rc < 0) { + mutex_unlock(&wq->wq_lock); + dev_warn(dev, "Writing WQ %d config failed: %d\n", wq->id, rc); + return rc; + } + + rc = idxd_wq_enable(wq); + if (rc < 0) { + mutex_unlock(&wq->wq_lock); + dev_warn(dev, "WQ %d enabling failed: %d\n", wq->id, rc); + return rc; + } + + rc = idxd_wq_map_portal(wq); + if (rc < 0) { + dev_warn(dev, "wq portal mapping failed: %d\n", rc); + rc = idxd_wq_disable(wq); + if (rc < 0) + dev_warn(dev, "IDXD wq disable failed\n"); + mutex_unlock(&wq->wq_lock); + return rc; + } + + wq->client_count = 0; + + if (wq->type == IDXD_WQT_KERNEL) { + rc = idxd_wq_init_percpu_ref(wq); + if (rc < 0) { + dev_dbg(dev, "percpu_ref setup failed\n"); + mutex_unlock(&wq->wq_lock); + return rc; + } + } + + if (is_idxd_wq_dmaengine(wq)) { + rc = idxd_register_dma_channel(wq); + if (rc < 0) { + dev_dbg(dev, "DMA channel register failed\n"); + mutex_unlock(&wq->wq_lock); + return rc; + } + } else if (is_idxd_wq_cdev(wq)) { + rc = idxd_wq_add_cdev(wq); + if (rc < 0) { + dev_dbg(dev, "Cdev creation failed\n"); + mutex_unlock(&wq->wq_lock); + return rc; + } + } + + mutex_unlock(&wq->wq_lock); + dev_info(dev, "wq %s enabled\n", dev_name(&wq->conf_dev)); + + return 0; +} + static int idxd_config_bus_probe(struct device *dev) { int rc; @@ -94,115 +215,8 @@ static int idxd_config_bus_probe(struct device *dev) return 0; } else if (is_idxd_wq_dev(dev)) { struct idxd_wq *wq = confdev_to_wq(dev); - struct idxd_device *idxd = wq->idxd; - - mutex_lock(&wq->wq_lock); - - if (idxd->state != IDXD_DEV_ENABLED) { - mutex_unlock(&wq->wq_lock); - dev_warn(dev, "Enabling while device not enabled.\n"); - return -EPERM; - } - - if (wq->state != IDXD_WQ_DISABLED) { - mutex_unlock(&wq->wq_lock); - dev_warn(dev, "WQ %d already enabled.\n", wq->id); - return -EBUSY; - } - - if (!wq->group) { - mutex_unlock(&wq->wq_lock); - dev_warn(dev, "WQ not attached to group.\n"); - return -EINVAL; - } - - if (strlen(wq->name) == 0) { - mutex_unlock(&wq->wq_lock); - dev_warn(dev, "WQ name not set.\n"); - return -EINVAL; - } - - /* Shared WQ checks */ - if (wq_shared(wq)) { - if (!device_swq_supported(idxd)) { - dev_warn(dev, - "PASID not enabled and shared WQ.\n"); - mutex_unlock(&wq->wq_lock); - return -ENXIO; - } - /* - * Shared wq with the threshold set to 0 means the user - * did not set the threshold or transitioned from a - * dedicated wq but did not set threshold. A value - * of 0 would effectively disable the shared wq. The - * driver does not allow a value of 0 to be set for - * threshold via sysfs. - */ - if (wq->threshold == 0) { - dev_warn(dev, - "Shared WQ and threshold 0.\n"); - mutex_unlock(&wq->wq_lock); - return -EINVAL; - } - } - - rc = idxd_wq_alloc_resources(wq); - if (rc < 0) { - mutex_unlock(&wq->wq_lock); - dev_warn(dev, "WQ resource alloc failed\n"); - return rc; - } - - spin_lock_irqsave(&idxd->dev_lock, flags); - rc = idxd_device_config(idxd); - spin_unlock_irqrestore(&idxd->dev_lock, flags); - if (rc < 0) { - mutex_unlock(&wq->wq_lock); - dev_warn(dev, "Writing WQ %d config failed: %d\n", - wq->id, rc); - return rc; - } - rc = idxd_wq_enable(wq); - if (rc < 0) { - mutex_unlock(&wq->wq_lock); - dev_warn(dev, "WQ %d enabling failed: %d\n", - wq->id, rc); - return rc; - } - - rc = idxd_wq_map_portal(wq); - if (rc < 0) { - dev_warn(dev, "wq portal mapping failed: %d\n", rc); - rc = idxd_wq_disable(wq); - if (rc < 0) - dev_warn(dev, "IDXD wq disable failed\n"); - mutex_unlock(&wq->wq_lock); - return rc; - } - - wq->client_count = 0; - - dev_info(dev, "wq %s enabled\n", dev_name(&wq->conf_dev)); - - if (is_idxd_wq_dmaengine(wq)) { - rc = idxd_register_dma_channel(wq); - if (rc < 0) { - dev_dbg(dev, "DMA channel register failed\n"); - mutex_unlock(&wq->wq_lock); - return rc; - } - } else if (is_idxd_wq_cdev(wq)) { - rc = idxd_wq_add_cdev(wq); - if (rc < 0) { - dev_dbg(dev, "Cdev creation failed\n"); - mutex_unlock(&wq->wq_lock); - return rc; - } - } - - mutex_unlock(&wq->wq_lock); - return 0; + return enable_wq(wq); } return -ENODEV; @@ -220,6 +234,9 @@ static void disable_wq(struct idxd_wq *wq) return; } + if (wq->type == IDXD_WQT_KERNEL) + idxd_wq_quiesce(wq); + if (is_idxd_wq_dmaengine(wq)) idxd_unregister_dma_channel(wq); else if (is_idxd_wq_cdev(wq)) -- Gitee From 330894b25da499d54870025fcd3e71ccfc0b2294 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 20 Apr 2021 11:46:28 -0700 Subject: [PATCH 12/85] dmaengine: idxd: add support for readonly config mode ANBZ: #22 commit 8c66bbdc4fbf3c297ebc8edf71f359e4a132c9db upstream. The read-only configuration mode is defined by the DSA spec as a mode of the device WQ configuration. When GENCAP register bit 31 is set to 0, the device is in RO mode and group configuration and some fields of the workqueue configuration registers are read-only and reflect the fixed configuration of the device. Add support for RO mode. The driver will load the values from the registers directly setup all the internally cached data structures based on the device configuration. Intel-SIG: commit 8c66bbdc4fbf dmaengine: idxd: add support for readonly config mode. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/161894438847.3202472.6317563824045432727.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 116 ++++++++++++++++++++++++++++++++++++++ drivers/dma/idxd/idxd.h | 1 + drivers/dma/idxd/init.c | 8 +++ drivers/dma/idxd/sysfs.c | 22 +++++--- 4 files changed, 138 insertions(+), 9 deletions(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 6d674fdedb4d..3ddb1c731080 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -884,3 +884,119 @@ int idxd_device_config(struct idxd_device *idxd) return 0; } + +static int idxd_wq_load_config(struct idxd_wq *wq) +{ + struct idxd_device *idxd = wq->idxd; + struct device *dev = &idxd->pdev->dev; + int wqcfg_offset; + int i; + + wqcfg_offset = WQCFG_OFFSET(idxd, wq->id, 0); + memcpy_fromio(wq->wqcfg, idxd->reg_base + wqcfg_offset, idxd->wqcfg_size); + + wq->size = wq->wqcfg->wq_size; + wq->threshold = wq->wqcfg->wq_thresh; + if (wq->wqcfg->priv) + wq->type = IDXD_WQT_KERNEL; + + /* The driver does not support shared WQ mode in read-only config yet */ + if (wq->wqcfg->mode == 0 || wq->wqcfg->pasid_en) + return -EOPNOTSUPP; + + set_bit(WQ_FLAG_DEDICATED, &wq->flags); + + wq->priority = wq->wqcfg->priority; + + for (i = 0; i < WQCFG_STRIDES(idxd); i++) { + wqcfg_offset = WQCFG_OFFSET(idxd, wq->id, i); + dev_dbg(dev, "WQ[%d][%d][%#x]: %#x\n", wq->id, i, wqcfg_offset, wq->wqcfg->bits[i]); + } + + return 0; +} + +static void idxd_group_load_config(struct idxd_group *group) +{ + struct idxd_device *idxd = group->idxd; + struct device *dev = &idxd->pdev->dev; + int i, j, grpcfg_offset; + + /* + * Load WQS bit fields + * Iterate through all 256 bits 64 bits at a time + */ + for (i = 0; i < GRPWQCFG_STRIDES; i++) { + struct idxd_wq *wq; + + grpcfg_offset = GRPWQCFG_OFFSET(idxd, group->id, i); + group->grpcfg.wqs[i] = ioread64(idxd->reg_base + grpcfg_offset); + dev_dbg(dev, "GRPCFG wq[%d:%d: %#x]: %#llx\n", + group->id, i, grpcfg_offset, group->grpcfg.wqs[i]); + + if (i * 64 >= idxd->max_wqs) + break; + + /* Iterate through all 64 bits and check for wq set */ + for (j = 0; j < 64; j++) { + int id = i * 64 + j; + + /* No need to check beyond max wqs */ + if (id >= idxd->max_wqs) + break; + + /* Set group assignment for wq if wq bit is set */ + if (group->grpcfg.wqs[i] & BIT(j)) { + wq = idxd->wqs[id]; + wq->group = group; + } + } + } + + grpcfg_offset = GRPENGCFG_OFFSET(idxd, group->id); + group->grpcfg.engines = ioread64(idxd->reg_base + grpcfg_offset); + dev_dbg(dev, "GRPCFG engs[%d: %#x]: %#llx\n", group->id, + grpcfg_offset, group->grpcfg.engines); + + /* Iterate through all 64 bits to check engines set */ + for (i = 0; i < 64; i++) { + if (i >= idxd->max_engines) + break; + + if (group->grpcfg.engines & BIT(i)) { + struct idxd_engine *engine = idxd->engines[i]; + + engine->group = group; + } + } + + grpcfg_offset = GRPFLGCFG_OFFSET(idxd, group->id); + group->grpcfg.flags.bits = ioread32(idxd->reg_base + grpcfg_offset); + dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#x\n", + group->id, grpcfg_offset, group->grpcfg.flags.bits); +} + +int idxd_device_load_config(struct idxd_device *idxd) +{ + union gencfg_reg reg; + int i, rc; + + reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET); + idxd->token_limit = reg.token_limit; + + for (i = 0; i < idxd->max_groups; i++) { + struct idxd_group *group = idxd->groups[i]; + + idxd_group_load_config(group); + } + + for (i = 0; i < idxd->max_wqs; i++) { + struct idxd_wq *wq = idxd->wqs[i]; + + rc = idxd_wq_load_config(wq); + if (rc < 0) + return rc; + } + + return 0; +} diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 1b539cbf3c14..940a2e1ddf12 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -384,6 +384,7 @@ void idxd_device_cleanup(struct idxd_device *idxd); int idxd_device_config(struct idxd_device *idxd); void idxd_device_wqs_clear_state(struct idxd_device *idxd); void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid); +int idxd_device_load_config(struct idxd_device *idxd); /* work queue control */ int idxd_wq_alloc_resources(struct idxd_wq *wq); diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index bf505ba3a119..8b5e4adc3ffc 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -482,6 +482,14 @@ static int idxd_probe(struct idxd_device *idxd) if (rc) goto err; + /* If the configs are readonly, then load them from device */ + if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { + dev_dbg(dev, "Loading RO device config\n"); + rc = idxd_device_load_config(idxd); + if (rc < 0) + goto err; + } + rc = idxd_setup_interrupts(idxd); if (rc) goto err; diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index dad5c0be9ae8..d45cb61f300b 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -110,7 +110,8 @@ static int enable_wq(struct idxd_wq *wq) } spin_lock_irqsave(&idxd->dev_lock, flags); - rc = idxd_device_config(idxd); + if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) + rc = idxd_device_config(idxd); spin_unlock_irqrestore(&idxd->dev_lock, flags); if (rc < 0) { mutex_unlock(&wq->wq_lock); @@ -170,7 +171,7 @@ static int enable_wq(struct idxd_wq *wq) static int idxd_config_bus_probe(struct device *dev) { - int rc; + int rc = 0; unsigned long flags; dev_dbg(dev, "%s called\n", __func__); @@ -188,7 +189,8 @@ static int idxd_config_bus_probe(struct device *dev) /* Perform IDXD configuration and enabling */ spin_lock_irqsave(&idxd->dev_lock, flags); - rc = idxd_device_config(idxd); + if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) + rc = idxd_device_config(idxd); spin_unlock_irqrestore(&idxd->dev_lock, flags); if (rc < 0) { module_put(THIS_MODULE); @@ -287,12 +289,14 @@ static int idxd_config_bus_remove(struct device *dev) idxd_unregister_dma_device(idxd); rc = idxd_device_disable(idxd); - for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = idxd->wqs[i]; - - mutex_lock(&wq->wq_lock); - idxd_wq_disable_cleanup(wq); - mutex_unlock(&wq->wq_lock); + if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { + for (i = 0; i < idxd->max_wqs; i++) { + struct idxd_wq *wq = idxd->wqs[i]; + + mutex_lock(&wq->wq_lock); + idxd_wq_disable_cleanup(wq); + mutex_unlock(&wq->wq_lock); + } } module_put(THIS_MODULE); if (rc < 0) -- Gitee From 1f1cc2aff904d2571fec5ffab397db0cc264442d Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 20 Apr 2021 11:46:34 -0700 Subject: [PATCH 13/85] dmaengine: idxd: add interrupt handle request and release support ANBZ: #22 commit eb15e7154fbfa3e61c777704b2ff28eb3a0d4796 upstream. DSA spec states that when Request Interrupt Handle and Release Interrupt Handle command bits are set in the CMDCAP register, these device commands must be supported by the driver. The interrupt handle is programmed in a descriptor. When Request Interrupt Handle is not supported, the interrupt handle is the index of the desired entry in the MSI-X table. When the command is supported, driver must use the command to obtain a handle to be programmed in the submitted descriptor. A requested handle may be revoked. After the handle is revoked, any use of the handle will result in Invalid Interrupt Handle error. Intel-SIG: commit eb15e7154fbf dmaengine: idxd: add interrupt handle request and release support. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/161894439422.3202472.17579543737810265471.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 71 ++++++++++++++++++++++++++++++++++++ drivers/dma/idxd/idxd.h | 13 +++++++ drivers/dma/idxd/init.c | 56 +++++++++++++++++++++++++++- drivers/dma/idxd/registers.h | 9 ++++- drivers/dma/idxd/submit.c | 35 ++++++++++++++---- drivers/dma/idxd/sysfs.c | 1 + 6 files changed, 176 insertions(+), 9 deletions(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 3ddb1c731080..54d5afec81cf 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -598,6 +598,77 @@ void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid) dev_dbg(dev, "pasid %d drained\n", pasid); } +int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle, + enum idxd_interrupt_type irq_type) +{ + struct device *dev = &idxd->pdev->dev; + u32 operand, status; + + if (!(idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE))) + return -EOPNOTSUPP; + + dev_dbg(dev, "get int handle, idx %d\n", idx); + + operand = idx & GENMASK(15, 0); + if (irq_type == IDXD_IRQ_IMS) + operand |= CMD_INT_HANDLE_IMS; + + dev_dbg(dev, "cmd: %u operand: %#x\n", IDXD_CMD_REQUEST_INT_HANDLE, operand); + + idxd_cmd_exec(idxd, IDXD_CMD_REQUEST_INT_HANDLE, operand, &status); + + if ((status & IDXD_CMDSTS_ERR_MASK) != IDXD_CMDSTS_SUCCESS) { + dev_dbg(dev, "request int handle failed: %#x\n", status); + return -ENXIO; + } + + *handle = (status >> IDXD_CMDSTS_RES_SHIFT) & GENMASK(15, 0); + + dev_dbg(dev, "int handle acquired: %u\n", *handle); + return 0; +} + +int idxd_device_release_int_handle(struct idxd_device *idxd, int handle, + enum idxd_interrupt_type irq_type) +{ + struct device *dev = &idxd->pdev->dev; + u32 operand, status; + union idxd_command_reg cmd; + unsigned long flags; + + if (!(idxd->hw.cmd_cap & BIT(IDXD_CMD_RELEASE_INT_HANDLE))) + return -EOPNOTSUPP; + + dev_dbg(dev, "release int handle, handle %d\n", handle); + + memset(&cmd, 0, sizeof(cmd)); + operand = handle & GENMASK(15, 0); + + if (irq_type == IDXD_IRQ_IMS) + operand |= CMD_INT_HANDLE_IMS; + + cmd.cmd = IDXD_CMD_RELEASE_INT_HANDLE; + cmd.operand = operand; + + dev_dbg(dev, "cmd: %u operand: %#x\n", IDXD_CMD_RELEASE_INT_HANDLE, operand); + + spin_lock_irqsave(&idxd->dev_lock, flags); + iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET); + + while (ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET) & IDXD_CMDSTS_ACTIVE) + cpu_relax(); + status = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET); + spin_unlock_irqrestore(&idxd->dev_lock, flags); + + if ((status & IDXD_CMDSTS_ERR_MASK) != IDXD_CMDSTS_SUCCESS) { + dev_dbg(dev, "release int handle failed: %#x\n", status); + return -ENXIO; + } + + dev_dbg(dev, "int handle released.\n"); + return 0; +} + /* Device configuration bits */ void idxd_msix_perm_setup(struct idxd_device *idxd) { diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 940a2e1ddf12..c1d4a1976206 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -160,6 +160,7 @@ struct idxd_hw { union group_cap_reg group_cap; union engine_cap_reg engine_cap; struct opcap opcap; + u32 cmd_cap; }; enum idxd_device_state { @@ -237,6 +238,8 @@ struct idxd_device { struct idxd_dma_dev *idxd_dma; struct workqueue_struct *wq; struct work_struct work; + + int *int_handles; }; /* IDXD software descriptor */ @@ -256,6 +259,7 @@ struct idxd_desc { struct list_head list; int id; int cpu; + unsigned int vector; struct idxd_wq *wq; }; @@ -330,6 +334,11 @@ enum idxd_portal_prot { IDXD_PORTAL_LIMITED, }; +enum idxd_interrupt_type { + IDXD_IRQ_MSIX = 0, + IDXD_IRQ_IMS, +}; + static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot) { return prot * 0x1000; @@ -385,6 +394,10 @@ int idxd_device_config(struct idxd_device *idxd); void idxd_device_wqs_clear_state(struct idxd_device *idxd); void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid); int idxd_device_load_config(struct idxd_device *idxd); +int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle, + enum idxd_interrupt_type irq_type); +int idxd_device_release_int_handle(struct idxd_device *idxd, int handle, + enum idxd_interrupt_type irq_type); /* work queue control */ int idxd_wq_alloc_resources(struct idxd_wq *wq); diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 8b5e4adc3ffc..1921f6c4956c 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -125,7 +125,25 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) dev_err(dev, "Failed to allocate irq %d.\n", irq_entry->vector); goto err_wq_irqs; } + dev_dbg(dev, "Allocated idxd-msix %d for vector %d\n", i, irq_entry->vector); + if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)) { + /* + * The MSIX vector enumeration starts at 1 with vector 0 being the + * misc interrupt that handles non I/O completion events. The + * interrupt handles are for IMS enumeration on guest. The misc + * interrupt vector does not require a handle and therefore we start + * the int_handles at index 0. Since 'i' starts at 1, the first + * int_handles index will be 0. + */ + rc = idxd_device_request_int_handle(idxd, i, &idxd->int_handles[i - 1], + IDXD_IRQ_MSIX); + if (rc < 0) { + free_irq(irq_entry->vector, irq_entry); + goto err_wq_irqs; + } + dev_dbg(dev, "int handle requested: %u\n", idxd->int_handles[i - 1]); + } } idxd_unmask_error_interrupts(idxd); @@ -136,6 +154,9 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) while (--i >= 0) { irq_entry = &idxd->irq_entries[i]; free_irq(irq_entry->vector, irq_entry); + if (i != 0) + idxd_device_release_int_handle(idxd, + idxd->int_handles[i], IDXD_IRQ_MSIX); } err_misc_irq: /* Disable error interrupt generation */ @@ -288,9 +309,15 @@ static int idxd_setup_internals(struct idxd_device *idxd) init_waitqueue_head(&idxd->cmd_waitq); + if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)) { + idxd->int_handles = devm_kcalloc(dev, idxd->max_wqs, sizeof(int), GFP_KERNEL); + if (!idxd->int_handles) + return -ENOMEM; + } + rc = idxd_setup_wqs(idxd); if (rc < 0) - return rc; + goto err_wqs; rc = idxd_setup_engines(idxd); if (rc < 0) @@ -317,6 +344,8 @@ static int idxd_setup_internals(struct idxd_device *idxd) err_engine: for (i = 0; i < idxd->max_wqs; i++) put_device(&idxd->wqs[i]->conf_dev); + err_wqs: + kfree(idxd->int_handles); return rc; } @@ -345,6 +374,12 @@ static void idxd_read_caps(struct idxd_device *idxd) /* reading generic capabilities */ idxd->hw.gen_cap.bits = ioread64(idxd->reg_base + IDXD_GENCAP_OFFSET); dev_dbg(dev, "gen_cap: %#llx\n", idxd->hw.gen_cap.bits); + + if (idxd->hw.gen_cap.cmd_cap) { + idxd->hw.cmd_cap = ioread32(idxd->reg_base + IDXD_CMDCAP_OFFSET); + dev_dbg(dev, "cmd_cap: %#x\n", idxd->hw.cmd_cap); + } + idxd->max_xfer_bytes = 1ULL << idxd->hw.gen_cap.max_xfer_shift; dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes); idxd->max_batch_size = 1U << idxd->hw.gen_cap.max_batch_shift; @@ -604,6 +639,24 @@ static void idxd_flush_work_list(struct idxd_irq_entry *ie) } } +static void idxd_release_int_handles(struct idxd_device *idxd) +{ + struct device *dev = &idxd->pdev->dev; + int i, rc; + + for (i = 0; i < idxd->num_wq_irqs; i++) { + if (idxd->hw.cmd_cap & BIT(IDXD_CMD_RELEASE_INT_HANDLE)) { + rc = idxd_device_release_int_handle(idxd, idxd->int_handles[i], + IDXD_IRQ_MSIX); + if (rc < 0) + dev_warn(dev, "irq handle %d release failed\n", + idxd->int_handles[i]); + else + dev_dbg(dev, "int handle requested: %u\n", idxd->int_handles[i]); + } + } +} + static void idxd_shutdown(struct pci_dev *pdev) { struct idxd_device *idxd = pci_get_drvdata(pdev); @@ -630,6 +683,7 @@ static void idxd_shutdown(struct pci_dev *pdev) } idxd_msix_perm_clear(idxd); + idxd_release_int_handles(idxd); pci_free_irq_vectors(pdev); pci_iounmap(pdev, idxd->reg_base); pci_disable_device(pdev); diff --git a/drivers/dma/idxd/registers.h b/drivers/dma/idxd/registers.h index 751ecb4f9f81..5cbf368c7367 100644 --- a/drivers/dma/idxd/registers.h +++ b/drivers/dma/idxd/registers.h @@ -24,8 +24,8 @@ union gen_cap_reg { u64 overlap_copy:1; u64 cache_control_mem:1; u64 cache_control_cache:1; + u64 cmd_cap:1; u64 rsvd:3; - u64 int_handle_req:1; u64 dest_readback:1; u64 drain_readback:1; u64 rsvd2:6; @@ -180,8 +180,11 @@ enum idxd_cmd { IDXD_CMD_DRAIN_PASID, IDXD_CMD_ABORT_PASID, IDXD_CMD_REQUEST_INT_HANDLE, + IDXD_CMD_RELEASE_INT_HANDLE, }; +#define CMD_INT_HANDLE_IMS 0x10000 + #define IDXD_CMDSTS_OFFSET 0xa8 union cmdsts_reg { struct { @@ -193,6 +196,8 @@ union cmdsts_reg { u32 bits; } __packed; #define IDXD_CMDSTS_ACTIVE 0x80000000 +#define IDXD_CMDSTS_ERR_MASK 0xff +#define IDXD_CMDSTS_RES_SHIFT 8 enum idxd_cmdsts_err { IDXD_CMDSTS_SUCCESS = 0, @@ -228,6 +233,8 @@ enum idxd_cmdsts_err { IDXD_CMDSTS_ERR_NO_HANDLE, }; +#define IDXD_CMDCAP_OFFSET 0xb0 + #define IDXD_SWERR_OFFSET 0xc0 #define IDXD_SWERR_VALID 0x00000001 #define IDXD_SWERR_OVERFLOW 0x00000002 diff --git a/drivers/dma/idxd/submit.c b/drivers/dma/idxd/submit.c index 02f9f51e29a6..19afb62abaff 100644 --- a/drivers/dma/idxd/submit.c +++ b/drivers/dma/idxd/submit.c @@ -22,11 +22,23 @@ static struct idxd_desc *__get_desc(struct idxd_wq *wq, int idx, int cpu) desc->hw->pasid = idxd->pasid; /* - * Descriptor completion vectors are 1-8 for MSIX. We will round - * robin through the 8 vectors. + * Descriptor completion vectors are 1...N for MSIX. We will round + * robin through the N vectors. */ wq->vec_ptr = (wq->vec_ptr % idxd->num_wq_irqs) + 1; - desc->hw->int_handle = wq->vec_ptr; + if (!idxd->int_handles) { + desc->hw->int_handle = wq->vec_ptr; + } else { + desc->vector = wq->vec_ptr; + /* + * int_handles are only for descriptor completion. However for device + * MSIX enumeration, vec 0 is used for misc interrupts. Therefore even + * though we are rotating through 1...N for descriptor interrupts, we + * need to acqurie the int_handles from 0..N-1. + */ + desc->hw->int_handle = idxd->int_handles[desc->vector - 1]; + } + return desc; } @@ -79,7 +91,6 @@ void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc) int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc) { struct idxd_device *idxd = wq->idxd; - int vec = desc->hw->int_handle; void __iomem *portal; int rc; @@ -117,9 +128,19 @@ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc) * Pending the descriptor to the lockless list for the irq_entry * that we designated the descriptor to. */ - if (desc->hw->flags & IDXD_OP_FLAG_RCI) - llist_add(&desc->llnode, - &idxd->irq_entries[vec].pending_llist); + if (desc->hw->flags & IDXD_OP_FLAG_RCI) { + int vec; + + /* + * If the driver is on host kernel, it would be the value + * assigned to interrupt handle, which is index for MSIX + * vector. If it's guest then can't use the int_handle since + * that is the index to IMS for the entire device. The guest + * device local index will be used. + */ + vec = !idxd->int_handles ? desc->hw->int_handle : desc->vector; + llist_add(&desc->llnode, &idxd->irq_entries[vec].pending_llist); + } return 0; } diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index d45cb61f300b..3f4ea4d0fae7 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -1600,6 +1600,7 @@ static void idxd_conf_device_release(struct device *dev) kfree(idxd->wqs); kfree(idxd->engines); kfree(idxd->irq_entries); + kfree(idxd->int_handles); ida_free(&idxd_ida, idxd->id); kfree(idxd); } -- Gitee From b4871938126b1b85c548541395cb641e62d7d5f5 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 20 Apr 2021 11:46:40 -0700 Subject: [PATCH 14/85] dmaengine: idxd: convert sprintf() to sysfs_emit() for all usages ANBZ: #22 commit 8241571fac9eeb7f3424ad343369eaa411919da3 upstream. Convert sprintf() to sysfs_emit() in order to check buffer overrun on sysfs outputs. Intel-SIG: commit 8241571fac9e dmaengine: idxd: convert sprintf() to sysfs_emit() for all usages. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/161894440044.3202472.13926639619695319753.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/sysfs.c | 116 +++++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 61 deletions(-) diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 3f4ea4d0fae7..0460d58e3941 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -350,9 +350,9 @@ static ssize_t engine_group_id_show(struct device *dev, container_of(dev, struct idxd_engine, conf_dev); if (engine->group) - return sprintf(buf, "%d\n", engine->group->id); + return sysfs_emit(buf, "%d\n", engine->group->id); else - return sprintf(buf, "%d\n", -1); + return sysfs_emit(buf, "%d\n", -1); } static ssize_t engine_group_id_store(struct device *dev, @@ -447,7 +447,7 @@ static ssize_t group_tokens_reserved_show(struct device *dev, struct idxd_group *group = container_of(dev, struct idxd_group, conf_dev); - return sprintf(buf, "%u\n", group->tokens_reserved); + return sysfs_emit(buf, "%u\n", group->tokens_reserved); } static ssize_t group_tokens_reserved_store(struct device *dev, @@ -495,7 +495,7 @@ static ssize_t group_tokens_allowed_show(struct device *dev, struct idxd_group *group = container_of(dev, struct idxd_group, conf_dev); - return sprintf(buf, "%u\n", group->tokens_allowed); + return sysfs_emit(buf, "%u\n", group->tokens_allowed); } static ssize_t group_tokens_allowed_store(struct device *dev, @@ -540,7 +540,7 @@ static ssize_t group_use_token_limit_show(struct device *dev, struct idxd_group *group = container_of(dev, struct idxd_group, conf_dev); - return sprintf(buf, "%u\n", group->use_token_limit); + return sysfs_emit(buf, "%u\n", group->use_token_limit); } static ssize_t group_use_token_limit_store(struct device *dev, @@ -583,7 +583,6 @@ static ssize_t group_engines_show(struct device *dev, struct idxd_group *group = container_of(dev, struct idxd_group, conf_dev); int i, rc = 0; - char *tmp = buf; struct idxd_device *idxd = group->idxd; for (i = 0; i < idxd->max_engines; i++) { @@ -593,12 +592,13 @@ static ssize_t group_engines_show(struct device *dev, continue; if (engine->group->id == group->id) - rc += sprintf(tmp + rc, "engine%d.%d ", - idxd->id, engine->id); + rc += sysfs_emit_at(buf, rc, "engine%d.%d ", idxd->id, engine->id); } + if (!rc) + return 0; rc--; - rc += sprintf(tmp + rc, "\n"); + rc += sysfs_emit_at(buf, rc, "\n"); return rc; } @@ -612,7 +612,6 @@ static ssize_t group_work_queues_show(struct device *dev, struct idxd_group *group = container_of(dev, struct idxd_group, conf_dev); int i, rc = 0; - char *tmp = buf; struct idxd_device *idxd = group->idxd; for (i = 0; i < idxd->max_wqs; i++) { @@ -622,12 +621,13 @@ static ssize_t group_work_queues_show(struct device *dev, continue; if (wq->group->id == group->id) - rc += sprintf(tmp + rc, "wq%d.%d ", - idxd->id, wq->id); + rc += sysfs_emit_at(buf, rc, "wq%d.%d ", idxd->id, wq->id); } + if (!rc) + return 0; rc--; - rc += sprintf(tmp + rc, "\n"); + rc += sysfs_emit_at(buf, rc, "\n"); return rc; } @@ -642,7 +642,7 @@ static ssize_t group_traffic_class_a_show(struct device *dev, struct idxd_group *group = container_of(dev, struct idxd_group, conf_dev); - return sprintf(buf, "%d\n", group->tc_a); + return sysfs_emit(buf, "%d\n", group->tc_a); } static ssize_t group_traffic_class_a_store(struct device *dev, @@ -683,7 +683,7 @@ static ssize_t group_traffic_class_b_show(struct device *dev, struct idxd_group *group = container_of(dev, struct idxd_group, conf_dev); - return sprintf(buf, "%d\n", group->tc_b); + return sysfs_emit(buf, "%d\n", group->tc_b); } static ssize_t group_traffic_class_b_store(struct device *dev, @@ -756,7 +756,7 @@ static ssize_t wq_clients_show(struct device *dev, { struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); - return sprintf(buf, "%d\n", wq->client_count); + return sysfs_emit(buf, "%d\n", wq->client_count); } static struct device_attribute dev_attr_wq_clients = @@ -769,12 +769,12 @@ static ssize_t wq_state_show(struct device *dev, switch (wq->state) { case IDXD_WQ_DISABLED: - return sprintf(buf, "disabled\n"); + return sysfs_emit(buf, "disabled\n"); case IDXD_WQ_ENABLED: - return sprintf(buf, "enabled\n"); + return sysfs_emit(buf, "enabled\n"); } - return sprintf(buf, "unknown\n"); + return sysfs_emit(buf, "unknown\n"); } static struct device_attribute dev_attr_wq_state = @@ -786,9 +786,9 @@ static ssize_t wq_group_id_show(struct device *dev, struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); if (wq->group) - return sprintf(buf, "%u\n", wq->group->id); + return sysfs_emit(buf, "%u\n", wq->group->id); else - return sprintf(buf, "-1\n"); + return sysfs_emit(buf, "-1\n"); } static ssize_t wq_group_id_store(struct device *dev, @@ -840,8 +840,7 @@ static ssize_t wq_mode_show(struct device *dev, struct device_attribute *attr, { struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); - return sprintf(buf, "%s\n", - wq_dedicated(wq) ? "dedicated" : "shared"); + return sysfs_emit(buf, "%s\n", wq_dedicated(wq) ? "dedicated" : "shared"); } static ssize_t wq_mode_store(struct device *dev, @@ -877,7 +876,7 @@ static ssize_t wq_size_show(struct device *dev, struct device_attribute *attr, { struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); - return sprintf(buf, "%u\n", wq->size); + return sysfs_emit(buf, "%u\n", wq->size); } static int total_claimed_wq_size(struct idxd_device *idxd) @@ -928,7 +927,7 @@ static ssize_t wq_priority_show(struct device *dev, { struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); - return sprintf(buf, "%u\n", wq->priority); + return sysfs_emit(buf, "%u\n", wq->priority); } static ssize_t wq_priority_store(struct device *dev, @@ -965,8 +964,7 @@ static ssize_t wq_block_on_fault_show(struct device *dev, { struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); - return sprintf(buf, "%u\n", - test_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags)); + return sysfs_emit(buf, "%u\n", test_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags)); } static ssize_t wq_block_on_fault_store(struct device *dev, @@ -1005,7 +1003,7 @@ static ssize_t wq_threshold_show(struct device *dev, { struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); - return sprintf(buf, "%u\n", wq->threshold); + return sysfs_emit(buf, "%u\n", wq->threshold); } static ssize_t wq_threshold_store(struct device *dev, @@ -1048,15 +1046,12 @@ static ssize_t wq_type_show(struct device *dev, switch (wq->type) { case IDXD_WQT_KERNEL: - return sprintf(buf, "%s\n", - idxd_wq_type_names[IDXD_WQT_KERNEL]); + return sysfs_emit(buf, "%s\n", idxd_wq_type_names[IDXD_WQT_KERNEL]); case IDXD_WQT_USER: - return sprintf(buf, "%s\n", - idxd_wq_type_names[IDXD_WQT_USER]); + return sysfs_emit(buf, "%s\n", idxd_wq_type_names[IDXD_WQT_USER]); case IDXD_WQT_NONE: default: - return sprintf(buf, "%s\n", - idxd_wq_type_names[IDXD_WQT_NONE]); + return sysfs_emit(buf, "%s\n", idxd_wq_type_names[IDXD_WQT_NONE]); } return -EINVAL; @@ -1097,7 +1092,7 @@ static ssize_t wq_name_show(struct device *dev, { struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); - return sprintf(buf, "%s\n", wq->name); + return sysfs_emit(buf, "%s\n", wq->name); } static ssize_t wq_name_store(struct device *dev, @@ -1167,7 +1162,7 @@ static ssize_t wq_max_transfer_size_show(struct device *dev, struct device_attri { struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); - return sprintf(buf, "%llu\n", wq->max_xfer_bytes); + return sysfs_emit(buf, "%llu\n", wq->max_xfer_bytes); } static ssize_t wq_max_transfer_size_store(struct device *dev, struct device_attribute *attr, @@ -1201,7 +1196,7 @@ static ssize_t wq_max_batch_size_show(struct device *dev, struct device_attribut { struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); - return sprintf(buf, "%u\n", wq->max_batch_size); + return sysfs_emit(buf, "%u\n", wq->max_batch_size); } static ssize_t wq_max_batch_size_store(struct device *dev, struct device_attribute *attr, @@ -1234,7 +1229,7 @@ static ssize_t wq_ats_disable_show(struct device *dev, struct device_attribute * { struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); - return sprintf(buf, "%u\n", wq->ats_dis); + return sysfs_emit(buf, "%u\n", wq->ats_dis); } static ssize_t wq_ats_disable_store(struct device *dev, struct device_attribute *attr, @@ -1311,7 +1306,7 @@ static ssize_t version_show(struct device *dev, struct device_attribute *attr, struct idxd_device *idxd = container_of(dev, struct idxd_device, conf_dev); - return sprintf(buf, "%#x\n", idxd->hw.version); + return sysfs_emit(buf, "%#x\n", idxd->hw.version); } static DEVICE_ATTR_RO(version); @@ -1322,7 +1317,7 @@ static ssize_t max_work_queues_size_show(struct device *dev, struct idxd_device *idxd = container_of(dev, struct idxd_device, conf_dev); - return sprintf(buf, "%u\n", idxd->max_wq_size); + return sysfs_emit(buf, "%u\n", idxd->max_wq_size); } static DEVICE_ATTR_RO(max_work_queues_size); @@ -1332,7 +1327,7 @@ static ssize_t max_groups_show(struct device *dev, struct idxd_device *idxd = container_of(dev, struct idxd_device, conf_dev); - return sprintf(buf, "%u\n", idxd->max_groups); + return sysfs_emit(buf, "%u\n", idxd->max_groups); } static DEVICE_ATTR_RO(max_groups); @@ -1342,7 +1337,7 @@ static ssize_t max_work_queues_show(struct device *dev, struct idxd_device *idxd = container_of(dev, struct idxd_device, conf_dev); - return sprintf(buf, "%u\n", idxd->max_wqs); + return sysfs_emit(buf, "%u\n", idxd->max_wqs); } static DEVICE_ATTR_RO(max_work_queues); @@ -1352,7 +1347,7 @@ static ssize_t max_engines_show(struct device *dev, struct idxd_device *idxd = container_of(dev, struct idxd_device, conf_dev); - return sprintf(buf, "%u\n", idxd->max_engines); + return sysfs_emit(buf, "%u\n", idxd->max_engines); } static DEVICE_ATTR_RO(max_engines); @@ -1362,7 +1357,7 @@ static ssize_t numa_node_show(struct device *dev, struct idxd_device *idxd = container_of(dev, struct idxd_device, conf_dev); - return sprintf(buf, "%d\n", dev_to_node(&idxd->pdev->dev)); + return sysfs_emit(buf, "%d\n", dev_to_node(&idxd->pdev->dev)); } static DEVICE_ATTR_RO(numa_node); @@ -1372,7 +1367,7 @@ static ssize_t max_batch_size_show(struct device *dev, struct idxd_device *idxd = container_of(dev, struct idxd_device, conf_dev); - return sprintf(buf, "%u\n", idxd->max_batch_size); + return sysfs_emit(buf, "%u\n", idxd->max_batch_size); } static DEVICE_ATTR_RO(max_batch_size); @@ -1383,7 +1378,7 @@ static ssize_t max_transfer_size_show(struct device *dev, struct idxd_device *idxd = container_of(dev, struct idxd_device, conf_dev); - return sprintf(buf, "%llu\n", idxd->max_xfer_bytes); + return sysfs_emit(buf, "%llu\n", idxd->max_xfer_bytes); } static DEVICE_ATTR_RO(max_transfer_size); @@ -1409,7 +1404,7 @@ static ssize_t gen_cap_show(struct device *dev, struct idxd_device *idxd = container_of(dev, struct idxd_device, conf_dev); - return sprintf(buf, "%#llx\n", idxd->hw.gen_cap.bits); + return sysfs_emit(buf, "%#llx\n", idxd->hw.gen_cap.bits); } static DEVICE_ATTR_RO(gen_cap); @@ -1419,8 +1414,7 @@ static ssize_t configurable_show(struct device *dev, struct idxd_device *idxd = container_of(dev, struct idxd_device, conf_dev); - return sprintf(buf, "%u\n", - test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)); + return sysfs_emit(buf, "%u\n", test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)); } static DEVICE_ATTR_RO(configurable); @@ -1440,7 +1434,7 @@ static ssize_t clients_show(struct device *dev, } spin_unlock_irqrestore(&idxd->dev_lock, flags); - return sprintf(buf, "%d\n", count); + return sysfs_emit(buf, "%d\n", count); } static DEVICE_ATTR_RO(clients); @@ -1450,7 +1444,7 @@ static ssize_t pasid_enabled_show(struct device *dev, struct idxd_device *idxd = container_of(dev, struct idxd_device, conf_dev); - return sprintf(buf, "%u\n", device_pasid_enabled(idxd)); + return sysfs_emit(buf, "%u\n", device_pasid_enabled(idxd)); } static DEVICE_ATTR_RO(pasid_enabled); @@ -1463,14 +1457,14 @@ static ssize_t state_show(struct device *dev, switch (idxd->state) { case IDXD_DEV_DISABLED: case IDXD_DEV_CONF_READY: - return sprintf(buf, "disabled\n"); + return sysfs_emit(buf, "disabled\n"); case IDXD_DEV_ENABLED: - return sprintf(buf, "enabled\n"); + return sysfs_emit(buf, "enabled\n"); case IDXD_DEV_HALTED: - return sprintf(buf, "halted\n"); + return sysfs_emit(buf, "halted\n"); } - return sprintf(buf, "unknown\n"); + return sysfs_emit(buf, "unknown\n"); } static DEVICE_ATTR_RO(state); @@ -1484,10 +1478,10 @@ static ssize_t errors_show(struct device *dev, spin_lock_irqsave(&idxd->dev_lock, flags); for (i = 0; i < 4; i++) - out += sprintf(buf + out, "%#018llx ", idxd->sw_err.bits[i]); + out += sysfs_emit_at(buf, out, "%#018llx ", idxd->sw_err.bits[i]); spin_unlock_irqrestore(&idxd->dev_lock, flags); out--; - out += sprintf(buf + out, "\n"); + out += sysfs_emit_at(buf, out, "\n"); return out; } static DEVICE_ATTR_RO(errors); @@ -1498,7 +1492,7 @@ static ssize_t max_tokens_show(struct device *dev, struct idxd_device *idxd = container_of(dev, struct idxd_device, conf_dev); - return sprintf(buf, "%u\n", idxd->max_tokens); + return sysfs_emit(buf, "%u\n", idxd->max_tokens); } static DEVICE_ATTR_RO(max_tokens); @@ -1508,7 +1502,7 @@ static ssize_t token_limit_show(struct device *dev, struct idxd_device *idxd = container_of(dev, struct idxd_device, conf_dev); - return sprintf(buf, "%u\n", idxd->token_limit); + return sysfs_emit(buf, "%u\n", idxd->token_limit); } static ssize_t token_limit_store(struct device *dev, @@ -1547,7 +1541,7 @@ static ssize_t cdev_major_show(struct device *dev, struct idxd_device *idxd = container_of(dev, struct idxd_device, conf_dev); - return sprintf(buf, "%u\n", idxd->major); + return sysfs_emit(buf, "%u\n", idxd->major); } static DEVICE_ATTR_RO(cdev_major); @@ -1556,7 +1550,7 @@ static ssize_t cmd_status_show(struct device *dev, { struct idxd_device *idxd = container_of(dev, struct idxd_device, conf_dev); - return sprintf(buf, "%#x\n", idxd->cmd_status); + return sysfs_emit(buf, "%#x\n", idxd->cmd_status); } static DEVICE_ATTR_RO(cmd_status); -- Gitee From bb2aa67ae60ed9cc5a82205432d1128dc408583a Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 20 Apr 2021 11:46:46 -0700 Subject: [PATCH 15/85] dmaengine: idxd: enable SVA feature for IOMMU ANBZ: #22 commit cf5f86a7d47df149857ba2fb72f9c6c9da46af2e upstream. Enable IOMMU_DEV_FEAT_SVA before attempt to bind pasid. This is needed according to iommu_sva_bind_device() comment. Currently Intel IOMMU code does this before bind call. It really needs to be controlled by the driver. Intel-SIG: commit cf5f86a7d47d dmaengine: idxd: enable SVA feature for IOMMU. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/161894440621.3202472.17644507396206848134.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/init.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 1921f6c4956c..4cd56f159e2a 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -501,11 +501,18 @@ static int idxd_probe(struct idxd_device *idxd) dev_dbg(dev, "IDXD reset complete\n"); if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) { - rc = idxd_enable_system_pasid(idxd); - if (rc < 0) - dev_warn(dev, "Failed to enable PASID. No SVA support: %d\n", rc); - else - set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); + rc = iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA); + if (rc == 0) { + rc = idxd_enable_system_pasid(idxd); + if (rc < 0) { + iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA); + dev_warn(dev, "Failed to enable PASID. No SVA support: %d\n", rc); + } else { + set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); + } + } else { + dev_warn(dev, "Unable to turn on SVA feature.\n"); + } } else if (!sva) { dev_warn(dev, "User forced SVA off via module param.\n"); } @@ -539,6 +546,7 @@ static int idxd_probe(struct idxd_device *idxd) err: if (device_pasid_enabled(idxd)) idxd_disable_system_pasid(idxd); + iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA); return rc; } @@ -699,6 +707,7 @@ static void idxd_remove(struct pci_dev *pdev) if (device_pasid_enabled(idxd)) idxd_disable_system_pasid(idxd); idxd_unregister_devices(idxd); + iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); } static struct pci_driver idxd_pci_driver = { -- Gitee From 7c4da818ccb0a02cccb05a7d7466caeea72ddb52 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 20 Apr 2021 11:46:51 -0700 Subject: [PATCH 16/85] dmaengine: idxd: support reporting of halt interrupt ANBZ: #22 commit 5b0c68c473a131c2acb21abad44b0047b200e185 upstream. Unmask the halt error interrupt so it gets reported to the interrupt handler. When halt state interrupt is received, quiesce the kernel WQs and unmap the portals to stop submission. Intel-SIG: commit 5b0c68c473a1 dmaengine: idxd: support reporting of halt interrupt. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/161894441167.3202472.9485946398140619501.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 15 +++++++++++++++ drivers/dma/idxd/idxd.h | 2 ++ drivers/dma/idxd/init.c | 12 ++++++++++++ drivers/dma/idxd/irq.c | 2 ++ drivers/dma/idxd/registers.h | 3 ++- 5 files changed, 33 insertions(+), 1 deletion(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 54d5afec81cf..3934e660d951 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -47,6 +47,7 @@ void idxd_unmask_error_interrupts(struct idxd_device *idxd) genctrl.bits = ioread32(idxd->reg_base + IDXD_GENCTRL_OFFSET); genctrl.softerr_int_en = 1; + genctrl.halt_int_en = 1; iowrite32(genctrl.bits, idxd->reg_base + IDXD_GENCTRL_OFFSET); } @@ -56,6 +57,7 @@ void idxd_mask_error_interrupts(struct idxd_device *idxd) genctrl.bits = ioread32(idxd->reg_base + IDXD_GENCTRL_OFFSET); genctrl.softerr_int_en = 0; + genctrl.halt_int_en = 0; iowrite32(genctrl.bits, idxd->reg_base + IDXD_GENCTRL_OFFSET); } @@ -312,6 +314,19 @@ void idxd_wq_unmap_portal(struct idxd_wq *wq) struct device *dev = &wq->idxd->pdev->dev; devm_iounmap(dev, wq->portal); + wq->portal = NULL; +} + +void idxd_wqs_unmap_portal(struct idxd_device *idxd) +{ + int i; + + for (i = 0; i < idxd->max_wqs; i++) { + struct idxd_wq *wq = idxd->wqs[i]; + + if (wq->portal) + idxd_wq_unmap_portal(wq); + } } int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid) diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index c1d4a1976206..d7185c6bfade 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -371,6 +371,7 @@ int idxd_register_devices(struct idxd_device *idxd); void idxd_unregister_devices(struct idxd_device *idxd); int idxd_register_driver(void); void idxd_unregister_driver(void); +void idxd_wqs_quiesce(struct idxd_device *idxd); /* device interrupt control */ void idxd_msix_perm_setup(struct idxd_device *idxd); @@ -400,6 +401,7 @@ int idxd_device_release_int_handle(struct idxd_device *idxd, int handle, enum idxd_interrupt_type irq_type); /* work queue control */ +void idxd_wqs_unmap_portal(struct idxd_device *idxd); int idxd_wq_alloc_resources(struct idxd_wq *wq); void idxd_wq_free_resources(struct idxd_wq *wq); int idxd_wq_enable(struct idxd_wq *wq); diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 4cd56f159e2a..6e325f185f15 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -647,6 +647,18 @@ static void idxd_flush_work_list(struct idxd_irq_entry *ie) } } +void idxd_wqs_quiesce(struct idxd_device *idxd) +{ + struct idxd_wq *wq; + int i; + + for (i = 0; i < idxd->max_wqs; i++) { + wq = idxd->wqs[i]; + if (wq->state == IDXD_WQ_ENABLED && wq->type == IDXD_WQT_KERNEL) + idxd_wq_quiesce(wq); + } +} + static void idxd_release_int_handles(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c index fc0781e3f36d..43eea5c9cbd4 100644 --- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -202,6 +202,8 @@ static int process_misc_interrupts(struct idxd_device *idxd, u32 cause) queue_work(idxd->wq, &idxd->work); } else { spin_lock_bh(&idxd->dev_lock); + idxd_wqs_quiesce(idxd); + idxd_wqs_unmap_portal(idxd); idxd_device_wqs_clear_state(idxd); dev_err(&idxd->pdev->dev, "idxd halted, need %s.\n", diff --git a/drivers/dma/idxd/registers.h b/drivers/dma/idxd/registers.h index 5cbf368c7367..6c11375cc56a 100644 --- a/drivers/dma/idxd/registers.h +++ b/drivers/dma/idxd/registers.h @@ -120,7 +120,8 @@ union gencfg_reg { union genctrl_reg { struct { u32 softerr_int_en:1; - u32 rsvd:31; + u32 halt_int_en:1; + u32 rsvd:30; }; u32 bits; } __packed; -- Gitee From 2bc28a97ac33ddf236e4a190a12a5740ea6571bc Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 20 Apr 2021 12:00:56 -0700 Subject: [PATCH 17/85] dmaengine: idxd: device cmd should use dedicated lock ANBZ: #22 commit 53b2ee7f637c4f1fa2f50dbdb210088e30c11d2b upstream. Create a dedicated lock for device command operations. Put the device command operation under finer grained locking instead of using the idxd->dev_lock. Intel-SIG: commit 53b2ee7f637c dmaengine: idxd: device cmd should use dedicated lock. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Suggested-by: Sanjay Kumar Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/161894525685.3210132.16160045731436382560.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 18 +++++++++--------- drivers/dma/idxd/idxd.h | 1 + drivers/dma/idxd/init.c | 1 + 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 3934e660d951..420b93fe5feb 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -465,13 +465,13 @@ int idxd_device_init_reset(struct idxd_device *idxd) memset(&cmd, 0, sizeof(cmd)); cmd.cmd = IDXD_CMD_RESET_DEVICE; dev_dbg(dev, "%s: sending reset for init.\n", __func__); - spin_lock_irqsave(&idxd->dev_lock, flags); + spin_lock_irqsave(&idxd->cmd_lock, flags); iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET); while (ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET) & IDXD_CMDSTS_ACTIVE) cpu_relax(); - spin_unlock_irqrestore(&idxd->dev_lock, flags); + spin_unlock_irqrestore(&idxd->cmd_lock, flags); return 0; } @@ -494,10 +494,10 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand, cmd.operand = operand; cmd.int_req = 1; - spin_lock_irqsave(&idxd->dev_lock, flags); + spin_lock_irqsave(&idxd->cmd_lock, flags); wait_event_lock_irq(idxd->cmd_waitq, !test_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags), - idxd->dev_lock); + idxd->cmd_lock); dev_dbg(&idxd->pdev->dev, "%s: sending cmd: %#x op: %#x\n", __func__, cmd_code, operand); @@ -511,9 +511,9 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand, * After command submitted, release lock and go to sleep until * the command completes via interrupt. */ - spin_unlock_irqrestore(&idxd->dev_lock, flags); + spin_unlock_irqrestore(&idxd->cmd_lock, flags); wait_for_completion(&done); - spin_lock_irqsave(&idxd->dev_lock, flags); + spin_lock_irqsave(&idxd->cmd_lock, flags); if (status) { *status = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET); idxd->cmd_status = *status & GENMASK(7, 0); @@ -522,7 +522,7 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand, __clear_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags); /* Wake up other pending commands */ wake_up(&idxd->cmd_waitq); - spin_unlock_irqrestore(&idxd->dev_lock, flags); + spin_unlock_irqrestore(&idxd->cmd_lock, flags); } int idxd_device_enable(struct idxd_device *idxd) @@ -667,13 +667,13 @@ int idxd_device_release_int_handle(struct idxd_device *idxd, int handle, dev_dbg(dev, "cmd: %u operand: %#x\n", IDXD_CMD_RELEASE_INT_HANDLE, operand); - spin_lock_irqsave(&idxd->dev_lock, flags); + spin_lock_irqsave(&idxd->cmd_lock, flags); iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET); while (ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET) & IDXD_CMDSTS_ACTIVE) cpu_relax(); status = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET); - spin_unlock_irqrestore(&idxd->dev_lock, flags); + spin_unlock_irqrestore(&idxd->cmd_lock, flags); if ((status & IDXD_CMDSTS_ERR_MASK) != IDXD_CMDSTS_SUCCESS) { dev_dbg(dev, "release int handle failed: %#x\n", status); diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index d7185c6bfade..87330b6940fa 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -204,6 +204,7 @@ struct idxd_device { void __iomem *reg_base; spinlock_t dev_lock; /* spinlock for device */ + spinlock_t cmd_lock; /* spinlock for device commands */ struct completion *cmd_done; struct idxd_group **groups; struct idxd_wq **wqs; diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 6e325f185f15..c7e2ab2295e7 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -449,6 +449,7 @@ static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_d } spin_lock_init(&idxd->dev_lock); + spin_lock_init(&idxd->cmd_lock); return idxd; } -- Gitee From 3b779d19ee58ead721f88836675a677b5a7b508b Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 20 Apr 2021 12:00:34 -0700 Subject: [PATCH 18/85] dmaengine: idxd: remove MSIX masking for interrupt handlers ANBZ: #22 commit a16104617d212d4b482568847b25172972b87e60 upstream. Remove interrupt masking and just let the hard irq handler keep firing for new events. This is less of a performance impact vs the MMIO readback inside the pci_msi_{mask,unmas}_irq(). Especially with a loaded system those flushes can be stuck behind large amounts of MMIO writes to flush. When guest kernel is running on top of VFIO mdev, mask/unmask causes a vmexit each time and is not desirable. Intel-SIG: commit a16104617d21 dmaengine: idxd: remove MSIX masking for interrupt handlers. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Suggested-by: Dan Williams Signed-off-by: Dave Jiang Reviewed-by: Dan Williams Link: https://lore.kernel.org/r/161894523436.3210025.1834640110556139277.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/idxd.h | 1 - drivers/dma/idxd/init.c | 4 ++-- drivers/dma/idxd/irq.c | 12 ------------ 3 files changed, 2 insertions(+), 15 deletions(-) diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 87330b6940fa..97c96ca6ab70 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -377,7 +377,6 @@ void idxd_wqs_quiesce(struct idxd_device *idxd); /* device interrupt control */ void idxd_msix_perm_setup(struct idxd_device *idxd); void idxd_msix_perm_clear(struct idxd_device *idxd); -irqreturn_t idxd_irq_handler(int vec, void *data); irqreturn_t idxd_misc_thread(int vec, void *data); irqreturn_t idxd_wq_thread(int irq, void *data); void idxd_mask_error_interrupts(struct idxd_device *idxd); diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index c7e2ab2295e7..5ef0caa057d3 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -102,7 +102,7 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) } irq_entry = &idxd->irq_entries[0]; - rc = request_threaded_irq(irq_entry->vector, idxd_irq_handler, idxd_misc_thread, + rc = request_threaded_irq(irq_entry->vector, NULL, idxd_misc_thread, 0, "idxd-misc", irq_entry); if (rc < 0) { dev_err(dev, "Failed to allocate misc interrupt.\n"); @@ -119,7 +119,7 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) init_llist_head(&idxd->irq_entries[i].pending_llist); INIT_LIST_HEAD(&idxd->irq_entries[i].work_list); - rc = request_threaded_irq(irq_entry->vector, idxd_irq_handler, + rc = request_threaded_irq(irq_entry->vector, NULL, idxd_wq_thread, 0, "idxd-portal", irq_entry); if (rc < 0) { dev_err(dev, "Failed to allocate irq %d.\n", irq_entry->vector); diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c index 43eea5c9cbd4..afee571e0194 100644 --- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -102,15 +102,6 @@ static int idxd_device_schedule_fault_process(struct idxd_device *idxd, return 0; } -irqreturn_t idxd_irq_handler(int vec, void *data) -{ - struct idxd_irq_entry *irq_entry = data; - struct idxd_device *idxd = irq_entry->idxd; - - idxd_mask_msix_vector(idxd, irq_entry->id); - return IRQ_WAKE_THREAD; -} - static int process_misc_interrupts(struct idxd_device *idxd, u32 cause) { struct device *dev = &idxd->pdev->dev; @@ -237,7 +228,6 @@ irqreturn_t idxd_misc_thread(int vec, void *data) iowrite32(cause, idxd->reg_base + IDXD_INTCAUSE_OFFSET); } - idxd_unmask_msix_vector(idxd, irq_entry->id); return IRQ_HANDLED; } @@ -394,8 +384,6 @@ irqreturn_t idxd_wq_thread(int irq, void *data) int processed; processed = idxd_desc_process(irq_entry); - idxd_unmask_msix_vector(irq_entry->idxd, irq_entry->id); - if (processed == 0) return IRQ_NONE; -- Gitee From 317fb71f73a9427970a3b451c5c0f0bf78b13565 Mon Sep 17 00:00:00 2001 From: Tom Zanussi Date: Sat, 24 Apr 2021 10:04:15 -0500 Subject: [PATCH 19/85] dmaengine: idxd: Add IDXD performance monitor support ANBZ: #22 commit 81dd4d4d6178306ab31db91bdc7353d485bdafce upstream. Implement the IDXD performance monitor capability (named 'perfmon' in the DSA (Data Streaming Accelerator) spec [1]), which supports the collection of information about key events occurring during DSA and IAX (Intel Analytics Accelerator) device execution, to assist in performance tuning and debugging. The idxd perfmon support is implemented as part of the IDXD driver and interfaces with the Linux perf framework. It has several features in common with the existing uncore pmu support: - it does not support sampling - does not support per-thread counting However it also has some unique features not present in the core and uncore support: - all general-purpose counters are identical, thus no event constraints - operation is always system-wide While the core perf subsystem assumes that all counters are by default per-cpu, the uncore pmus are socket-scoped and use a cpu mask to restrict counting to one cpu from each socket. IDXD counters use a similar strategy but expand the scope even further; since IDXD counters are system-wide and can be read from any cpu, the IDXD perf driver picks a single cpu to do the work (with cpu hotplug notifiers to choose a different cpu if the chosen one is taken off-line). More specifically, the perf userspace tool by default opens a counter for each cpu for an event. However, if it finds a cpumask file associated with the pmu under sysfs, as is the case with the uncore pmus, it will open counters only on the cpus specified by the cpumask. Since perfmon only needs to open a single counter per event for a given IDXD device, the perfmon driver will create a sysfs cpumask file for the device and insert the first cpu of the system into it. When a user uses perf to open an event, perf will open a single counter on the cpu specified by the cpu mask. This amounts to the default system-wide rather than per-cpu counting mentioned previously for perfmon pmu events. In order to keep the cpu mask up-to-date, the driver implements cpu hotplug support for multiple devices, as IDXD usually enumerates and registers more than one idxd device. The perfmon driver implements basic perfmon hardware capability discovery and configuration, and is initialized by the IDXD driver's probe function. During initialization, the driver retrieves the total number of supported performance counters, the pmu ID, and the device type from idxd device, and registers itself under the Linux perf framework. The perf userspace tool can be used to monitor single or multiple events depending on the given configuration, as well as event groups, which are also supported by the perfmon driver. The user configures events using the perf tool command-line interface by specifying the event and corresponding event category, along with an optional set of filters that can be used to restrict counting to specific work queues, traffic classes, page and transfer sizes, and engines (See [1] for specifics). With the configuration specified by the user, the perf tool issues a system call passing that information to the kernel, which uses it to initialize the specified event(s). The event(s) are opened and started, and following termination of the perf command, they're stopped. At that point, the perfmon driver will read the latest count for the event(s), calculate the difference between the latest counter values and previously tracked counter values, and display the final incremental count as the event count for the cycle. An overflow handler registered on the IDXD irq path is used to account for counter overflows, which are signaled by an overflow interrupt. Below are a couple of examples of perf usage for monitoring DSA events. The following monitors all events in the 'engine' category. Becuuse no filters are specified, this captures all engine events for the workload, which in this case is 19 iterations of the work generated by the kernel dmatest module. Details describing the events can be found in Appendix D of [1], Performance Monitoring Events, but briefly they are: event 0x1: total input data processed, in 32-byte units event 0x2: total data written, in 32-byte units event 0x4: number of work descriptors that read the source event 0x8: number of work descriptors that write the destination event 0x10: number of work descriptors dispatched from batch descriptors event 0x20: number of work descriptors dispatched from work queues # perf stat -e dsa0/event=0x1,event_category=0x1/, dsa0/event=0x2,event_category=0x1/, dsa0/event=0x4,event_category=0x1/, dsa0/event=0x8,event_category=0x1/, dsa0/event=0x10,event_category=0x1/, dsa0/event=0x20,event_category=0x1/ modprobe dmatest channel=dma0chan0 timeout=2000 iterations=19 run=1 wait=1 Performance counter stats for 'system wide': 5,332 dsa0/event=0x1,event_category=0x1/ 5,327 dsa0/event=0x2,event_category=0x1/ 19 dsa0/event=0x4,event_category=0x1/ 19 dsa0/event=0x8,event_category=0x1/ 0 dsa0/event=0x10,event_category=0x1/ 19 dsa0/event=0x20,event_category=0x1/ 21.977436186 seconds time elapsed The command below illustrates filter usage with a simple example. It specifies that MEM_MOVE operations should be counted for the DSA device dsa0 (event 0x8 corresponds to the EV_MEM_MOVE event - Number of Memory Move Descriptors, which is part of event category 0x3 - Operations. The detailed category and event IDs are available in Appendix D, Performance Monitoring Events, of [1]). In addition to the event and event category, a number of filters are also specified (the detailed filter values are available in Chapter 6.4 (Filter Support) of [1]), which will restrict counting to only those events that meet all of the filter criteria. In this case, the filters specify that only MEM_MOVE operations that are serviced by work queue wq0 and specifically engine number engine0 and traffic class tc0 having sizes between 0 and 4k and page size of between 0 and 1G result in a counter hit; anything else will be filtered out and not appear in the final count. Note that filters are optional - any filter not specified is assumed to be all ones and will pass anything. # perf stat -e dsa0/filter_wq=0x1,filter_tc=0x1,filter_sz=0x7, filter_eng=0x1,event=0x8,event_category=0x3/ modprobe dmatest channel=dma0chan0 timeout=2000 iterations=19 run=1 wait=1 Performance counter stats for 'system wide': 19 dsa0/filter_wq=0x1,filter_tc=0x1,filter_sz=0x7, filter_eng=0x1,event=0x8,event_category=0x3/ 21.865914091 seconds time elapsed The output above reflects that the unspecified workload resulted in the counting of 19 MEM_MOVE operation events that met the filter criteria. [1]: https://software.intel.com/content/www/us/en/develop/download/intel-data-streaming-accelerator-preliminary-architecture-specification.html [ Based on work originally by Jing Lin. ] Intel-SIG: commit 81dd4d4d6178 dmaengine: idxd: Add IDXD performance monitor support. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reviewed-by: Dave Jiang Reviewed-by: Kan Liang Signed-off-by: Tom Zanussi Link: https://lore.kernel.org/r/0c5080a7d541904c4ad42b848c76a1ce056ddac7.1619276133.git.zanussi@kernel.org Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- .../sysfs-bus-event_source-devices-dsa | 30 + drivers/dma/Kconfig | 12 + drivers/dma/idxd/Makefile | 2 + drivers/dma/idxd/idxd.h | 45 ++ drivers/dma/idxd/perfmon.c | 662 ++++++++++++++++++ drivers/dma/idxd/perfmon.h | 119 ++++ drivers/dma/idxd/registers.h | 108 +++ include/linux/cpuhotplug.h | 1 + 8 files changed, 979 insertions(+) create mode 100644 Documentation/ABI/testing/sysfs-bus-event_source-devices-dsa create mode 100644 drivers/dma/idxd/perfmon.c create mode 100644 drivers/dma/idxd/perfmon.h diff --git a/Documentation/ABI/testing/sysfs-bus-event_source-devices-dsa b/Documentation/ABI/testing/sysfs-bus-event_source-devices-dsa new file mode 100644 index 000000000000..3c7d132281b0 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-bus-event_source-devices-dsa @@ -0,0 +1,30 @@ +What: /sys/bus/event_source/devices/dsa*/format +Date: April 2021 +KernelVersion: 5.13 +Contact: Tom Zanussi +Description: Read-only. Attribute group to describe the magic bits + that go into perf_event_attr.config or + perf_event_attr.config1 for the IDXD DSA pmu. (See also + ABI/testing/sysfs-bus-event_source-devices-format). + + Each attribute in this group defines a bit range in + perf_event_attr.config or perf_event_attr.config1. + All supported attributes are listed below (See the + IDXD DSA Spec for possible attribute values):: + + event_category = "config:0-3" - event category + event = "config:4-31" - event ID + + filter_wq = "config1:0-31" - workqueue filter + filter_tc = "config1:32-39" - traffic class filter + filter_pgsz = "config1:40-43" - page size filter + filter_sz = "config1:44-51" - transfer size filter + filter_eng = "config1:52-59" - engine filter + +What: /sys/bus/event_source/devices/dsa*/cpumask +Date: April 2021 +KernelVersion: 5.13 +Contact: Tom Zanussi +Description: Read-only. This file always returns the cpu to which the + IDXD DSA pmu is bound for access to all dsa pmu + performance monitoring events. diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 35ee4a775c31..1d522ae93c6e 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -307,6 +307,18 @@ config INTEL_IDXD_SVM depends on PCI_PASID depends on PCI_IOV +config INTEL_IDXD_PERFMON + bool "Intel Data Accelerators performance monitor support" + depends on INTEL_IDXD + help + Enable performance monitor (pmu) support for the Intel(R) + data accelerators present in Intel Xeon CPU. With this + enabled, perf can be used to monitor the DSA (Intel Data + Streaming Accelerator) events described in the Intel DSA + spec. + + If unsure, say N. + config INTEL_IOATDMA tristate "Intel I/OAT DMA support" depends on PCI && X86_64 diff --git a/drivers/dma/idxd/Makefile b/drivers/dma/idxd/Makefile index 8978b898d777..6d11558756f8 100644 --- a/drivers/dma/idxd/Makefile +++ b/drivers/dma/idxd/Makefile @@ -1,2 +1,4 @@ obj-$(CONFIG_INTEL_IDXD) += idxd.o idxd-y := init.o irq.o device.o sysfs.o submit.o dma.o cdev.o + +idxd-$(CONFIG_INTEL_IDXD_PERFMON) += perfmon.o diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 97c96ca6ab70..26482c7d4c3a 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include "registers.h" #define IDXD_DRIVER_VERSION "1.00" @@ -29,6 +31,7 @@ enum idxd_type { }; #define IDXD_NAME_SIZE 128 +#define IDXD_PMU_EVENT_MAX 64 struct idxd_device_driver { struct device_driver drv; @@ -61,6 +64,31 @@ struct idxd_group { int tc_b; }; +struct idxd_pmu { + struct idxd_device *idxd; + + struct perf_event *event_list[IDXD_PMU_EVENT_MAX]; + int n_events; + + DECLARE_BITMAP(used_mask, IDXD_PMU_EVENT_MAX); + + struct pmu pmu; + char name[IDXD_NAME_SIZE]; + int cpu; + + int n_counters; + int counter_width; + int n_event_categories; + + bool per_counter_caps_supported; + unsigned long supported_event_categories; + + unsigned long supported_filters; + int n_filters; + + struct hlist_node cpuhp_node; +}; + #define IDXD_MAX_PRIORITY 0xf enum idxd_wq_state { @@ -241,6 +269,8 @@ struct idxd_device { struct work_struct work; int *int_handles; + + struct idxd_pmu *idxd_pmu; }; /* IDXD software descriptor */ @@ -437,4 +467,19 @@ int idxd_cdev_get_major(struct idxd_device *idxd); int idxd_wq_add_cdev(struct idxd_wq *wq); void idxd_wq_del_cdev(struct idxd_wq *wq); +/* perfmon */ +#if IS_ENABLED(CONFIG_INTEL_IDXD_PERFMON) +int perfmon_pmu_init(struct idxd_device *idxd); +void perfmon_pmu_remove(struct idxd_device *idxd); +void perfmon_counter_overflow(struct idxd_device *idxd); +void perfmon_init(void); +void perfmon_exit(void); +#else +static inline int perfmon_pmu_init(struct idxd_device *idxd) { return 0; } +static inline void perfmon_pmu_remove(struct idxd_device *idxd) {} +static inline void perfmon_counter_overflow(struct idxd_device *idxd) {} +static inline void perfmon_init(void) {} +static inline void perfmon_exit(void) {} +#endif + #endif diff --git a/drivers/dma/idxd/perfmon.c b/drivers/dma/idxd/perfmon.c new file mode 100644 index 000000000000..d73004f47cf4 --- /dev/null +++ b/drivers/dma/idxd/perfmon.c @@ -0,0 +1,662 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2020 Intel Corporation. All rights rsvd. */ + +#include +#include +#include "idxd.h" +#include "perfmon.h" + +static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, + char *buf); + +static cpumask_t perfmon_dsa_cpu_mask; +static bool cpuhp_set_up; +static enum cpuhp_state cpuhp_slot; + +/* + * perf userspace reads this attribute to determine which cpus to open + * counters on. It's connected to perfmon_dsa_cpu_mask, which is + * maintained by the cpu hotplug handlers. + */ +static DEVICE_ATTR_RO(cpumask); + +static struct attribute *perfmon_cpumask_attrs[] = { + &dev_attr_cpumask.attr, + NULL, +}; + +static struct attribute_group cpumask_attr_group = { + .attrs = perfmon_cpumask_attrs, +}; + +/* + * These attributes specify the bits in the config word that the perf + * syscall uses to pass the event ids and categories to perfmon. + */ +DEFINE_PERFMON_FORMAT_ATTR(event_category, "config:0-3"); +DEFINE_PERFMON_FORMAT_ATTR(event, "config:4-31"); + +/* + * These attributes specify the bits in the config1 word that the perf + * syscall uses to pass filter data to perfmon. + */ +DEFINE_PERFMON_FORMAT_ATTR(filter_wq, "config1:0-31"); +DEFINE_PERFMON_FORMAT_ATTR(filter_tc, "config1:32-39"); +DEFINE_PERFMON_FORMAT_ATTR(filter_pgsz, "config1:40-43"); +DEFINE_PERFMON_FORMAT_ATTR(filter_sz, "config1:44-51"); +DEFINE_PERFMON_FORMAT_ATTR(filter_eng, "config1:52-59"); + +#define PERFMON_FILTERS_START 2 +#define PERFMON_FILTERS_MAX 5 + +static struct attribute *perfmon_format_attrs[] = { + &format_attr_idxd_event_category.attr, + &format_attr_idxd_event.attr, + &format_attr_idxd_filter_wq.attr, + &format_attr_idxd_filter_tc.attr, + &format_attr_idxd_filter_pgsz.attr, + &format_attr_idxd_filter_sz.attr, + &format_attr_idxd_filter_eng.attr, + NULL, +}; + +static struct attribute_group perfmon_format_attr_group = { + .name = "format", + .attrs = perfmon_format_attrs, +}; + +static const struct attribute_group *perfmon_attr_groups[] = { + &perfmon_format_attr_group, + &cpumask_attr_group, + NULL, +}; + +static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return cpumap_print_to_pagebuf(true, buf, &perfmon_dsa_cpu_mask); +} + +static bool is_idxd_event(struct idxd_pmu *idxd_pmu, struct perf_event *event) +{ + return &idxd_pmu->pmu == event->pmu; +} + +static int perfmon_collect_events(struct idxd_pmu *idxd_pmu, + struct perf_event *leader, + bool do_grp) +{ + struct perf_event *event; + int n, max_count; + + max_count = idxd_pmu->n_counters; + n = idxd_pmu->n_events; + + if (n >= max_count) + return -EINVAL; + + if (is_idxd_event(idxd_pmu, leader)) { + idxd_pmu->event_list[n] = leader; + idxd_pmu->event_list[n]->hw.idx = n; + n++; + } + + if (!do_grp) + return n; + + for_each_sibling_event(event, leader) { + if (!is_idxd_event(idxd_pmu, event) || + event->state <= PERF_EVENT_STATE_OFF) + continue; + + if (n >= max_count) + return -EINVAL; + + idxd_pmu->event_list[n] = event; + idxd_pmu->event_list[n]->hw.idx = n; + n++; + } + + return n; +} + +static void perfmon_assign_hw_event(struct idxd_pmu *idxd_pmu, + struct perf_event *event, int idx) +{ + struct idxd_device *idxd = idxd_pmu->idxd; + struct hw_perf_event *hwc = &event->hw; + + hwc->idx = idx; + hwc->config_base = ioread64(CNTRCFG_REG(idxd, idx)); + hwc->event_base = ioread64(CNTRCFG_REG(idxd, idx)); +} + +static int perfmon_assign_event(struct idxd_pmu *idxd_pmu, + struct perf_event *event) +{ + int i; + + for (i = 0; i < IDXD_PMU_EVENT_MAX; i++) + if (!test_and_set_bit(i, idxd_pmu->used_mask)) + return i; + + return -EINVAL; +} + +/* + * Check whether there are enough counters to satisfy that all the + * events in the group can actually be scheduled at the same time. + * + * To do this, create a fake idxd_pmu object so the event collection + * and assignment functions can be used without affecting the internal + * state of the real idxd_pmu object. + */ +static int perfmon_validate_group(struct idxd_pmu *pmu, + struct perf_event *event) +{ + struct perf_event *leader = event->group_leader; + struct idxd_pmu *fake_pmu; + int i, ret = 0, n, idx; + + fake_pmu = kzalloc(sizeof(*fake_pmu), GFP_KERNEL); + if (!fake_pmu) + return -ENOMEM; + + fake_pmu->pmu.name = pmu->pmu.name; + fake_pmu->n_counters = pmu->n_counters; + + n = perfmon_collect_events(fake_pmu, leader, true); + if (n < 0) { + ret = n; + goto out; + } + + fake_pmu->n_events = n; + n = perfmon_collect_events(fake_pmu, event, false); + if (n < 0) { + ret = n; + goto out; + } + + fake_pmu->n_events = n; + + for (i = 0; i < n; i++) { + event = fake_pmu->event_list[i]; + + idx = perfmon_assign_event(fake_pmu, event); + if (idx < 0) { + ret = idx; + goto out; + } + } +out: + kfree(fake_pmu); + + return ret; +} + +static int perfmon_pmu_event_init(struct perf_event *event) +{ + struct idxd_device *idxd; + int ret = 0; + + idxd = event_to_idxd(event); + event->hw.idx = -1; + + if (event->attr.type != event->pmu->type) + return -ENOENT; + + /* sampling not supported */ + if (event->attr.sample_period) + return -EINVAL; + + if (event->cpu < 0) + return -EINVAL; + + if (event->pmu != &idxd->idxd_pmu->pmu) + return -EINVAL; + + event->hw.event_base = ioread64(PERFMON_TABLE_OFFSET(idxd)); + event->cpu = idxd->idxd_pmu->cpu; + event->hw.config = event->attr.config; + + if (event->group_leader != event) + /* non-group events have themselves as leader */ + ret = perfmon_validate_group(idxd->idxd_pmu, event); + + return ret; +} + +static inline u64 perfmon_pmu_read_counter(struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + struct idxd_device *idxd; + int cntr = hwc->idx; + + idxd = event_to_idxd(event); + + return ioread64(CNTRDATA_REG(idxd, cntr)); +} + +static void perfmon_pmu_event_update(struct perf_event *event) +{ + struct idxd_device *idxd = event_to_idxd(event); + u64 prev_raw_count, new_raw_count, delta, p, n; + int shift = 64 - idxd->idxd_pmu->counter_width; + struct hw_perf_event *hwc = &event->hw; + + do { + prev_raw_count = local64_read(&hwc->prev_count); + new_raw_count = perfmon_pmu_read_counter(event); + } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count, + new_raw_count) != prev_raw_count); + + n = (new_raw_count << shift); + p = (prev_raw_count << shift); + + delta = ((n - p) >> shift); + + local64_add(delta, &event->count); +} + +void perfmon_counter_overflow(struct idxd_device *idxd) +{ + int i, n_counters, max_loop = OVERFLOW_SIZE; + struct perf_event *event; + unsigned long ovfstatus; + + n_counters = min(idxd->idxd_pmu->n_counters, OVERFLOW_SIZE); + + ovfstatus = ioread32(OVFSTATUS_REG(idxd)); + + /* + * While updating overflowed counters, other counters behind + * them could overflow and be missed in a given pass. + * Normally this could happen at most n_counters times, but in + * theory a tiny counter width could result in continual + * overflows and endless looping. max_loop provides a + * failsafe in that highly unlikely case. + */ + while (ovfstatus && max_loop--) { + /* Figure out which counter(s) overflowed */ + for_each_set_bit(i, &ovfstatus, n_counters) { + unsigned long ovfstatus_clear = 0; + + /* Update event->count for overflowed counter */ + event = idxd->idxd_pmu->event_list[i]; + perfmon_pmu_event_update(event); + /* Writing 1 to OVFSTATUS bit clears it */ + set_bit(i, &ovfstatus_clear); + iowrite32(ovfstatus_clear, OVFSTATUS_REG(idxd)); + } + + ovfstatus = ioread32(OVFSTATUS_REG(idxd)); + } + + /* + * Should never happen. If so, it means a counter(s) looped + * around twice while this handler was running. + */ + WARN_ON_ONCE(ovfstatus); +} + +static inline void perfmon_reset_config(struct idxd_device *idxd) +{ + iowrite32(CONFIG_RESET, PERFRST_REG(idxd)); + iowrite32(0, OVFSTATUS_REG(idxd)); + iowrite32(0, PERFFRZ_REG(idxd)); +} + +static inline void perfmon_reset_counters(struct idxd_device *idxd) +{ + iowrite32(CNTR_RESET, PERFRST_REG(idxd)); +} + +static inline void perfmon_reset(struct idxd_device *idxd) +{ + perfmon_reset_config(idxd); + perfmon_reset_counters(idxd); +} + +static void perfmon_pmu_event_start(struct perf_event *event, int mode) +{ + u32 flt_wq, flt_tc, flt_pg_sz, flt_xfer_sz, flt_eng = 0; + u64 cntr_cfg, cntrdata, event_enc, event_cat = 0; + struct hw_perf_event *hwc = &event->hw; + union filter_cfg flt_cfg; + union event_cfg event_cfg; + struct idxd_device *idxd; + int cntr; + + idxd = event_to_idxd(event); + + event->hw.idx = hwc->idx; + cntr = hwc->idx; + + /* Obtain event category and event value from user space */ + event_cfg.val = event->attr.config; + flt_cfg.val = event->attr.config1; + event_cat = event_cfg.event_cat; + event_enc = event_cfg.event_enc; + + /* Obtain filter configuration from user space */ + flt_wq = flt_cfg.wq; + flt_tc = flt_cfg.tc; + flt_pg_sz = flt_cfg.pg_sz; + flt_xfer_sz = flt_cfg.xfer_sz; + flt_eng = flt_cfg.eng; + + if (flt_wq && test_bit(FLT_WQ, &idxd->idxd_pmu->supported_filters)) + iowrite32(flt_wq, FLTCFG_REG(idxd, cntr, FLT_WQ)); + if (flt_tc && test_bit(FLT_TC, &idxd->idxd_pmu->supported_filters)) + iowrite32(flt_tc, FLTCFG_REG(idxd, cntr, FLT_TC)); + if (flt_pg_sz && test_bit(FLT_PG_SZ, &idxd->idxd_pmu->supported_filters)) + iowrite32(flt_pg_sz, FLTCFG_REG(idxd, cntr, FLT_PG_SZ)); + if (flt_xfer_sz && test_bit(FLT_XFER_SZ, &idxd->idxd_pmu->supported_filters)) + iowrite32(flt_xfer_sz, FLTCFG_REG(idxd, cntr, FLT_XFER_SZ)); + if (flt_eng && test_bit(FLT_ENG, &idxd->idxd_pmu->supported_filters)) + iowrite32(flt_eng, FLTCFG_REG(idxd, cntr, FLT_ENG)); + + /* Read the start value */ + cntrdata = ioread64(CNTRDATA_REG(idxd, cntr)); + local64_set(&event->hw.prev_count, cntrdata); + + /* Set counter to event/category */ + cntr_cfg = event_cat << CNTRCFG_CATEGORY_SHIFT; + cntr_cfg |= event_enc << CNTRCFG_EVENT_SHIFT; + /* Set interrupt on overflow and counter enable bits */ + cntr_cfg |= (CNTRCFG_IRQ_OVERFLOW | CNTRCFG_ENABLE); + + iowrite64(cntr_cfg, CNTRCFG_REG(idxd, cntr)); +} + +static void perfmon_pmu_event_stop(struct perf_event *event, int mode) +{ + struct hw_perf_event *hwc = &event->hw; + struct idxd_device *idxd; + int i, cntr = hwc->idx; + u64 cntr_cfg; + + idxd = event_to_idxd(event); + + /* remove this event from event list */ + for (i = 0; i < idxd->idxd_pmu->n_events; i++) { + if (event != idxd->idxd_pmu->event_list[i]) + continue; + + for (++i; i < idxd->idxd_pmu->n_events; i++) + idxd->idxd_pmu->event_list[i - 1] = idxd->idxd_pmu->event_list[i]; + --idxd->idxd_pmu->n_events; + break; + } + + cntr_cfg = ioread64(CNTRCFG_REG(idxd, cntr)); + cntr_cfg &= ~CNTRCFG_ENABLE; + iowrite64(cntr_cfg, CNTRCFG_REG(idxd, cntr)); + + if (mode == PERF_EF_UPDATE) + perfmon_pmu_event_update(event); + + event->hw.idx = -1; + clear_bit(cntr, idxd->idxd_pmu->used_mask); +} + +static void perfmon_pmu_event_del(struct perf_event *event, int mode) +{ + perfmon_pmu_event_stop(event, PERF_EF_UPDATE); +} + +static int perfmon_pmu_event_add(struct perf_event *event, int flags) +{ + struct idxd_device *idxd = event_to_idxd(event); + struct idxd_pmu *idxd_pmu = idxd->idxd_pmu; + struct hw_perf_event *hwc = &event->hw; + int idx, n; + + n = perfmon_collect_events(idxd_pmu, event, false); + if (n < 0) + return n; + + hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; + if (!(flags & PERF_EF_START)) + hwc->state |= PERF_HES_ARCH; + + idx = perfmon_assign_event(idxd_pmu, event); + if (idx < 0) + return idx; + + perfmon_assign_hw_event(idxd_pmu, event, idx); + + if (flags & PERF_EF_START) + perfmon_pmu_event_start(event, 0); + + idxd_pmu->n_events = n; + + return 0; +} + +static void enable_perfmon_pmu(struct idxd_device *idxd) +{ + iowrite32(COUNTER_UNFREEZE, PERFFRZ_REG(idxd)); +} + +static void disable_perfmon_pmu(struct idxd_device *idxd) +{ + iowrite32(COUNTER_FREEZE, PERFFRZ_REG(idxd)); +} + +static void perfmon_pmu_enable(struct pmu *pmu) +{ + struct idxd_device *idxd = pmu_to_idxd(pmu); + + enable_perfmon_pmu(idxd); +} + +static void perfmon_pmu_disable(struct pmu *pmu) +{ + struct idxd_device *idxd = pmu_to_idxd(pmu); + + disable_perfmon_pmu(idxd); +} + +static void skip_filter(int i) +{ + int j; + + for (j = i; j < PERFMON_FILTERS_MAX; j++) + perfmon_format_attrs[PERFMON_FILTERS_START + j] = + perfmon_format_attrs[PERFMON_FILTERS_START + j + 1]; +} + +static void idxd_pmu_init(struct idxd_pmu *idxd_pmu) +{ + int i; + + for (i = 0 ; i < PERFMON_FILTERS_MAX; i++) { + if (!test_bit(i, &idxd_pmu->supported_filters)) + skip_filter(i); + } + + idxd_pmu->pmu.name = idxd_pmu->name; + idxd_pmu->pmu.attr_groups = perfmon_attr_groups; + idxd_pmu->pmu.task_ctx_nr = perf_invalid_context; + idxd_pmu->pmu.event_init = perfmon_pmu_event_init; + idxd_pmu->pmu.pmu_enable = perfmon_pmu_enable, + idxd_pmu->pmu.pmu_disable = perfmon_pmu_disable, + idxd_pmu->pmu.add = perfmon_pmu_event_add; + idxd_pmu->pmu.del = perfmon_pmu_event_del; + idxd_pmu->pmu.start = perfmon_pmu_event_start; + idxd_pmu->pmu.stop = perfmon_pmu_event_stop; + idxd_pmu->pmu.read = perfmon_pmu_event_update; + idxd_pmu->pmu.capabilities = PERF_PMU_CAP_NO_EXCLUDE; + idxd_pmu->pmu.module = THIS_MODULE; +} + +void perfmon_pmu_remove(struct idxd_device *idxd) +{ + if (!idxd->idxd_pmu) + return; + + cpuhp_state_remove_instance(cpuhp_slot, &idxd->idxd_pmu->cpuhp_node); + perf_pmu_unregister(&idxd->idxd_pmu->pmu); + kfree(idxd->idxd_pmu); + idxd->idxd_pmu = NULL; +} + +static int perf_event_cpu_online(unsigned int cpu, struct hlist_node *node) +{ + struct idxd_pmu *idxd_pmu; + + idxd_pmu = hlist_entry_safe(node, typeof(*idxd_pmu), cpuhp_node); + + /* select the first online CPU as the designated reader */ + if (cpumask_empty(&perfmon_dsa_cpu_mask)) { + cpumask_set_cpu(cpu, &perfmon_dsa_cpu_mask); + idxd_pmu->cpu = cpu; + } + + return 0; +} + +static int perf_event_cpu_offline(unsigned int cpu, struct hlist_node *node) +{ + struct idxd_pmu *idxd_pmu; + unsigned int target; + + idxd_pmu = hlist_entry_safe(node, typeof(*idxd_pmu), cpuhp_node); + + if (!cpumask_test_and_clear_cpu(cpu, &perfmon_dsa_cpu_mask)) + return 0; + + target = cpumask_any_but(cpu_online_mask, cpu); + + /* migrate events if there is a valid target */ + if (target < nr_cpu_ids) + cpumask_set_cpu(target, &perfmon_dsa_cpu_mask); + else + target = -1; + + perf_pmu_migrate_context(&idxd_pmu->pmu, cpu, target); + + return 0; +} + +int perfmon_pmu_init(struct idxd_device *idxd) +{ + union idxd_perfcap perfcap; + struct idxd_pmu *idxd_pmu; + int rc = -ENODEV; + + /* + * perfmon module initialization failed, nothing to do + */ + if (!cpuhp_set_up) + return -ENODEV; + + /* + * If perfmon_offset or num_counters is 0, it means perfmon is + * not supported on this hardware. + */ + if (idxd->perfmon_offset == 0) + return -ENODEV; + + idxd_pmu = kzalloc(sizeof(*idxd_pmu), GFP_KERNEL); + if (!idxd_pmu) + return -ENOMEM; + + idxd_pmu->idxd = idxd; + idxd->idxd_pmu = idxd_pmu; + + if (idxd->data->type == IDXD_TYPE_DSA) { + rc = sprintf(idxd_pmu->name, "dsa%d", idxd->id); + if (rc < 0) + goto free; + } else if (idxd->data->type == IDXD_TYPE_IAX) { + rc = sprintf(idxd_pmu->name, "iax%d", idxd->id); + if (rc < 0) + goto free; + } else { + goto free; + } + + perfmon_reset(idxd); + + perfcap.bits = ioread64(PERFCAP_REG(idxd)); + + /* + * If total perf counter is 0, stop further registration. + * This is necessary in order to support driver running on + * guest which does not have pmon support. + */ + if (perfcap.num_perf_counter == 0) + goto free; + + /* A counter width of 0 means it can't count */ + if (perfcap.counter_width == 0) + goto free; + + /* Overflow interrupt and counter freeze support must be available */ + if (!perfcap.overflow_interrupt || !perfcap.counter_freeze) + goto free; + + /* Number of event categories cannot be 0 */ + if (perfcap.num_event_category == 0) + goto free; + + /* + * We don't support per-counter capabilities for now. + */ + if (perfcap.cap_per_counter) + goto free; + + idxd_pmu->n_event_categories = perfcap.num_event_category; + idxd_pmu->supported_event_categories = perfcap.global_event_category; + idxd_pmu->per_counter_caps_supported = perfcap.cap_per_counter; + + /* check filter capability. If 0, then filters are not supported */ + idxd_pmu->supported_filters = perfcap.filter; + if (perfcap.filter) + idxd_pmu->n_filters = hweight8(perfcap.filter); + + /* Store the total number of counters categories, and counter width */ + idxd_pmu->n_counters = perfcap.num_perf_counter; + idxd_pmu->counter_width = perfcap.counter_width; + + idxd_pmu_init(idxd_pmu); + + rc = perf_pmu_register(&idxd_pmu->pmu, idxd_pmu->name, -1); + if (rc) + goto free; + + rc = cpuhp_state_add_instance(cpuhp_slot, &idxd_pmu->cpuhp_node); + if (rc) { + perf_pmu_unregister(&idxd->idxd_pmu->pmu); + goto free; + } +out: + return rc; +free: + kfree(idxd_pmu); + idxd->idxd_pmu = NULL; + + goto out; +} + +void __init perfmon_init(void) +{ + int rc = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, + "driver/dma/idxd/perf:online", + perf_event_cpu_online, + perf_event_cpu_offline); + if (WARN_ON(rc < 0)) + return; + + cpuhp_slot = rc; + cpuhp_set_up = true; +} + +void __exit perfmon_exit(void) +{ + if (cpuhp_set_up) + cpuhp_remove_multi_state(cpuhp_slot); +} diff --git a/drivers/dma/idxd/perfmon.h b/drivers/dma/idxd/perfmon.h new file mode 100644 index 000000000000..9a081a1bc605 --- /dev/null +++ b/drivers/dma/idxd/perfmon.h @@ -0,0 +1,119 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2020 Intel Corporation. All rights rsvd. */ + +#ifndef _PERFMON_H_ +#define _PERFMON_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "registers.h" + +static inline struct idxd_pmu *event_to_pmu(struct perf_event *event) +{ + struct idxd_pmu *idxd_pmu; + struct pmu *pmu; + + pmu = event->pmu; + idxd_pmu = container_of(pmu, struct idxd_pmu, pmu); + + return idxd_pmu; +} + +static inline struct idxd_device *event_to_idxd(struct perf_event *event) +{ + struct idxd_pmu *idxd_pmu; + struct pmu *pmu; + + pmu = event->pmu; + idxd_pmu = container_of(pmu, struct idxd_pmu, pmu); + + return idxd_pmu->idxd; +} + +static inline struct idxd_device *pmu_to_idxd(struct pmu *pmu) +{ + struct idxd_pmu *idxd_pmu; + + idxd_pmu = container_of(pmu, struct idxd_pmu, pmu); + + return idxd_pmu->idxd; +} + +enum dsa_perf_events { + DSA_PERF_EVENT_WQ = 0, + DSA_PERF_EVENT_ENGINE, + DSA_PERF_EVENT_ADDR_TRANS, + DSA_PERF_EVENT_OP, + DSA_PERF_EVENT_COMPL, + DSA_PERF_EVENT_MAX, +}; + +enum filter_enc { + FLT_WQ = 0, + FLT_TC, + FLT_PG_SZ, + FLT_XFER_SZ, + FLT_ENG, + FLT_MAX, +}; + +#define CONFIG_RESET 0x0000000000000001 +#define CNTR_RESET 0x0000000000000002 +#define CNTR_ENABLE 0x0000000000000001 +#define INTR_OVFL 0x0000000000000002 + +#define COUNTER_FREEZE 0x00000000FFFFFFFF +#define COUNTER_UNFREEZE 0x0000000000000000 +#define OVERFLOW_SIZE 32 + +#define CNTRCFG_ENABLE BIT(0) +#define CNTRCFG_IRQ_OVERFLOW BIT(1) +#define CNTRCFG_CATEGORY_SHIFT 8 +#define CNTRCFG_EVENT_SHIFT 32 + +#define PERFMON_TABLE_OFFSET(_idxd) \ +({ \ + typeof(_idxd) __idxd = (_idxd); \ + ((__idxd)->reg_base + (__idxd)->perfmon_offset); \ +}) +#define PERFMON_REG_OFFSET(idxd, offset) \ + (PERFMON_TABLE_OFFSET(idxd) + (offset)) + +#define PERFCAP_REG(idxd) (PERFMON_REG_OFFSET(idxd, IDXD_PERFCAP_OFFSET)) +#define PERFRST_REG(idxd) (PERFMON_REG_OFFSET(idxd, IDXD_PERFRST_OFFSET)) +#define OVFSTATUS_REG(idxd) (PERFMON_REG_OFFSET(idxd, IDXD_OVFSTATUS_OFFSET)) +#define PERFFRZ_REG(idxd) (PERFMON_REG_OFFSET(idxd, IDXD_PERFFRZ_OFFSET)) + +#define FLTCFG_REG(idxd, cntr, flt) \ + (PERFMON_REG_OFFSET(idxd, IDXD_FLTCFG_OFFSET) + ((cntr) * 32) + ((flt) * 4)) + +#define CNTRCFG_REG(idxd, cntr) \ + (PERFMON_REG_OFFSET(idxd, IDXD_CNTRCFG_OFFSET) + ((cntr) * 8)) +#define CNTRDATA_REG(idxd, cntr) \ + (PERFMON_REG_OFFSET(idxd, IDXD_CNTRDATA_OFFSET) + ((cntr) * 8)) +#define CNTRCAP_REG(idxd, cntr) \ + (PERFMON_REG_OFFSET(idxd, IDXD_CNTRCAP_OFFSET) + ((cntr) * 8)) + +#define EVNTCAP_REG(idxd, category) \ + (PERFMON_REG_OFFSET(idxd, IDXD_EVNTCAP_OFFSET) + ((category) * 8)) + +#define DEFINE_PERFMON_FORMAT_ATTR(_name, _format) \ +static ssize_t __perfmon_idxd_##_name##_show(struct kobject *kobj, \ + struct kobj_attribute *attr, \ + char *page) \ +{ \ + BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \ + return sprintf(page, _format "\n"); \ +} \ +static struct kobj_attribute format_attr_idxd_##_name = \ + __ATTR(_name, 0444, __perfmon_idxd_##_name##_show, NULL) + +#endif diff --git a/drivers/dma/idxd/registers.h b/drivers/dma/idxd/registers.h index 6c11375cc56a..c970c3f025f0 100644 --- a/drivers/dma/idxd/registers.h +++ b/drivers/dma/idxd/registers.h @@ -386,4 +386,112 @@ union wqcfg { #define GRPENGCFG_OFFSET(idxd_dev, n) ((idxd_dev)->grpcfg_offset + (n) * GRPCFG_SIZE + 32) #define GRPFLGCFG_OFFSET(idxd_dev, n) ((idxd_dev)->grpcfg_offset + (n) * GRPCFG_SIZE + 40) +/* Following is performance monitor registers */ +#define IDXD_PERFCAP_OFFSET 0x0 +union idxd_perfcap { + struct { + u64 num_perf_counter:6; + u64 rsvd1:2; + u64 counter_width:8; + u64 num_event_category:4; + u64 global_event_category:16; + u64 filter:8; + u64 rsvd2:8; + u64 cap_per_counter:1; + u64 writeable_counter:1; + u64 counter_freeze:1; + u64 overflow_interrupt:1; + u64 rsvd3:8; + }; + u64 bits; +} __packed; + +#define IDXD_EVNTCAP_OFFSET 0x80 +union idxd_evntcap { + struct { + u64 events:28; + u64 rsvd:36; + }; + u64 bits; +} __packed; + +struct idxd_event { + union { + struct { + u32 event_category:4; + u32 events:28; + }; + u32 val; + }; +} __packed; + +#define IDXD_CNTRCAP_OFFSET 0x800 +struct idxd_cntrcap { + union { + struct { + u32 counter_width:8; + u32 rsvd:20; + u32 num_events:4; + }; + u32 val; + }; + struct idxd_event events[]; +} __packed; + +#define IDXD_PERFRST_OFFSET 0x10 +union idxd_perfrst { + struct { + u32 perfrst_config:1; + u32 perfrst_counter:1; + u32 rsvd:30; + }; + u32 val; +} __packed; + +#define IDXD_OVFSTATUS_OFFSET 0x30 +#define IDXD_PERFFRZ_OFFSET 0x20 +#define IDXD_CNTRCFG_OFFSET 0x100 +union idxd_cntrcfg { + struct { + u64 enable:1; + u64 interrupt_ovf:1; + u64 global_freeze_ovf:1; + u64 rsvd1:5; + u64 event_category:4; + u64 rsvd2:20; + u64 events:28; + u64 rsvd3:4; + }; + u64 val; +} __packed; + +#define IDXD_FLTCFG_OFFSET 0x300 + +#define IDXD_CNTRDATA_OFFSET 0x200 +union idxd_cntrdata { + struct { + u64 event_count_value; + }; + u64 val; +} __packed; + +union event_cfg { + struct { + u64 event_cat:4; + u64 event_enc:28; + }; + u64 val; +} __packed; + +union filter_cfg { + struct { + u64 wq:32; + u64 tc:8; + u64 pg_sz:4; + u64 xfer_sz:8; + u64 eng:8; + }; + u64 val; +} __packed; + #endif diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 295c29108a2c..a54fa0f7006f 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -168,6 +168,7 @@ enum cpuhp_state { CPUHP_AP_PERF_X86_RAPL_ONLINE, CPUHP_AP_PERF_X86_CQM_ONLINE, CPUHP_AP_PERF_X86_CSTATE_ONLINE, + CPUHP_AP_PERF_X86_IDXD_ONLINE, CPUHP_AP_PERF_S390_CF_ONLINE, CPUHP_AP_PERF_S390_SF_ONLINE, CPUHP_AP_PERF_ARM_CCI_ONLINE, -- Gitee From ad9fc1c95cd1e514e7805d88bbda612f4acc41c9 Mon Sep 17 00:00:00 2001 From: Tom Zanussi Date: Sat, 24 Apr 2021 10:04:16 -0500 Subject: [PATCH 20/85] dmaengine: idxd: Enable IDXD performance monitor support ANBZ: #22 commit 0bde4444ec44b8e64bbd4af72fcaef58bcdbd4ce upstream. Add the code needed in the main IDXD driver to interface with the IDXD perfmon implementation. [ Based on work originally by Jing Lin. ] Intel-SIG: commit 0bde4444ec44 dmaengine: idxd: Enable IDXD performance monitor support. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reviewed-by: Dave Jiang Reviewed-by: Kan Liang Signed-off-by: Tom Zanussi Link: https://lore.kernel.org/r/a5564a5583911565d31c2af9234218c5166c4b2c.1619276133.git.zanussi@kernel.org Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/init.c | 9 +++++++++ drivers/dma/idxd/irq.c | 5 +---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 5ef0caa057d3..de2d8cee063b 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -21,6 +21,7 @@ #include "../dmaengine.h" #include "registers.h" #include "idxd.h" +#include "perfmon.h" MODULE_VERSION(IDXD_DRIVER_VERSION); MODULE_LICENSE("GPL v2"); @@ -541,6 +542,10 @@ static int idxd_probe(struct idxd_device *idxd) idxd->major = idxd_cdev_get_major(idxd); + rc = perfmon_pmu_init(idxd); + if (rc < 0) + dev_warn(dev, "Failed to initialize perfmon. No PMU support: %d\n", rc); + dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id); return 0; @@ -720,6 +725,7 @@ static void idxd_remove(struct pci_dev *pdev) if (device_pasid_enabled(idxd)) idxd_disable_system_pasid(idxd); idxd_unregister_devices(idxd); + perfmon_pmu_remove(idxd); iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); } @@ -749,6 +755,8 @@ static int __init idxd_init_module(void) else support_enqcmd = true; + perfmon_init(); + err = idxd_register_bus_type(); if (err < 0) return err; @@ -783,5 +791,6 @@ static void __exit idxd_exit_module(void) pci_unregister_driver(&idxd_pci_driver); idxd_cdev_remove(); idxd_unregister_bus_type(); + perfmon_exit(); } module_exit(idxd_exit_module); diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c index afee571e0194..ae68e1e5487a 100644 --- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -156,11 +156,8 @@ static int process_misc_interrupts(struct idxd_device *idxd, u32 cause) } if (cause & IDXD_INTC_PERFMON_OVFL) { - /* - * Driver does not utilize perfmon counter overflow interrupt - * yet. - */ val |= IDXD_INTC_PERFMON_OVFL; + perfmon_counter_overflow(idxd); } val ^= cause; -- Gitee From 80f138e669ff7f3081201ef07e058a8760d2a1f6 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Mon, 26 Apr 2021 16:09:19 -0700 Subject: [PATCH 21/85] dmaengine: idxd: add engine 'struct device' missing bus type assignment ANBZ: #22 commit 1c4841ccbd2b185587010d6178aac11953f61d4c upstream. engine 'struct device' setup is missing assigning the bus type. Add it to dsa_bus_type. Intel-SIG: commit 1c4841ccbd2b dmaengine: idxd: add engine 'struct device' missing bus type assignment. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Fixes: 75b911309060 ("dmaengine: idxd: fix engine conf_dev lifetime") Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/161947841562.984844.17505646725993659651.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/init.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index de2d8cee063b..eaee1acbf002 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -242,6 +242,7 @@ static int idxd_setup_engines(struct idxd_device *idxd) engine->idxd = idxd; device_initialize(&engine->conf_dev); engine->conf_dev.parent = &idxd->conf_dev; + engine->conf_dev.bus = &dsa_bus_type; engine->conf_dev.type = &idxd_engine_device_type; rc = dev_set_name(&engine->conf_dev, "engine%d.%d", idxd->id, engine->id); if (rc < 0) { -- Gitee From d8a3ee9d38b9631629881ee42f51a2af0163dd2c Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 25 May 2021 12:23:37 -0700 Subject: [PATCH 22/85] dmaengine: idxd: Add missing cleanup for early error out in probe call ANBZ: #22 commit ddf742d4f3f12a6ba1b8e6ecbbf3ae736942f970 upstream. The probe call stack is missing some cleanup when things fail in the middle. Add the appropriate cleanup routines to make sure we exit gracefully. Intel-SIG: commit ddf742d4f3f1 dmaengine: idxd: Add missing cleanup for early error out in probe call. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Fixes: a39c7cd0438e ("dmaengine: idxd: removal of pcim managed mmio mapping") Reported-by: Nikhil Rao Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162197061707.392656.15760573520817310791.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/init.c | 61 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index eaee1acbf002..442d55c11a5f 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -168,6 +168,32 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) return rc; } +static void idxd_cleanup_interrupts(struct idxd_device *idxd) +{ + struct pci_dev *pdev = idxd->pdev; + struct idxd_irq_entry *irq_entry; + int i, msixcnt; + + msixcnt = pci_msix_vec_count(pdev); + if (msixcnt <= 0) + return; + + irq_entry = &idxd->irq_entries[0]; + free_irq(irq_entry->vector, irq_entry); + + for (i = 1; i < msixcnt; i++) { + + irq_entry = &idxd->irq_entries[i]; + if (idxd->hw.cmd_cap & BIT(IDXD_CMD_RELEASE_INT_HANDLE)) + idxd_device_release_int_handle(idxd, idxd->int_handles[i], + IDXD_IRQ_MSIX); + free_irq(irq_entry->vector, irq_entry); + } + + idxd_mask_error_interrupts(idxd); + pci_free_irq_vectors(pdev); +} + static int idxd_setup_wqs(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; @@ -304,6 +330,19 @@ static int idxd_setup_groups(struct idxd_device *idxd) return rc; } +static void idxd_cleanup_internals(struct idxd_device *idxd) +{ + int i; + + for (i = 0; i < idxd->max_groups; i++) + put_device(&idxd->groups[i]->conf_dev); + for (i = 0; i < idxd->max_engines; i++) + put_device(&idxd->engines[i]->conf_dev); + for (i = 0; i < idxd->max_wqs; i++) + put_device(&idxd->wqs[i]->conf_dev); + destroy_workqueue(idxd->wq); +} + static int idxd_setup_internals(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; @@ -532,12 +571,12 @@ static int idxd_probe(struct idxd_device *idxd) dev_dbg(dev, "Loading RO device config\n"); rc = idxd_device_load_config(idxd); if (rc < 0) - goto err; + goto err_config; } rc = idxd_setup_interrupts(idxd); if (rc) - goto err; + goto err_config; dev_dbg(dev, "IDXD interrupt setup complete.\n"); @@ -550,6 +589,8 @@ static int idxd_probe(struct idxd_device *idxd) dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id); return 0; + err_config: + idxd_cleanup_internals(idxd); err: if (device_pasid_enabled(idxd)) idxd_disable_system_pasid(idxd); @@ -557,6 +598,18 @@ static int idxd_probe(struct idxd_device *idxd) return rc; } +static void idxd_cleanup(struct idxd_device *idxd) +{ + struct device *dev = &idxd->pdev->dev; + + perfmon_pmu_remove(idxd); + idxd_cleanup_interrupts(idxd); + idxd_cleanup_internals(idxd); + if (device_pasid_enabled(idxd)) + idxd_disable_system_pasid(idxd); + iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA); +} + static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { struct device *dev = &pdev->dev; @@ -609,7 +662,7 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) rc = idxd_register_devices(idxd); if (rc) { dev_err(dev, "IDXD sysfs setup failed\n"); - goto err; + goto err_dev_register; } idxd->state = IDXD_DEV_CONF_READY; @@ -619,6 +672,8 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) return 0; + err_dev_register: + idxd_cleanup(idxd); err: pci_iounmap(pdev, idxd->reg_base); err_iomap: -- Gitee From 958d8fd42d5bbade5b1f81c52a4615b8aac4e229 Mon Sep 17 00:00:00 2001 From: Jiapeng Chong Date: Wed, 2 Jun 2021 18:07:26 +0800 Subject: [PATCH 23/85] dmaengine: idxd: Fix missing error code in idxd_cdev_open() ANBZ: #22 commit 99b18e88a1cf737ae924123d63b46d9a3d17b1af upstream. The error code is missing in this code scenario, add the error code '-EINVAL' to the return value 'rc'. Eliminate the follow smatch warning: drivers/dma/idxd/cdev.c:113 idxd_cdev_open() warn: missing error code 'rc'. Intel-SIG: commit 99b18e88a1cf dmaengine: idxd: Fix missing error code in idxd_cdev_open(). Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reported-by: Abaci Robot Signed-off-by: Jiapeng Chong Acked-by: Dave Jiang Link: https://lore.kernel.org/r/1622628446-87909-1-git-send-email-jiapeng.chong@linux.alibaba.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/cdev.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index 302cba5ff779..d4419bf1fede 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -110,6 +110,7 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp) pasid = iommu_sva_get_pasid(sva); if (pasid == IOMMU_PASID_INVALID) { iommu_sva_unbind_device(sva); + rc = -EINVAL; goto failed; } -- Gitee From eacb3b389da1d762e2213dd2526237172a796895 Mon Sep 17 00:00:00 2001 From: Jiapeng Chong Date: Thu, 6 May 2021 19:00:47 +0800 Subject: [PATCH 24/85] dmaengine: idxd: Remove redundant variable cdev_ctx ANBZ: #22 commit 58cb138e20296dffe3b2be2beb64e5aa22846b84 upstream. Variable cdev_ctx is set to '&ictx[wq->idxd->data->type]' but this value is not used, hence it is a redundant assignment and can be removed. Clean up the following clang-analyzer warning: drivers/dma/idxd/cdev.c:300:2: warning: Value stored to 'cdev_ctx' is never read [clang-analyzer-deadcode.DeadStores]. Intel-SIG: commit 58cb138e2029 dmaengine: idxd: Remove redundant variable cdev_ctx. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reported-by: Abaci Robot Signed-off-by: Jiapeng Chong Acked-by: Dave Jiang Link: https://lore.kernel.org/r/1620298847-33127-1-git-send-email-jiapeng.chong@linux.alibaba.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/cdev.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index d4419bf1fede..e9def577c697 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -296,9 +296,7 @@ int idxd_wq_add_cdev(struct idxd_wq *wq) void idxd_wq_del_cdev(struct idxd_wq *wq) { struct idxd_cdev *idxd_cdev; - struct idxd_cdev_context *cdev_ctx; - cdev_ctx = &ictx[wq->idxd->data->type]; idxd_cdev = wq->idxd_cdev; wq->idxd_cdev = NULL; cdev_device_del(&idxd_cdev->cdev, &idxd_cdev->dev); -- Gitee From 57413014b2d4cac11c5aa06790f8d73fecf92599 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Sun, 9 May 2021 17:38:25 -0700 Subject: [PATCH 25/85] dmaengine: idxd: remove devm allocation for idxd->int_handles ANBZ: #22 commit 33f9f3c33e9336e5f49501c9632584c3d1f4f3a5 upstream. Allocation of idxd->int_handles was merged incorrectly for the 5.13 merge window. The devm_kcalloc should've been regular kcalloc due to devm_* removal series for the driver. Intel-SIG: commit 33f9f3c33e93 dmaengine: idxd: remove devm allocation for idxd->int_handles. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Fixes: eb15e7154fbf ("dmaengine: idxd: add interrupt handle request and release support") Reported-by: Dan Carpenter Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162060710518.130816.11349798049329202863.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/init.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 442d55c11a5f..c8ae41d36040 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -351,7 +351,8 @@ static int idxd_setup_internals(struct idxd_device *idxd) init_waitqueue_head(&idxd->cmd_waitq); if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)) { - idxd->int_handles = devm_kcalloc(dev, idxd->max_wqs, sizeof(int), GFP_KERNEL); + idxd->int_handles = kcalloc_node(idxd->max_wqs, sizeof(int), GFP_KERNEL, + dev_to_node(dev)); if (!idxd->int_handles) return -ENOMEM; } -- Gitee From 701e11f9064fd583c6d1e40c3dcc8aefbf70b003 Mon Sep 17 00:00:00 2001 From: Tony Luck Date: Wed, 21 Jul 2021 12:25:20 -0700 Subject: [PATCH 26/85] dmaengine: idxd: Change license on idxd.h to LGPL ANBZ: #22 commit 25905f602fdb0cfa147017056636768a7aa1ff6f upstream. This file was given GPL-2.0 license. But LGPL-2.1 makes more sense as it needs to be used by libraries outside of the kernel source tree. Intel-SIG: commit 25905f602fdb dmaengine: idxd: Change license on idxd.h to LGPL. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Signed-off-by: Tony Luck Signed-off-by: Linus Torvalds [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- include/uapi/linux/idxd.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/uapi/linux/idxd.h b/include/uapi/linux/idxd.h index e33997b4d750..edc346a77c91 100644 --- a/include/uapi/linux/idxd.h +++ b/include/uapi/linux/idxd.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* SPDX-License-Identifier: LGPL-2.1 WITH Linux-syscall-note */ /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ #ifndef _USR_IDXD_H_ #define _USR_IDXD_H_ -- Gitee From 9720dfe993965fe853bbabc41f401e7e1bf93b88 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 24 Jun 2021 12:08:21 -0700 Subject: [PATCH 27/85] dmaengine: idxd: add missing percpu ref put on failure ANBZ: #22 commit ac24a2dc06cd773895d2fba0378c2538b8176565 upstream. When enqcmds() fails, exit path is missing a percpu_ref_put(). This can cause failure on shutdown path when the driver is attempting to quiesce the wq. Add missing percpu_ref_put() call on the error exit path. Intel-SIG: commit ac24a2dc06cd dmaengine: idxd: add missing percpu ref put on failure. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Fixes: 93a40a6d7428 ("dmaengine: idxd: add percpu_ref to descriptor submission path") Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162456170168.1121236.7240941044089212312.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/submit.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/dma/idxd/submit.c b/drivers/dma/idxd/submit.c index 19afb62abaff..b0f1ddf75d31 100644 --- a/drivers/dma/idxd/submit.c +++ b/drivers/dma/idxd/submit.c @@ -118,8 +118,10 @@ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc) * device is not accepting descriptor at all. */ rc = enqcmds(portal, desc->hw); - if (rc < 0) + if (rc < 0) { + percpu_ref_put(&wq->wq_active); return rc; + } } percpu_ref_put(&wq->wq_active); -- Gitee From 31a921d1ac8c74284163a29ea82767a664dfcafb Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 14 Jul 2021 11:38:41 -0700 Subject: [PATCH 28/85] dmaengine: idxd: fix desc->vector that isn't being updated ANBZ: #22 commit 8ba89a3c7967808f33478a8573277cf6a7412c4c upstream. Missing update for desc->vector when the wq vector gets updated. This causes the desc->vector to always be at 0. Intel-SIG: commit 8ba89a3c7967 dmaengine: idxd: fix desc->vector that isn't being updated. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Fixes: da435aedb00a ("dmaengine: idxd: fix array index when int_handles are being used") Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162628784374.353761.4736602409627820431.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/submit.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/dma/idxd/submit.c b/drivers/dma/idxd/submit.c index b0f1ddf75d31..0f0c1fbdc116 100644 --- a/drivers/dma/idxd/submit.c +++ b/drivers/dma/idxd/submit.c @@ -25,11 +25,10 @@ static struct idxd_desc *__get_desc(struct idxd_wq *wq, int idx, int cpu) * Descriptor completion vectors are 1...N for MSIX. We will round * robin through the N vectors. */ - wq->vec_ptr = (wq->vec_ptr % idxd->num_wq_irqs) + 1; + wq->vec_ptr = desc->vector = (wq->vec_ptr % idxd->num_wq_irqs) + 1; if (!idxd->int_handles) { desc->hw->int_handle = wq->vec_ptr; } else { - desc->vector = wq->vec_ptr; /* * int_handles are only for descriptor completion. However for device * MSIX enumeration, vec 0 is used for misc interrupts. Therefore even -- Gitee From 0c7416d46f9018eb368491efea2179563857b6d1 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 24 Jun 2021 12:09:29 -0700 Subject: [PATCH 29/85] dmaengine: idxd: fix array index when int_handles are being used ANBZ: #22 commit da435aedb00a4ef61019ff11ae0c08ffb9b1fb18 upstream. The index to the irq vector should be local and has no relation to the assigned interrupt handle. Assign the MSIX interrupt index that is programmed for the descriptor. The interrupt handle only matters when it comes to hardware descriptor programming. Intel-SIG: commit da435aedb00a dmaengine: idxd: fix array index when int_handles are being used. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Fixes: eb15e7154fbf ("dmaengine: idxd: add interrupt handle request and release support") Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162456176939.1121476.3366256009925001897.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/submit.c | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/drivers/dma/idxd/submit.c b/drivers/dma/idxd/submit.c index 0f0c1fbdc116..cb3e65218781 100644 --- a/drivers/dma/idxd/submit.c +++ b/drivers/dma/idxd/submit.c @@ -129,19 +129,8 @@ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc) * Pending the descriptor to the lockless list for the irq_entry * that we designated the descriptor to. */ - if (desc->hw->flags & IDXD_OP_FLAG_RCI) { - int vec; - - /* - * If the driver is on host kernel, it would be the value - * assigned to interrupt handle, which is index for MSIX - * vector. If it's guest then can't use the int_handle since - * that is the index to IMS for the entire device. The guest - * device local index will be used. - */ - vec = !idxd->int_handles ? desc->hw->int_handle : desc->vector; - llist_add(&desc->llnode, &idxd->irq_entries[vec].pending_llist); - } + if (desc->hw->flags & IDXD_OP_FLAG_RCI) + llist_add(&desc->llnode, &idxd->irq_entries[desc->vector].pending_llist); return 0; } -- Gitee From 034a73e3001f8e353c688001fae99b1076148885 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 24 Jun 2021 13:39:33 -0700 Subject: [PATCH 30/85] dmaengine: idxd: assign MSIX vectors to each WQ rather than roundrobin ANBZ: #22 commit 6cfd9e62e3297993f9f9d2d15f3acb14a0e8abbf upstream. IOPS increased when changing MSIX vector to per WQ from roundrobin. Allows descriptor to be completed by the submitter improves caching locality. Intel-SIG: commit 6cfd9e62e329 dmaengine: idxd: assign MSIX vectors to each WQ rather than roundrobin. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Deviation from upstream: Merge commit da435aedb00a dmaengine: idxd: fix array index when int_handles are being used. Suggested-by: Konstantin Ananyev Signed-off-by: Dave Jiang Acked-by: Konstantin Ananyev Link: https://lore.kernel.org/r/162456717326.1130457.15258077196523268356.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/idxd.h | 2 -- drivers/dma/idxd/submit.c | 22 +++++++--------------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 26482c7d4c3a..de61bf98110c 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -153,7 +153,6 @@ struct idxd_wq { enum idxd_wq_state state; unsigned long flags; union wqcfg *wqcfg; - u32 vec_ptr; /* interrupt steering */ struct dsa_hw_desc **hw_descs; int num_descs; union { @@ -290,7 +289,6 @@ struct idxd_desc { struct list_head list; int id; int cpu; - unsigned int vector; struct idxd_wq *wq; }; diff --git a/drivers/dma/idxd/submit.c b/drivers/dma/idxd/submit.c index cb3e65218781..425eeea9577b 100644 --- a/drivers/dma/idxd/submit.c +++ b/drivers/dma/idxd/submit.c @@ -22,21 +22,13 @@ static struct idxd_desc *__get_desc(struct idxd_wq *wq, int idx, int cpu) desc->hw->pasid = idxd->pasid; /* - * Descriptor completion vectors are 1...N for MSIX. We will round - * robin through the N vectors. + * On host, MSIX vecotr 0 is used for misc interrupt. Therefore when we match + * vector 1:1 to the WQ id, we need to add 1 */ - wq->vec_ptr = desc->vector = (wq->vec_ptr % idxd->num_wq_irqs) + 1; - if (!idxd->int_handles) { - desc->hw->int_handle = wq->vec_ptr; - } else { - /* - * int_handles are only for descriptor completion. However for device - * MSIX enumeration, vec 0 is used for misc interrupts. Therefore even - * though we are rotating through 1...N for descriptor interrupts, we - * need to acqurie the int_handles from 0..N-1. - */ - desc->hw->int_handle = idxd->int_handles[desc->vector - 1]; - } + if (!idxd->int_handles) + desc->hw->int_handle = wq->id + 1; + else + desc->hw->int_handle = idxd->int_handles[wq->id]; return desc; } @@ -130,7 +122,7 @@ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc) * that we designated the descriptor to. */ if (desc->hw->flags & IDXD_OP_FLAG_RCI) - llist_add(&desc->llnode, &idxd->irq_entries[desc->vector].pending_llist); + llist_add(&desc->llnode, &idxd->irq_entries[wq->id + 1].pending_llist); return 0; } -- Gitee From 21ee8e72b3189530711677b1a509c844e21b19dd Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 24 Jun 2021 13:43:32 -0700 Subject: [PATCH 31/85] dmaengine: idxd: fix setup sequence for MSIXPERM table ANBZ: #22 commit d5c10e0fc8645342fe5c9796b00c84ab078cd713 upstream. The MSIX permission table should be programmed BEFORE request_irq() happens. This prevents any possibility of an interrupt happening before the MSIX perm table is setup, however slight. Intel-SIG: commit d5c10e0fc864 dmaengine: idxd: fix setup sequence for MSIXPERM table. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Fixes: 6df0e6c57dfc ("dmaengine: idxd: clear MSIX permission entry on shutdown") Sign-off-by: Dave Jiang Link: https://lore.kernel.org/r/162456741222.1138073.1298447364671237896.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/init.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index c8ae41d36040..4e32a4dcc3ab 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -102,6 +102,8 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) spin_lock_init(&idxd->irq_entries[i].list_lock); } + idxd_msix_perm_setup(idxd); + irq_entry = &idxd->irq_entries[0]; rc = request_threaded_irq(irq_entry->vector, NULL, idxd_misc_thread, 0, "idxd-misc", irq_entry); @@ -148,7 +150,6 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) } idxd_unmask_error_interrupts(idxd); - idxd_msix_perm_setup(idxd); return 0; err_wq_irqs: @@ -162,6 +163,7 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) err_misc_irq: /* Disable error interrupt generation */ idxd_mask_error_interrupts(idxd); + idxd_msix_perm_clear(idxd); err_irq_entries: pci_free_irq_vectors(pdev); dev_err(dev, "No usable interrupts\n"); -- Gitee From 225d1b4e633edb5198b8d59c52f29a149edbfb27 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 14 Jul 2021 14:57:19 -0700 Subject: [PATCH 32/85] dmaengine: idxd: fix sequence for pci driver remove() and shutdown() ANBZ: #22 commit 7eb25da161befbc9a80e94e1bd90d6c06aa645cf upstream. ->shutdown() call should only be responsible for quiescing the device. Currently it is doing PCI device tear down. This causes issue when things like MMIO mapping is removed while idxd_unregister_devices() will trigger removal of idxd device sub-driver and still initiates MMIO writes to the device. Another issue is with the unregistering of idxd 'struct device', the memory context gets freed. So the teardown calls are accessing freed memory and can cause kernel oops. Move all the teardown bits that doesn't belong in shutdown to ->remove() call. Move unregistering of the idxd conf_dev 'struct device' to after doing all the teardown to free all the memory that's no longer needed. Intel-SIG: commit 7eb25da161be dmaengine: idxd: fix sequence for pci driver remove() and shutdown(). Incremental backporting patches for DSA/IAX on Intel Xeon platform. Fixes: 47c16ac27d4c ("dmaengine: idxd: fix idxd conf_dev 'struct device' lifetime") Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162629983901.395844.17964803190905549615.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/init.c | 26 +++++++++++++++++--------- drivers/dma/idxd/sysfs.c | 2 -- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 4e32a4dcc3ab..c0f4c0422f32 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -760,32 +760,40 @@ static void idxd_shutdown(struct pci_dev *pdev) for (i = 0; i < msixcnt; i++) { irq_entry = &idxd->irq_entries[i]; synchronize_irq(irq_entry->vector); - free_irq(irq_entry->vector, irq_entry); if (i == 0) continue; idxd_flush_pending_llist(irq_entry); idxd_flush_work_list(irq_entry); } - - idxd_msix_perm_clear(idxd); - idxd_release_int_handles(idxd); - pci_free_irq_vectors(pdev); - pci_iounmap(pdev, idxd->reg_base); - pci_disable_device(pdev); - destroy_workqueue(idxd->wq); + flush_workqueue(idxd->wq); } static void idxd_remove(struct pci_dev *pdev) { struct idxd_device *idxd = pci_get_drvdata(pdev); + struct idxd_irq_entry *irq_entry; + int msixcnt = pci_msix_vec_count(pdev); + int i; dev_dbg(&pdev->dev, "%s called\n", __func__); idxd_shutdown(pdev); if (device_pasid_enabled(idxd)) idxd_disable_system_pasid(idxd); idxd_unregister_devices(idxd); - perfmon_pmu_remove(idxd); + + for (i = 0; i < msixcnt; i++) { + irq_entry = &idxd->irq_entries[i]; + free_irq(irq_entry->vector, irq_entry); + } + idxd_msix_perm_clear(idxd); + idxd_release_int_handles(idxd); + pci_free_irq_vectors(pdev); + pci_iounmap(pdev, idxd->reg_base); iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); + pci_disable_device(pdev); + destroy_workqueue(idxd->wq); + perfmon_pmu_remove(idxd); + device_unregister(&idxd->conf_dev); } static struct pci_driver idxd_pci_driver = { diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 0460d58e3941..bb4df63906a7 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -1744,8 +1744,6 @@ void idxd_unregister_devices(struct idxd_device *idxd) device_unregister(&group->conf_dev); } - - device_unregister(&idxd->conf_dev); } int idxd_register_bus_type(void) -- Gitee From e9f813e19f2c3a32b1e808de51cee506ce26adc7 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 14 Jul 2021 11:50:06 -0700 Subject: [PATCH 33/85] dmaengine: idxd: fix submission race window ANBZ: #22 commit 6b4b87f2c31ac1af4f244990a7cbfb50d3f3e33f upstream. Konstantin observed that when descriptors are submitted, the descriptor is added to the pending list after the submission. This creates a race window with the slight possibility that the descriptor can complete before it gets added to the pending list and this window would cause the completion handler to miss processing the descriptor. To address the issue, the addition of the descriptor to the pending list must be done before it gets submitted to the hardware. However, submitting to swq with ENQCMDS instruction can cause a failure with the condition of either wq is full or wq is not "active". With the descriptor allocation being the gate to the wq capacity, it is not possible to hit a retry with ENQCMDS submission to the swq. The only possible failure can happen is when wq is no longer "active" due to hw error and therefore we are moving towards taking down the portal. Given this is a rare condition and there's no longer concern over I/O performance, the driver can walk the completion lists in order to retrieve and abort the descriptor. The error path will set the descriptor to aborted status. It will take the work list lock to prevent further processing of worklist. It will do a delete_all on the pending llist to retrieve all descriptors on the pending llist. The delete_all action does not require a lock. It will walk through the acquired llist to find the aborted descriptor while add all remaining descriptors to the work list since it holds the lock. If it does not find the aborted descriptor on the llist, it will walk through the work list. And if it still does not find the descriptor, then it means the interrupt handler has removed the desc from the llist but is pending on the work list lock and will process it once the error path releases the lock. Intel-SIG: commit 6b4b87f2c31a dmaengine: idxd: fix submission race window. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Deviation from upstream: Merge commit 88c5d0a2b9b0 Merge branch 'fixes' into next. Fixes: eb15e7154fbf ("dmaengine: idxd: add interrupt handle request and release support") Reported-by: Konstantin Ananyev Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162628855747.360485.10101925573082466530.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/idxd.h | 14 ++++++++ drivers/dma/idxd/irq.c | 27 +++++++++----- drivers/dma/idxd/submit.c | 75 ++++++++++++++++++++++++++++++++++----- 3 files changed, 99 insertions(+), 17 deletions(-) diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index de61bf98110c..551413eb0ffb 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -292,6 +292,14 @@ struct idxd_desc { struct idxd_wq *wq; }; +/* + * This is software defined error for the completion status. We overload the error code + * that will never appear in completion status and only SWERR register. + */ +enum idxd_completion_status { + IDXD_COMP_DESC_ABORT = 0xff, +}; + #define confdev_to_idxd(dev) container_of(dev, struct idxd_device, conf_dev) #define confdev_to_wq(dev) container_of(dev, struct idxd_wq, conf_dev) @@ -480,4 +488,10 @@ static inline void perfmon_init(void) {} static inline void perfmon_exit(void) {} #endif +static inline void complete_desc(struct idxd_desc *desc, enum idxd_complete_type reason) +{ + idxd_dma_complete_txd(desc, reason); + idxd_free_desc(desc->wq, desc); +} + #endif diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c index ae68e1e5487a..4e3a7198c0ca 100644 --- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -245,12 +245,6 @@ static inline bool match_fault(struct idxd_desc *desc, u64 fault_addr) return false; } -static inline void complete_desc(struct idxd_desc *desc, enum idxd_complete_type reason) -{ - idxd_dma_complete_txd(desc, reason); - idxd_free_desc(desc->wq, desc); -} - static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry, enum irq_work_type wtype, int *processed, u64 data) @@ -272,8 +266,16 @@ static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry, reason = IDXD_COMPLETE_DEV_FAIL; llist_for_each_entry_safe(desc, t, head, llnode) { - if (desc->completion->status) { - if ((desc->completion->status & DSA_COMP_STATUS_MASK) != DSA_COMP_SUCCESS) + u8 status = desc->completion->status & DSA_COMP_STATUS_MASK; + + if (status) { + if (unlikely(status == IDXD_COMP_DESC_ABORT)) { + complete_desc(desc, IDXD_COMPLETE_ABORT); + (*processed)++; + continue; + } + + if (unlikely(status != DSA_COMP_SUCCESS)) match_fault(desc, data); complete_desc(desc, reason); (*processed)++; @@ -329,7 +331,14 @@ static int irq_process_work_list(struct idxd_irq_entry *irq_entry, spin_unlock_irqrestore(&irq_entry->list_lock, flags); list_for_each_entry(desc, &flist, list) { - if ((desc->completion->status & DSA_COMP_STATUS_MASK) != DSA_COMP_SUCCESS) + u8 status = desc->completion->status & DSA_COMP_STATUS_MASK; + + if (unlikely(status == IDXD_COMP_DESC_ABORT)) { + complete_desc(desc, IDXD_COMPLETE_ABORT); + continue; + } + + if (unlikely(status != DSA_COMP_SUCCESS)) match_fault(desc, data); complete_desc(desc, reason); } diff --git a/drivers/dma/idxd/submit.c b/drivers/dma/idxd/submit.c index 425eeea9577b..6ef704dd4d0b 100644 --- a/drivers/dma/idxd/submit.c +++ b/drivers/dma/idxd/submit.c @@ -79,9 +79,64 @@ void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc) sbitmap_queue_clear(&wq->sbq, desc->id, cpu); } +static struct idxd_desc *list_abort_desc(struct idxd_wq *wq, struct idxd_irq_entry *ie, + struct idxd_desc *desc) +{ + struct idxd_desc *d, *n; + + lockdep_assert_held(&ie->list_lock); + list_for_each_entry_safe(d, n, &ie->work_list, list) { + if (d == desc) { + list_del(&d->list); + return d; + } + } + + /* + * At this point, the desc needs to be aborted is held by the completion + * handler where it has taken it off the pending list but has not added to the + * work list. It will be cleaned up by the interrupt handler when it sees the + * IDXD_COMP_DESC_ABORT for completion status. + */ + return NULL; +} + +static void llist_abort_desc(struct idxd_wq *wq, struct idxd_irq_entry *ie, + struct idxd_desc *desc) +{ + struct idxd_desc *d, *t, *found = NULL; + struct llist_node *head; + unsigned long flags; + + desc->completion->status = IDXD_COMP_DESC_ABORT; + /* + * Grab the list lock so it will block the irq thread handler. This allows the + * abort code to locate the descriptor need to be aborted. + */ + spin_lock_irqsave(&ie->list_lock, flags); + head = llist_del_all(&ie->pending_llist); + if (head) { + llist_for_each_entry_safe(d, t, head, llnode) { + if (d == desc) { + found = desc; + continue; + } + list_add_tail(&desc->list, &ie->work_list); + } + } + + if (!found) + found = list_abort_desc(wq, ie, desc); + spin_unlock_irqrestore(&ie->list_lock, flags); + + if (found) + complete_desc(found, IDXD_COMPLETE_ABORT); +} + int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc) { struct idxd_device *idxd = wq->idxd; + struct idxd_irq_entry *ie = NULL; void __iomem *portal; int rc; @@ -99,6 +154,16 @@ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc) * even on UP because the recipient is a device. */ wmb(); + + /* + * Pending the descriptor to the lockless list for the irq_entry + * that we designated the descriptor to. + */ + if (desc->hw->flags & IDXD_OP_FLAG_RCI) { + ie = &idxd->irq_entries[wq->id + 1]; + llist_add(&desc->llnode, &ie->pending_llist); + } + if (wq_dedicated(wq)) { iosubmit_cmds512(portal, desc->hw, 1); } else { @@ -111,18 +176,12 @@ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc) rc = enqcmds(portal, desc->hw); if (rc < 0) { percpu_ref_put(&wq->wq_active); + if (ie) + llist_abort_desc(wq, ie, desc); return rc; } } percpu_ref_put(&wq->wq_active); - - /* - * Pending the descriptor to the lockless list for the irq_entry - * that we designated the descriptor to. - */ - if (desc->hw->flags & IDXD_OP_FLAG_RCI) - llist_add(&desc->llnode, &idxd->irq_entries[wq->id + 1].pending_llist); - return 0; } -- Gitee From cbda1b1041667d220c09e1788a033e31b06c2095 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Thu, 8 Jul 2021 07:08:26 +0200 Subject: [PATCH 34/85] dmaengine: idxd: Simplify code and axe the use of a deprecated API ANBZ: #22 commit 53b50458110d829c8fc45e7803a335a515698fd8 upstream. The wrappers in include/linux/pci-dma-compat.h should go away. Replace 'pci_set_dma_mask/pci_set_consistent_dma_mask' by an equivalent and less verbose 'dma_set_mask_and_coherent()' call. Even if the code may look different, it should have exactly the same run-time behavior. If pci_set_dma_mask(64) fails and pci_set_dma_mask(32) succeeds, then pci_set_consistent_dma_mask(64) will also fail. Intel-SIG: commit 53b50458110d dmaengine: idxd: Simplify code and axe the use of a deprecated API. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Signed-off-by: Christophe JAILLET Acked-by: Dave Jiang Link: https://lore.kernel.org/r/70c8a3bc67e41c5fefb526ecd64c5174c1e2dc76.1625720835.git.christophe.jaillet@wanadoo.fr Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/init.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index c0f4c0422f32..75ac6a4bc9d1 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -639,15 +639,9 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) } dev_dbg(dev, "Set DMA masks\n"); - rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); if (rc) - rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); - if (rc) - goto err; - - rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); - if (rc) - rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); if (rc) goto err; -- Gitee From d80d10472d1d6e68c0cce111d23231fea1e03ee9 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 4 Jun 2021 17:06:21 -0700 Subject: [PATCH 35/85] dmanegine: idxd: cleanup all device related bits after disabling device ANBZ: #22 commit 0dcfe41e9a4ca759ccc87a48e3bb9cc3b08ff1e8 upstream. The previous state cleanup patch only performed wq state cleanups. This does not go far enough as when device is disabled or reset, the state for groups and engines must also be cleaned up. Add additional state cleanup beyond wq cleanup. Tie those cleanups directly to device disable and reset, and wq disable and reset. Intel-SIG: commit 0dcfe41e9a4c dmanegine: idxd: cleanup all device related bits after disabling device. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Fixes: da32b28c95a7 ("dmaengine: idxd: cleanup workqueue config after disabling") Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162285154108.2096632.5572805472362321307.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 88 +++++++++++++++++++++++++++++---------- drivers/dma/idxd/idxd.h | 6 +-- drivers/dma/idxd/irq.c | 4 +- drivers/dma/idxd/sysfs.c | 22 +++------- 4 files changed, 74 insertions(+), 46 deletions(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 420b93fe5feb..928c2c8817f0 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -15,6 +15,8 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand, u32 *status); +static void idxd_device_wqs_clear_state(struct idxd_device *idxd); +static void idxd_wq_disable_cleanup(struct idxd_wq *wq); /* Interrupt control bits */ void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id) @@ -234,7 +236,7 @@ int idxd_wq_enable(struct idxd_wq *wq) return 0; } -int idxd_wq_disable(struct idxd_wq *wq) +int idxd_wq_disable(struct idxd_wq *wq, bool reset_config) { struct idxd_device *idxd = wq->idxd; struct device *dev = &idxd->pdev->dev; @@ -255,6 +257,8 @@ int idxd_wq_disable(struct idxd_wq *wq) return -ENXIO; } + if (reset_config) + idxd_wq_disable_cleanup(wq); wq->state = IDXD_WQ_DISABLED; dev_dbg(dev, "WQ %d disabled\n", wq->id); return 0; @@ -289,6 +293,7 @@ void idxd_wq_reset(struct idxd_wq *wq) operand = BIT(wq->id % 16) | ((wq->id / 16) << 16); idxd_cmd_exec(idxd, IDXD_CMD_RESET_WQ, operand, NULL); + idxd_wq_disable_cleanup(wq); wq->state = IDXD_WQ_DISABLED; } @@ -337,7 +342,7 @@ int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid) unsigned int offset; unsigned long flags; - rc = idxd_wq_disable(wq); + rc = idxd_wq_disable(wq, false); if (rc < 0) return rc; @@ -364,7 +369,7 @@ int idxd_wq_disable_pasid(struct idxd_wq *wq) unsigned int offset; unsigned long flags; - rc = idxd_wq_disable(wq); + rc = idxd_wq_disable(wq, false); if (rc < 0) return rc; @@ -383,11 +388,11 @@ int idxd_wq_disable_pasid(struct idxd_wq *wq) return 0; } -void idxd_wq_disable_cleanup(struct idxd_wq *wq) +static void idxd_wq_disable_cleanup(struct idxd_wq *wq) { struct idxd_device *idxd = wq->idxd; - lockdep_assert_held(&idxd->dev_lock); + lockdep_assert_held(&wq->wq_lock); memset(wq->wqcfg, 0, idxd->wqcfg_size); wq->type = IDXD_WQT_NONE; wq->size = 0; @@ -548,22 +553,6 @@ int idxd_device_enable(struct idxd_device *idxd) return 0; } -void idxd_device_wqs_clear_state(struct idxd_device *idxd) -{ - int i; - - lockdep_assert_held(&idxd->dev_lock); - - for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = idxd->wqs[i]; - - if (wq->state == IDXD_WQ_ENABLED) { - idxd_wq_disable_cleanup(wq); - wq->state = IDXD_WQ_DISABLED; - } - } -} - int idxd_device_disable(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; @@ -585,7 +574,7 @@ int idxd_device_disable(struct idxd_device *idxd) } spin_lock_irqsave(&idxd->dev_lock, flags); - idxd_device_wqs_clear_state(idxd); + idxd_device_clear_state(idxd); idxd->state = IDXD_DEV_CONF_READY; spin_unlock_irqrestore(&idxd->dev_lock, flags); return 0; @@ -597,7 +586,7 @@ void idxd_device_reset(struct idxd_device *idxd) idxd_cmd_exec(idxd, IDXD_CMD_RESET_DEVICE, 0, NULL); spin_lock_irqsave(&idxd->dev_lock, flags); - idxd_device_wqs_clear_state(idxd); + idxd_device_clear_state(idxd); idxd->state = IDXD_DEV_CONF_READY; spin_unlock_irqrestore(&idxd->dev_lock, flags); } @@ -685,6 +674,59 @@ int idxd_device_release_int_handle(struct idxd_device *idxd, int handle, } /* Device configuration bits */ +static void idxd_engines_clear_state(struct idxd_device *idxd) +{ + struct idxd_engine *engine; + int i; + + lockdep_assert_held(&idxd->dev_lock); + for (i = 0; i < idxd->max_engines; i++) { + engine = idxd->engines[i]; + engine->group = NULL; + } +} + +static void idxd_groups_clear_state(struct idxd_device *idxd) +{ + struct idxd_group *group; + int i; + + lockdep_assert_held(&idxd->dev_lock); + for (i = 0; i < idxd->max_groups; i++) { + group = idxd->groups[i]; + memset(&group->grpcfg, 0, sizeof(group->grpcfg)); + group->num_engines = 0; + group->num_wqs = 0; + group->use_token_limit = false; + group->tokens_allowed = 0; + group->tokens_reserved = 0; + group->tc_a = -1; + group->tc_b = -1; + } +} + +static void idxd_device_wqs_clear_state(struct idxd_device *idxd) +{ + int i; + + lockdep_assert_held(&idxd->dev_lock); + for (i = 0; i < idxd->max_wqs; i++) { + struct idxd_wq *wq = idxd->wqs[i]; + + if (wq->state == IDXD_WQ_ENABLED) { + idxd_wq_disable_cleanup(wq); + wq->state = IDXD_WQ_DISABLED; + } + } +} + +void idxd_device_clear_state(struct idxd_device *idxd) +{ + idxd_groups_clear_state(idxd); + idxd_engines_clear_state(idxd); + idxd_device_wqs_clear_state(idxd); +} + void idxd_msix_perm_setup(struct idxd_device *idxd) { union msix_perm mperm; diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 551413eb0ffb..d875b3d41ed2 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -426,9 +426,8 @@ int idxd_device_init_reset(struct idxd_device *idxd); int idxd_device_enable(struct idxd_device *idxd); int idxd_device_disable(struct idxd_device *idxd); void idxd_device_reset(struct idxd_device *idxd); -void idxd_device_cleanup(struct idxd_device *idxd); +void idxd_device_clear_state(struct idxd_device *idxd); int idxd_device_config(struct idxd_device *idxd); -void idxd_device_wqs_clear_state(struct idxd_device *idxd); void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid); int idxd_device_load_config(struct idxd_device *idxd); int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle, @@ -441,12 +440,11 @@ void idxd_wqs_unmap_portal(struct idxd_device *idxd); int idxd_wq_alloc_resources(struct idxd_wq *wq); void idxd_wq_free_resources(struct idxd_wq *wq); int idxd_wq_enable(struct idxd_wq *wq); -int idxd_wq_disable(struct idxd_wq *wq); +int idxd_wq_disable(struct idxd_wq *wq, bool reset_config); void idxd_wq_drain(struct idxd_wq *wq); void idxd_wq_reset(struct idxd_wq *wq); int idxd_wq_map_portal(struct idxd_wq *wq); void idxd_wq_unmap_portal(struct idxd_wq *wq); -void idxd_wq_disable_cleanup(struct idxd_wq *wq); int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid); int idxd_wq_disable_pasid(struct idxd_wq *wq); void idxd_wq_quiesce(struct idxd_wq *wq); diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c index 4e3a7198c0ca..2924819ca8f3 100644 --- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -59,7 +59,7 @@ static void idxd_device_reinit(struct work_struct *work) return; out: - idxd_device_wqs_clear_state(idxd); + idxd_device_clear_state(idxd); } static void idxd_device_fault_work(struct work_struct *work) @@ -192,7 +192,7 @@ static int process_misc_interrupts(struct idxd_device *idxd, u32 cause) spin_lock_bh(&idxd->dev_lock); idxd_wqs_quiesce(idxd); idxd_wqs_unmap_portal(idxd); - idxd_device_wqs_clear_state(idxd); + idxd_device_clear_state(idxd); dev_err(&idxd->pdev->dev, "idxd halted, need %s.\n", gensts.reset_type == IDXD_DEVICE_RESET_FLR ? diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index bb4df63906a7..528cde54724b 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -129,7 +129,7 @@ static int enable_wq(struct idxd_wq *wq) rc = idxd_wq_map_portal(wq); if (rc < 0) { dev_warn(dev, "wq portal mapping failed: %d\n", rc); - rc = idxd_wq_disable(wq); + rc = idxd_wq_disable(wq, false); if (rc < 0) dev_warn(dev, "IDXD wq disable failed\n"); mutex_unlock(&wq->wq_lock); @@ -262,8 +262,6 @@ static void disable_wq(struct idxd_wq *wq) static int idxd_config_bus_remove(struct device *dev) { - int rc; - dev_dbg(dev, "%s called for %s\n", __func__, dev_name(dev)); /* disable workqueue here */ @@ -288,22 +286,12 @@ static int idxd_config_bus_remove(struct device *dev) } idxd_unregister_dma_device(idxd); - rc = idxd_device_disable(idxd); - if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { - for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = idxd->wqs[i]; - - mutex_lock(&wq->wq_lock); - idxd_wq_disable_cleanup(wq); - mutex_unlock(&wq->wq_lock); - } - } + idxd_device_disable(idxd); + if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) + idxd_device_reset(idxd); module_put(THIS_MODULE); - if (rc < 0) - dev_warn(dev, "Device disable failed\n"); - else - dev_info(dev, "Device %s disabled\n", dev_name(dev)); + dev_info(dev, "Device %s disabled\n", dev_name(dev)); } return 0; -- Gitee From 4f92ca708d736e77b5d4b04123da64c3e83e602e Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 3 Jun 2021 14:57:35 -0700 Subject: [PATCH 36/85] dmaengine: idxd: Add wq occupancy information to sysfs attribute ANBZ: #22 commit e753a64bee753136087dfd70b37fdd199e942ea9 upstream. Add occupancy information to wq sysfs attribute. Attribute will show wq occupancy data if "WQ Occupancy Support" field in WQCAP is 1. It displays the number of entries currently in this WQ. This is provided as an estimate and should not be relied on to determine whether there is space in the WQ. The data is to provide information to user apps for flow control. Intel-SIG: commit e753a64bee75 dmaengine: idxd: Add wq occupancy information to sysfs attribute. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162275745546.1857062.8765615879420582018.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- .../ABI/stable/sysfs-driver-dma-idxd | 7 +++++++ drivers/dma/idxd/registers.h | 3 +++ drivers/dma/idxd/sysfs.c | 19 +++++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/Documentation/ABI/stable/sysfs-driver-dma-idxd b/Documentation/ABI/stable/sysfs-driver-dma-idxd index 55285c136cf0..ebd521314803 100644 --- a/Documentation/ABI/stable/sysfs-driver-dma-idxd +++ b/Documentation/ABI/stable/sysfs-driver-dma-idxd @@ -211,6 +211,13 @@ Contact: dmaengine@vger.kernel.org Description: Indicate whether ATS disable is turned on for the workqueue. 0 indicates ATS is on, and 1 indicates ATS is off for the workqueue. +What: /sys/bus/dsa/devices/wq./occupancy +Date May 25, 2021 +KernelVersion: 5.14.0 +Contact: dmaengine@vger.kernel.org +Description: Show the current number of entries in this WQ if WQ Occupancy + Support bit WQ capabilities is 1. + What: /sys/bus/dsa/devices/engine./group_id Date: Oct 25, 2019 KernelVersion: 5.6.0 diff --git a/drivers/dma/idxd/registers.h b/drivers/dma/idxd/registers.h index c970c3f025f0..7343a8f48819 100644 --- a/drivers/dma/idxd/registers.h +++ b/drivers/dma/idxd/registers.h @@ -349,6 +349,9 @@ union wqcfg { } __packed; #define WQCFG_PASID_IDX 2 +#define WQCFG_OCCUP_IDX 6 + +#define WQCFG_OCCUP_MASK 0xffff /* * This macro calculates the offset into the WQCFG register diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 528cde54724b..33c27df40f1e 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -1246,6 +1246,24 @@ static ssize_t wq_ats_disable_store(struct device *dev, struct device_attribute static struct device_attribute dev_attr_wq_ats_disable = __ATTR(ats_disable, 0644, wq_ats_disable_show, wq_ats_disable_store); +static ssize_t wq_occupancy_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct idxd_wq *wq = confdev_to_wq(dev); + struct idxd_device *idxd = wq->idxd; + u32 occup, offset; + + if (!idxd->hw.wq_cap.occupancy) + return -EOPNOTSUPP; + + offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_OCCUP_IDX); + occup = ioread32(idxd->reg_base + offset) & WQCFG_OCCUP_MASK; + + return sysfs_emit(buf, "%u\n", occup); +} + +static struct device_attribute dev_attr_wq_occupancy = + __ATTR(occupancy, 0444, wq_occupancy_show, NULL); + static struct attribute *idxd_wq_attributes[] = { &dev_attr_wq_clients.attr, &dev_attr_wq_state.attr, @@ -1261,6 +1279,7 @@ static struct attribute *idxd_wq_attributes[] = { &dev_attr_wq_max_transfer_size.attr, &dev_attr_wq_max_batch_size.attr, &dev_attr_wq_ats_disable.attr, + &dev_attr_wq_occupancy.attr, NULL, }; -- Gitee From 36d2b0b33c272a0db11575a26702fd64742a3c2e Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 3 Jun 2021 11:01:37 -0700 Subject: [PATCH 37/85] dmaengine: idxd: have command status always set ANBZ: #22 commit 53499d1fc11267e078557fc68ce943c1eb3b7a37 upstream. The cached command status is only set when the write back status is is passed in. Move the variable set outside of the check so it is always set. Intel-SIG: commit 53499d1fc112 dmaengine: idxd: have command status always set. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Fixes: 0d5c10b4c84d ("dmaengine: idxd: add work queue drain support") Reported-by: Ramesh Thomas Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162274329740.1822314.3443875665504707588.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 928c2c8817f0..c8cf1de72176 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -486,6 +486,7 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand, union idxd_command_reg cmd; DECLARE_COMPLETION_ONSTACK(done); unsigned long flags; + u32 stat; if (idxd_device_is_halted(idxd)) { dev_warn(&idxd->pdev->dev, "Device is HALTED!\n"); @@ -518,11 +519,11 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand, */ spin_unlock_irqrestore(&idxd->cmd_lock, flags); wait_for_completion(&done); + stat = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET); spin_lock_irqsave(&idxd->cmd_lock, flags); - if (status) { - *status = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET); - idxd->cmd_status = *status & GENMASK(7, 0); - } + if (status) + *status = stat; + idxd->cmd_status = stat & GENMASK(7, 0); __clear_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags); /* Wake up other pending commands */ -- Gitee From 22d8e711fd8afc23a25e892595967f90fc70458f Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Fri, 25 Jun 2021 10:38:10 +0200 Subject: [PATCH 38/85] dmaengine: idxd: depends on !UML ANBZ: #22 commit b2296eeac91555bd13f774efa7ab7d4b12fb71ef upstream. Now that UML has PCI support, this driver must depend also on !UML since it pokes at X86_64 architecture internals that don't exist on ARCH=um. Intel-SIG: commit b2296eeac915 dmaengine: idxd: depends on !UML. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reported-by: kernel test robot Signed-off-by: Johannes Berg Acked-by: Dave Jiang Acked-By: Anton Ivanov Link: https://lore.kernel.org/r/20210625103810.fe877ae0aef4.If240438e3f50ae226f3f755fc46ea498c6858393@changeid Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 1d522ae93c6e..1a7c46615464 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -285,7 +285,7 @@ config INTEL_IDMA64 config INTEL_IDXD tristate "Intel Data Accelerators support" - depends on PCI && X86_64 + depends on PCI && X86_64 && !UML depends on PCI_MSI depends on SBITMAP select DMA_ENGINE -- Gitee From a90c6094798e3f6cbfcbf38b89ab0689df6844a8 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Jul 2021 11:43:09 -0700 Subject: [PATCH 39/85] dmaengine: idxd: add driver register helper ANBZ: #22 commit 3ecfc9135e6c82183d121c5578ed5d6f07a53ec8 upstream. Add helper functions for dsa-driver registration similar to other bus-types. In particular, do not require dsa-drivers to open-code the bus, owner, and mod_name fields. Let registration and unregistration operate on the 'struct idxd_device_driver' instead of the raw / embedded 'struct device_driver'. Intel-SIG: commit 3ecfc9135e6c dmaengine: idxd: add driver register helper. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reviewed-by: Dan Williams Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162637458949.744545.14996726325385482050.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/idxd.h | 7 +++++++ drivers/dma/idxd/init.c | 17 +++++++++++++++++ drivers/dma/idxd/sysfs.c | 7 ++----- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index d875b3d41ed2..8db19b899709 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -402,6 +402,13 @@ static inline int idxd_wq_refcount(struct idxd_wq *wq) return wq->client_count; }; +int __must_check __idxd_driver_register(struct idxd_device_driver *idxd_drv, + struct module *module, const char *mod_name); +#define idxd_driver_register(driver) \ + __idxd_driver_register(driver, THIS_MODULE, KBUILD_MODNAME) + +void idxd_driver_unregister(struct idxd_device_driver *idxd_drv); + int idxd_register_bus_type(void); void idxd_unregister_bus_type(void); int idxd_register_devices(struct idxd_device *idxd); diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 75ac6a4bc9d1..b15817751d5f 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -855,3 +855,20 @@ static void __exit idxd_exit_module(void) perfmon_exit(); } module_exit(idxd_exit_module); + +int __idxd_driver_register(struct idxd_device_driver *idxd_drv, struct module *owner, + const char *mod_name) +{ + struct device_driver *drv = &idxd_drv->drv; + + drv->bus = &dsa_bus_type; + drv->owner = owner; + drv->mod_name = mod_name; + + return driver_register(drv); +} + +void idxd_driver_unregister(struct idxd_device_driver *idxd_drv) +{ + driver_unregister(&idxd_drv->drv); +} diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 33c27df40f1e..bf229b12d527 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -313,21 +313,18 @@ struct bus_type dsa_bus_type = { static struct idxd_device_driver dsa_drv = { .drv = { .name = "dsa", - .bus = &dsa_bus_type, - .owner = THIS_MODULE, - .mod_name = KBUILD_MODNAME, }, }; /* IDXD generic driver setup */ int idxd_register_driver(void) { - return driver_register(&dsa_drv.drv); + return idxd_driver_register(&dsa_drv); } void idxd_unregister_driver(void) { - driver_unregister(&dsa_drv.drv); + idxd_driver_unregister(&dsa_drv); } /* IDXD engine attributes */ -- Gitee From 1f62a60c657b09766e472c38ca6ba503573f3a40 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Jul 2021 11:43:15 -0700 Subject: [PATCH 40/85] dmaengine: idxd: add driver name ANBZ: #22 commit da5a11d75d6837c9c5ef40810f66ce9d2db6ca5e upstream. Add name field in idxd_device_driver so we don't have to touch the 'struct device_driver' during declaration. Intel-SIG: commit da5a11d75d68 dmaengine: idxd: add driver name. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reviewed-by: Dan Williams Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162637459517.744545.7572915135318813722.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/idxd.h | 1 + drivers/dma/idxd/init.c | 1 + drivers/dma/idxd/sysfs.c | 4 +--- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 8db19b899709..e8721ff028c2 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -34,6 +34,7 @@ enum idxd_type { #define IDXD_PMU_EVENT_MAX 64 struct idxd_device_driver { + const char *name; struct device_driver drv; }; diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index b15817751d5f..6403d55c7ff7 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -861,6 +861,7 @@ int __idxd_driver_register(struct idxd_device_driver *idxd_drv, struct module *o { struct device_driver *drv = &idxd_drv->drv; + drv->name = idxd_drv->name; drv->bus = &dsa_bus_type; drv->owner = owner; drv->mod_name = mod_name; diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index bf229b12d527..60779f57c118 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -311,9 +311,7 @@ struct bus_type dsa_bus_type = { }; static struct idxd_device_driver dsa_drv = { - .drv = { - .name = "dsa", - }, + .name = "dsa", }; /* IDXD generic driver setup */ -- Gitee From e113f1ea1135a895533142854bfe81f170c3afda Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Jul 2021 11:43:20 -0700 Subject: [PATCH 41/85] dmaengine: idxd: add 'struct idxd_dev' as wrapper for conf_dev ANBZ: #22 commit 700af3a0a26cbac87e4a0ae1dfa79597d0056d5f upstream. Add a 'struct idxd_dev' that wraps the 'struct device' for idxd conf_dev that registers with the dsa bus. This is introduced in order to deal with multiple different types of 'devices' that are registered on the dsa_bus when the compat driver needs to route them to the correct driver to attach. The bind() call now can determine the type of device and then do the appropriate driver matching. Intel-SIG: commit 700af3a0a26c dmaengine: idxd: add 'struct idxd_dev' as wrapper for conf_dev. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reviewed-by Dan Williams Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162637460065.744545.584492831446090984.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/cdev.c | 11 +- drivers/dma/idxd/dma.c | 4 +- drivers/dma/idxd/idxd.h | 82 ++++++++++++-- drivers/dma/idxd/init.c | 98 +++++++++------- drivers/dma/idxd/irq.c | 2 +- drivers/dma/idxd/sysfs.c | 239 ++++++++++++++++++--------------------- 6 files changed, 251 insertions(+), 185 deletions(-) diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index e9def577c697..18a003b93812 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -41,7 +41,7 @@ struct idxd_user_context { static void idxd_cdev_dev_release(struct device *dev) { - struct idxd_cdev *idxd_cdev = container_of(dev, struct idxd_cdev, dev); + struct idxd_cdev *idxd_cdev = dev_to_cdev(dev); struct idxd_cdev_context *cdev_ctx; struct idxd_wq *wq = idxd_cdev->wq; @@ -256,9 +256,10 @@ int idxd_wq_add_cdev(struct idxd_wq *wq) if (!idxd_cdev) return -ENOMEM; + idxd_cdev->idxd_dev.type = IDXD_DEV_CDEV; idxd_cdev->wq = wq; cdev = &idxd_cdev->cdev; - dev = &idxd_cdev->dev; + dev = cdev_dev(idxd_cdev); cdev_ctx = &ictx[wq->idxd->data->type]; minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL); if (minor < 0) { @@ -268,7 +269,7 @@ int idxd_wq_add_cdev(struct idxd_wq *wq) idxd_cdev->minor = minor; device_initialize(dev); - dev->parent = &wq->conf_dev; + dev->parent = wq_confdev(wq); dev->bus = &dsa_bus_type; dev->type = &idxd_cdev_device_type; dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor); @@ -299,8 +300,8 @@ void idxd_wq_del_cdev(struct idxd_wq *wq) idxd_cdev = wq->idxd_cdev; wq->idxd_cdev = NULL; - cdev_device_del(&idxd_cdev->cdev, &idxd_cdev->dev); - put_device(&idxd_cdev->dev); + cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev)); + put_device(cdev_dev(idxd_cdev)); } int idxd_cdev_register(void) diff --git a/drivers/dma/idxd/dma.c b/drivers/dma/idxd/dma.c index 77439b645044..2e52f9a50519 100644 --- a/drivers/dma/idxd/dma.c +++ b/drivers/dma/idxd/dma.c @@ -245,7 +245,7 @@ int idxd_register_dma_channel(struct idxd_wq *wq) wq->idxd_chan = idxd_chan; idxd_chan->wq = wq; - get_device(&wq->conf_dev); + get_device(wq_confdev(wq)); return 0; } @@ -260,5 +260,5 @@ void idxd_unregister_dma_channel(struct idxd_wq *wq) list_del(&chan->device_node); kfree(wq->idxd_chan); wq->idxd_chan = NULL; - put_device(&wq->conf_dev); + put_device(wq_confdev(wq)); } diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index e8721ff028c2..ae60fcc7b625 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -17,8 +17,24 @@ extern struct kmem_cache *idxd_desc_pool; -struct idxd_device; struct idxd_wq; +struct idxd_dev; + +enum idxd_dev_type { + IDXD_DEV_NONE = -1, + IDXD_DEV_DSA = 0, + IDXD_DEV_IAX, + IDXD_DEV_WQ, + IDXD_DEV_GROUP, + IDXD_DEV_ENGINE, + IDXD_DEV_CDEV, + IDXD_DEV_MAX_TYPE, +}; + +struct idxd_dev { + struct device conf_dev; + enum idxd_dev_type type; +}; #define IDXD_REG_TIMEOUT 50 #define IDXD_DRAIN_TIMEOUT 5000 @@ -52,7 +68,7 @@ struct idxd_irq_entry { }; struct idxd_group { - struct device conf_dev; + struct idxd_dev idxd_dev; struct idxd_device *idxd; struct grpcfg grpcfg; int id; @@ -111,7 +127,7 @@ enum idxd_wq_type { struct idxd_cdev { struct idxd_wq *wq; struct cdev cdev; - struct device dev; + struct idxd_dev idxd_dev; int minor; }; @@ -139,7 +155,7 @@ struct idxd_wq { void __iomem *portal; struct percpu_ref wq_active; struct completion wq_dead; - struct device conf_dev; + struct idxd_dev idxd_dev; struct idxd_cdev *idxd_cdev; struct wait_queue_head err_queue; struct idxd_device *idxd; @@ -174,7 +190,7 @@ struct idxd_wq { }; struct idxd_engine { - struct device conf_dev; + struct idxd_dev idxd_dev; int id; struct idxd_group *group; struct idxd_device *idxd; @@ -218,7 +234,7 @@ struct idxd_driver_data { }; struct idxd_device { - struct device conf_dev; + struct idxd_dev idxd_dev; struct idxd_driver_data *data; struct list_head list; struct idxd_hw hw; @@ -301,8 +317,58 @@ enum idxd_completion_status { IDXD_COMP_DESC_ABORT = 0xff, }; -#define confdev_to_idxd(dev) container_of(dev, struct idxd_device, conf_dev) -#define confdev_to_wq(dev) container_of(dev, struct idxd_wq, conf_dev) +#define idxd_confdev(idxd) &idxd->idxd_dev.conf_dev +#define wq_confdev(wq) &wq->idxd_dev.conf_dev +#define engine_confdev(engine) &engine->idxd_dev.conf_dev +#define group_confdev(group) &group->idxd_dev.conf_dev +#define cdev_dev(cdev) &cdev->idxd_dev.conf_dev + +#define confdev_to_idxd_dev(dev) container_of(dev, struct idxd_dev, conf_dev) + +static inline struct idxd_device *confdev_to_idxd(struct device *dev) +{ + struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); + + return container_of(idxd_dev, struct idxd_device, idxd_dev); +} + +static inline struct idxd_wq *confdev_to_wq(struct device *dev) +{ + struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); + + return container_of(idxd_dev, struct idxd_wq, idxd_dev); +} + +static inline struct idxd_engine *confdev_to_engine(struct device *dev) +{ + struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); + + return container_of(idxd_dev, struct idxd_engine, idxd_dev); +} + +static inline struct idxd_group *confdev_to_group(struct device *dev) +{ + struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); + + return container_of(idxd_dev, struct idxd_group, idxd_dev); +} + +static inline struct idxd_cdev *dev_to_cdev(struct device *dev) +{ + struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); + + return container_of(idxd_dev, struct idxd_cdev, idxd_dev); +} + +static inline void idxd_dev_set_type(struct idxd_dev *idev, int type) +{ + if (type >= IDXD_DEV_MAX_TYPE) { + idev->type = IDXD_DEV_NONE; + return; + } + + idev->type = type; +} extern struct bus_type dsa_bus_type; extern struct bus_type iax_bus_type; diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 6403d55c7ff7..f500076882d2 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -200,6 +200,7 @@ static int idxd_setup_wqs(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; struct idxd_wq *wq; + struct device *conf_dev; int i, rc; idxd->wqs = kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *), @@ -214,15 +215,17 @@ static int idxd_setup_wqs(struct idxd_device *idxd) goto err; } + idxd_dev_set_type(&wq->idxd_dev, IDXD_DEV_WQ); + conf_dev = wq_confdev(wq); wq->id = i; wq->idxd = idxd; - device_initialize(&wq->conf_dev); - wq->conf_dev.parent = &idxd->conf_dev; - wq->conf_dev.bus = &dsa_bus_type; - wq->conf_dev.type = &idxd_wq_device_type; - rc = dev_set_name(&wq->conf_dev, "wq%d.%d", idxd->id, wq->id); + device_initialize(wq_confdev(wq)); + conf_dev->parent = idxd_confdev(idxd); + conf_dev->bus = &dsa_bus_type; + conf_dev->type = &idxd_wq_device_type; + rc = dev_set_name(conf_dev, "wq%d.%d", idxd->id, wq->id); if (rc < 0) { - put_device(&wq->conf_dev); + put_device(conf_dev); goto err; } @@ -233,7 +236,7 @@ static int idxd_setup_wqs(struct idxd_device *idxd) wq->max_batch_size = idxd->max_batch_size; wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev)); if (!wq->wqcfg) { - put_device(&wq->conf_dev); + put_device(conf_dev); rc = -ENOMEM; goto err; } @@ -243,8 +246,11 @@ static int idxd_setup_wqs(struct idxd_device *idxd) return 0; err: - while (--i >= 0) - put_device(&idxd->wqs[i]->conf_dev); + while (--i >= 0) { + wq = idxd->wqs[i]; + conf_dev = wq_confdev(wq); + put_device(conf_dev); + } return rc; } @@ -252,6 +258,7 @@ static int idxd_setup_engines(struct idxd_device *idxd) { struct idxd_engine *engine; struct device *dev = &idxd->pdev->dev; + struct device *conf_dev; int i, rc; idxd->engines = kcalloc_node(idxd->max_engines, sizeof(struct idxd_engine *), @@ -266,15 +273,17 @@ static int idxd_setup_engines(struct idxd_device *idxd) goto err; } + idxd_dev_set_type(&engine->idxd_dev, IDXD_DEV_ENGINE); + conf_dev = engine_confdev(engine); engine->id = i; engine->idxd = idxd; - device_initialize(&engine->conf_dev); - engine->conf_dev.parent = &idxd->conf_dev; - engine->conf_dev.bus = &dsa_bus_type; - engine->conf_dev.type = &idxd_engine_device_type; - rc = dev_set_name(&engine->conf_dev, "engine%d.%d", idxd->id, engine->id); + device_initialize(conf_dev); + conf_dev->parent = idxd_confdev(idxd); + conf_dev->bus = &dsa_bus_type; + conf_dev->type = &idxd_engine_device_type; + rc = dev_set_name(conf_dev, "engine%d.%d", idxd->id, engine->id); if (rc < 0) { - put_device(&engine->conf_dev); + put_device(conf_dev); goto err; } @@ -284,14 +293,18 @@ static int idxd_setup_engines(struct idxd_device *idxd) return 0; err: - while (--i >= 0) - put_device(&idxd->engines[i]->conf_dev); + while (--i >= 0) { + engine = idxd->engines[i]; + conf_dev = engine_confdev(engine); + put_device(conf_dev); + } return rc; } static int idxd_setup_groups(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; + struct device *conf_dev; struct idxd_group *group; int i, rc; @@ -307,15 +320,17 @@ static int idxd_setup_groups(struct idxd_device *idxd) goto err; } + idxd_dev_set_type(&group->idxd_dev, IDXD_DEV_GROUP); + conf_dev = group_confdev(group); group->id = i; group->idxd = idxd; - device_initialize(&group->conf_dev); - group->conf_dev.parent = &idxd->conf_dev; - group->conf_dev.bus = &dsa_bus_type; - group->conf_dev.type = &idxd_group_device_type; - rc = dev_set_name(&group->conf_dev, "group%d.%d", idxd->id, group->id); + device_initialize(conf_dev); + conf_dev->parent = idxd_confdev(idxd); + conf_dev->bus = &dsa_bus_type; + conf_dev->type = &idxd_group_device_type; + rc = dev_set_name(conf_dev, "group%d.%d", idxd->id, group->id); if (rc < 0) { - put_device(&group->conf_dev); + put_device(conf_dev); goto err; } @@ -327,8 +342,10 @@ static int idxd_setup_groups(struct idxd_device *idxd) return 0; err: - while (--i >= 0) - put_device(&idxd->groups[i]->conf_dev); + while (--i >= 0) { + group = idxd->groups[i]; + put_device(group_confdev(group)); + } return rc; } @@ -337,11 +354,11 @@ static void idxd_cleanup_internals(struct idxd_device *idxd) int i; for (i = 0; i < idxd->max_groups; i++) - put_device(&idxd->groups[i]->conf_dev); + put_device(group_confdev(idxd->groups[i])); for (i = 0; i < idxd->max_engines; i++) - put_device(&idxd->engines[i]->conf_dev); + put_device(engine_confdev(idxd->engines[i])); for (i = 0; i < idxd->max_wqs; i++) - put_device(&idxd->wqs[i]->conf_dev); + put_device(wq_confdev(idxd->wqs[i])); destroy_workqueue(idxd->wq); } @@ -381,13 +398,13 @@ static int idxd_setup_internals(struct idxd_device *idxd) err_wkq_create: for (i = 0; i < idxd->max_groups; i++) - put_device(&idxd->groups[i]->conf_dev); + put_device(group_confdev(idxd->groups[i])); err_group: for (i = 0; i < idxd->max_engines; i++) - put_device(&idxd->engines[i]->conf_dev); + put_device(engine_confdev(idxd->engines[i])); err_engine: for (i = 0; i < idxd->max_wqs; i++) - put_device(&idxd->wqs[i]->conf_dev); + put_device(wq_confdev(idxd->wqs[i])); err_wqs: kfree(idxd->int_handles); return rc; @@ -469,6 +486,7 @@ static void idxd_read_caps(struct idxd_device *idxd) static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_data *data) { struct device *dev = &pdev->dev; + struct device *conf_dev; struct idxd_device *idxd; int rc; @@ -476,19 +494,21 @@ static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_d if (!idxd) return NULL; + conf_dev = idxd_confdev(idxd); idxd->pdev = pdev; idxd->data = data; + idxd_dev_set_type(&idxd->idxd_dev, idxd->data->type); idxd->id = ida_alloc(&idxd_ida, GFP_KERNEL); if (idxd->id < 0) return NULL; - device_initialize(&idxd->conf_dev); - idxd->conf_dev.parent = dev; - idxd->conf_dev.bus = &dsa_bus_type; - idxd->conf_dev.type = idxd->data->dev_type; - rc = dev_set_name(&idxd->conf_dev, "%s%d", idxd->data->name_prefix, idxd->id); + device_initialize(conf_dev); + conf_dev->parent = dev; + conf_dev->bus = &dsa_bus_type; + conf_dev->type = idxd->data->dev_type; + rc = dev_set_name(conf_dev, "%s%d", idxd->data->name_prefix, idxd->id); if (rc < 0) { - put_device(&idxd->conf_dev); + put_device(conf_dev); return NULL; } @@ -674,7 +694,7 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) err: pci_iounmap(pdev, idxd->reg_base); err_iomap: - put_device(&idxd->conf_dev); + put_device(idxd_confdev(idxd)); err_idxd_alloc: pci_disable_device(pdev); return rc; @@ -787,7 +807,7 @@ static void idxd_remove(struct pci_dev *pdev) pci_disable_device(pdev); destroy_workqueue(idxd->wq); perfmon_pmu_remove(idxd); - device_unregister(&idxd->conf_dev); + device_unregister(idxd_confdev(idxd)); } static struct pci_driver idxd_pci_driver = { diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c index 2924819ca8f3..be65d55e1fc4 100644 --- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -51,7 +51,7 @@ static void idxd_device_reinit(struct work_struct *work) rc = idxd_wq_enable(wq); if (rc < 0) { dev_warn(dev, "Unable to re-enable wq %s\n", - dev_name(&wq->conf_dev)); + dev_name(wq_confdev(wq))); } } } diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 60779f57c118..f603b11141c4 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -164,7 +164,7 @@ static int enable_wq(struct idxd_wq *wq) } mutex_unlock(&wq->wq_lock); - dev_info(dev, "wq %s enabled\n", dev_name(&wq->conf_dev)); + dev_info(dev, "wq %s enabled\n", dev_name(wq_confdev(wq))); return 0; } @@ -230,7 +230,7 @@ static void disable_wq(struct idxd_wq *wq) struct device *dev = &idxd->pdev->dev; mutex_lock(&wq->wq_lock); - dev_dbg(dev, "%s removing WQ %s\n", __func__, dev_name(&wq->conf_dev)); + dev_dbg(dev, "%s removing WQ %s\n", __func__, dev_name(wq_confdev(wq))); if (wq->state == IDXD_WQ_DISABLED) { mutex_unlock(&wq->wq_lock); return; @@ -257,7 +257,7 @@ static void disable_wq(struct idxd_wq *wq) wq->client_count = 0; mutex_unlock(&wq->wq_lock); - dev_info(dev, "wq %s disabled\n", dev_name(&wq->conf_dev)); + dev_info(dev, "wq %s disabled\n", dev_name(wq_confdev(wq))); } static int idxd_config_bus_remove(struct device *dev) @@ -274,15 +274,15 @@ static int idxd_config_bus_remove(struct device *dev) int i; dev_dbg(dev, "%s removing dev %s\n", __func__, - dev_name(&idxd->conf_dev)); + dev_name(idxd_confdev(idxd))); for (i = 0; i < idxd->max_wqs; i++) { struct idxd_wq *wq = idxd->wqs[i]; if (wq->state == IDXD_WQ_DISABLED) continue; dev_warn(dev, "Active wq %d on disable %s.\n", i, - dev_name(&idxd->conf_dev)); - device_release_driver(&wq->conf_dev); + dev_name(wq_confdev(wq))); + device_release_driver(wq_confdev(wq)); } idxd_unregister_dma_device(idxd); @@ -329,8 +329,7 @@ void idxd_unregister_driver(void) static ssize_t engine_group_id_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_engine *engine = - container_of(dev, struct idxd_engine, conf_dev); + struct idxd_engine *engine = confdev_to_engine(dev); if (engine->group) return sysfs_emit(buf, "%d\n", engine->group->id); @@ -342,8 +341,7 @@ static ssize_t engine_group_id_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_engine *engine = - container_of(dev, struct idxd_engine, conf_dev); + struct idxd_engine *engine = confdev_to_engine(dev); struct idxd_device *idxd = engine->idxd; long id; int rc; @@ -397,7 +395,7 @@ static const struct attribute_group *idxd_engine_attribute_groups[] = { static void idxd_conf_engine_release(struct device *dev) { - struct idxd_engine *engine = container_of(dev, struct idxd_engine, conf_dev); + struct idxd_engine *engine = confdev_to_engine(dev); kfree(engine); } @@ -427,8 +425,7 @@ static ssize_t group_tokens_reserved_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_group *group = - container_of(dev, struct idxd_group, conf_dev); + struct idxd_group *group = confdev_to_group(dev); return sysfs_emit(buf, "%u\n", group->tokens_reserved); } @@ -437,8 +434,7 @@ static ssize_t group_tokens_reserved_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_group *group = - container_of(dev, struct idxd_group, conf_dev); + struct idxd_group *group = confdev_to_group(dev); struct idxd_device *idxd = group->idxd; unsigned long val; int rc; @@ -475,8 +471,7 @@ static ssize_t group_tokens_allowed_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_group *group = - container_of(dev, struct idxd_group, conf_dev); + struct idxd_group *group = confdev_to_group(dev); return sysfs_emit(buf, "%u\n", group->tokens_allowed); } @@ -485,8 +480,7 @@ static ssize_t group_tokens_allowed_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_group *group = - container_of(dev, struct idxd_group, conf_dev); + struct idxd_group *group = confdev_to_group(dev); struct idxd_device *idxd = group->idxd; unsigned long val; int rc; @@ -520,8 +514,7 @@ static ssize_t group_use_token_limit_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_group *group = - container_of(dev, struct idxd_group, conf_dev); + struct idxd_group *group = confdev_to_group(dev); return sysfs_emit(buf, "%u\n", group->use_token_limit); } @@ -530,8 +523,7 @@ static ssize_t group_use_token_limit_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_group *group = - container_of(dev, struct idxd_group, conf_dev); + struct idxd_group *group = confdev_to_group(dev); struct idxd_device *idxd = group->idxd; unsigned long val; int rc; @@ -563,8 +555,7 @@ static struct device_attribute dev_attr_group_use_token_limit = static ssize_t group_engines_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_group *group = - container_of(dev, struct idxd_group, conf_dev); + struct idxd_group *group = confdev_to_group(dev); int i, rc = 0; struct idxd_device *idxd = group->idxd; @@ -592,8 +583,7 @@ static struct device_attribute dev_attr_group_engines = static ssize_t group_work_queues_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_group *group = - container_of(dev, struct idxd_group, conf_dev); + struct idxd_group *group = confdev_to_group(dev); int i, rc = 0; struct idxd_device *idxd = group->idxd; @@ -622,8 +612,7 @@ static ssize_t group_traffic_class_a_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_group *group = - container_of(dev, struct idxd_group, conf_dev); + struct idxd_group *group = confdev_to_group(dev); return sysfs_emit(buf, "%d\n", group->tc_a); } @@ -632,8 +621,7 @@ static ssize_t group_traffic_class_a_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_group *group = - container_of(dev, struct idxd_group, conf_dev); + struct idxd_group *group = confdev_to_group(dev); struct idxd_device *idxd = group->idxd; long val; int rc; @@ -663,8 +651,7 @@ static ssize_t group_traffic_class_b_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_group *group = - container_of(dev, struct idxd_group, conf_dev); + struct idxd_group *group = confdev_to_group(dev); return sysfs_emit(buf, "%d\n", group->tc_b); } @@ -673,8 +660,7 @@ static ssize_t group_traffic_class_b_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_group *group = - container_of(dev, struct idxd_group, conf_dev); + struct idxd_group *group = confdev_to_group(dev); struct idxd_device *idxd = group->idxd; long val; int rc; @@ -722,7 +708,7 @@ static const struct attribute_group *idxd_group_attribute_groups[] = { static void idxd_conf_group_release(struct device *dev) { - struct idxd_group *group = container_of(dev, struct idxd_group, conf_dev); + struct idxd_group *group = confdev_to_group(dev); kfree(group); } @@ -737,7 +723,7 @@ struct device_type idxd_group_device_type = { static ssize_t wq_clients_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); return sysfs_emit(buf, "%d\n", wq->client_count); } @@ -748,7 +734,7 @@ static struct device_attribute dev_attr_wq_clients = static ssize_t wq_state_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); switch (wq->state) { case IDXD_WQ_DISABLED: @@ -766,7 +752,7 @@ static struct device_attribute dev_attr_wq_state = static ssize_t wq_group_id_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); if (wq->group) return sysfs_emit(buf, "%u\n", wq->group->id); @@ -778,7 +764,7 @@ static ssize_t wq_group_id_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); struct idxd_device *idxd = wq->idxd; long id; int rc; @@ -821,7 +807,7 @@ static struct device_attribute dev_attr_wq_group_id = static ssize_t wq_mode_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); return sysfs_emit(buf, "%s\n", wq_dedicated(wq) ? "dedicated" : "shared"); } @@ -830,7 +816,7 @@ static ssize_t wq_mode_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); struct idxd_device *idxd = wq->idxd; if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) @@ -857,7 +843,7 @@ static struct device_attribute dev_attr_wq_mode = static ssize_t wq_size_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); return sysfs_emit(buf, "%u\n", wq->size); } @@ -880,7 +866,7 @@ static ssize_t wq_size_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); unsigned long size; struct idxd_device *idxd = wq->idxd; int rc; @@ -908,7 +894,7 @@ static struct device_attribute dev_attr_wq_size = static ssize_t wq_priority_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); return sysfs_emit(buf, "%u\n", wq->priority); } @@ -917,7 +903,7 @@ static ssize_t wq_priority_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); unsigned long prio; struct idxd_device *idxd = wq->idxd; int rc; @@ -945,7 +931,7 @@ static struct device_attribute dev_attr_wq_priority = static ssize_t wq_block_on_fault_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); return sysfs_emit(buf, "%u\n", test_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags)); } @@ -954,7 +940,7 @@ static ssize_t wq_block_on_fault_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); struct idxd_device *idxd = wq->idxd; bool bof; int rc; @@ -984,7 +970,7 @@ static struct device_attribute dev_attr_wq_block_on_fault = static ssize_t wq_threshold_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); return sysfs_emit(buf, "%u\n", wq->threshold); } @@ -993,7 +979,7 @@ static ssize_t wq_threshold_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); struct idxd_device *idxd = wq->idxd; unsigned int val; int rc; @@ -1025,7 +1011,7 @@ static struct device_attribute dev_attr_wq_threshold = static ssize_t wq_type_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); switch (wq->type) { case IDXD_WQT_KERNEL: @@ -1044,7 +1030,7 @@ static ssize_t wq_type_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); enum idxd_wq_type old_type; if (wq->state != IDXD_WQ_DISABLED) @@ -1073,7 +1059,7 @@ static struct device_attribute dev_attr_wq_type = static ssize_t wq_name_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); return sysfs_emit(buf, "%s\n", wq->name); } @@ -1082,7 +1068,7 @@ static ssize_t wq_name_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); if (wq->state != IDXD_WQ_DISABLED) return -EPERM; @@ -1109,7 +1095,7 @@ static struct device_attribute dev_attr_wq_name = static ssize_t wq_cdev_minor_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); int minor = -1; mutex_lock(&wq->wq_lock); @@ -1143,7 +1129,7 @@ static int __get_sysfs_u64(const char *buf, u64 *val) static ssize_t wq_max_transfer_size_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); return sysfs_emit(buf, "%llu\n", wq->max_xfer_bytes); } @@ -1151,7 +1137,7 @@ static ssize_t wq_max_transfer_size_show(struct device *dev, struct device_attri static ssize_t wq_max_transfer_size_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); struct idxd_device *idxd = wq->idxd; u64 xfer_size; int rc; @@ -1177,7 +1163,7 @@ static struct device_attribute dev_attr_wq_max_transfer_size = static ssize_t wq_max_batch_size_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); return sysfs_emit(buf, "%u\n", wq->max_batch_size); } @@ -1185,7 +1171,7 @@ static ssize_t wq_max_batch_size_show(struct device *dev, struct device_attribut static ssize_t wq_max_batch_size_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); struct idxd_device *idxd = wq->idxd; u64 batch_size; int rc; @@ -1210,7 +1196,7 @@ static struct device_attribute dev_attr_wq_max_batch_size = static ssize_t wq_ats_disable_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); return sysfs_emit(buf, "%u\n", wq->ats_dis); } @@ -1218,7 +1204,7 @@ static ssize_t wq_ats_disable_show(struct device *dev, struct device_attribute * static ssize_t wq_ats_disable_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); struct idxd_device *idxd = wq->idxd; bool ats_dis; int rc; @@ -1289,7 +1275,7 @@ static const struct attribute_group *idxd_wq_attribute_groups[] = { static void idxd_conf_wq_release(struct device *dev) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); kfree(wq->wqcfg); kfree(wq); @@ -1305,8 +1291,7 @@ struct device_type idxd_wq_device_type = { static ssize_t version_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%#x\n", idxd->hw.version); } @@ -1316,8 +1301,7 @@ static ssize_t max_work_queues_size_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%u\n", idxd->max_wq_size); } @@ -1326,8 +1310,7 @@ static DEVICE_ATTR_RO(max_work_queues_size); static ssize_t max_groups_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%u\n", idxd->max_groups); } @@ -1336,8 +1319,7 @@ static DEVICE_ATTR_RO(max_groups); static ssize_t max_work_queues_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%u\n", idxd->max_wqs); } @@ -1346,8 +1328,7 @@ static DEVICE_ATTR_RO(max_work_queues); static ssize_t max_engines_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%u\n", idxd->max_engines); } @@ -1356,8 +1337,7 @@ static DEVICE_ATTR_RO(max_engines); static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%d\n", dev_to_node(&idxd->pdev->dev)); } @@ -1366,8 +1346,7 @@ static DEVICE_ATTR_RO(numa_node); static ssize_t max_batch_size_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%u\n", idxd->max_batch_size); } @@ -1377,8 +1356,7 @@ static ssize_t max_transfer_size_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%llu\n", idxd->max_xfer_bytes); } @@ -1387,8 +1365,7 @@ static DEVICE_ATTR_RO(max_transfer_size); static ssize_t op_cap_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); int i, rc = 0; for (i = 0; i < 4; i++) @@ -1403,8 +1380,7 @@ static DEVICE_ATTR_RO(op_cap); static ssize_t gen_cap_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%#llx\n", idxd->hw.gen_cap.bits); } @@ -1413,8 +1389,7 @@ static DEVICE_ATTR_RO(gen_cap); static ssize_t configurable_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%u\n", test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)); } @@ -1423,8 +1398,7 @@ static DEVICE_ATTR_RO(configurable); static ssize_t clients_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); unsigned long flags; int count = 0, i; @@ -1443,8 +1417,7 @@ static DEVICE_ATTR_RO(clients); static ssize_t pasid_enabled_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%u\n", device_pasid_enabled(idxd)); } @@ -1453,8 +1426,7 @@ static DEVICE_ATTR_RO(pasid_enabled); static ssize_t state_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); switch (idxd->state) { case IDXD_DEV_DISABLED: @@ -1473,8 +1445,7 @@ static DEVICE_ATTR_RO(state); static ssize_t errors_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); int i, out = 0; unsigned long flags; @@ -1491,8 +1462,7 @@ static DEVICE_ATTR_RO(errors); static ssize_t max_tokens_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%u\n", idxd->max_tokens); } @@ -1501,8 +1471,7 @@ static DEVICE_ATTR_RO(max_tokens); static ssize_t token_limit_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%u\n", idxd->token_limit); } @@ -1511,8 +1480,7 @@ static ssize_t token_limit_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); unsigned long val; int rc; @@ -1540,8 +1508,7 @@ static DEVICE_ATTR_RW(token_limit); static ssize_t cdev_major_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%u\n", idxd->major); } @@ -1550,7 +1517,7 @@ static DEVICE_ATTR_RO(cdev_major); static ssize_t cmd_status_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%#x\n", idxd->cmd_status); } @@ -1590,7 +1557,7 @@ static const struct attribute_group *idxd_attribute_groups[] = { static void idxd_conf_device_release(struct device *dev) { - struct idxd_device *idxd = container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); kfree(idxd->groups); kfree(idxd->wqs); @@ -1615,12 +1582,12 @@ struct device_type iax_device_type = { static int idxd_register_engine_devices(struct idxd_device *idxd) { + struct idxd_engine *engine; int i, j, rc; for (i = 0; i < idxd->max_engines; i++) { - struct idxd_engine *engine = idxd->engines[i]; - - rc = device_add(&engine->conf_dev); + engine = idxd->engines[i]; + rc = device_add(engine_confdev(engine)); if (rc < 0) goto cleanup; } @@ -1629,22 +1596,26 @@ static int idxd_register_engine_devices(struct idxd_device *idxd) cleanup: j = i - 1; - for (; i < idxd->max_engines; i++) - put_device(&idxd->engines[i]->conf_dev); + for (; i < idxd->max_engines; i++) { + engine = idxd->engines[i]; + put_device(engine_confdev(engine)); + } - while (j--) - device_unregister(&idxd->engines[j]->conf_dev); + while (j--) { + engine = idxd->engines[j]; + device_unregister(engine_confdev(engine)); + } return rc; } static int idxd_register_group_devices(struct idxd_device *idxd) { + struct idxd_group *group; int i, j, rc; for (i = 0; i < idxd->max_groups; i++) { - struct idxd_group *group = idxd->groups[i]; - - rc = device_add(&group->conf_dev); + group = idxd->groups[i]; + rc = device_add(group_confdev(group)); if (rc < 0) goto cleanup; } @@ -1653,22 +1624,26 @@ static int idxd_register_group_devices(struct idxd_device *idxd) cleanup: j = i - 1; - for (; i < idxd->max_groups; i++) - put_device(&idxd->groups[i]->conf_dev); + for (; i < idxd->max_groups; i++) { + group = idxd->groups[i]; + put_device(group_confdev(group)); + } - while (j--) - device_unregister(&idxd->groups[j]->conf_dev); + while (j--) { + group = idxd->groups[j]; + device_unregister(group_confdev(group)); + } return rc; } static int idxd_register_wq_devices(struct idxd_device *idxd) { + struct idxd_wq *wq; int i, rc, j; for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = idxd->wqs[i]; - - rc = device_add(&wq->conf_dev); + wq = idxd->wqs[i]; + rc = device_add(wq_confdev(wq)); if (rc < 0) goto cleanup; } @@ -1677,11 +1652,15 @@ static int idxd_register_wq_devices(struct idxd_device *idxd) cleanup: j = i - 1; - for (; i < idxd->max_wqs; i++) - put_device(&idxd->wqs[i]->conf_dev); + for (; i < idxd->max_wqs; i++) { + wq = idxd->wqs[i]; + put_device(wq_confdev(wq)); + } - while (j--) - device_unregister(&idxd->wqs[j]->conf_dev); + while (j--) { + wq = idxd->wqs[j]; + device_unregister(wq_confdev(wq)); + } return rc; } @@ -1690,7 +1669,7 @@ int idxd_register_devices(struct idxd_device *idxd) struct device *dev = &idxd->pdev->dev; int rc, i; - rc = device_add(&idxd->conf_dev); + rc = device_add(idxd_confdev(idxd)); if (rc < 0) return rc; @@ -1716,12 +1695,12 @@ int idxd_register_devices(struct idxd_device *idxd) err_group: for (i = 0; i < idxd->max_engines; i++) - device_unregister(&idxd->engines[i]->conf_dev); + device_unregister(engine_confdev(idxd->engines[i])); err_engine: for (i = 0; i < idxd->max_wqs; i++) - device_unregister(&idxd->wqs[i]->conf_dev); + device_unregister(wq_confdev(idxd->wqs[i])); err_wq: - device_del(&idxd->conf_dev); + device_del(idxd_confdev(idxd)); return rc; } @@ -1732,19 +1711,19 @@ void idxd_unregister_devices(struct idxd_device *idxd) for (i = 0; i < idxd->max_wqs; i++) { struct idxd_wq *wq = idxd->wqs[i]; - device_unregister(&wq->conf_dev); + device_unregister(wq_confdev(wq)); } for (i = 0; i < idxd->max_engines; i++) { struct idxd_engine *engine = idxd->engines[i]; - device_unregister(&engine->conf_dev); + device_unregister(engine_confdev(engine)); } for (i = 0; i < idxd->max_groups; i++) { struct idxd_group *group = idxd->groups[i]; - device_unregister(&group->conf_dev); + device_unregister(group_confdev(group)); } } -- Gitee From df5847de0c90ea20b829cab8eb00c8e5fa79d803 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Jul 2021 11:43:26 -0700 Subject: [PATCH 42/85] dmaengine: idxd: remove IDXD_DEV_CONF_READY ANBZ: #22 commit f52058ae11523304a337de249c4c07ba5076f288 upstream. The IDXD_DEV_CONF_READY state flag is no longer needed. The current implementation uses this flag to stop the device from doing configuration until the pci driver probe has completed. With the driver architecture going towards multiple sub-driver attached to the dsa_bus, this is no longer feasible. The sub-drivers will be allowed to probe and return with failure when they are not ready to complete the probe rather than using a state flag to gate the probing. There is no expectation that the devices auto-attach to a driver. Userspace configuration is expected to setup the device before enabling. Intel-SIG: commit f52058ae1152 dmaengine: idxd: remove IDXD_DEV_CONF_READY. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reviewed-by: Dan Williams Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162637460633.744545.8902095097471365420.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 4 ++-- drivers/dma/idxd/idxd.h | 1 - drivers/dma/idxd/init.c | 2 -- drivers/dma/idxd/sysfs.c | 14 -------------- 4 files changed, 2 insertions(+), 19 deletions(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index c8cf1de72176..4a2af9799239 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -576,7 +576,7 @@ int idxd_device_disable(struct idxd_device *idxd) spin_lock_irqsave(&idxd->dev_lock, flags); idxd_device_clear_state(idxd); - idxd->state = IDXD_DEV_CONF_READY; + idxd->state = IDXD_DEV_DISABLED; spin_unlock_irqrestore(&idxd->dev_lock, flags); return 0; } @@ -588,7 +588,7 @@ void idxd_device_reset(struct idxd_device *idxd) idxd_cmd_exec(idxd, IDXD_CMD_RESET_DEVICE, 0, NULL); spin_lock_irqsave(&idxd->dev_lock, flags); idxd_device_clear_state(idxd); - idxd->state = IDXD_DEV_CONF_READY; + idxd->state = IDXD_DEV_DISABLED; spin_unlock_irqrestore(&idxd->dev_lock, flags); } diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index ae60fcc7b625..9fc1a88f336d 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -210,7 +210,6 @@ struct idxd_hw { enum idxd_device_state { IDXD_DEV_HALTED = -1, IDXD_DEV_DISABLED = 0, - IDXD_DEV_CONF_READY, IDXD_DEV_ENABLED, }; diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index f500076882d2..c22225b14c5d 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -682,8 +682,6 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_dev_register; } - idxd->state = IDXD_DEV_CONF_READY; - dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n", idxd->hw.version); diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index f603b11141c4..2a978055e22b 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -22,17 +22,9 @@ static int idxd_config_bus_match(struct device *dev, int matched = 0; if (is_idxd_dev(dev)) { - struct idxd_device *idxd = confdev_to_idxd(dev); - - if (idxd->state != IDXD_DEV_CONF_READY) - return 0; matched = 1; } else if (is_idxd_wq_dev(dev)) { struct idxd_wq *wq = confdev_to_wq(dev); - struct idxd_device *idxd = wq->idxd; - - if (idxd->state < IDXD_DEV_CONF_READY) - return 0; if (wq->state != IDXD_WQ_DISABLED) { dev_dbg(dev, "%s not disabled\n", dev_name(dev)); @@ -179,11 +171,6 @@ static int idxd_config_bus_probe(struct device *dev) if (is_idxd_dev(dev)) { struct idxd_device *idxd = confdev_to_idxd(dev); - if (idxd->state != IDXD_DEV_CONF_READY) { - dev_warn(dev, "Device not ready for config\n"); - return -EBUSY; - } - if (!try_module_get(THIS_MODULE)) return -ENXIO; @@ -1430,7 +1417,6 @@ static ssize_t state_show(struct device *dev, switch (idxd->state) { case IDXD_DEV_DISABLED: - case IDXD_DEV_CONF_READY: return sysfs_emit(buf, "disabled\n"); case IDXD_DEV_ENABLED: return sysfs_emit(buf, "enabled\n"); -- Gitee From 699fbbe7195b70b03982abfc1a58127aa7b5bc16 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Jul 2021 11:43:31 -0700 Subject: [PATCH 43/85] dmaengine: idxd: move wq_enable() to device.c ANBZ: #22 commit 1f2bb40337f0df1d9af80793e9fdacff7706e654 upstream. Move the wq_enable() function to device.c in preparation of setting up the idxd internal sub-driver framework. No logic changes. Intel-SIG: commit 1f2bb40337f0 dmaengine: idxd: move wq_enable() to device.c. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reviewed-by: Dan Williams Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162637461176.744545.3806109011554118998.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 124 ++++++++++++++++++++++++++++++++++++++ drivers/dma/idxd/idxd.h | 1 + drivers/dma/idxd/sysfs.c | 124 +------------------------------------- 3 files changed, 126 insertions(+), 123 deletions(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 4a2af9799239..b1c509bcfa31 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -1129,3 +1129,127 @@ int idxd_device_load_config(struct idxd_device *idxd) return 0; } + +static int __drv_enable_wq(struct idxd_wq *wq) +{ + struct idxd_device *idxd = wq->idxd; + struct device *dev = &idxd->pdev->dev; + unsigned long flags; + int rc = -ENXIO; + + lockdep_assert_held(&wq->wq_lock); + + if (idxd->state != IDXD_DEV_ENABLED) + goto err; + + if (wq->state != IDXD_WQ_DISABLED) { + dev_dbg(dev, "wq %d already enabled.\n", wq->id); + rc = -EBUSY; + goto err; + } + + if (!wq->group) { + dev_dbg(dev, "wq %d not attached to group.\n", wq->id); + goto err; + } + + if (strlen(wq->name) == 0) { + dev_dbg(dev, "wq %d name not set.\n", wq->id); + goto err; + } + + /* Shared WQ checks */ + if (wq_shared(wq)) { + if (!device_swq_supported(idxd)) { + dev_dbg(dev, "PASID not enabled and shared wq.\n"); + goto err; + } + /* + * Shared wq with the threshold set to 0 means the user + * did not set the threshold or transitioned from a + * dedicated wq but did not set threshold. A value + * of 0 would effectively disable the shared wq. The + * driver does not allow a value of 0 to be set for + * threshold via sysfs. + */ + if (wq->threshold == 0) { + dev_dbg(dev, "Shared wq and threshold 0.\n"); + goto err; + } + } + + rc = idxd_wq_alloc_resources(wq); + if (rc < 0) { + dev_dbg(dev, "wq resource alloc failed\n"); + goto err; + } + + spin_lock_irqsave(&idxd->dev_lock, flags); + if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) + rc = idxd_device_config(idxd); + spin_unlock_irqrestore(&idxd->dev_lock, flags); + if (rc < 0) { + dev_dbg(dev, "Writing wq %d config failed: %d\n", wq->id, rc); + goto err; + } + + rc = idxd_wq_enable(wq); + if (rc < 0) { + dev_dbg(dev, "wq %d enabling failed: %d\n", wq->id, rc); + goto err; + } + + rc = idxd_wq_map_portal(wq); + if (rc < 0) { + dev_dbg(dev, "wq %d portal mapping failed: %d\n", wq->id, rc); + goto err_map_portal; + } + + wq->client_count = 0; + + if (wq->type == IDXD_WQT_KERNEL) { + rc = idxd_wq_init_percpu_ref(wq); + if (rc < 0) { + dev_dbg(dev, "wq %d percpu_ref setup failed\n", wq->id); + goto err_cpu_ref; + } + } + + if (is_idxd_wq_dmaengine(wq)) { + rc = idxd_register_dma_channel(wq); + if (rc < 0) { + dev_dbg(dev, "wq %d DMA channel register failed\n", wq->id); + goto err_client; + } + } else if (is_idxd_wq_cdev(wq)) { + rc = idxd_wq_add_cdev(wq); + if (rc < 0) { + dev_dbg(dev, "wq %d cdev creation failed\n", wq->id); + goto err_client; + } + } + + dev_info(dev, "wq %s enabled\n", dev_name(wq_confdev(wq))); + return 0; + +err_client: + idxd_wq_quiesce(wq); +err_cpu_ref: + idxd_wq_unmap_portal(wq); +err_map_portal: + rc = idxd_wq_disable(wq, false); + if (rc < 0) + dev_dbg(dev, "wq %s disable failed\n", dev_name(wq_confdev(wq))); +err: + return rc; +} + +int drv_enable_wq(struct idxd_wq *wq) +{ + int rc; + + mutex_lock(&wq->wq_lock); + rc = __drv_enable_wq(wq); + mutex_unlock(&wq->wq_lock); + return rc; +} diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 9fc1a88f336d..551a61fa1aff 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -495,6 +495,7 @@ void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id); void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id); /* device control */ +int drv_enable_wq(struct idxd_wq *wq); int idxd_device_init_reset(struct idxd_device *idxd); int idxd_device_enable(struct idxd_device *idxd); int idxd_device_disable(struct idxd_device *idxd); diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 2a978055e22b..3e8cc07ebcdc 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -39,128 +39,6 @@ static int idxd_config_bus_match(struct device *dev, return matched; } -static int enable_wq(struct idxd_wq *wq) -{ - struct idxd_device *idxd = wq->idxd; - struct device *dev = &idxd->pdev->dev; - unsigned long flags; - int rc; - - mutex_lock(&wq->wq_lock); - - if (idxd->state != IDXD_DEV_ENABLED) { - mutex_unlock(&wq->wq_lock); - dev_warn(dev, "Enabling while device not enabled.\n"); - return -EPERM; - } - - if (wq->state != IDXD_WQ_DISABLED) { - mutex_unlock(&wq->wq_lock); - dev_warn(dev, "WQ %d already enabled.\n", wq->id); - return -EBUSY; - } - - if (!wq->group) { - mutex_unlock(&wq->wq_lock); - dev_warn(dev, "WQ not attached to group.\n"); - return -EINVAL; - } - - if (strlen(wq->name) == 0) { - mutex_unlock(&wq->wq_lock); - dev_warn(dev, "WQ name not set.\n"); - return -EINVAL; - } - - /* Shared WQ checks */ - if (wq_shared(wq)) { - if (!device_swq_supported(idxd)) { - dev_warn(dev, "PASID not enabled and shared WQ.\n"); - mutex_unlock(&wq->wq_lock); - return -ENXIO; - } - /* - * Shared wq with the threshold set to 0 means the user - * did not set the threshold or transitioned from a - * dedicated wq but did not set threshold. A value - * of 0 would effectively disable the shared wq. The - * driver does not allow a value of 0 to be set for - * threshold via sysfs. - */ - if (wq->threshold == 0) { - dev_warn(dev, "Shared WQ and threshold 0.\n"); - mutex_unlock(&wq->wq_lock); - return -EINVAL; - } - } - - rc = idxd_wq_alloc_resources(wq); - if (rc < 0) { - mutex_unlock(&wq->wq_lock); - dev_warn(dev, "WQ resource alloc failed\n"); - return rc; - } - - spin_lock_irqsave(&idxd->dev_lock, flags); - if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) - rc = idxd_device_config(idxd); - spin_unlock_irqrestore(&idxd->dev_lock, flags); - if (rc < 0) { - mutex_unlock(&wq->wq_lock); - dev_warn(dev, "Writing WQ %d config failed: %d\n", wq->id, rc); - return rc; - } - - rc = idxd_wq_enable(wq); - if (rc < 0) { - mutex_unlock(&wq->wq_lock); - dev_warn(dev, "WQ %d enabling failed: %d\n", wq->id, rc); - return rc; - } - - rc = idxd_wq_map_portal(wq); - if (rc < 0) { - dev_warn(dev, "wq portal mapping failed: %d\n", rc); - rc = idxd_wq_disable(wq, false); - if (rc < 0) - dev_warn(dev, "IDXD wq disable failed\n"); - mutex_unlock(&wq->wq_lock); - return rc; - } - - wq->client_count = 0; - - if (wq->type == IDXD_WQT_KERNEL) { - rc = idxd_wq_init_percpu_ref(wq); - if (rc < 0) { - dev_dbg(dev, "percpu_ref setup failed\n"); - mutex_unlock(&wq->wq_lock); - return rc; - } - } - - if (is_idxd_wq_dmaengine(wq)) { - rc = idxd_register_dma_channel(wq); - if (rc < 0) { - dev_dbg(dev, "DMA channel register failed\n"); - mutex_unlock(&wq->wq_lock); - return rc; - } - } else if (is_idxd_wq_cdev(wq)) { - rc = idxd_wq_add_cdev(wq); - if (rc < 0) { - dev_dbg(dev, "Cdev creation failed\n"); - mutex_unlock(&wq->wq_lock); - return rc; - } - } - - mutex_unlock(&wq->wq_lock); - dev_info(dev, "wq %s enabled\n", dev_name(wq_confdev(wq))); - - return 0; -} - static int idxd_config_bus_probe(struct device *dev) { int rc = 0; @@ -205,7 +83,7 @@ static int idxd_config_bus_probe(struct device *dev) } else if (is_idxd_wq_dev(dev)) { struct idxd_wq *wq = confdev_to_wq(dev); - return enable_wq(wq); + return drv_enable_wq(wq); } return -ENODEV; -- Gitee From 04485bb9c44c5fade24824cafe350523f64f3f65 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Jul 2021 11:43:37 -0700 Subject: [PATCH 44/85] dmaengine: idxd: move wq_disable() to device.c ANBZ: #22 commit 69e4f8be596d897679e44e86a323629537c02975 upstream. Move the wq_disable() function to device.c in preparation of setting up the idxd internal sub-driver framework. No logic changes. Intel-SIG: commit 69e4f8be596d dmaengine: idxd: move wq_disable() to device.c. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reviewed-by: Dan Williams Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162637461775.744545.9644048686618957886.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 37 +++++++++++++++++++++++++++++++++++++ drivers/dma/idxd/idxd.h | 1 + drivers/dma/idxd/sysfs.c | 38 +------------------------------------- 3 files changed, 39 insertions(+), 37 deletions(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index b1c509bcfa31..8d8e249931a9 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -1253,3 +1253,40 @@ int drv_enable_wq(struct idxd_wq *wq) mutex_unlock(&wq->wq_lock); return rc; } + +static void __drv_disable_wq(struct idxd_wq *wq) +{ + struct idxd_device *idxd = wq->idxd; + struct device *dev = &idxd->pdev->dev; + + lockdep_assert_held(&wq->wq_lock); + + if (wq->type == IDXD_WQT_KERNEL) + idxd_wq_quiesce(wq); + + if (is_idxd_wq_dmaengine(wq)) + idxd_unregister_dma_channel(wq); + else if (is_idxd_wq_cdev(wq)) + idxd_wq_del_cdev(wq); + + if (idxd_wq_refcount(wq)) + dev_warn(dev, "Clients has claim on wq %d: %d\n", + wq->id, idxd_wq_refcount(wq)); + + idxd_wq_unmap_portal(wq); + + idxd_wq_drain(wq); + idxd_wq_reset(wq); + + idxd_wq_free_resources(wq); + wq->client_count = 0; + + dev_info(dev, "wq %s disabled\n", dev_name(wq_confdev(wq))); +} + +void drv_disable_wq(struct idxd_wq *wq) +{ + mutex_lock(&wq->wq_lock); + __drv_disable_wq(wq); + mutex_unlock(&wq->wq_lock); +} diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 551a61fa1aff..e96ddbfc4569 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -496,6 +496,7 @@ void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id); /* device control */ int drv_enable_wq(struct idxd_wq *wq); +void drv_disable_wq(struct idxd_wq *wq); int idxd_device_init_reset(struct idxd_device *idxd); int idxd_device_enable(struct idxd_device *idxd); int idxd_device_disable(struct idxd_device *idxd); diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 3e8cc07ebcdc..9967fad58a01 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -89,42 +89,6 @@ static int idxd_config_bus_probe(struct device *dev) return -ENODEV; } -static void disable_wq(struct idxd_wq *wq) -{ - struct idxd_device *idxd = wq->idxd; - struct device *dev = &idxd->pdev->dev; - - mutex_lock(&wq->wq_lock); - dev_dbg(dev, "%s removing WQ %s\n", __func__, dev_name(wq_confdev(wq))); - if (wq->state == IDXD_WQ_DISABLED) { - mutex_unlock(&wq->wq_lock); - return; - } - - if (wq->type == IDXD_WQT_KERNEL) - idxd_wq_quiesce(wq); - - if (is_idxd_wq_dmaengine(wq)) - idxd_unregister_dma_channel(wq); - else if (is_idxd_wq_cdev(wq)) - idxd_wq_del_cdev(wq); - - if (idxd_wq_refcount(wq)) - dev_warn(dev, "Clients has claim on wq %d: %d\n", - wq->id, idxd_wq_refcount(wq)); - - idxd_wq_unmap_portal(wq); - - idxd_wq_drain(wq); - idxd_wq_reset(wq); - - idxd_wq_free_resources(wq); - wq->client_count = 0; - mutex_unlock(&wq->wq_lock); - - dev_info(dev, "wq %s disabled\n", dev_name(wq_confdev(wq))); -} - static int idxd_config_bus_remove(struct device *dev) { dev_dbg(dev, "%s called for %s\n", __func__, dev_name(dev)); @@ -133,7 +97,7 @@ static int idxd_config_bus_remove(struct device *dev) if (is_idxd_wq_dev(dev)) { struct idxd_wq *wq = confdev_to_wq(dev); - disable_wq(wq); + drv_disable_wq(wq); } else if (is_idxd_dev(dev)) { struct idxd_device *idxd = confdev_to_idxd(dev); int i; -- Gitee From 2375f5dfc439e916fac130d6e89170803151a743 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Jul 2021 11:43:43 -0700 Subject: [PATCH 45/85] dmaengine: idxd: remove bus shutdown ANBZ: #22 commit 3a5cc01647f07431b342e9703cda0542457ec467 upstream. Remove ->shutdown() function for the dsa bus as it does not do anything and is not necessary. Intel-SIG: commit 3a5cc01647f0 dmaengine: idxd: remove bus shutdown. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reviewed-by: Dan Williams Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162637462319.744545.10383189484257042066.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/sysfs.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 9967fad58a01..c3c869d8119a 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -126,17 +126,11 @@ static int idxd_config_bus_remove(struct device *dev) return 0; } -static void idxd_config_bus_shutdown(struct device *dev) -{ - dev_dbg(dev, "%s called\n", __func__); -} - struct bus_type dsa_bus_type = { .name = "dsa", .match = idxd_config_bus_match, .probe = idxd_config_bus_probe, .remove = idxd_config_bus_remove, - .shutdown = idxd_config_bus_shutdown, }; static struct idxd_device_driver dsa_drv = { -- Gitee From cf7561cc5fc1f7285a3b0547643b0e47c542b09d Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Jul 2021 11:43:49 -0700 Subject: [PATCH 46/85] dmaengine: idxd: remove iax_bus_type prototype ANBZ: #22 commit 1c264299431e9a105f3974ad49b6bccc3f03540f upstream. Remove unused iax_bus_type prototype declaration. Should have been removed when iax_bus_type was removed. Intel-SIG: commit 1c264299431e dmaengine: idxd: remove iax_bus_type prototype. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reviewed-by: Dan Williams Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162637462909.744545.7106049898386277608.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/idxd.h | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index e96ddbfc4569..4c3d3eb94450 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -370,7 +370,6 @@ static inline void idxd_dev_set_type(struct idxd_dev *idev, int type) } extern struct bus_type dsa_bus_type; -extern struct bus_type iax_bus_type; extern bool support_enqcmd; extern struct ida idxd_ida; -- Gitee From 3dae9e9eeb574c860c664108bd7f28bfeeba1641 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Jul 2021 11:43:55 -0700 Subject: [PATCH 47/85] dmaengine: idxd: fix bus_probe() and bus_remove() for dsa_bus ANBZ: #22 commit fcc2281b142bf14e3534d6b1150991194f8d1d44 upstream. Current implementation have put all the code that should be in a driver probe/remove in the bus probe/remove function. Add ->probe() and ->remove() support for the dsa_drv and move all those code out of bus probe/remove. The change does not split out the distinction between device sub-driver and wq sub-driver. It only cleans up the bus calls. The split out will be addressed in follow on patches. Intel-SIG: commit fcc2281b142b dmaengine: idxd: fix bus_probe() and bus_remove() for dsa_bus. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reviewed-by: Dan Williams Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162637463586.744545.5806250155539938643.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/idxd.h | 24 +++++---- drivers/dma/idxd/sysfs.c | 108 ++++++++++++++++++++------------------- 2 files changed, 69 insertions(+), 63 deletions(-) diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 4c3d3eb94450..493958ecc208 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -51,6 +51,8 @@ enum idxd_type { struct idxd_device_driver { const char *name; + int (*probe)(struct idxd_dev *idxd_dev); + void (*remove)(struct idxd_dev *idxd_dev); struct device_driver drv; }; @@ -323,19 +325,21 @@ enum idxd_completion_status { #define cdev_dev(cdev) &cdev->idxd_dev.conf_dev #define confdev_to_idxd_dev(dev) container_of(dev, struct idxd_dev, conf_dev) +#define idxd_dev_to_idxd(idxd_dev) container_of(idxd_dev, struct idxd_device, idxd_dev) +#define idxd_dev_to_wq(idxd_dev) container_of(idxd_dev, struct idxd_wq, idxd_dev) static inline struct idxd_device *confdev_to_idxd(struct device *dev) { struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); - return container_of(idxd_dev, struct idxd_device, idxd_dev); + return idxd_dev_to_idxd(idxd_dev); } static inline struct idxd_wq *confdev_to_wq(struct device *dev) { struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); - return container_of(idxd_dev, struct idxd_wq, idxd_dev); + return idxd_dev_to_wq(idxd_dev); } static inline struct idxd_engine *confdev_to_engine(struct device *dev) @@ -379,24 +383,24 @@ extern struct device_type idxd_wq_device_type; extern struct device_type idxd_engine_device_type; extern struct device_type idxd_group_device_type; -static inline bool is_dsa_dev(struct device *dev) +static inline bool is_dsa_dev(struct idxd_dev *idxd_dev) { - return dev->type == &dsa_device_type; + return idxd_dev->type == IDXD_DEV_DSA; } -static inline bool is_iax_dev(struct device *dev) +static inline bool is_iax_dev(struct idxd_dev *idxd_dev) { - return dev->type == &iax_device_type; + return idxd_dev->type == IDXD_DEV_IAX; } -static inline bool is_idxd_dev(struct device *dev) +static inline bool is_idxd_dev(struct idxd_dev *idxd_dev) { - return is_dsa_dev(dev) || is_iax_dev(dev); + return is_dsa_dev(idxd_dev) || is_iax_dev(idxd_dev); } -static inline bool is_idxd_wq_dev(struct device *dev) +static inline bool is_idxd_wq_dev(struct idxd_dev *idxd_dev) { - return dev->type == &idxd_wq_device_type; + return idxd_dev->type == IDXD_DEV_WQ; } static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq) diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index c3c869d8119a..f82416eec926 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -19,69 +19,80 @@ static char *idxd_wq_type_names[] = { static int idxd_config_bus_match(struct device *dev, struct device_driver *drv) { - int matched = 0; + struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); - if (is_idxd_dev(dev)) { - matched = 1; - } else if (is_idxd_wq_dev(dev)) { - struct idxd_wq *wq = confdev_to_wq(dev); + return (is_idxd_dev(idxd_dev) || is_idxd_wq_dev(idxd_dev)); +} - if (wq->state != IDXD_WQ_DISABLED) { - dev_dbg(dev, "%s not disabled\n", dev_name(dev)); - return 0; - } - matched = 1; - } +static int idxd_config_bus_probe(struct device *dev) +{ + struct idxd_device_driver *idxd_drv = + container_of(dev->driver, struct idxd_device_driver, drv); + struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); - if (matched) - dev_dbg(dev, "%s matched\n", dev_name(dev)); + return idxd_drv->probe(idxd_dev); +} - return matched; +static int idxd_config_bus_remove(struct device *dev) +{ + struct idxd_device_driver *idxd_drv = + container_of(dev->driver, struct idxd_device_driver, drv); + struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); + + idxd_drv->remove(idxd_dev); + return 0; } -static int idxd_config_bus_probe(struct device *dev) +struct bus_type dsa_bus_type = { + .name = "dsa", + .match = idxd_config_bus_match, + .probe = idxd_config_bus_probe, + .remove = idxd_config_bus_remove, +}; + +static int idxd_dsa_drv_probe(struct idxd_dev *idxd_dev) { - int rc = 0; + struct device *dev = &idxd_dev->conf_dev; unsigned long flags; + int rc; - dev_dbg(dev, "%s called\n", __func__); - - if (is_idxd_dev(dev)) { - struct idxd_device *idxd = confdev_to_idxd(dev); + if (is_idxd_dev(idxd_dev)) { + struct idxd_device *idxd = idxd_dev_to_idxd(idxd_dev); - if (!try_module_get(THIS_MODULE)) + if (idxd->state != IDXD_DEV_DISABLED) return -ENXIO; - /* Perform IDXD configuration and enabling */ + /* Device configuration */ spin_lock_irqsave(&idxd->dev_lock, flags); if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) rc = idxd_device_config(idxd); spin_unlock_irqrestore(&idxd->dev_lock, flags); if (rc < 0) { - module_put(THIS_MODULE); - dev_warn(dev, "Device config failed: %d\n", rc); + dev_dbg(dev, "Device config failed: %d\n", rc); return rc; } - /* start device */ + /* Start device */ rc = idxd_device_enable(idxd); if (rc < 0) { - module_put(THIS_MODULE); dev_warn(dev, "Device enable failed: %d\n", rc); return rc; } - dev_info(dev, "Device %s enabled\n", dev_name(dev)); - + /* Setup DMA device without channels */ rc = idxd_register_dma_device(idxd); if (rc < 0) { - module_put(THIS_MODULE); dev_dbg(dev, "Failed to register dmaengine device\n"); + idxd_device_disable(idxd); return rc; } + + dev_info(dev, "Device %s enabled\n", dev_name(dev)); return 0; - } else if (is_idxd_wq_dev(dev)) { - struct idxd_wq *wq = confdev_to_wq(dev); + } + + if (is_idxd_wq_dev(idxd_dev)) { + struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); return drv_enable_wq(wq); } @@ -89,21 +100,14 @@ static int idxd_config_bus_probe(struct device *dev) return -ENODEV; } -static int idxd_config_bus_remove(struct device *dev) +static void idxd_dsa_drv_remove(struct idxd_dev *idxd_dev) { - dev_dbg(dev, "%s called for %s\n", __func__, dev_name(dev)); - - /* disable workqueue here */ - if (is_idxd_wq_dev(dev)) { - struct idxd_wq *wq = confdev_to_wq(dev); + struct device *dev = &idxd_dev->conf_dev; - drv_disable_wq(wq); - } else if (is_idxd_dev(dev)) { - struct idxd_device *idxd = confdev_to_idxd(dev); + if (is_idxd_dev(idxd_dev)) { + struct idxd_device *idxd = idxd_dev_to_idxd(idxd_dev); int i; - dev_dbg(dev, "%s removing dev %s\n", __func__, - dev_name(idxd_confdev(idxd))); for (i = 0; i < idxd->max_wqs; i++) { struct idxd_wq *wq = idxd->wqs[i]; @@ -118,23 +122,21 @@ static int idxd_config_bus_remove(struct device *dev) idxd_device_disable(idxd); if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) idxd_device_reset(idxd); - module_put(THIS_MODULE); - - dev_info(dev, "Device %s disabled\n", dev_name(dev)); + return; } - return 0; -} + if (is_idxd_wq_dev(idxd_dev)) { + struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); -struct bus_type dsa_bus_type = { - .name = "dsa", - .match = idxd_config_bus_match, - .probe = idxd_config_bus_probe, - .remove = idxd_config_bus_remove, -}; + drv_disable_wq(wq); + return; + } +} static struct idxd_device_driver dsa_drv = { .name = "dsa", + .probe = idxd_dsa_drv_probe, + .remove = idxd_dsa_drv_remove, }; /* IDXD generic driver setup */ -- Gitee From 9c2f3bcb57075c73243743e25924caaa25634521 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Jul 2021 11:44:01 -0700 Subject: [PATCH 48/85] dmaengine: idxd: move probe() bits for idxd 'struct device' to device.c ANBZ: #22 commit bd42805b5da33b9c75f3ce0ae9d6ff0ec3f2cd6b upstream. Move the code related to a ->probe() function for the idxd 'struct device' to device.c to prep for the idxd device sub-driver in device.c. Intel-SIG: commit bd42805b5da3 dmaengine: idxd: move probe() bits for idxd 'struct device' to device.c. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reviewed-by: Dan Williams Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162637464189.744545.17423830646786162194.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 37 ++++++++++++++++++++++++++++++++++++ drivers/dma/idxd/idxd.h | 1 + drivers/dma/idxd/sysfs.c | 40 ++------------------------------------- 3 files changed, 40 insertions(+), 38 deletions(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 8d8e249931a9..b9aa209efee4 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -1290,3 +1290,40 @@ void drv_disable_wq(struct idxd_wq *wq) __drv_disable_wq(wq); mutex_unlock(&wq->wq_lock); } + +int idxd_device_drv_probe(struct idxd_dev *idxd_dev) +{ + struct idxd_device *idxd = idxd_dev_to_idxd(idxd_dev); + unsigned long flags; + int rc = 0; + + /* + * Device should be in disabled state for the idxd_drv to load. If it's in + * enabled state, then the device was altered outside of driver's control. + * If the state is in halted state, then we don't want to proceed. + */ + if (idxd->state != IDXD_DEV_DISABLED) + return -ENXIO; + + /* Device configuration */ + spin_lock_irqsave(&idxd->dev_lock, flags); + if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) + rc = idxd_device_config(idxd); + spin_unlock_irqrestore(&idxd->dev_lock, flags); + if (rc < 0) + return -ENXIO; + + /* Start device */ + rc = idxd_device_enable(idxd); + if (rc < 0) + return rc; + + /* Setup DMA device without channels */ + rc = idxd_register_dma_device(idxd); + if (rc < 0) { + idxd_device_disable(idxd); + return rc; + } + + return 0; +} diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 493958ecc208..dbbd36feb462 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -498,6 +498,7 @@ void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id); void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id); /* device control */ +int idxd_device_drv_probe(struct idxd_dev *idxd_dev); int drv_enable_wq(struct idxd_wq *wq); void drv_disable_wq(struct idxd_wq *wq); int idxd_device_init_reset(struct idxd_device *idxd); diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index f82416eec926..221a61e3bb9c 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -52,44 +52,8 @@ struct bus_type dsa_bus_type = { static int idxd_dsa_drv_probe(struct idxd_dev *idxd_dev) { - struct device *dev = &idxd_dev->conf_dev; - unsigned long flags; - int rc; - - if (is_idxd_dev(idxd_dev)) { - struct idxd_device *idxd = idxd_dev_to_idxd(idxd_dev); - - if (idxd->state != IDXD_DEV_DISABLED) - return -ENXIO; - - /* Device configuration */ - spin_lock_irqsave(&idxd->dev_lock, flags); - if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) - rc = idxd_device_config(idxd); - spin_unlock_irqrestore(&idxd->dev_lock, flags); - if (rc < 0) { - dev_dbg(dev, "Device config failed: %d\n", rc); - return rc; - } - - /* Start device */ - rc = idxd_device_enable(idxd); - if (rc < 0) { - dev_warn(dev, "Device enable failed: %d\n", rc); - return rc; - } - - /* Setup DMA device without channels */ - rc = idxd_register_dma_device(idxd); - if (rc < 0) { - dev_dbg(dev, "Failed to register dmaengine device\n"); - idxd_device_disable(idxd); - return rc; - } - - dev_info(dev, "Device %s enabled\n", dev_name(dev)); - return 0; - } + if (is_idxd_dev(idxd_dev)) + return idxd_device_drv_probe(idxd_dev); if (is_idxd_wq_dev(idxd_dev)) { struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); -- Gitee From f00686aa1088dd846f57762a8a62dc9a81ed0bab Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Jul 2021 11:44:07 -0700 Subject: [PATCH 49/85] dmaengine: idxd: idxd: move remove() bits for idxd 'struct device' to device.c ANBZ: #22 commit 745e92a6d816277fcbd231bda5ad2d882b22fe52 upstream. Move the code related to a ->remove() function for the idxd 'struct device' to device.c to prep for the idxd device sub-driver in device.c. Intel-SIG: commit 745e92a6d816 dmaengine: idxd: idxd: move remove() bits for idxd 'struct device' to device.c. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reviewed-by: Dan Williams Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162637464768.744545.15797285510999151668.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 22 ++++++++++++++++++++++ drivers/dma/idxd/idxd.h | 1 + drivers/dma/idxd/sysfs.c | 20 +------------------- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index b9aa209efee4..d5a0b6fff3b9 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -1327,3 +1327,25 @@ int idxd_device_drv_probe(struct idxd_dev *idxd_dev) return 0; } + +void idxd_device_drv_remove(struct idxd_dev *idxd_dev) +{ + struct device *dev = &idxd_dev->conf_dev; + struct idxd_device *idxd = idxd_dev_to_idxd(idxd_dev); + int i; + + for (i = 0; i < idxd->max_wqs; i++) { + struct idxd_wq *wq = idxd->wqs[i]; + struct device *wq_dev = wq_confdev(wq); + + if (wq->state == IDXD_WQ_DISABLED) + continue; + dev_warn(dev, "Active wq %d on disable %s.\n", i, dev_name(wq_dev)); + device_release_driver(wq_dev); + } + + idxd_unregister_dma_device(idxd); + idxd_device_disable(idxd); + if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) + idxd_device_reset(idxd); +} diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index dbbd36feb462..1c8abba13470 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -499,6 +499,7 @@ void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id); /* device control */ int idxd_device_drv_probe(struct idxd_dev *idxd_dev); +void idxd_device_drv_remove(struct idxd_dev *idxd_dev); int drv_enable_wq(struct idxd_wq *wq); void drv_disable_wq(struct idxd_wq *wq); int idxd_device_init_reset(struct idxd_device *idxd); diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 221a61e3bb9c..abea8aca6799 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -66,26 +66,8 @@ static int idxd_dsa_drv_probe(struct idxd_dev *idxd_dev) static void idxd_dsa_drv_remove(struct idxd_dev *idxd_dev) { - struct device *dev = &idxd_dev->conf_dev; - if (is_idxd_dev(idxd_dev)) { - struct idxd_device *idxd = idxd_dev_to_idxd(idxd_dev); - int i; - - for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = idxd->wqs[i]; - - if (wq->state == IDXD_WQ_DISABLED) - continue; - dev_warn(dev, "Active wq %d on disable %s.\n", i, - dev_name(wq_confdev(wq))); - device_release_driver(wq_confdev(wq)); - } - - idxd_unregister_dma_device(idxd); - idxd_device_disable(idxd); - if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) - idxd_device_reset(idxd); + idxd_device_drv_remove(idxd_dev); return; } -- Gitee From 458ca64b2778d66519fd216da08249a292234a77 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Jul 2021 11:44:13 -0700 Subject: [PATCH 50/85] dmanegine: idxd: open code the dsa_drv registration ANBZ: #22 commit c05257b5600bb35a580ecdb25695efff26326d59 upstream. Don't need a wrapper to register the driver. Just do it directly. Intel-SIG: commit c05257b5600b dmanegine: idxd: open code the dsa_drv registration. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reviewed-by: Dan Williams Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162637465319.744545.16325178432532362906.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/idxd.h | 2 ++ drivers/dma/idxd/init.c | 10 +++++----- drivers/dma/idxd/sysfs.c | 13 +------------ 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 1c8abba13470..7fc26b7727c0 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -56,6 +56,8 @@ struct idxd_device_driver { struct device_driver drv; }; +extern struct idxd_device_driver dsa_drv; + struct idxd_irq_entry { struct idxd_device *idxd; int id; diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index c22225b14c5d..5b628e6c04bf 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -840,9 +840,9 @@ static int __init idxd_init_module(void) if (err < 0) return err; - err = idxd_register_driver(); + err = idxd_driver_register(&dsa_drv); if (err < 0) - goto err_idxd_driver_register; + goto err_dsa_driver_register; err = idxd_cdev_register(); if (err) @@ -857,8 +857,8 @@ static int __init idxd_init_module(void) err_pci_register: idxd_cdev_remove(); err_cdev_register: - idxd_unregister_driver(); -err_idxd_driver_register: + idxd_driver_unregister(&dsa_drv); +err_dsa_driver_register: idxd_unregister_bus_type(); return err; } @@ -866,7 +866,7 @@ module_init(idxd_init_module); static void __exit idxd_exit_module(void) { - idxd_unregister_driver(); + idxd_driver_unregister(&dsa_drv); pci_unregister_driver(&idxd_pci_driver); idxd_cdev_remove(); idxd_unregister_bus_type(); diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index abea8aca6799..9f2d06c2aa98 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -79,23 +79,12 @@ static void idxd_dsa_drv_remove(struct idxd_dev *idxd_dev) } } -static struct idxd_device_driver dsa_drv = { +struct idxd_device_driver dsa_drv = { .name = "dsa", .probe = idxd_dsa_drv_probe, .remove = idxd_dsa_drv_remove, }; -/* IDXD generic driver setup */ -int idxd_register_driver(void) -{ - return idxd_driver_register(&dsa_drv); -} - -void idxd_unregister_driver(void) -{ - idxd_driver_unregister(&dsa_drv); -} - /* IDXD engine attributes */ static ssize_t engine_group_id_show(struct device *dev, struct device_attribute *attr, char *buf) -- Gitee From 2afd9802b991d86b44bb7bd7515e6a9a3e6aa34b Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Jul 2021 11:44:18 -0700 Subject: [PATCH 51/85] dmaengine: idxd: add type to driver in order to allow device matching ANBZ: #22 commit 5fee6567ec387088ec965ee60c63051bbe102cac upstream. Add an array of support device types to the idxd_device_driver definition in order to enable simple matching of device type to a given driver. The deprecated / omnibus dsa_drv driver specifies IDXD_DEV_NONE as its only role is to service legacy userspace (old accel-config) directed bind requests and route them to them the proper driver. It need not attach to a device when the bus is autoprobed. The accel-config tooling is being updated to drop its dependency on this deprecated bind scheme. Intel-SIG: commit 5fee6567ec38 dmaengine: idxd: add type to driver in order to allow device matching. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reviewed-by: Dan Willliams Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162637465882.744545.17456174666211577867.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/idxd.h | 1 + drivers/dma/idxd/init.c | 5 +++++ drivers/dma/idxd/sysfs.c | 16 +++++++++++++++- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 7fc26b7727c0..4bb5a65ec237 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -51,6 +51,7 @@ enum idxd_type { struct idxd_device_driver { const char *name; + enum idxd_dev_type *type; int (*probe)(struct idxd_dev *idxd_dev); void (*remove)(struct idxd_dev *idxd_dev); struct device_driver drv; diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 5b628e6c04bf..544ff7137292 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -879,6 +879,11 @@ int __idxd_driver_register(struct idxd_device_driver *idxd_drv, struct module *o { struct device_driver *drv = &idxd_drv->drv; + if (!idxd_drv->type) { + pr_debug("driver type not set (%ps)\n", __builtin_return_address(0)); + return -EINVAL; + } + drv->name = idxd_drv->name; drv->bus = &dsa_bus_type; drv->owner = owner; diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 9f2d06c2aa98..8d48903df131 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -19,9 +19,18 @@ static char *idxd_wq_type_names[] = { static int idxd_config_bus_match(struct device *dev, struct device_driver *drv) { + struct idxd_device_driver *idxd_drv = + container_of(drv, struct idxd_device_driver, drv); struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); + int i = 0; + + while (idxd_drv->type[i] != IDXD_DEV_NONE) { + if (idxd_dev->type == idxd_drv->type[i]) + return 1; + i++; + } - return (is_idxd_dev(idxd_dev) || is_idxd_wq_dev(idxd_dev)); + return 0; } static int idxd_config_bus_probe(struct device *dev) @@ -79,10 +88,15 @@ static void idxd_dsa_drv_remove(struct idxd_dev *idxd_dev) } } +static enum idxd_dev_type dev_types[] = { + IDXD_DEV_NONE, +}; + struct idxd_device_driver dsa_drv = { .name = "dsa", .probe = idxd_dsa_drv_probe, .remove = idxd_dsa_drv_remove, + .type = dev_types, }; /* IDXD engine attributes */ -- Gitee From b386aebc1a0c45732e041ae4f38135069c9ce836 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Jul 2021 11:44:24 -0700 Subject: [PATCH 52/85] dmaengine: idxd: create idxd_device sub-driver ANBZ: #22 commit 034b3290ba257f1a3c8730f3fba72e11645e7b50 upstream. The original architecture of /sys/bus/dsa invented a scheme whereby a single entry in the list of bus drivers, /sys/bus/drivers/dsa, handled all device types and internally routed them to different drivers. Those internal drivers were invisible to userspace. Now, as /sys/bus/dsa wants to grow support for alternate drivers for a given device, for example vfio-mdev instead of kernel-internal-dmaengine, a proper bus device-driver model is needed. The first step in that process is separating the existing omnibus/implicit "dsa" driver into proper individual drivers registered on /sys/bus/dsa. Establish the idxd_drv driver that control the enabling and disabling of the accelerator device. Intel-SIG: commit 034b3290ba25 dmaengine: idxd: create idxd_device sub-driver. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reviewed-by: Dan Williams Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162637466439.744545.15210886092627144577.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 13 +++++++++++++ drivers/dma/idxd/idxd.h | 3 +++ drivers/dma/idxd/init.c | 7 +++++++ 3 files changed, 23 insertions(+) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index d5a0b6fff3b9..12ae3f1639f1 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -1349,3 +1349,16 @@ void idxd_device_drv_remove(struct idxd_dev *idxd_dev) if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) idxd_device_reset(idxd); } + +static enum idxd_dev_type dev_types[] = { + IDXD_DEV_DSA, + IDXD_DEV_IAX, + IDXD_DEV_NONE, +}; + +struct idxd_device_driver idxd_drv = { + .type = dev_types, + .probe = idxd_device_drv_probe, + .remove = idxd_device_drv_remove, + .name = "idxd", +}; diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 4bb5a65ec237..a356d227f755 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -58,6 +58,7 @@ struct idxd_device_driver { }; extern struct idxd_device_driver dsa_drv; +extern struct idxd_device_driver idxd_drv; struct idxd_irq_entry { struct idxd_device *idxd; @@ -501,6 +502,8 @@ void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id); void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id); /* device control */ +int idxd_register_idxd_drv(void); +void idxd_unregister_idxd_drv(void); int idxd_device_drv_probe(struct idxd_dev *idxd_dev); void idxd_device_drv_remove(struct idxd_dev *idxd_dev); int drv_enable_wq(struct idxd_wq *wq); diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 544ff7137292..c19b03c17ab9 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -840,6 +840,10 @@ static int __init idxd_init_module(void) if (err < 0) return err; + err = idxd_driver_register(&idxd_drv); + if (err < 0) + goto err_idxd_driver_register; + err = idxd_driver_register(&dsa_drv); if (err < 0) goto err_dsa_driver_register; @@ -859,6 +863,8 @@ static int __init idxd_init_module(void) err_cdev_register: idxd_driver_unregister(&dsa_drv); err_dsa_driver_register: + idxd_driver_unregister(&idxd_drv); +err_idxd_driver_register: idxd_unregister_bus_type(); return err; } @@ -866,6 +872,7 @@ module_init(idxd_init_module); static void __exit idxd_exit_module(void) { + idxd_driver_unregister(&idxd_drv); idxd_driver_unregister(&dsa_drv); pci_unregister_driver(&idxd_pci_driver); idxd_cdev_remove(); -- Gitee From 94f44b2869553bafd12cf54ebb5bddb81e57ee47 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Jul 2021 11:44:30 -0700 Subject: [PATCH 53/85] dmaengine: idxd: create dmaengine driver for wq 'device' ANBZ: #22 commit 0cda4f6986a3824cac500f66326ff267bf37110f upstream. The original architecture of /sys/bus/dsa invented a scheme whereby a single entry in the list of bus drivers, /sys/bus/drivers/dsa, handled all device types and internally routed them to different drivers. Those internal drivers were invisible to userspace. Now, as /sys/bus/dsa wants to grow support for alternate drivers for a given device, for example vfio-mdev instead of kernel-internal-dmaengine, a proper bus device-driver model is needed. The first step in that process is separating the existing omnibus/implicit "dsa" driver into proper individual drivers registered on /sys/bus/dsa. Establish the idxd_dmaengine_drv driver that controls the enabling and disabling of the wq and also register and unregister the dma channel. idxd_wq_alloc_resources() and idxd_wq_free_resources() also get moved to the dmaengine driver. The resources (dma descriptors allocation and setup) are only used by the dmaengine driver and should only happen when it loads. The char dev driver (cdev) related bits are left in the __drv_enable_wq() and __drv_disable_wq() calls to be moved when we split out the char dev driver just like how the dmaengine driver is split out. WQ autoload support is not expected currently. With the amount of configuration needed for the device, the wq is always expected to be enabled by a tool (or via sysfs) rather than auto enabled at driver load. Intel-SIG: commit 0cda4f6986a3 dmaengine: idxd: create dmaengine driver for wq 'device'. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reviewed-by: Dan Williams Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162637467033.744545.12330636655625405394.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 40 +++----------------- drivers/dma/idxd/dma.c | 77 +++++++++++++++++++++++++++++++++++++++ drivers/dma/idxd/idxd.h | 3 ++ drivers/dma/idxd/init.c | 7 ++++ 4 files changed, 92 insertions(+), 35 deletions(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 12ae3f1639f1..4dcc9431ae3d 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -1130,7 +1130,7 @@ int idxd_device_load_config(struct idxd_device *idxd) return 0; } -static int __drv_enable_wq(struct idxd_wq *wq) +int __drv_enable_wq(struct idxd_wq *wq) { struct idxd_device *idxd = wq->idxd; struct device *dev = &idxd->pdev->dev; @@ -1178,12 +1178,7 @@ static int __drv_enable_wq(struct idxd_wq *wq) } } - rc = idxd_wq_alloc_resources(wq); - if (rc < 0) { - dev_dbg(dev, "wq resource alloc failed\n"); - goto err; - } - + rc = 0; spin_lock_irqsave(&idxd->dev_lock, flags); if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) rc = idxd_device_config(idxd); @@ -1207,21 +1202,7 @@ static int __drv_enable_wq(struct idxd_wq *wq) wq->client_count = 0; - if (wq->type == IDXD_WQT_KERNEL) { - rc = idxd_wq_init_percpu_ref(wq); - if (rc < 0) { - dev_dbg(dev, "wq %d percpu_ref setup failed\n", wq->id); - goto err_cpu_ref; - } - } - - if (is_idxd_wq_dmaengine(wq)) { - rc = idxd_register_dma_channel(wq); - if (rc < 0) { - dev_dbg(dev, "wq %d DMA channel register failed\n", wq->id); - goto err_client; - } - } else if (is_idxd_wq_cdev(wq)) { + if (is_idxd_wq_cdev(wq)) { rc = idxd_wq_add_cdev(wq); if (rc < 0) { dev_dbg(dev, "wq %d cdev creation failed\n", wq->id); @@ -1229,12 +1210,9 @@ static int __drv_enable_wq(struct idxd_wq *wq) } } - dev_info(dev, "wq %s enabled\n", dev_name(wq_confdev(wq))); return 0; err_client: - idxd_wq_quiesce(wq); -err_cpu_ref: idxd_wq_unmap_portal(wq); err_map_portal: rc = idxd_wq_disable(wq, false); @@ -1254,19 +1232,14 @@ int drv_enable_wq(struct idxd_wq *wq) return rc; } -static void __drv_disable_wq(struct idxd_wq *wq) +void __drv_disable_wq(struct idxd_wq *wq) { struct idxd_device *idxd = wq->idxd; struct device *dev = &idxd->pdev->dev; lockdep_assert_held(&wq->wq_lock); - if (wq->type == IDXD_WQT_KERNEL) - idxd_wq_quiesce(wq); - - if (is_idxd_wq_dmaengine(wq)) - idxd_unregister_dma_channel(wq); - else if (is_idxd_wq_cdev(wq)) + if (is_idxd_wq_cdev(wq)) idxd_wq_del_cdev(wq); if (idxd_wq_refcount(wq)) @@ -1278,10 +1251,7 @@ static void __drv_disable_wq(struct idxd_wq *wq) idxd_wq_drain(wq); idxd_wq_reset(wq); - idxd_wq_free_resources(wq); wq->client_count = 0; - - dev_info(dev, "wq %s disabled\n", dev_name(wq_confdev(wq))); } void drv_disable_wq(struct idxd_wq *wq) diff --git a/drivers/dma/idxd/dma.c b/drivers/dma/idxd/dma.c index 2e52f9a50519..7e3281700183 100644 --- a/drivers/dma/idxd/dma.c +++ b/drivers/dma/idxd/dma.c @@ -262,3 +262,80 @@ void idxd_unregister_dma_channel(struct idxd_wq *wq) wq->idxd_chan = NULL; put_device(wq_confdev(wq)); } + +static int idxd_dmaengine_drv_probe(struct idxd_dev *idxd_dev) +{ + struct device *dev = &idxd_dev->conf_dev; + struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); + struct idxd_device *idxd = wq->idxd; + int rc; + + if (idxd->state != IDXD_DEV_ENABLED) + return -ENXIO; + + mutex_lock(&wq->wq_lock); + wq->type = IDXD_WQT_KERNEL; + rc = __drv_enable_wq(wq); + if (rc < 0) { + dev_dbg(dev, "Enable wq %d failed: %d\n", wq->id, rc); + rc = -ENXIO; + goto err; + } + + rc = idxd_wq_alloc_resources(wq); + if (rc < 0) { + dev_dbg(dev, "WQ resource alloc failed\n"); + goto err_res_alloc; + } + + rc = idxd_wq_init_percpu_ref(wq); + if (rc < 0) { + dev_dbg(dev, "percpu_ref setup failed\n"); + goto err_ref; + } + + rc = idxd_register_dma_channel(wq); + if (rc < 0) { + dev_dbg(dev, "Failed to register dma channel\n"); + goto err_dma; + } + + mutex_unlock(&wq->wq_lock); + return 0; + +err_dma: + idxd_wq_quiesce(wq); +err_ref: + idxd_wq_free_resources(wq); +err_res_alloc: + __drv_disable_wq(wq); +err: + wq->type = IDXD_WQT_NONE; + mutex_unlock(&wq->wq_lock); + return rc; +} + +static void idxd_dmaengine_drv_remove(struct idxd_dev *idxd_dev) +{ + struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); + + mutex_lock(&wq->wq_lock); + idxd_wq_quiesce(wq); + idxd_unregister_dma_channel(wq); + __drv_disable_wq(wq); + idxd_wq_free_resources(wq); + wq->type = IDXD_WQT_NONE; + mutex_unlock(&wq->wq_lock); +} + +static enum idxd_dev_type dev_types[] = { + IDXD_DEV_WQ, + IDXD_DEV_NONE, +}; + +struct idxd_device_driver idxd_dmaengine_drv = { + .probe = idxd_dmaengine_drv_probe, + .remove = idxd_dmaengine_drv_remove, + .name = "dmaengine", + .type = dev_types, +}; diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index a356d227f755..a840c328bec9 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -59,6 +59,7 @@ struct idxd_device_driver { extern struct idxd_device_driver dsa_drv; extern struct idxd_device_driver idxd_drv; +extern struct idxd_device_driver idxd_dmaengine_drv; struct idxd_irq_entry { struct idxd_device *idxd; @@ -507,7 +508,9 @@ void idxd_unregister_idxd_drv(void); int idxd_device_drv_probe(struct idxd_dev *idxd_dev); void idxd_device_drv_remove(struct idxd_dev *idxd_dev); int drv_enable_wq(struct idxd_wq *wq); +int __drv_enable_wq(struct idxd_wq *wq); void drv_disable_wq(struct idxd_wq *wq); +void __drv_disable_wq(struct idxd_wq *wq); int idxd_device_init_reset(struct idxd_device *idxd); int idxd_device_enable(struct idxd_device *idxd); int idxd_device_disable(struct idxd_device *idxd); diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index c19b03c17ab9..6f38128ce400 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -844,6 +844,10 @@ static int __init idxd_init_module(void) if (err < 0) goto err_idxd_driver_register; + err = idxd_driver_register(&idxd_dmaengine_drv); + if (err < 0) + goto err_idxd_dmaengine_driver_register; + err = idxd_driver_register(&dsa_drv); if (err < 0) goto err_dsa_driver_register; @@ -863,6 +867,8 @@ static int __init idxd_init_module(void) err_cdev_register: idxd_driver_unregister(&dsa_drv); err_dsa_driver_register: + idxd_driver_unregister(&idxd_dmaengine_drv); +err_idxd_dmaengine_driver_register: idxd_driver_unregister(&idxd_drv); err_idxd_driver_register: idxd_unregister_bus_type(); @@ -872,6 +878,7 @@ module_init(idxd_init_module); static void __exit idxd_exit_module(void) { + idxd_driver_unregister(&idxd_dmaengine_drv); idxd_driver_unregister(&idxd_drv); idxd_driver_unregister(&dsa_drv); pci_unregister_driver(&idxd_pci_driver); -- Gitee From 259d6fdd78f4f75486fe508f5818be0e43fa08a8 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Jul 2021 11:44:35 -0700 Subject: [PATCH 54/85] dmaengine: idxd: create user driver for wq 'device' ANBZ: #22 commit 448c3de8ac8353fc4447738ae3c56c4eb6c2131d upstream. The original architecture of /sys/bus/dsa invented a scheme whereby a single entry in the list of bus drivers, /sys/bus/drivers/dsa, handled all device types and internally routed them to different drivers. Those internal drivers were invisible to userspace. Now, as /sys/bus/dsa wants to grow support for alternate drivers for a given device, for example vfio-mdev instead of kernel-internal-dmaengine, a proper bus device-driver model is needed. The first step in that process is separating the existing omnibus/implicit "dsa" driver into proper individual drivers registered on /sys/bus/dsa. Establish the idxd_user_drv driver that controls the enabling and disabling of the wq and also register and unregister a char device to allow user space to mmap the descriptor submission portal. The cdev related bits are moved to the cdev driver probe/remove and out of the drv_enabe/disable_wq() calls. These bits are exclusive to the cdev operation and not part of the generic enable/disable of the wq device. Intel-SIG: commit 448c3de8ac83 dmaengine: idxd: create user driver for wq 'device'. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reviewed-by: Dan Williams Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162637467578.744545.10203997610072341376.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/cdev.c | 53 +++++++++++++++++++++++++++++++++++++++ drivers/dma/idxd/device.c | 14 ----------- drivers/dma/idxd/idxd.h | 1 + drivers/dma/idxd/init.c | 7 ++++++ 4 files changed, 61 insertions(+), 14 deletions(-) diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index 18a003b93812..b67bbf24242a 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -304,6 +304,59 @@ void idxd_wq_del_cdev(struct idxd_wq *wq) put_device(cdev_dev(idxd_cdev)); } +static int idxd_user_drv_probe(struct idxd_dev *idxd_dev) +{ + struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); + struct idxd_device *idxd = wq->idxd; + int rc; + + if (idxd->state != IDXD_DEV_ENABLED) + return -ENXIO; + + mutex_lock(&wq->wq_lock); + wq->type = IDXD_WQT_USER; + rc = __drv_enable_wq(wq); + if (rc < 0) + goto err; + + rc = idxd_wq_add_cdev(wq); + if (rc < 0) + goto err_cdev; + + mutex_unlock(&wq->wq_lock); + return 0; + +err_cdev: + __drv_disable_wq(wq); +err: + wq->type = IDXD_WQT_NONE; + mutex_unlock(&wq->wq_lock); + return rc; +} + +static void idxd_user_drv_remove(struct idxd_dev *idxd_dev) +{ + struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); + + mutex_lock(&wq->wq_lock); + idxd_wq_del_cdev(wq); + __drv_disable_wq(wq); + wq->type = IDXD_WQT_NONE; + mutex_unlock(&wq->wq_lock); +} + +static enum idxd_dev_type dev_types[] = { + IDXD_DEV_WQ, + IDXD_DEV_NONE, +}; + +struct idxd_device_driver idxd_user_drv = { + .probe = idxd_user_drv_probe, + .remove = idxd_user_drv_remove, + .name = "user", + .type = dev_types, +}; + int idxd_cdev_register(void) { int rc, i; diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 4dcc9431ae3d..9bbc28d9a9eb 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -1201,19 +1201,8 @@ int __drv_enable_wq(struct idxd_wq *wq) } wq->client_count = 0; - - if (is_idxd_wq_cdev(wq)) { - rc = idxd_wq_add_cdev(wq); - if (rc < 0) { - dev_dbg(dev, "wq %d cdev creation failed\n", wq->id); - goto err_client; - } - } - return 0; -err_client: - idxd_wq_unmap_portal(wq); err_map_portal: rc = idxd_wq_disable(wq, false); if (rc < 0) @@ -1239,9 +1228,6 @@ void __drv_disable_wq(struct idxd_wq *wq) lockdep_assert_held(&wq->wq_lock); - if (is_idxd_wq_cdev(wq)) - idxd_wq_del_cdev(wq); - if (idxd_wq_refcount(wq)) dev_warn(dev, "Clients has claim on wq %d: %d\n", wq->id, idxd_wq_refcount(wq)); diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index a840c328bec9..bacec9b93a7e 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -60,6 +60,7 @@ struct idxd_device_driver { extern struct idxd_device_driver dsa_drv; extern struct idxd_device_driver idxd_drv; extern struct idxd_device_driver idxd_dmaengine_drv; +extern struct idxd_device_driver idxd_user_drv; struct idxd_irq_entry { struct idxd_device *idxd; diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 6f38128ce400..33a80f700ff8 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -848,6 +848,10 @@ static int __init idxd_init_module(void) if (err < 0) goto err_idxd_dmaengine_driver_register; + err = idxd_driver_register(&idxd_user_drv); + if (err < 0) + goto err_idxd_user_driver_register; + err = idxd_driver_register(&dsa_drv); if (err < 0) goto err_dsa_driver_register; @@ -867,6 +871,8 @@ static int __init idxd_init_module(void) err_cdev_register: idxd_driver_unregister(&dsa_drv); err_dsa_driver_register: + idxd_driver_unregister(&idxd_user_drv); +err_idxd_user_driver_register: idxd_driver_unregister(&idxd_dmaengine_drv); err_idxd_dmaengine_driver_register: idxd_driver_unregister(&idxd_drv); @@ -878,6 +884,7 @@ module_init(idxd_init_module); static void __exit idxd_exit_module(void) { + idxd_driver_unregister(&idxd_user_drv); idxd_driver_unregister(&idxd_dmaengine_drv); idxd_driver_unregister(&idxd_drv); idxd_driver_unregister(&dsa_drv); -- Gitee From 12a1f1bfb63b82339593f8c63f2e05387595aaeb Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Jul 2021 11:44:41 -0700 Subject: [PATCH 55/85] dmaengine: dsa: move dsa_bus_type out of idxd driver to standalone ANBZ: #22 commit d9e5481fca74f870cf2fc2f90a0e77e85c0b5b86 upstream. In preparation for dsa_drv compat support to be built-in, move the bus code to its own compilation unit. A follow-on patch adds the compat implementation. Recall that the compat implementation allows for the deprecated / omnibus dsa_drv binding scheme rather than the idiomatic organization of a full fledged bus driver per driver type. Intel-SIG: commit d9e5481fca74 dmaengine: dsa: move dsa_bus_type out of idxd driver to standalone. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reviewed-by: Dan Williams Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162637468142.744545.2811632736881720857.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/Kconfig | 4 ++ drivers/dma/Makefile | 2 +- drivers/dma/idxd/Makefile | 5 +++ drivers/dma/idxd/bus.c | 92 +++++++++++++++++++++++++++++++++++++++ drivers/dma/idxd/init.c | 30 +------------ drivers/dma/idxd/sysfs.c | 43 ------------------ 6 files changed, 103 insertions(+), 73 deletions(-) create mode 100644 drivers/dma/idxd/bus.c diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 1a7c46615464..589958761fe7 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -283,6 +283,10 @@ config INTEL_IDMA64 Enable DMA support for Intel Low Power Subsystem such as found on Intel Skylake PCH. +config INTEL_IDXD_BUS + tristate + default INTEL_IDXD + config INTEL_IDXD tristate "Intel Data Accelerators support" depends on PCI && X86_64 && !UML diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index 948a8da05f8b..8928816a4f30 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -42,7 +42,7 @@ obj-$(CONFIG_IMX_DMA) += imx-dma.o obj-$(CONFIG_IMX_SDMA) += imx-sdma.o obj-$(CONFIG_INTEL_IDMA64) += idma64.o obj-$(CONFIG_INTEL_IOATDMA) += ioat/ -obj-$(CONFIG_INTEL_IDXD) += idxd/ +obj-y += idxd/ obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o obj-$(CONFIG_K3_DMA) += k3dma.o obj-$(CONFIG_LPC18XX_DMAMUX) += lpc18xx-dmamux.o diff --git a/drivers/dma/idxd/Makefile b/drivers/dma/idxd/Makefile index 6d11558756f8..8c29ed4d48c3 100644 --- a/drivers/dma/idxd/Makefile +++ b/drivers/dma/idxd/Makefile @@ -1,4 +1,9 @@ +ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE=IDXD + obj-$(CONFIG_INTEL_IDXD) += idxd.o idxd-y := init.o irq.o device.o sysfs.o submit.o dma.o cdev.o idxd-$(CONFIG_INTEL_IDXD_PERFMON) += perfmon.o + +obj-$(CONFIG_INTEL_IDXD_BUS) += idxd_bus.o +idxd_bus-y := bus.o diff --git a/drivers/dma/idxd/bus.c b/drivers/dma/idxd/bus.c new file mode 100644 index 000000000000..02837f0fb3e4 --- /dev/null +++ b/drivers/dma/idxd/bus.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2021 Intel Corporation. All rights rsvd. */ +#include +#include +#include +#include +#include "idxd.h" + + +int __idxd_driver_register(struct idxd_device_driver *idxd_drv, struct module *owner, + const char *mod_name) +{ + struct device_driver *drv = &idxd_drv->drv; + + if (!idxd_drv->type) { + pr_debug("driver type not set (%ps)\n", __builtin_return_address(0)); + return -EINVAL; + } + + drv->name = idxd_drv->name; + drv->bus = &dsa_bus_type; + drv->owner = owner; + drv->mod_name = mod_name; + + return driver_register(drv); +} +EXPORT_SYMBOL_GPL(__idxd_driver_register); + +void idxd_driver_unregister(struct idxd_device_driver *idxd_drv) +{ + driver_unregister(&idxd_drv->drv); +} +EXPORT_SYMBOL_GPL(idxd_driver_unregister); + +static int idxd_config_bus_match(struct device *dev, + struct device_driver *drv) +{ + struct idxd_device_driver *idxd_drv = + container_of(drv, struct idxd_device_driver, drv); + struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); + int i = 0; + + while (idxd_drv->type[i] != IDXD_DEV_NONE) { + if (idxd_dev->type == idxd_drv->type[i]) + return 1; + i++; + } + + return 0; +} + +static int idxd_config_bus_probe(struct device *dev) +{ + struct idxd_device_driver *idxd_drv = + container_of(dev->driver, struct idxd_device_driver, drv); + struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); + + return idxd_drv->probe(idxd_dev); +} + +static int idxd_config_bus_remove(struct device *dev) +{ + struct idxd_device_driver *idxd_drv = + container_of(dev->driver, struct idxd_device_driver, drv); + struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); + + idxd_drv->remove(idxd_dev); + return 0; +} + +struct bus_type dsa_bus_type = { + .name = "dsa", + .match = idxd_config_bus_match, + .probe = idxd_config_bus_probe, + .remove = idxd_config_bus_remove, +}; +EXPORT_SYMBOL_GPL(dsa_bus_type); + +static int __init dsa_bus_init(void) +{ + return bus_register(&dsa_bus_type); +} +module_init(dsa_bus_init); + +static void __exit dsa_bus_exit(void) +{ + bus_unregister(&dsa_bus_type); +} +module_exit(dsa_bus_exit); + +MODULE_DESCRIPTION("IDXD driver dsa_bus_type driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 33a80f700ff8..9b797fcdfd7b 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -26,6 +26,7 @@ MODULE_VERSION(IDXD_DRIVER_VERSION); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Intel Corporation"); +MODULE_IMPORT_NS(IDXD); static bool sva = true; module_param(sva, bool, 0644); @@ -836,10 +837,6 @@ static int __init idxd_init_module(void) perfmon_init(); - err = idxd_register_bus_type(); - if (err < 0) - return err; - err = idxd_driver_register(&idxd_drv); if (err < 0) goto err_idxd_driver_register; @@ -877,7 +874,6 @@ static int __init idxd_init_module(void) err_idxd_dmaengine_driver_register: idxd_driver_unregister(&idxd_drv); err_idxd_driver_register: - idxd_unregister_bus_type(); return err; } module_init(idxd_init_module); @@ -890,30 +886,6 @@ static void __exit idxd_exit_module(void) idxd_driver_unregister(&dsa_drv); pci_unregister_driver(&idxd_pci_driver); idxd_cdev_remove(); - idxd_unregister_bus_type(); perfmon_exit(); } module_exit(idxd_exit_module); - -int __idxd_driver_register(struct idxd_device_driver *idxd_drv, struct module *owner, - const char *mod_name) -{ - struct device_driver *drv = &idxd_drv->drv; - - if (!idxd_drv->type) { - pr_debug("driver type not set (%ps)\n", __builtin_return_address(0)); - return -EINVAL; - } - - drv->name = idxd_drv->name; - drv->bus = &dsa_bus_type; - drv->owner = owner; - drv->mod_name = mod_name; - - return driver_register(drv); -} - -void idxd_driver_unregister(struct idxd_device_driver *idxd_drv) -{ - driver_unregister(&idxd_drv->drv); -} diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 8d48903df131..633f4947ed32 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -16,49 +16,6 @@ static char *idxd_wq_type_names[] = { [IDXD_WQT_USER] = "user", }; -static int idxd_config_bus_match(struct device *dev, - struct device_driver *drv) -{ - struct idxd_device_driver *idxd_drv = - container_of(drv, struct idxd_device_driver, drv); - struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); - int i = 0; - - while (idxd_drv->type[i] != IDXD_DEV_NONE) { - if (idxd_dev->type == idxd_drv->type[i]) - return 1; - i++; - } - - return 0; -} - -static int idxd_config_bus_probe(struct device *dev) -{ - struct idxd_device_driver *idxd_drv = - container_of(dev->driver, struct idxd_device_driver, drv); - struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); - - return idxd_drv->probe(idxd_dev); -} - -static int idxd_config_bus_remove(struct device *dev) -{ - struct idxd_device_driver *idxd_drv = - container_of(dev->driver, struct idxd_device_driver, drv); - struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); - - idxd_drv->remove(idxd_dev); - return 0; -} - -struct bus_type dsa_bus_type = { - .name = "dsa", - .match = idxd_config_bus_match, - .probe = idxd_config_bus_probe, - .remove = idxd_config_bus_remove, -}; - static int idxd_dsa_drv_probe(struct idxd_dev *idxd_dev) { if (is_idxd_dev(idxd_dev)) -- Gitee From dbd1ee9a181b72b9f3580971b602ba802b7fcb07 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 15 Jul 2021 11:44:47 -0700 Subject: [PATCH 56/85] dmaengine: idxd: move dsa_drv support to compatible mode ANBZ: #22 commit 6e7f3ee97bbe2c7d7a53b7dbd7a08a579e03c8c9 upstream. The original architecture of /sys/bus/dsa invented a scheme whereby a single entry in the list of bus drivers, /sys/bus/drivers/dsa, handled all device types and internally routed them to different different drivers. Those internal drivers were invisible to userspace. With the idxd driver transitioned to a proper bus device-driver model, the legacy behavior needs to be preserved due to it being exposed to user space via sysfs. Create a compat driver to provide the legacy behavior for /sys/bus/dsa/drivers/dsa. This should satisfy user tool accel-config v3.2 or ealier where this behavior is expected. If the distro has a newer accel-config then the legacy mode does not need to be enabled. When the compat driver binds the device (i.e. dsa0) to the dsa driver, it will be bound to the new idxd_drv. The wq device (i.e. wq0.0) will be bound to either the dmaengine_drv or the user_drv. The dsa_drv becomes a routing mechansim for the new drivers. It will not support additional external drivers that are implemented later. Intel-SIG: commit 6e7f3ee97bbe dmaengine: idxd: move dsa_drv support to compatible mode. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reviewed-by: Dan Williams Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162637468705.744545.4399080971745974435.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/Kconfig | 17 ++++++ drivers/dma/idxd/Makefile | 3 + drivers/dma/idxd/cdev.c | 1 + drivers/dma/idxd/compat.c | 114 ++++++++++++++++++++++++++++++++++++++ drivers/dma/idxd/device.c | 1 + drivers/dma/idxd/dma.c | 1 + drivers/dma/idxd/idxd.h | 10 +++- drivers/dma/idxd/init.c | 7 --- drivers/dma/idxd/sysfs.c | 40 ------------- 9 files changed, 146 insertions(+), 48 deletions(-) create mode 100644 drivers/dma/idxd/compat.c diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 589958761fe7..f340bcc6d476 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -301,6 +301,23 @@ config INTEL_IDXD If unsure, say N. +config INTEL_IDXD_COMPAT + bool "Legacy behavior for idxd driver" + depends on PCI && X86_64 + select INTEL_IDXD_BUS + help + Compatible driver to support old /sys/bus/dsa/drivers/dsa behavior. + The old behavior performed driver bind/unbind for device and wq + devices all under the dsa driver. The compat driver will emulate + the legacy behavior in order to allow existing support apps (i.e. + accel-config) to continue function. It is expected that accel-config + v3.2 and earlier will need the compat mode. A distro with later + accel-config version can disable this compat config. + + Say Y if you have old applications that require such behavior. + + If unsure, say N. + # Config symbol that collects all the dependencies that's necessary to # support shared virtual memory for the devices supported by idxd. config INTEL_IDXD_SVM diff --git a/drivers/dma/idxd/Makefile b/drivers/dma/idxd/Makefile index 8c29ed4d48c3..a1e9f2b3a37c 100644 --- a/drivers/dma/idxd/Makefile +++ b/drivers/dma/idxd/Makefile @@ -7,3 +7,6 @@ idxd-$(CONFIG_INTEL_IDXD_PERFMON) += perfmon.o obj-$(CONFIG_INTEL_IDXD_BUS) += idxd_bus.o idxd_bus-y := bus.o + +obj-$(CONFIG_INTEL_IDXD_COMPAT) += idxd_compat.o +idxd_compat-y := compat.o diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index b67bbf24242a..f6a4603517ba 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -356,6 +356,7 @@ struct idxd_device_driver idxd_user_drv = { .name = "user", .type = dev_types, }; +EXPORT_SYMBOL_GPL(idxd_user_drv); int idxd_cdev_register(void) { diff --git a/drivers/dma/idxd/compat.c b/drivers/dma/idxd/compat.c new file mode 100644 index 000000000000..d67746ee0c1a --- /dev/null +++ b/drivers/dma/idxd/compat.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2021 Intel Corporation. All rights rsvd. */ +#include +#include +#include +#include +#include +#include "idxd.h" + +extern int device_driver_attach(struct device_driver *drv, struct device *dev); +extern void device_driver_detach(struct device *dev); + +#define DRIVER_ATTR_IGNORE_LOCKDEP(_name, _mode, _show, _store) \ + struct driver_attribute driver_attr_##_name = \ + __ATTR_IGNORE_LOCKDEP(_name, _mode, _show, _store) + +static ssize_t unbind_store(struct device_driver *drv, const char *buf, size_t count) +{ + struct bus_type *bus = drv->bus; + struct device *dev; + int rc = -ENODEV; + + dev = bus_find_device_by_name(bus, NULL, buf); + if (dev && dev->driver) { + device_driver_detach(dev); + rc = count; + } + + return rc; +} +static DRIVER_ATTR_IGNORE_LOCKDEP(unbind, 0200, NULL, unbind_store); + +static ssize_t bind_store(struct device_driver *drv, const char *buf, size_t count) +{ + struct bus_type *bus = drv->bus; + struct device *dev; + struct device_driver *alt_drv; + int rc = -ENODEV; + struct idxd_dev *idxd_dev; + + dev = bus_find_device_by_name(bus, NULL, buf); + if (!dev || dev->driver || drv != &dsa_drv.drv) + return -ENODEV; + + idxd_dev = confdev_to_idxd_dev(dev); + if (is_idxd_dev(idxd_dev)) { + alt_drv = driver_find("idxd", bus); + if (!alt_drv) + return -ENODEV; + } else if (is_idxd_wq_dev(idxd_dev)) { + struct idxd_wq *wq = confdev_to_wq(dev); + + if (is_idxd_wq_kernel(wq)) { + alt_drv = driver_find("dmaengine", bus); + if (!alt_drv) + return -ENODEV; + } else if (is_idxd_wq_user(wq)) { + alt_drv = driver_find("user", bus); + if (!alt_drv) + return -ENODEV; + } else { + return -ENODEV; + } + } + + rc = device_driver_attach(alt_drv, dev); + if (rc < 0) + return rc; + + return count; +} +static DRIVER_ATTR_IGNORE_LOCKDEP(bind, 0200, NULL, bind_store); + +static struct attribute *dsa_drv_compat_attrs[] = { + &driver_attr_bind.attr, + &driver_attr_unbind.attr, + NULL, +}; + +static const struct attribute_group dsa_drv_compat_attr_group = { + .attrs = dsa_drv_compat_attrs, +}; + +static const struct attribute_group *dsa_drv_compat_groups[] = { + &dsa_drv_compat_attr_group, + NULL, +}; + +static int idxd_dsa_drv_probe(struct idxd_dev *idxd_dev) +{ + return -ENODEV; +} + +static void idxd_dsa_drv_remove(struct idxd_dev *idxd_dev) +{ +} + +static enum idxd_dev_type dev_types[] = { + IDXD_DEV_NONE, +}; + +struct idxd_device_driver dsa_drv = { + .name = "dsa", + .probe = idxd_dsa_drv_probe, + .remove = idxd_dsa_drv_remove, + .type = dev_types, + .drv = { + .suppress_bind_attrs = true, + .groups = dsa_drv_compat_groups, + }, +}; + +module_idxd_driver(dsa_drv); +MODULE_IMPORT_NS(IDXD); diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 9bbc28d9a9eb..99350ac9a292 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -1318,3 +1318,4 @@ struct idxd_device_driver idxd_drv = { .remove = idxd_device_drv_remove, .name = "idxd", }; +EXPORT_SYMBOL_GPL(idxd_drv); diff --git a/drivers/dma/idxd/dma.c b/drivers/dma/idxd/dma.c index 7e3281700183..2fd7ec29a08f 100644 --- a/drivers/dma/idxd/dma.c +++ b/drivers/dma/idxd/dma.c @@ -339,3 +339,4 @@ struct idxd_device_driver idxd_dmaengine_drv = { .name = "dmaengine", .type = dev_types, }; +EXPORT_SYMBOL_GPL(idxd_dmaengine_drv); diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index bacec9b93a7e..d0874d8877d9 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -416,11 +416,16 @@ static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq) return false; } -static inline bool is_idxd_wq_cdev(struct idxd_wq *wq) +static inline bool is_idxd_wq_user(struct idxd_wq *wq) { return wq->type == IDXD_WQT_USER; } +static inline bool is_idxd_wq_kernel(struct idxd_wq *wq) +{ + return wq->type == IDXD_WQT_KERNEL; +} + static inline bool wq_dedicated(struct idxd_wq *wq) { return test_bit(WQ_FLAG_DEDICATED, &wq->flags); @@ -484,6 +489,9 @@ int __must_check __idxd_driver_register(struct idxd_device_driver *idxd_drv, void idxd_driver_unregister(struct idxd_device_driver *idxd_drv); +#define module_idxd_driver(__idxd_driver) \ + module_driver(__idxd_driver, idxd_driver_register, idxd_driver_unregister) + int idxd_register_bus_type(void); void idxd_unregister_bus_type(void); int idxd_register_devices(struct idxd_device *idxd); diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 9b797fcdfd7b..8db56f98059f 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -849,10 +849,6 @@ static int __init idxd_init_module(void) if (err < 0) goto err_idxd_user_driver_register; - err = idxd_driver_register(&dsa_drv); - if (err < 0) - goto err_dsa_driver_register; - err = idxd_cdev_register(); if (err) goto err_cdev_register; @@ -866,8 +862,6 @@ static int __init idxd_init_module(void) err_pci_register: idxd_cdev_remove(); err_cdev_register: - idxd_driver_unregister(&dsa_drv); -err_dsa_driver_register: idxd_driver_unregister(&idxd_user_drv); err_idxd_user_driver_register: idxd_driver_unregister(&idxd_dmaengine_drv); @@ -883,7 +877,6 @@ static void __exit idxd_exit_module(void) idxd_driver_unregister(&idxd_user_drv); idxd_driver_unregister(&idxd_dmaengine_drv); idxd_driver_unregister(&idxd_drv); - idxd_driver_unregister(&dsa_drv); pci_unregister_driver(&idxd_pci_driver); idxd_cdev_remove(); perfmon_exit(); diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 633f4947ed32..b883e9f16e7f 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -16,46 +16,6 @@ static char *idxd_wq_type_names[] = { [IDXD_WQT_USER] = "user", }; -static int idxd_dsa_drv_probe(struct idxd_dev *idxd_dev) -{ - if (is_idxd_dev(idxd_dev)) - return idxd_device_drv_probe(idxd_dev); - - if (is_idxd_wq_dev(idxd_dev)) { - struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); - - return drv_enable_wq(wq); - } - - return -ENODEV; -} - -static void idxd_dsa_drv_remove(struct idxd_dev *idxd_dev) -{ - if (is_idxd_dev(idxd_dev)) { - idxd_device_drv_remove(idxd_dev); - return; - } - - if (is_idxd_wq_dev(idxd_dev)) { - struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); - - drv_disable_wq(wq); - return; - } -} - -static enum idxd_dev_type dev_types[] = { - IDXD_DEV_NONE, -}; - -struct idxd_device_driver dsa_drv = { - .name = "dsa", - .probe = idxd_dsa_drv_probe, - .remove = idxd_dsa_drv_remove, - .type = dev_types, -}; - /* IDXD engine attributes */ static ssize_t engine_group_id_show(struct device *dev, struct device_attribute *attr, char *buf) -- Gitee From 6f028bd7285238df88f4c5277364693e6554b534 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 14 Jul 2021 16:25:00 -0700 Subject: [PATCH 57/85] dmaengine: idxd: remove fault processing code ANBZ: #22 commit 0e96454ca26cc5c594ec792f7e5168cce726f7cf upstream. Kernel memory are pinned and will not cause faults. Since the driver does not support interrupts for user descriptors, no fault errors are expected to come through the misc interrupt. Remove dead code. Intel-SIG: commit 0e96454ca26c dmaengine: idxd: remove fault processing code. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162630502789.631986.10591230961790023856.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/irq.c | 95 ++---------------------------------------- 1 file changed, 4 insertions(+), 91 deletions(-) diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c index be65d55e1fc4..e018459b534f 100644 --- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -23,10 +23,8 @@ struct idxd_fault { }; static int irq_process_work_list(struct idxd_irq_entry *irq_entry, - enum irq_work_type wtype, int *processed, u64 data); static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry, - enum irq_work_type wtype, int *processed, u64 data); static void idxd_device_reinit(struct work_struct *work) @@ -62,46 +60,6 @@ static void idxd_device_reinit(struct work_struct *work) idxd_device_clear_state(idxd); } -static void idxd_device_fault_work(struct work_struct *work) -{ - struct idxd_fault *fault = container_of(work, struct idxd_fault, work); - struct idxd_irq_entry *ie; - int i; - int processed; - int irqcnt = fault->idxd->num_wq_irqs + 1; - - for (i = 1; i < irqcnt; i++) { - ie = &fault->idxd->irq_entries[i]; - irq_process_work_list(ie, IRQ_WORK_PROCESS_FAULT, - &processed, fault->addr); - if (processed) - break; - - irq_process_pending_llist(ie, IRQ_WORK_PROCESS_FAULT, - &processed, fault->addr); - if (processed) - break; - } - - kfree(fault); -} - -static int idxd_device_schedule_fault_process(struct idxd_device *idxd, - u64 fault_addr) -{ - struct idxd_fault *fault; - - fault = kmalloc(sizeof(*fault), GFP_ATOMIC); - if (!fault) - return -ENOMEM; - - fault->addr = fault_addr; - fault->idxd = idxd; - INIT_WORK(&fault->work, idxd_device_fault_work); - queue_work(idxd->wq, &fault->work); - return 0; -} - static int process_misc_interrupts(struct idxd_device *idxd, u32 cause) { struct device *dev = &idxd->pdev->dev; @@ -168,15 +126,6 @@ static int process_misc_interrupts(struct idxd_device *idxd, u32 cause) if (!err) return 0; - /* - * This case should rarely happen and typically is due to software - * programming error by the driver. - */ - if (idxd->sw_err.valid && - idxd->sw_err.desc_valid && - idxd->sw_err.fault_addr) - idxd_device_schedule_fault_process(idxd, idxd->sw_err.fault_addr); - gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET); if (gensts.state == IDXD_DEVICE_STATE_HALT) { idxd->state = IDXD_DEV_HALTED; @@ -228,43 +177,19 @@ irqreturn_t idxd_misc_thread(int vec, void *data) return IRQ_HANDLED; } -static inline bool match_fault(struct idxd_desc *desc, u64 fault_addr) -{ - /* - * Completion address can be bad as well. Check fault address match for descriptor - * and completion address. - */ - if ((u64)desc->hw == fault_addr || (u64)desc->completion == fault_addr) { - struct idxd_device *idxd = desc->wq->idxd; - struct device *dev = &idxd->pdev->dev; - - dev_warn(dev, "desc with fault address: %#llx\n", fault_addr); - return true; - } - - return false; -} - static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry, - enum irq_work_type wtype, int *processed, u64 data) { struct idxd_desc *desc, *t; struct llist_node *head; int queued = 0; unsigned long flags; - enum idxd_complete_type reason; *processed = 0; head = llist_del_all(&irq_entry->pending_llist); if (!head) goto out; - if (wtype == IRQ_WORK_NORMAL) - reason = IDXD_COMPLETE_NORMAL; - else - reason = IDXD_COMPLETE_DEV_FAIL; - llist_for_each_entry_safe(desc, t, head, llnode) { u8 status = desc->completion->status & DSA_COMP_STATUS_MASK; @@ -275,9 +200,7 @@ static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry, continue; } - if (unlikely(status != DSA_COMP_SUCCESS)) - match_fault(desc, data); - complete_desc(desc, reason); + complete_desc(desc, IDXD_COMPLETE_NORMAL); (*processed)++; } else { spin_lock_irqsave(&irq_entry->list_lock, flags); @@ -293,20 +216,14 @@ static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry, } static int irq_process_work_list(struct idxd_irq_entry *irq_entry, - enum irq_work_type wtype, int *processed, u64 data) { int queued = 0; unsigned long flags; LIST_HEAD(flist); struct idxd_desc *desc, *n; - enum idxd_complete_type reason; *processed = 0; - if (wtype == IRQ_WORK_NORMAL) - reason = IDXD_COMPLETE_NORMAL; - else - reason = IDXD_COMPLETE_DEV_FAIL; /* * This lock protects list corruption from access of list outside of the irq handler @@ -338,9 +255,7 @@ static int irq_process_work_list(struct idxd_irq_entry *irq_entry, continue; } - if (unlikely(status != DSA_COMP_SUCCESS)) - match_fault(desc, data); - complete_desc(desc, reason); + complete_desc(desc, IDXD_COMPLETE_NORMAL); } return queued; @@ -370,14 +285,12 @@ static int idxd_desc_process(struct idxd_irq_entry *irq_entry) * 5. Repeat until no more descriptors. */ do { - rc = irq_process_work_list(irq_entry, IRQ_WORK_NORMAL, - &processed, 0); + rc = irq_process_work_list(irq_entry, &processed, 0); total += processed; if (rc != 0) continue; - rc = irq_process_pending_llist(irq_entry, IRQ_WORK_NORMAL, - &processed, 0); + rc = irq_process_pending_llist(irq_entry, &processed, 0); total += processed; } while (rc != 0); -- Gitee From 4a0d62cf8a7ba5cb15c72005bff5eaef25bc5301 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 20 Jul 2021 13:42:10 -0700 Subject: [PATCH 58/85] dmaengine: idxd: Set defaults for GRPCFG traffic class ANBZ: #22 commit ade8a86b512cf8db0d0e975a971ce356953cfcb3 upstream. Set GRPCFG traffic class to value of 1 for best performance on current generation of accelerators. Also add override option to allow experimentation. Sysfs knobs are disabled for DSA/IAX gen1 devices. Intel-SIG: commit ade8a86b512c dmaengine: idxd: Set defaults for GRPCFG traffic class. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162681373005.1968485.3761065664382799202.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- Documentation/admin-guide/kernel-parameters.txt | 5 +++++ drivers/dma/idxd/idxd.h | 1 + drivers/dma/idxd/init.c | 13 +++++++++++-- drivers/dma/idxd/registers.h | 3 +++ drivers/dma/idxd/sysfs.c | 6 ++++++ 5 files changed, 26 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index a734227e8512..b8e1c0acf4b7 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -1687,6 +1687,11 @@ support for the idxd driver. By default it is set to true (1). + idxd.tc_override= [HW] + Format: + Allow override of default traffic class configuration + for the device. By default it is set to false (0). + ieee754= [MIPS] Select IEEE Std 754 conformance mode Format: { strict | legacy | 2008 | relaxed } Default: strict diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index d0874d8877d9..4e4dc0110e77 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -16,6 +16,7 @@ #define IDXD_DRIVER_VERSION "1.00" extern struct kmem_cache *idxd_desc_pool; +extern bool tc_override; struct idxd_wq; struct idxd_dev; diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 8db56f98059f..eb09bc591c31 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -32,6 +32,10 @@ static bool sva = true; module_param(sva, bool, 0644); MODULE_PARM_DESC(sva, "Toggle SVA support on/off"); +bool tc_override; +module_param(tc_override, bool, 0644); +MODULE_PARM_DESC(tc_override, "Override traffic class defaults"); + #define DRV_NAME "idxd" bool support_enqcmd; @@ -336,8 +340,13 @@ static int idxd_setup_groups(struct idxd_device *idxd) } idxd->groups[i] = group; - group->tc_a = -1; - group->tc_b = -1; + if (idxd->hw.version < DEVICE_VERSION_2 && !tc_override) { + group->tc_a = 1; + group->tc_b = 1; + } else { + group->tc_a = -1; + group->tc_b = -1; + } } return 0; diff --git a/drivers/dma/idxd/registers.h b/drivers/dma/idxd/registers.h index 7343a8f48819..ffc7550a77ee 100644 --- a/drivers/dma/idxd/registers.h +++ b/drivers/dma/idxd/registers.h @@ -7,6 +7,9 @@ #define PCI_DEVICE_ID_INTEL_DSA_SPR0 0x0b25 #define PCI_DEVICE_ID_INTEL_IAX_SPR0 0x0cfe +#define DEVICE_VERSION_1 0x100 +#define DEVICE_VERSION_2 0x200 + #define IDXD_MMIO_BAR 0 #define IDXD_WQ_BAR 2 #define IDXD_PORTAL_SIZE PAGE_SIZE diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index b883e9f16e7f..881a12596d4b 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -327,6 +327,9 @@ static ssize_t group_traffic_class_a_store(struct device *dev, if (idxd->state == IDXD_DEV_ENABLED) return -EPERM; + if (idxd->hw.version < DEVICE_VERSION_2 && !tc_override) + return -EPERM; + if (val < 0 || val > 7) return -EINVAL; @@ -366,6 +369,9 @@ static ssize_t group_traffic_class_b_store(struct device *dev, if (idxd->state == IDXD_DEV_ENABLED) return -EPERM; + if (idxd->hw.version < DEVICE_VERSION_2 && !tc_override) + return -EPERM; + if (val < 0 || val > 7) return -EINVAL; -- Gitee From 7f76ab11e3a2fe6d3fc380879b16270d47aec9a6 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 21 Jul 2021 11:35:03 -0700 Subject: [PATCH 59/85] dmaengine: idxd: fix uninit var for alt_drv ANBZ: #22 commit 568b2126466f926a10be0b53b40c2d6ae056d8d6 upstream. 0-day detected uninitialized alt_drv variable in the bind_store() function. The branch can be taken when device is not idxd device or wq 'struct device'. Init alt_drv to NULL. Intel-SIG: commit 568b2126466f dmaengine: idxd: fix uninit var for alt_drv. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Fixes: 6e7f3ee97bbe ("dmaengine: idxd: move dsa_drv support to compatible mode") Reported-by: kernel test robot Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162689250332.2114335.636367120454420852.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/compat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/dma/idxd/compat.c b/drivers/dma/idxd/compat.c index d67746ee0c1a..d7616c240dcd 100644 --- a/drivers/dma/idxd/compat.c +++ b/drivers/dma/idxd/compat.c @@ -34,7 +34,7 @@ static ssize_t bind_store(struct device_driver *drv, const char *buf, size_t cou { struct bus_type *bus = drv->bus; struct device *dev; - struct device_driver *alt_drv; + struct device_driver *alt_drv = NULL; int rc = -ENODEV; struct idxd_dev *idxd_dev; -- Gitee From 069136da2567b9a59ccd8fa0ae28fe91191bbc9f Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 22 Jul 2021 10:54:10 -0700 Subject: [PATCH 60/85] dmaengine: idxd: fix wq slot allocation index check ANBZ: #22 commit 673d812d30be67942762bb9e8548abb26a3ba4a7 upstream. The sbitmap wait and allocate routine checks the index that is returned from sbitmap_queue_get(). It should be idxd >= 0 as 0 is also a valid index. This fixes issue where submission path hangs when WQ size is 1. Intel-SIG: commit 673d812d30be dmaengine: idxd: fix wq slot allocation index check. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Fixes: 0705107fcc80 ("dmaengine: idxd: move submission to sbitmap_queue") Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162697645067.3478714.506720687816951762.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/submit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/dma/idxd/submit.c b/drivers/dma/idxd/submit.c index 6ef704dd4d0b..65b0130ab2db 100644 --- a/drivers/dma/idxd/submit.c +++ b/drivers/dma/idxd/submit.c @@ -59,7 +59,7 @@ struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype) if (signal_pending_state(TASK_INTERRUPTIBLE, current)) break; idx = sbitmap_queue_get(sbq, &cpu); - if (idx > 0) + if (idx >= 0) break; schedule(); } -- Gitee From cbde7aa5238e3079126a6eeb5fb60310512f6eac Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 20 Jul 2021 13:42:04 -0700 Subject: [PATCH 61/85] dmaengine: idxd: rotate portal address for better performance ANBZ: #22 commit a9c171527a3403cae6c1907744b1bc9ca301f912 upstream. The device submission portal is on a 4k page and any of those 64bit aligned address on the page can be used for descriptor submission. By rotating the offset through the 4k range and prevent successive writes to the same MMIO address, performance improvement is observed through testing. Intel-SIG: commit a9c171527a34 dmaengine: idxd: rotate portal address for better performance. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162681372446.1968485.10634280461681015569.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 1 + drivers/dma/idxd/idxd.h | 20 ++++++++++++++++++++ drivers/dma/idxd/submit.c | 2 +- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 99350ac9a292..41f67a195eb6 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -320,6 +320,7 @@ void idxd_wq_unmap_portal(struct idxd_wq *wq) devm_iounmap(dev, wq->portal); wq->portal = NULL; + wq->portal_offset = 0; } void idxd_wqs_unmap_portal(struct idxd_device *idxd) diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 4e4dc0110e77..94983bced189 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -11,6 +11,7 @@ #include #include #include +#include #include "registers.h" #define IDXD_DRIVER_VERSION "1.00" @@ -162,6 +163,7 @@ struct idxd_dma_chan { struct idxd_wq { void __iomem *portal; + u32 portal_offset; struct percpu_ref wq_active; struct completion wq_dead; struct idxd_dev idxd_dev; @@ -468,6 +470,24 @@ static inline int idxd_get_wq_portal_full_offset(int wq_id, return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot); } +#define IDXD_PORTAL_MASK (PAGE_SIZE - 1) + +/* + * Even though this function can be accessed by multiple threads, it is safe to use. + * At worst the address gets used more than once before it gets incremented. We don't + * hit a threshold until iops becomes many million times a second. So the occasional + * reuse of the same address is tolerable compare to using an atomic variable. This is + * safe on a system that has atomic load/store for 32bit integers. Given that this is an + * Intel iEP device, that should not be a problem. + */ +static inline void __iomem *idxd_wq_portal_addr(struct idxd_wq *wq) +{ + int ofs = wq->portal_offset; + + wq->portal_offset = (ofs + sizeof(struct dsa_raw_desc)) & IDXD_PORTAL_MASK; + return wq->portal + ofs; +} + static inline void idxd_wq_get(struct idxd_wq *wq) { wq->client_count++; diff --git a/drivers/dma/idxd/submit.c b/drivers/dma/idxd/submit.c index 65b0130ab2db..92ae9a157cc9 100644 --- a/drivers/dma/idxd/submit.c +++ b/drivers/dma/idxd/submit.c @@ -146,7 +146,7 @@ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc) if (!percpu_ref_tryget_live(&wq->wq_active)) return -ENXIO; - portal = wq->portal; + portal = idxd_wq_portal_addr(wq); /* * The wmb() flushes writes to coherent DMA data before -- Gitee From 45fa2eac713355168e52c7ae76f83f9d39e76834 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 20 Jul 2021 13:42:15 -0700 Subject: [PATCH 62/85] dmanegine: idxd: add software command status ANBZ: #22 commit 125d10373ad991888c9e94d2da49bcc5ccba2127 upstream. Enabling device and wq returns standard errno and that does not provide enough details to indicate what exactly failed. The hardware command status is only 8bits. Expand the command status to 32bits and use the upper 16 bits to define software errors to provide more details on the exact failure. Bit 31 will be used to indicate the error is software set as the driver is using some of the spec defined hardware error as well. Intel-SIG: commit 125d10373ad9 dmanegine: idxd: add software command status. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Cc: Ramesh Thomas Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162681373579.1968485.5891788397526827892.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- .../ABI/stable/sysfs-driver-dma-idxd | 2 ++ drivers/dma/idxd/cdev.c | 5 +++- drivers/dma/idxd/device.c | 22 +++++++++++++++--- drivers/dma/idxd/dma.c | 4 ++++ drivers/dma/idxd/idxd.h | 2 +- drivers/dma/idxd/sysfs.c | 11 ++++++++- include/uapi/linux/idxd.h | 23 +++++++++++++++++++ 7 files changed, 63 insertions(+), 6 deletions(-) diff --git a/Documentation/ABI/stable/sysfs-driver-dma-idxd b/Documentation/ABI/stable/sysfs-driver-dma-idxd index ebd521314803..c67c63bacd3c 100644 --- a/Documentation/ABI/stable/sysfs-driver-dma-idxd +++ b/Documentation/ABI/stable/sysfs-driver-dma-idxd @@ -128,6 +128,8 @@ Date: Aug 28, 2020 KernelVersion: 5.10.0 Contact: dmaengine@vger.kernel.org Description: The last executed device administrative command's status/error. + Also last configuration error overloaded. + Writing to it will clear the status. What: /sys/bus/dsa/devices/wq./block_on_fault Date: Oct 27, 2020 diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index f6a4603517ba..4d2ecdb130e7 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -320,9 +320,12 @@ static int idxd_user_drv_probe(struct idxd_dev *idxd_dev) goto err; rc = idxd_wq_add_cdev(wq); - if (rc < 0) + if (rc < 0) { + idxd->cmd_status = IDXD_SCMD_CDEV_ERR; goto err_cdev; + } + idxd->cmd_status = 0; mutex_unlock(&wq->wq_lock); return 0; diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 41f67a195eb6..86fa4b4590f9 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -840,6 +840,7 @@ static int idxd_wq_config_write(struct idxd_wq *wq) wq->wqcfg->wq_size = wq->size; if (wq->size == 0) { + idxd->cmd_status = IDXD_SCMD_WQ_NO_SIZE; dev_warn(dev, "Incorrect work queue size: 0\n"); return -EINVAL; } @@ -975,6 +976,7 @@ static int idxd_wqs_setup(struct idxd_device *idxd) continue; if (wq_shared(wq) && !device_swq_supported(idxd)) { + idxd->cmd_status = IDXD_SCMD_WQ_NO_SWQ_SUPPORT; dev_warn(dev, "No shared wq support but configured.\n"); return -EINVAL; } @@ -983,8 +985,10 @@ static int idxd_wqs_setup(struct idxd_device *idxd) configured++; } - if (configured == 0) + if (configured == 0) { + idxd->cmd_status = IDXD_SCMD_WQ_NONE_CONFIGURED; return -EINVAL; + } return 0; } @@ -1140,21 +1144,26 @@ int __drv_enable_wq(struct idxd_wq *wq) lockdep_assert_held(&wq->wq_lock); - if (idxd->state != IDXD_DEV_ENABLED) + if (idxd->state != IDXD_DEV_ENABLED) { + idxd->cmd_status = IDXD_SCMD_DEV_NOT_ENABLED; goto err; + } if (wq->state != IDXD_WQ_DISABLED) { dev_dbg(dev, "wq %d already enabled.\n", wq->id); + idxd->cmd_status = IDXD_SCMD_WQ_ENABLED; rc = -EBUSY; goto err; } if (!wq->group) { dev_dbg(dev, "wq %d not attached to group.\n", wq->id); + idxd->cmd_status = IDXD_SCMD_WQ_NO_GRP; goto err; } if (strlen(wq->name) == 0) { + idxd->cmd_status = IDXD_SCMD_WQ_NO_NAME; dev_dbg(dev, "wq %d name not set.\n", wq->id); goto err; } @@ -1162,6 +1171,7 @@ int __drv_enable_wq(struct idxd_wq *wq) /* Shared WQ checks */ if (wq_shared(wq)) { if (!device_swq_supported(idxd)) { + idxd->cmd_status = IDXD_SCMD_WQ_NO_SVM; dev_dbg(dev, "PASID not enabled and shared wq.\n"); goto err; } @@ -1174,6 +1184,7 @@ int __drv_enable_wq(struct idxd_wq *wq) * threshold via sysfs. */ if (wq->threshold == 0) { + idxd->cmd_status = IDXD_SCMD_WQ_NO_THRESH; dev_dbg(dev, "Shared wq and threshold 0.\n"); goto err; } @@ -1197,6 +1208,7 @@ int __drv_enable_wq(struct idxd_wq *wq) rc = idxd_wq_map_portal(wq); if (rc < 0) { + idxd->cmd_status = IDXD_SCMD_WQ_PORTAL_ERR; dev_dbg(dev, "wq %d portal mapping failed: %d\n", wq->id, rc); goto err_map_portal; } @@ -1259,8 +1271,10 @@ int idxd_device_drv_probe(struct idxd_dev *idxd_dev) * enabled state, then the device was altered outside of driver's control. * If the state is in halted state, then we don't want to proceed. */ - if (idxd->state != IDXD_DEV_DISABLED) + if (idxd->state != IDXD_DEV_DISABLED) { + idxd->cmd_status = IDXD_SCMD_DEV_ENABLED; return -ENXIO; + } /* Device configuration */ spin_lock_irqsave(&idxd->dev_lock, flags); @@ -1279,9 +1293,11 @@ int idxd_device_drv_probe(struct idxd_dev *idxd_dev) rc = idxd_register_dma_device(idxd); if (rc < 0) { idxd_device_disable(idxd); + idxd->cmd_status = IDXD_SCMD_DEV_DMA_ERR; return rc; } + idxd->cmd_status = 0; return 0; } diff --git a/drivers/dma/idxd/dma.c b/drivers/dma/idxd/dma.c index 2fd7ec29a08f..a195225687bb 100644 --- a/drivers/dma/idxd/dma.c +++ b/drivers/dma/idxd/dma.c @@ -284,22 +284,26 @@ static int idxd_dmaengine_drv_probe(struct idxd_dev *idxd_dev) rc = idxd_wq_alloc_resources(wq); if (rc < 0) { + idxd->cmd_status = IDXD_SCMD_WQ_RES_ALLOC_ERR; dev_dbg(dev, "WQ resource alloc failed\n"); goto err_res_alloc; } rc = idxd_wq_init_percpu_ref(wq); if (rc < 0) { + idxd->cmd_status = IDXD_SCMD_PERCPU_ERR; dev_dbg(dev, "percpu_ref setup failed\n"); goto err_ref; } rc = idxd_register_dma_channel(wq); if (rc < 0) { + idxd->cmd_status = IDXD_SCMD_DMA_CHAN_ERR; dev_dbg(dev, "Failed to register dma channel\n"); goto err_dma; } + idxd->cmd_status = 0; mutex_unlock(&wq->wq_lock); return 0; diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 94983bced189..bfcb03329f77 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -252,7 +252,7 @@ struct idxd_device { unsigned long flags; int id; int major; - u8 cmd_status; + u32 cmd_status; struct pci_dev *pdev; void __iomem *reg_base; diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 881a12596d4b..4c01587c9d4a 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -1217,7 +1217,16 @@ static ssize_t cmd_status_show(struct device *dev, return sysfs_emit(buf, "%#x\n", idxd->cmd_status); } -static DEVICE_ATTR_RO(cmd_status); + +static ssize_t cmd_status_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct idxd_device *idxd = confdev_to_idxd(dev); + + idxd->cmd_status = 0; + return count; +} +static DEVICE_ATTR_RW(cmd_status); static struct attribute *idxd_device_attributes[] = { &dev_attr_version.attr, diff --git a/include/uapi/linux/idxd.h b/include/uapi/linux/idxd.h index edc346a77c91..ca24c25252fb 100644 --- a/include/uapi/linux/idxd.h +++ b/include/uapi/linux/idxd.h @@ -9,6 +9,29 @@ #include #endif +/* Driver command error status */ +enum idxd_scmd_stat { + IDXD_SCMD_DEV_ENABLED = 0x80000010, + IDXD_SCMD_DEV_NOT_ENABLED = 0x80000020, + IDXD_SCMD_WQ_ENABLED = 0x80000021, + IDXD_SCMD_DEV_DMA_ERR = 0x80020000, + IDXD_SCMD_WQ_NO_GRP = 0x80030000, + IDXD_SCMD_WQ_NO_NAME = 0x80040000, + IDXD_SCMD_WQ_NO_SVM = 0x80050000, + IDXD_SCMD_WQ_NO_THRESH = 0x80060000, + IDXD_SCMD_WQ_PORTAL_ERR = 0x80070000, + IDXD_SCMD_WQ_RES_ALLOC_ERR = 0x80080000, + IDXD_SCMD_PERCPU_ERR = 0x80090000, + IDXD_SCMD_DMA_CHAN_ERR = 0x800a0000, + IDXD_SCMD_CDEV_ERR = 0x800b0000, + IDXD_SCMD_WQ_NO_SWQ_SUPPORT = 0x800c0000, + IDXD_SCMD_WQ_NONE_CONFIGURED = 0x800d0000, + IDXD_SCMD_WQ_NO_SIZE = 0x800e0000, +}; + +#define IDXD_SCMD_SOFTERR_MASK 0x80000000 +#define IDXD_SCMD_SOFTERR_SHIFT 16 + /* Descriptor flags */ #define IDXD_OP_FLAG_FENCE 0x0001 #define IDXD_OP_FLAG_BOF 0x0002 -- Gitee From 4cd57e22d23ed0d78349e358c67cee696c8c03fb Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 22 Jul 2021 13:10:51 -0700 Subject: [PATCH 63/85] dmaengine: idxd: fix abort status check ANBZ: #22 commit b60bb6e2bfc192091b8f792781b83b5e0f9324f6 upstream. Coverity static analysis of linux-next found issue. The check (status == IDXD_COMP_DESC_ABORT) is always false since status was previously masked with 0x7f and IDXD_COMP_DESC_ABORT is 0xff. Intel-SIG: commit b60bb6e2bfc1 dmaengine: idxd: fix abort status check. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Fixes: 6b4b87f2c31a ("dmaengine: idxd: fix submission race window") Reported-by: Colin Ian King Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162698465160.3560828.18173186265683415384.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/irq.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c index e018459b534f..65dc7bbb0a13 100644 --- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -194,7 +194,11 @@ static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry, u8 status = desc->completion->status & DSA_COMP_STATUS_MASK; if (status) { - if (unlikely(status == IDXD_COMP_DESC_ABORT)) { + /* + * Check against the original status as ABORT is software defined + * and 0xff, which DSA_COMP_STATUS_MASK can mask out. + */ + if (unlikely(desc->completion->status == IDXD_COMP_DESC_ABORT)) { complete_desc(desc, IDXD_COMPLETE_ABORT); (*processed)++; continue; @@ -250,7 +254,11 @@ static int irq_process_work_list(struct idxd_irq_entry *irq_entry, list_for_each_entry(desc, &flist, list) { u8 status = desc->completion->status & DSA_COMP_STATUS_MASK; - if (unlikely(status == IDXD_COMP_DESC_ABORT)) { + /* + * Check against the original status as ABORT is software defined + * and 0xff, which DSA_COMP_STATUS_MASK can mask out. + */ + if (unlikely(desc->completion->status == IDXD_COMP_DESC_ABORT)) { complete_desc(desc, IDXD_COMPLETE_ABORT); continue; } -- Gitee From 4de44c2b0bac2c681d99e4c34b4e80b5095ef098 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Thu, 29 Jul 2021 14:04:01 +0200 Subject: [PATCH 64/85] dmaengine: idxd: Fix a possible NULL pointer dereference ANBZ: #22 commit e9c5b0b53ccca81dd0a35c62309e243a57c7959d upstream. 'device_driver_attach()' dereferences its first argument (i.e. 'alt_drv') so it must not be NULL. Simplify the error handling logic about NULL 'alt_drv' in order to be more robust and future-proof. Intel-SIG: commit e9c5b0b53ccc dmaengine: idxd: Fix a possible NULL pointer dereference. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Fixes: 568b2126466f ("dmaengine: idxd: fix uninit var for alt_drv") Fixes: 6e7f3ee97bbe ("dmaengine: idxd: move dsa_drv support to compatible mode") Signed-off-by: Christophe JAILLET Acked-by: Dave Jiang Link: https://lore.kernel.org/r/77f0dc4f3966591d1f0cffb614a94085f8895a85.1627560174.git.christophe.jaillet@wanadoo.fr Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/compat.c | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/drivers/dma/idxd/compat.c b/drivers/dma/idxd/compat.c index d7616c240dcd..3df21615f888 100644 --- a/drivers/dma/idxd/compat.c +++ b/drivers/dma/idxd/compat.c @@ -45,23 +45,16 @@ static ssize_t bind_store(struct device_driver *drv, const char *buf, size_t cou idxd_dev = confdev_to_idxd_dev(dev); if (is_idxd_dev(idxd_dev)) { alt_drv = driver_find("idxd", bus); - if (!alt_drv) - return -ENODEV; } else if (is_idxd_wq_dev(idxd_dev)) { struct idxd_wq *wq = confdev_to_wq(dev); - if (is_idxd_wq_kernel(wq)) { + if (is_idxd_wq_kernel(wq)) alt_drv = driver_find("dmaengine", bus); - if (!alt_drv) - return -ENODEV; - } else if (is_idxd_wq_user(wq)) { + else if (is_idxd_wq_user(wq)) alt_drv = driver_find("user", bus); - if (!alt_drv) - return -ENODEV; - } else { - return -ENODEV; - } } + if (!alt_drv) + return -ENODEV; rc = device_driver_attach(alt_drv, dev); if (rc < 0) -- Gitee From aac77bfafa96136487863df815e5ccc09e35ea39 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Mon, 2 Aug 2021 10:58:20 -0700 Subject: [PATCH 65/85] dmaengine: idxd: Remove unused status variable in irq_process_work_list() ANBZ: #22 commit 53cbf462f6b5c9de364efdf443ffb74ed082463a upstream. status is no longer used within this block: drivers/dma/idxd/irq.c:255:6: warning: unused variable 'status' [-Wunused-variable] u8 status = desc->completion->status & DSA_COMP_STATUS_MASK; ^ 1 warning generated. Intel-SIG: commit 53cbf462f6b5 dmaengine: idxd: Remove unused status variable in irq_process_work_list(). Incremental backporting patches for DSA/IAX on Intel Xeon platform. Fixes: b60bb6e2bfc1 ("dmaengine: idxd: fix abort status check") Signed-off-by: Nathan Chancellor Acked-by: Dave Jiang Link: https://lore.kernel.org/r/20210802175820.3153920-1-nathan@kernel.org Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/irq.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c index 65dc7bbb0a13..91e46ca3a0ad 100644 --- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -252,8 +252,6 @@ static int irq_process_work_list(struct idxd_irq_entry *irq_entry, spin_unlock_irqrestore(&irq_entry->list_lock, flags); list_for_each_entry(desc, &flist, list) { - u8 status = desc->completion->status & DSA_COMP_STATUS_MASK; - /* * Check against the original status as ABORT is software defined * and 0xff, which DSA_COMP_STATUS_MASK can mask out. -- Gitee From c4a2da6631945c16536bc56790456a74e0d5b43b Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 3 Aug 2021 15:32:06 -0700 Subject: [PATCH 66/85] dmaengine: idxd: add capability check for 'block on fault' attribute ANBZ: #22 commit 81c2f79c2104c5b48f01da674bc2f7d4bc600db4 upstream. The device general capability has a bit that indicate whether 'block on fault' is supported. Add check to wq sysfs knob to check if cap exists before allowing user to toggle. Intel-SIG: commit 81c2f79c2104 dmaengine: idxd: add capability check for 'block on fault' attribute. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162802992615.3084999.12539468940404102898.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/sysfs.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 4c01587c9d4a..a88886d0f27b 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -642,6 +642,9 @@ static ssize_t wq_block_on_fault_store(struct device *dev, bool bof; int rc; + if (!idxd->hw.gen_cap.block_on_fault) + return -EOPNOTSUPP; + if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) return -EPERM; -- Gitee From 57625ebae7f332836a5cb9f678c79c855983dcc5 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 3 Aug 2021 15:37:15 -0700 Subject: [PATCH 67/85] dmaengine: idxd: clear block on fault flag when clear wq ANBZ: #22 commit bd2f4ae5e019efcfadd6b491204fd60adf14f4a3 upstream. The block on fault flag is not cleared when we disable or reset wq. This causes it to remain set if the user does not clear it on the next configuration load. Add clear of flag in dxd_wq_disable_cleanup() routine. Intel-SIG: commit bd2f4ae5e019 dmaengine: idxd: clear block on fault flag when clear wq. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Fixes: da32b28c95a7 ("dmaengine: idxd: cleanup workqueue config after disabling") Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162803023553.3086015.8158952172068868803.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 86fa4b4590f9..21f0d732b76e 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -402,6 +402,7 @@ static void idxd_wq_disable_cleanup(struct idxd_wq *wq) wq->priority = 0; wq->ats_dis = 0; clear_bit(WQ_FLAG_DEDICATED, &wq->flags); + clear_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags); memset(wq->name, 0, WQ_NAME_SIZE); } -- Gitee From 28e35fe8ef66ad872aeda237c54de3dbd2f345ef Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 3 Aug 2021 15:29:30 -0700 Subject: [PATCH 68/85] dmaengine: idxd: make I/O interrupt handler one shot ANBZ: #22 commit d803c8b9f3f2b8e5c047f2d0a27a9ea3ef91510f upstream. The interrupt thread handler currently loops forever to process outstanding completions. This causes either an "irq X: nobody cared" kernel splat or the NMI watchdog kicks in due to running too long in the function. The irq thread handler is expected to run again after exiting if there are interrupts fired while the thread handler is running. So the handler code can process all the completed I/O in a single pass and exit without losing the follow on completed I/O. Intel-SIG: commit d803c8b9f3f2 dmaengine: idxd: make I/O interrupt handler one shot. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reviewed-by: Dan Williams Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162802977005.3084234.11836261157026497585.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/irq.c | 59 ++++++------------------------------------ 1 file changed, 8 insertions(+), 51 deletions(-) diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c index 91e46ca3a0ad..11addb394793 100644 --- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -22,11 +22,6 @@ struct idxd_fault { struct idxd_device *idxd; }; -static int irq_process_work_list(struct idxd_irq_entry *irq_entry, - int *processed, u64 data); -static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry, - int *processed, u64 data); - static void idxd_device_reinit(struct work_struct *work) { struct idxd_device *idxd = container_of(work, struct idxd_device, work); @@ -177,18 +172,15 @@ irqreturn_t idxd_misc_thread(int vec, void *data) return IRQ_HANDLED; } -static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry, - int *processed, u64 data) +static void irq_process_pending_llist(struct idxd_irq_entry *irq_entry) { struct idxd_desc *desc, *t; struct llist_node *head; - int queued = 0; unsigned long flags; - *processed = 0; head = llist_del_all(&irq_entry->pending_llist); if (!head) - goto out; + return; llist_for_each_entry_safe(desc, t, head, llnode) { u8 status = desc->completion->status & DSA_COMP_STATUS_MASK; @@ -200,35 +192,25 @@ static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry, */ if (unlikely(desc->completion->status == IDXD_COMP_DESC_ABORT)) { complete_desc(desc, IDXD_COMPLETE_ABORT); - (*processed)++; continue; } complete_desc(desc, IDXD_COMPLETE_NORMAL); - (*processed)++; } else { spin_lock_irqsave(&irq_entry->list_lock, flags); list_add_tail(&desc->list, &irq_entry->work_list); spin_unlock_irqrestore(&irq_entry->list_lock, flags); - queued++; } } - - out: - return queued; } -static int irq_process_work_list(struct idxd_irq_entry *irq_entry, - int *processed, u64 data) +static void irq_process_work_list(struct idxd_irq_entry *irq_entry) { - int queued = 0; unsigned long flags; LIST_HEAD(flist); struct idxd_desc *desc, *n; - *processed = 0; - /* * This lock protects list corruption from access of list outside of the irq handler * thread. @@ -236,16 +218,13 @@ static int irq_process_work_list(struct idxd_irq_entry *irq_entry, spin_lock_irqsave(&irq_entry->list_lock, flags); if (list_empty(&irq_entry->work_list)) { spin_unlock_irqrestore(&irq_entry->list_lock, flags); - return 0; + return; } list_for_each_entry_safe(desc, n, &irq_entry->work_list, list) { if (desc->completion->status) { list_del(&desc->list); - (*processed)++; list_add_tail(&desc->list, &flist); - } else { - queued++; } } @@ -263,13 +242,11 @@ static int irq_process_work_list(struct idxd_irq_entry *irq_entry, complete_desc(desc, IDXD_COMPLETE_NORMAL); } - - return queued; } -static int idxd_desc_process(struct idxd_irq_entry *irq_entry) +irqreturn_t idxd_wq_thread(int irq, void *data) { - int rc, processed, total = 0; + struct idxd_irq_entry *irq_entry = data; /* * There are two lists we are processing. The pending_llist is where @@ -288,29 +265,9 @@ static int idxd_desc_process(struct idxd_irq_entry *irq_entry) * and process the completed entries. * 4. If the entry is still waiting on hardware, list_add_tail() to * the work_list. - * 5. Repeat until no more descriptors. */ - do { - rc = irq_process_work_list(irq_entry, &processed, 0); - total += processed; - if (rc != 0) - continue; - - rc = irq_process_pending_llist(irq_entry, &processed, 0); - total += processed; - } while (rc != 0); - - return total; -} - -irqreturn_t idxd_wq_thread(int irq, void *data) -{ - struct idxd_irq_entry *irq_entry = data; - int processed; - - processed = idxd_desc_process(irq_entry); - if (processed == 0) - return IRQ_NONE; + irq_process_work_list(irq_entry); + irq_process_pending_llist(irq_entry); return IRQ_HANDLED; } -- Gitee From 8957f2de2db10b70d4ed121047219f6837f3abef Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 6 Aug 2021 08:36:43 -0700 Subject: [PATCH 69/85] dmaengine: idxd: remove interrupt flag for completion list spinlock ANBZ: #22 commit 9fce3b3a0ab4cad407a27b5e36603c23f1b5b278 upstream. The list lock is never acquired in interrupt context. Therefore there is no need to disable interrupts. Remove interrupt flags for lock operations. Intel-SIG: commit 9fce3b3a0ab4 dmaengine: idxd: remove interrupt flag for completion list spinlock. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reviewed-by: Dan Williams Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162826417450.3454650.3733188117742416238.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/irq.c | 12 +++++------- drivers/dma/idxd/submit.c | 5 ++--- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c index 11addb394793..d221c2e37460 100644 --- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -176,7 +176,6 @@ static void irq_process_pending_llist(struct idxd_irq_entry *irq_entry) { struct idxd_desc *desc, *t; struct llist_node *head; - unsigned long flags; head = llist_del_all(&irq_entry->pending_llist); if (!head) @@ -197,17 +196,16 @@ static void irq_process_pending_llist(struct idxd_irq_entry *irq_entry) complete_desc(desc, IDXD_COMPLETE_NORMAL); } else { - spin_lock_irqsave(&irq_entry->list_lock, flags); + spin_lock(&irq_entry->list_lock); list_add_tail(&desc->list, &irq_entry->work_list); - spin_unlock_irqrestore(&irq_entry->list_lock, flags); + spin_unlock(&irq_entry->list_lock); } } } static void irq_process_work_list(struct idxd_irq_entry *irq_entry) { - unsigned long flags; LIST_HEAD(flist); struct idxd_desc *desc, *n; @@ -215,9 +213,9 @@ static void irq_process_work_list(struct idxd_irq_entry *irq_entry) * This lock protects list corruption from access of list outside of the irq handler * thread. */ - spin_lock_irqsave(&irq_entry->list_lock, flags); + spin_lock(&irq_entry->list_lock); if (list_empty(&irq_entry->work_list)) { - spin_unlock_irqrestore(&irq_entry->list_lock, flags); + spin_unlock(&irq_entry->list_lock); return; } @@ -228,7 +226,7 @@ static void irq_process_work_list(struct idxd_irq_entry *irq_entry) } } - spin_unlock_irqrestore(&irq_entry->list_lock, flags); + spin_unlock(&irq_entry->list_lock); list_for_each_entry(desc, &flist, list) { /* diff --git a/drivers/dma/idxd/submit.c b/drivers/dma/idxd/submit.c index 92ae9a157cc9..4b514c63af15 100644 --- a/drivers/dma/idxd/submit.c +++ b/drivers/dma/idxd/submit.c @@ -106,14 +106,13 @@ static void llist_abort_desc(struct idxd_wq *wq, struct idxd_irq_entry *ie, { struct idxd_desc *d, *t, *found = NULL; struct llist_node *head; - unsigned long flags; desc->completion->status = IDXD_COMP_DESC_ABORT; /* * Grab the list lock so it will block the irq thread handler. This allows the * abort code to locate the descriptor need to be aborted. */ - spin_lock_irqsave(&ie->list_lock, flags); + spin_lock(&ie->list_lock); head = llist_del_all(&ie->pending_llist); if (head) { llist_for_each_entry_safe(d, t, head, llnode) { @@ -127,7 +126,7 @@ static void llist_abort_desc(struct idxd_wq *wq, struct idxd_irq_entry *ie, if (!found) found = list_abort_desc(wq, ie, desc); - spin_unlock_irqrestore(&ie->list_lock, flags); + spin_unlock(&ie->list_lock); if (found) complete_desc(found, IDXD_COMPLETE_ABORT); -- Gitee From 79c123662ad83baf1ea49c44c1b148c3f905920e Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 6 Aug 2021 10:37:40 -0700 Subject: [PATCH 70/85] dmaengine: idxd: make submit failure path consistent on desc freeing ANBZ: #22 commit 0b030f54f094fcd42f4a607a675c1851129a58c8 upstream. The submission path for dmaengine API does not do descriptor freeing on failure. Also, with the abort mechanism, the freeing of descriptor happens when the abort callback is completed. Therefore free descriptor on all error paths for submission call to make things consistent. Also remove the double free that would happen on abort in idxd_dma_tx_submit() call. Intel-SIG: commit 0b030f54f094 dmaengine: idxd: make submit failure path consistent on desc freeing. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Fixes: 6b4b87f2c31a ("dmaengine: idxd: fix submission race window") Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162827146072.3459011.10255348500504659810.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/dma.c | 4 +--- drivers/dma/idxd/submit.c | 11 +++++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/drivers/dma/idxd/dma.c b/drivers/dma/idxd/dma.c index a195225687bb..5c0a4d8a31f5 100644 --- a/drivers/dma/idxd/dma.c +++ b/drivers/dma/idxd/dma.c @@ -149,10 +149,8 @@ static dma_cookie_t idxd_dma_tx_submit(struct dma_async_tx_descriptor *tx) cookie = dma_cookie_assign(tx); rc = idxd_submit_desc(wq, desc); - if (rc < 0) { - idxd_free_desc(wq, desc); + if (rc < 0) return rc; - } return cookie; } diff --git a/drivers/dma/idxd/submit.c b/drivers/dma/idxd/submit.c index 4b514c63af15..de76fb4abac2 100644 --- a/drivers/dma/idxd/submit.c +++ b/drivers/dma/idxd/submit.c @@ -139,11 +139,15 @@ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc) void __iomem *portal; int rc; - if (idxd->state != IDXD_DEV_ENABLED) + if (idxd->state != IDXD_DEV_ENABLED) { + idxd_free_desc(wq, desc); return -EIO; + } - if (!percpu_ref_tryget_live(&wq->wq_active)) + if (!percpu_ref_tryget_live(&wq->wq_active)) { + idxd_free_desc(wq, desc); return -ENXIO; + } portal = idxd_wq_portal_addr(wq); @@ -175,8 +179,11 @@ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc) rc = enqcmds(portal, desc->hw); if (rc < 0) { percpu_ref_put(&wq->wq_active); + /* abort operation frees the descriptor */ if (ie) llist_abort_desc(wq, ie, desc); + else + idxd_free_desc(wq, desc); return rc; } } -- Gitee From 97d534ae553d2f507dc5b66147caccf2c099800a Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 6 Aug 2021 10:38:37 -0700 Subject: [PATCH 71/85] dmaengine: idxd: set descriptor allocation size to threshold for swq ANBZ: #22 commit 9806eb5c79579e200f88660e043706eb490547e6 upstream. Since submission is sent to limited portal, the actual wq size for shared wq is set by the threshold rather than the wq size. When the wq type is shared, set the allocated descriptors to the threshold. Intel-SIG: commit 9806eb5c7957 dmaengine: idxd: set descriptor allocation size to threshold for swq. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162827151733.3459223.3829837172226042408.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 21f0d732b76e..e093cf225a5c 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -141,8 +141,8 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq) if (wq->type != IDXD_WQT_KERNEL) return 0; - wq->num_descs = wq->size; - num_descs = wq->size; + num_descs = wq_dedicated(wq) ? wq->size : wq->threshold; + wq->num_descs = num_descs; rc = alloc_hw_descs(wq, num_descs); if (rc < 0) -- Gitee From d0678617d8e5d33060e7b6e1ec8bbc4b3a003b54 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 19 Aug 2021 09:34:06 -0700 Subject: [PATCH 72/85] dmaengine: idxd: fix setting up priv mode for dwq ANBZ: #22 commit d8071323c5632bdf0a8ef9b9e5662fac43649f9d upstream. DSA spec says WQ priv bit is 0 if the Privileged Mode Enable field of the PCI Express PASID capability is 0 and pasid is enabled. Make sure that the WQCFG priv field is set correctly according to usage type. Reject config if setting up kernel WQ type and no support. Also add the correct priv setup for a descriptor. Intel-SIG: commit d8071323c563 dmaengine: idxd: fix setting up priv mode for dwq. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Fixes: 484f910e93b4 ("dmaengine: idxd: fix wq config registers offset programming") Cc: Ramesh Thomas Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162939084657.903168.14160019185148244596.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/Kconfig | 1 + drivers/dma/idxd/device.c | 29 ++++++++++++++++++++++++++++- drivers/dma/idxd/dma.c | 6 +++++- include/uapi/linux/idxd.h | 1 + 4 files changed, 35 insertions(+), 2 deletions(-) diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index f340bcc6d476..99a31b51f01e 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -291,6 +291,7 @@ config INTEL_IDXD tristate "Intel Data Accelerators support" depends on PCI && X86_64 && !UML depends on PCI_MSI + depends on PCI_PASID depends on SBITMAP select DMA_ENGINE help diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index e093cf225a5c..241df74fc047 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -818,6 +818,15 @@ static int idxd_groups_config_write(struct idxd_device *idxd) return 0; } +static bool idxd_device_pasid_priv_enabled(struct idxd_device *idxd) +{ + struct pci_dev *pdev = idxd->pdev; + + if (pdev->pasid_enabled && (pdev->pasid_features & PCI_PASID_CAP_PRIV)) + return true; + return false; +} + static int idxd_wq_config_write(struct idxd_wq *wq) { struct idxd_device *idxd = wq->idxd; @@ -850,7 +859,6 @@ static int idxd_wq_config_write(struct idxd_wq *wq) wq->wqcfg->wq_thresh = wq->threshold; /* byte 8-11 */ - wq->wqcfg->priv = !!(wq->type == IDXD_WQT_KERNEL); if (wq_dedicated(wq)) wq->wqcfg->mode = 1; @@ -860,6 +868,25 @@ static int idxd_wq_config_write(struct idxd_wq *wq) wq->wqcfg->pasid = idxd->pasid; } + /* + * Here the priv bit is set depending on the WQ type. priv = 1 if the + * WQ type is kernel to indicate privileged access. This setting only + * matters for dedicated WQ. According to the DSA spec: + * If the WQ is in dedicated mode, WQ PASID Enable is 1, and the + * Privileged Mode Enable field of the PCI Express PASID capability + * is 0, this field must be 0. + * + * In the case of a dedicated kernel WQ that is not able to support + * the PASID cap, then the configuration will be rejected. + */ + wq->wqcfg->priv = !!(wq->type == IDXD_WQT_KERNEL); + if (wq_dedicated(wq) && wq->wqcfg->pasid_en && + !idxd_device_pasid_priv_enabled(idxd) && + wq->type == IDXD_WQT_KERNEL) { + idxd->cmd_status = IDXD_SCMD_WQ_NO_PRIV; + return -EOPNOTSUPP; + } + wq->wqcfg->priority = wq->priority; if (idxd->hw.gen_cap.block_on_fault && diff --git a/drivers/dma/idxd/dma.c b/drivers/dma/idxd/dma.c index 5c0a4d8a31f5..e0f056c1d1f5 100644 --- a/drivers/dma/idxd/dma.c +++ b/drivers/dma/idxd/dma.c @@ -69,7 +69,11 @@ static inline void idxd_prep_desc_common(struct idxd_wq *wq, hw->src_addr = addr_f1; hw->dst_addr = addr_f2; hw->xfer_size = len; - hw->priv = !!(wq->type == IDXD_WQT_KERNEL); + /* + * For dedicated WQ, this field is ignored and HW will use the WQCFG.priv + * field instead. This field should be set to 1 for kernel descriptors. + */ + hw->priv = 1; hw->completion_addr = compl; } diff --git a/include/uapi/linux/idxd.h b/include/uapi/linux/idxd.h index ca24c25252fb..c750eac09fc9 100644 --- a/include/uapi/linux/idxd.h +++ b/include/uapi/linux/idxd.h @@ -27,6 +27,7 @@ enum idxd_scmd_stat { IDXD_SCMD_WQ_NO_SWQ_SUPPORT = 0x800c0000, IDXD_SCMD_WQ_NONE_CONFIGURED = 0x800d0000, IDXD_SCMD_WQ_NO_SIZE = 0x800e0000, + IDXD_SCMD_WQ_NO_PRIV = 0x800f0000, }; #define IDXD_SCMD_SOFTERR_MASK 0x80000000 -- Gitee From ecd4d7eab6697451f5c9b348e8f6113704d7f74f Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 24 Aug 2021 14:24:39 -0700 Subject: [PATCH 73/85] dmaengine: idxd: remove interrupt disable for cmd_lock ANBZ: #22 commit f9f4082dbc56c40093bcb5c1f62c04a916eca9a2 upstream. The cmd_lock spinlock is not being used in hard interrupt context. There is no need to disable irq when acquiring the lock. Convert all cmd_lock acquisition to plain spin_lock() calls. Intel-SIG: commit f9f4082dbc56 dmaengine: idxd: remove interrupt disable for cmd_lock. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reviewed-by: Dan Williams Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162984027930.1939209.15758413737332339204.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 241df74fc047..4f6516d7555f 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -462,7 +462,6 @@ int idxd_device_init_reset(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; union idxd_command_reg cmd; - unsigned long flags; if (idxd_device_is_halted(idxd)) { dev_warn(&idxd->pdev->dev, "Device is HALTED!\n"); @@ -472,13 +471,13 @@ int idxd_device_init_reset(struct idxd_device *idxd) memset(&cmd, 0, sizeof(cmd)); cmd.cmd = IDXD_CMD_RESET_DEVICE; dev_dbg(dev, "%s: sending reset for init.\n", __func__); - spin_lock_irqsave(&idxd->cmd_lock, flags); + spin_lock(&idxd->cmd_lock); iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET); while (ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET) & IDXD_CMDSTS_ACTIVE) cpu_relax(); - spin_unlock_irqrestore(&idxd->cmd_lock, flags); + spin_unlock(&idxd->cmd_lock); return 0; } @@ -487,7 +486,6 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand, { union idxd_command_reg cmd; DECLARE_COMPLETION_ONSTACK(done); - unsigned long flags; u32 stat; if (idxd_device_is_halted(idxd)) { @@ -502,7 +500,7 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand, cmd.operand = operand; cmd.int_req = 1; - spin_lock_irqsave(&idxd->cmd_lock, flags); + spin_lock(&idxd->cmd_lock); wait_event_lock_irq(idxd->cmd_waitq, !test_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags), idxd->cmd_lock); @@ -519,10 +517,10 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand, * After command submitted, release lock and go to sleep until * the command completes via interrupt. */ - spin_unlock_irqrestore(&idxd->cmd_lock, flags); + spin_unlock(&idxd->cmd_lock); wait_for_completion(&done); stat = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET); - spin_lock_irqsave(&idxd->cmd_lock, flags); + spin_lock(&idxd->cmd_lock); if (status) *status = stat; idxd->cmd_status = stat & GENMASK(7, 0); @@ -530,7 +528,7 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand, __clear_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags); /* Wake up other pending commands */ wake_up(&idxd->cmd_waitq); - spin_unlock_irqrestore(&idxd->cmd_lock, flags); + spin_unlock(&idxd->cmd_lock); } int idxd_device_enable(struct idxd_device *idxd) @@ -641,7 +639,6 @@ int idxd_device_release_int_handle(struct idxd_device *idxd, int handle, struct device *dev = &idxd->pdev->dev; u32 operand, status; union idxd_command_reg cmd; - unsigned long flags; if (!(idxd->hw.cmd_cap & BIT(IDXD_CMD_RELEASE_INT_HANDLE))) return -EOPNOTSUPP; @@ -659,13 +656,13 @@ int idxd_device_release_int_handle(struct idxd_device *idxd, int handle, dev_dbg(dev, "cmd: %u operand: %#x\n", IDXD_CMD_RELEASE_INT_HANDLE, operand); - spin_lock_irqsave(&idxd->cmd_lock, flags); + spin_lock(&idxd->cmd_lock); iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET); while (ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET) & IDXD_CMDSTS_ACTIVE) cpu_relax(); status = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET); - spin_unlock_irqrestore(&idxd->cmd_lock, flags); + spin_unlock(&idxd->cmd_lock); if ((status & IDXD_CMDSTS_ERR_MASK) != IDXD_CMDSTS_SUCCESS) { dev_dbg(dev, "release int handle failed: %#x\n", status); -- Gitee From f879327c30fe5b1b11ff9980138c0d1ad109a278 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 24 Aug 2021 14:24:27 -0700 Subject: [PATCH 74/85] dmaengine: idxd: remove interrupt disable for dev_lock ANBZ: #22 commit cf84a4b968f38383534bcd0484385c9254828b2c upstream. The spinlock is not being used in hard interrupt context. There is no need to disable irq when acquiring the lock. The interrupt thread handler also is not in bottom half context, therefore we can also remove disabling of the bh. Convert all dev_lock acquisition to plain spin_lock() calls. Intel-SIG: commit cf84a4b968f3 dmaengine: idxd: remove interrupt disable for dev_lock. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reviewed-by: Dan Williams Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/162984026772.1939166.11504067782824765879.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/cdev.c | 5 ++--- drivers/dma/idxd/device.c | 31 ++++++++++++------------------- drivers/dma/idxd/irq.c | 8 ++++---- drivers/dma/idxd/sysfs.c | 10 ++++------ 4 files changed, 22 insertions(+), 32 deletions(-) diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index 4d2ecdb130e7..b9b2b4a4124e 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -218,14 +218,13 @@ static __poll_t idxd_cdev_poll(struct file *filp, struct idxd_user_context *ctx = filp->private_data; struct idxd_wq *wq = ctx->wq; struct idxd_device *idxd = wq->idxd; - unsigned long flags; __poll_t out = 0; poll_wait(filp, &wq->err_queue, wait); - spin_lock_irqsave(&idxd->dev_lock, flags); + spin_lock(&idxd->dev_lock); if (idxd->sw_err.valid) out = EPOLLIN | EPOLLRDNORM; - spin_unlock_irqrestore(&idxd->dev_lock, flags); + spin_unlock(&idxd->dev_lock); return out; } diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 4f6516d7555f..83a5ff2ecf2a 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -341,19 +341,18 @@ int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid) int rc; union wqcfg wqcfg; unsigned int offset; - unsigned long flags; rc = idxd_wq_disable(wq, false); if (rc < 0) return rc; offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX); - spin_lock_irqsave(&idxd->dev_lock, flags); + spin_lock(&idxd->dev_lock); wqcfg.bits[WQCFG_PASID_IDX] = ioread32(idxd->reg_base + offset); wqcfg.pasid_en = 1; wqcfg.pasid = pasid; iowrite32(wqcfg.bits[WQCFG_PASID_IDX], idxd->reg_base + offset); - spin_unlock_irqrestore(&idxd->dev_lock, flags); + spin_unlock(&idxd->dev_lock); rc = idxd_wq_enable(wq); if (rc < 0) @@ -368,19 +367,18 @@ int idxd_wq_disable_pasid(struct idxd_wq *wq) int rc; union wqcfg wqcfg; unsigned int offset; - unsigned long flags; rc = idxd_wq_disable(wq, false); if (rc < 0) return rc; offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX); - spin_lock_irqsave(&idxd->dev_lock, flags); + spin_lock(&idxd->dev_lock); wqcfg.bits[WQCFG_PASID_IDX] = ioread32(idxd->reg_base + offset); wqcfg.pasid_en = 0; wqcfg.pasid = 0; iowrite32(wqcfg.bits[WQCFG_PASID_IDX], idxd->reg_base + offset); - spin_unlock_irqrestore(&idxd->dev_lock, flags); + spin_unlock(&idxd->dev_lock); rc = idxd_wq_enable(wq); if (rc < 0) @@ -558,7 +556,6 @@ int idxd_device_disable(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; u32 status; - unsigned long flags; if (!idxd_is_enabled(idxd)) { dev_dbg(dev, "Device is not enabled\n"); @@ -574,22 +571,20 @@ int idxd_device_disable(struct idxd_device *idxd) return -ENXIO; } - spin_lock_irqsave(&idxd->dev_lock, flags); + spin_lock(&idxd->dev_lock); idxd_device_clear_state(idxd); idxd->state = IDXD_DEV_DISABLED; - spin_unlock_irqrestore(&idxd->dev_lock, flags); + spin_unlock(&idxd->dev_lock); return 0; } void idxd_device_reset(struct idxd_device *idxd) { - unsigned long flags; - idxd_cmd_exec(idxd, IDXD_CMD_RESET_DEVICE, 0, NULL); - spin_lock_irqsave(&idxd->dev_lock, flags); + spin_lock(&idxd->dev_lock); idxd_device_clear_state(idxd); idxd->state = IDXD_DEV_DISABLED; - spin_unlock_irqrestore(&idxd->dev_lock, flags); + spin_unlock(&idxd->dev_lock); } void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid) @@ -1164,7 +1159,6 @@ int __drv_enable_wq(struct idxd_wq *wq) { struct idxd_device *idxd = wq->idxd; struct device *dev = &idxd->pdev->dev; - unsigned long flags; int rc = -ENXIO; lockdep_assert_held(&wq->wq_lock); @@ -1216,10 +1210,10 @@ int __drv_enable_wq(struct idxd_wq *wq) } rc = 0; - spin_lock_irqsave(&idxd->dev_lock, flags); + spin_lock(&idxd->dev_lock); if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) rc = idxd_device_config(idxd); - spin_unlock_irqrestore(&idxd->dev_lock, flags); + spin_unlock(&idxd->dev_lock); if (rc < 0) { dev_dbg(dev, "Writing wq %d config failed: %d\n", wq->id, rc); goto err; @@ -1288,7 +1282,6 @@ void drv_disable_wq(struct idxd_wq *wq) int idxd_device_drv_probe(struct idxd_dev *idxd_dev) { struct idxd_device *idxd = idxd_dev_to_idxd(idxd_dev); - unsigned long flags; int rc = 0; /* @@ -1302,10 +1295,10 @@ int idxd_device_drv_probe(struct idxd_dev *idxd_dev) } /* Device configuration */ - spin_lock_irqsave(&idxd->dev_lock, flags); + spin_lock(&idxd->dev_lock); if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) rc = idxd_device_config(idxd); - spin_unlock_irqrestore(&idxd->dev_lock, flags); + spin_unlock(&idxd->dev_lock); if (rc < 0) return -ENXIO; diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c index d221c2e37460..ca88fa7a328e 100644 --- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -64,7 +64,7 @@ static int process_misc_interrupts(struct idxd_device *idxd, u32 cause) bool err = false; if (cause & IDXD_INTC_ERR) { - spin_lock_bh(&idxd->dev_lock); + spin_lock(&idxd->dev_lock); for (i = 0; i < 4; i++) idxd->sw_err.bits[i] = ioread64(idxd->reg_base + IDXD_SWERR_OFFSET + i * sizeof(u64)); @@ -89,7 +89,7 @@ static int process_misc_interrupts(struct idxd_device *idxd, u32 cause) } } - spin_unlock_bh(&idxd->dev_lock); + spin_unlock(&idxd->dev_lock); val |= IDXD_INTC_ERR; for (i = 0; i < 4; i++) @@ -133,7 +133,7 @@ static int process_misc_interrupts(struct idxd_device *idxd, u32 cause) INIT_WORK(&idxd->work, idxd_device_reinit); queue_work(idxd->wq, &idxd->work); } else { - spin_lock_bh(&idxd->dev_lock); + spin_lock(&idxd->dev_lock); idxd_wqs_quiesce(idxd); idxd_wqs_unmap_portal(idxd); idxd_device_clear_state(idxd); @@ -141,7 +141,7 @@ static int process_misc_interrupts(struct idxd_device *idxd, u32 cause) "idxd halted, need %s.\n", gensts.reset_type == IDXD_DEVICE_RESET_FLR ? "FLR" : "system reset"); - spin_unlock_bh(&idxd->dev_lock); + spin_unlock(&idxd->dev_lock); return -ENXIO; } } diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index a88886d0f27b..a9025be940db 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -1099,16 +1099,15 @@ static ssize_t clients_show(struct device *dev, struct device_attribute *attr, char *buf) { struct idxd_device *idxd = confdev_to_idxd(dev); - unsigned long flags; int count = 0, i; - spin_lock_irqsave(&idxd->dev_lock, flags); + spin_lock(&idxd->dev_lock); for (i = 0; i < idxd->max_wqs; i++) { struct idxd_wq *wq = idxd->wqs[i]; count += wq->client_count; } - spin_unlock_irqrestore(&idxd->dev_lock, flags); + spin_unlock(&idxd->dev_lock); return sysfs_emit(buf, "%d\n", count); } @@ -1146,12 +1145,11 @@ static ssize_t errors_show(struct device *dev, { struct idxd_device *idxd = confdev_to_idxd(dev); int i, out = 0; - unsigned long flags; - spin_lock_irqsave(&idxd->dev_lock, flags); + spin_lock(&idxd->dev_lock); for (i = 0; i < 4; i++) out += sysfs_emit_at(buf, out, "%#018llx ", idxd->sw_err.bits[i]); - spin_unlock_irqrestore(&idxd->dev_lock, flags); + spin_unlock(&idxd->dev_lock); out--; out += sysfs_emit_at(buf, out, "\n"); return out; -- Gitee From 555f623801c4f1e0d936e01e24c6023b7bc81f95 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 29 Sep 2021 12:15:38 -0700 Subject: [PATCH 75/85] dmaengine: idxd: move out percpu_ref_exit() to ensure it's outside submission ANBZ: #22 commit 85f604af9c83a4656b1d07bec73298c3ba7d7c1e upstream. percpu_ref_tryget_live() is safe to call as long as ref is between init and exit according to the function comment. Move percpu_ref_exit() so it is called after the dma channel is no longer valid to ensure this holds true. Intel-SIG: commit 85f604af9c83 dmaengine: idxd: move out percpu_ref_exit() to ensure it's outside submission. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Fixes: 93a40a6d7428 ("dmaengine: idxd: add percpu_ref to descriptor submission path") Suggested-by: Kevin Tian Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/163294293832.914350.10326422026738506152.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 1 - drivers/dma/idxd/dma.c | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 83a5ff2ecf2a..cbbfa17d8d11 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -427,7 +427,6 @@ void idxd_wq_quiesce(struct idxd_wq *wq) { percpu_ref_kill(&wq->wq_active); wait_for_completion(&wq->wq_dead); - percpu_ref_exit(&wq->wq_active); } /* Device control bits */ diff --git a/drivers/dma/idxd/dma.c b/drivers/dma/idxd/dma.c index e0f056c1d1f5..b90b085d18cf 100644 --- a/drivers/dma/idxd/dma.c +++ b/drivers/dma/idxd/dma.c @@ -311,6 +311,7 @@ static int idxd_dmaengine_drv_probe(struct idxd_dev *idxd_dev) err_dma: idxd_wq_quiesce(wq); + percpu_ref_exit(&wq->wq_active); err_ref: idxd_wq_free_resources(wq); err_res_alloc: @@ -329,6 +330,7 @@ static void idxd_dmaengine_drv_remove(struct idxd_dev *idxd_dev) idxd_wq_quiesce(wq); idxd_unregister_dma_channel(wq); __drv_disable_wq(wq); + percpu_ref_exit(&wq->wq_active); idxd_wq_free_resources(wq); wq->type = IDXD_WQT_NONE; mutex_unlock(&wq->wq_lock); -- Gitee From 3e816bb672bdb4a51c9db69a8a8b86b3518e4839 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 12 Oct 2021 11:01:59 -0700 Subject: [PATCH 76/85] dmaengine: idxd: check GENCAP config support for gencfg register ANBZ: #22 commit 79c4c3db7d86b9bec94562275efc82e58f3d0132 upstream. DSA spec 1.2 has moved the GENCFG register under the GENCAP configuration support with respect to writability. Add check in driver before writing to GENCFG register. Intel-SIG: commit 79c4c3db7d86 dmaengine: idxd: check GENCAP config support for gencfg register. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/163406171896.1303830.11217958011385656998.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index cbbfa17d8d11..27612329f510 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -791,7 +791,7 @@ static int idxd_groups_config_write(struct idxd_device *idxd) struct device *dev = &idxd->pdev->dev; /* Setup bandwidth token limit */ - if (idxd->token_limit) { + if (idxd->hw.gen_cap.config_en && idxd->token_limit) { reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET); reg.token_limit = idxd->token_limit; iowrite32(reg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET); -- Gitee From 369db243d22fd89d7a502d445756bcd82035e5d4 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 12 Oct 2021 11:01:19 -0700 Subject: [PATCH 77/85] dmaengine: idxd: remove gen cap field per spec 1.2 update ANBZ: #22 commit c5b64b6826e020041ef4f68a281dee2f33c827d6 upstream. Remove max_descs_per_engine field. The recently released DSA spec 1.2 [1] has removed this field and made it reserved. [1]: https://software.intel.com/content/dam/develop/external/us/en/documents-tps/341204-intel-data-streaming-accelerator-spec.pdf Intel-SIG: commit c5b64b6826e0 dmaengine: idxd: remove gen cap field per spec 1.2 update. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/163406167978.1303649.1798682437841822837.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/registers.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/dma/idxd/registers.h b/drivers/dma/idxd/registers.h index ffc7550a77ee..eeb11e6eb25b 100644 --- a/drivers/dma/idxd/registers.h +++ b/drivers/dma/idxd/registers.h @@ -36,8 +36,7 @@ union gen_cap_reg { u64 max_batch_shift:4; u64 max_ims_mult:6; u64 config_en:1; - u64 max_descs_per_engine:8; - u64 rsvd3:24; + u64 rsvd3:32; }; u64 bits; } __packed; -- Gitee From 27e6afcfc3008c78f339d5d2b27be656914f8607 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 20 Oct 2021 09:27:25 -0700 Subject: [PATCH 78/85] dmaengine: idxd: remove kernel wq type set when load configuration ANBZ: #22 commit 15af840831f69baa9efb0d50007459d2015397a5 upstream. Remove setting of wq type on guest kernel during configuration load on RO device config. The user will set the kernel wq type and this setting based on config is not necessary. Intel-SIG: commit 15af840831f6 dmaengine: idxd: remove kernel wq type set when load configuration. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/163474724511.2607444.1876715711451990426.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 27612329f510..ef3281fc3f8b 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -1050,8 +1050,6 @@ static int idxd_wq_load_config(struct idxd_wq *wq) wq->size = wq->wqcfg->wq_size; wq->threshold = wq->wqcfg->wq_thresh; - if (wq->wqcfg->priv) - wq->type = IDXD_WQT_KERNEL; /* The driver does not support shared WQ mode in read-only config yet */ if (wq->wqcfg->mode == 0 || wq->wqcfg->pasid_en) -- Gitee From 82eab012e96537b7599a44da7a5290ce0cbf35ea Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 21 Sep 2021 13:15:58 -0700 Subject: [PATCH 79/85] dmanegine: idxd: fix resource free ordering on driver removal ANBZ: #22 commit 98da0106aac0d3c5d4a3c95d238f1ff88957bbfc upstream. Fault triggers on ioread32() when pci driver unbind is envoked. The placement of idxd sub-driver removal causes the probing of the device mmio region after the mmio mapping being torn down. The driver needs the sub-drivers to be unbound but not release the idxd context until all shutdown activities has been done. Move the sub-driver unregistering up before the remove() calls shutdown(). But take a device ref on the idxd->conf_dev so that the memory does not get freed in ->release(). When all cleanup activities has been done, release the ref to allow the idxd memory to be freed. [57159.542766] RIP: 0010:ioread32+0x27/0x60 [57159.547097] Code: 00 66 90 48 81 ff ff ff 03 00 77 1e 48 81 ff 00 00 01 00 76 05 0f b7 d7 ed c3 8b 15 03 50 41 01 b8 ff ff ff ff 85 d2 75 04 c3 <8b> 07 c3 55 83 ea 01 48 89 fe 48 c7 c7 00 70 5f 82 48 89 e5 48 83 [57159.566647] RSP: 0018:ffffc900011abb60 EFLAGS: 00010292 [57159.572295] RAX: ffffc900011e0000 RBX: ffff888107d39800 RCX: 0000000000000000 [57159.579842] RDX: 0000000000000000 RSI: ffffffff82b1e448 RDI: ffffc900011e0090 [57159.587421] RBP: ffffc900011abb88 R08: 0000000000000000 R09: 0000000000000001 [57159.594972] R10: 0000000000000001 R11: 0000000000000000 R12: ffff8881019840d0 [57159.602533] R13: ffff8881097e9000 R14: ffffffffa08542a0 R15: 00000000000003a8 [57159.610093] FS: 00007f991e0a8740(0000) GS:ffff888459900000(0000) knlGS:00000000000 00000 [57159.618614] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [57159.624814] CR2: ffffc900011e0090 CR3: 000000010862a002 CR4: 00000000003706e0 [57159.632397] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [57159.639973] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [57159.647601] Call Trace: [57159.650502] ? idxd_device_disable+0x41/0x110 [idxd] [57159.655948] idxd_device_drv_remove+0x2b/0x80 [idxd] [57159.661374] idxd_config_bus_remove+0x16/0x20 [57159.666191] __device_release_driver+0x163/0x240 [57159.671320] device_release_driver+0x2b/0x40 [57159.676052] bus_remove_device+0xf5/0x160 [57159.680524] device_del+0x19c/0x400 [57159.684440] device_unregister+0x18/0x60 [57159.688792] idxd_remove+0x140/0x1c0 [idxd] [57159.693406] pci_device_remove+0x3e/0xb0 [57159.697758] __device_release_driver+0x163/0x240 [57159.702788] device_driver_detach+0x43/0xb0 [57159.707424] unbind_store+0x11e/0x130 [57159.711537] drv_attr_store+0x24/0x30 [57159.715646] sysfs_kf_write+0x4b/0x60 [57159.719710] kernfs_fop_write_iter+0x153/0x1e0 [57159.724563] new_sync_write+0x120/0x1b0 [57159.728812] vfs_write+0x23e/0x350 [57159.732624] ksys_write+0x70/0xf0 [57159.736335] __x64_sys_write+0x1a/0x20 [57159.740492] do_syscall_64+0x3b/0x90 [57159.744465] entry_SYSCALL_64_after_hwframe+0x44/0xae [57159.749908] RIP: 0033:0x7f991e19c387 [57159.753898] Code: 0d 00 f7 d8 64 89 02 48 c7 c0 ff ff ff ff eb b7 0f 1f 00 f3 0f 1e fa 64 8b 04 25 18 00 00 00 85 c0 75 10 b8 01 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 51 c3 48 83 ec 28 48 89 54 24 18 48 89 74 24 [57159.773564] RSP: 002b:00007ffc2ce2d6a8 EFLAGS: 00000246 ORIG_RAX: 0000000000000001 [57159.781550] RAX: ffffffffffffffda RBX: 000000000000000c RCX: 00007f991e19c387 [57159.789133] RDX: 000000000000000c RSI: 000055ee2630e140 RDI: 0000000000000001 [57159.796695] RBP: 000055ee2630e140 R08: 0000000000000000 R09: 00007f991e2324e0 [57159.804246] R10: 00007f991e2323e0 R11: 0000000000000246 R12: 000000000000000c [57159.811800] R13: 00007f991e26f520 R14: 000000000000000c R15: 00007f991e26f700 [57159.819373] Modules linked in: idxd bridge stp llc bnep sunrpc nls_iso8859_1 intel_ rapl_msr intel_rapl_common x86_pkg_temp_thermal intel_powerclamp coretemp snd_hda_code c_realtek iTCO_wdt 8250_dw snd_hda_codec_generic kvm_intel ledtrig_audio iTCO_vendor_s upport snd_hda_intel snd_intel_dspcfg ppdev kvm snd_hda_codec intel_wmi_thunderbolt sn d_hwdep irqbypass iwlwifi btusb snd_hda_core rapl btrtl intel_cstate snd_seq btbcm snd _seq_device btintel snd_pcm cfg80211 bluetooth pcspkr psmouse input_leds snd_timer int el_lpss_pci mei_me intel_lpss snd ecdh_generic ecc mei ucsi_acpi i2c_i801 idma64 i2c_s mbus virt_dma soundcore typec_ucsi typec wmi parport_pc parport video mac_hid acpi_pad sch_fq_codel drm ip_tables x_tables crct10dif_pclmul crc32_pclmul ghash_clmulni_intel usbkbd hid_generic usbmouse aesni_intel usbhid crypto_simd cryptd e1000e hid serio_ra w ahci libahci pinctrl_sunrisepoint fuse msr autofs4 [last unloaded: idxd] [57159.904082] CR2: ffffc900011e0090 [57159.907877] ---[ end trace b4e32f49ce9176a4 ]--- Intel-SIG: commit 98da0106aac0 dmanegine: idxd: fix resource free ordering on driver removal. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Fixes: 49c4959f04b5 ("dmaengine: idxd: fix sequence for pci driver remove() and shutdown()") Reported-by: Ziye Yang Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/163225535868.4152687.9318737776682088722.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/init.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index eb09bc591c31..7bf03f371ce1 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -797,11 +797,19 @@ static void idxd_remove(struct pci_dev *pdev) int msixcnt = pci_msix_vec_count(pdev); int i; - dev_dbg(&pdev->dev, "%s called\n", __func__); + idxd_unregister_devices(idxd); + /* + * When ->release() is called for the idxd->conf_dev, it frees all the memory related + * to the idxd context. The driver still needs those bits in order to do the rest of + * the cleanup. However, we do need to unbound the idxd sub-driver. So take a ref + * on the device here to hold off the freeing while allowing the idxd sub-driver + * to unbind. + */ + get_device(idxd_confdev(idxd)); + device_unregister(idxd_confdev(idxd)); idxd_shutdown(pdev); if (device_pasid_enabled(idxd)) idxd_disable_system_pasid(idxd); - idxd_unregister_devices(idxd); for (i = 0; i < msixcnt; i++) { irq_entry = &idxd->irq_entries[i]; @@ -815,7 +823,7 @@ static void idxd_remove(struct pci_dev *pdev) pci_disable_device(pdev); destroy_workqueue(idxd->wq); perfmon_pmu_remove(idxd); - device_unregister(idxd_confdev(idxd)); + put_device(idxd_confdev(idxd)); } static struct pci_driver idxd_pci_driver = { -- Gitee From 85a127eed57f98c98a862d2e9bc0f2f476373380 Mon Sep 17 00:00:00 2001 From: Bixuan Cui Date: Wed, 8 Sep 2021 17:28:26 +0800 Subject: [PATCH 80/85] dmaengine: idxd: Use list_move_tail instead of list_del/list_add_tail ANBZ: #22 commit ee5c6f0ca219b65f5085043d481d9b6f045693d5 upstream. Using list_move_tail() instead of list_del() + list_add_tail() Intel-SIG: commit ee5c6f0ca219 dmaengine: idxd: Use list_move_tail instead of list_del/list_add_tail. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Reported-by: Hulk Robot Signed-off-by: Bixuan Cui Acked-by: Dave Jiang Link: https://lore.kernel.org/r/20210908092826.67765-1-cuibixuan@huawei.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/irq.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c index ca88fa7a328e..79fcfc4883e4 100644 --- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -221,8 +221,7 @@ static void irq_process_work_list(struct idxd_irq_entry *irq_entry) list_for_each_entry_safe(desc, n, &irq_entry->work_list, list) { if (desc->completion->status) { - list_del(&desc->list); - list_add_tail(&desc->list, &flist); + list_move_tail(&desc->list, &flist); } } -- Gitee From 7590f58c8a5d60dc6302a2ded14ba5e489939ee1 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 8 Sep 2021 16:04:03 -0700 Subject: [PATCH 81/85] dmaengine: idxd: add halt interrupt support ANBZ: #22 commit 88d97ea82cbe352851a8654ee952d3a694c8c2c6 upstream. Add halt interrupt support. Given that the misc interrupt handler already check halt state, the driver just need to run the halt handling code when receiving the halt interrupt. Intel-SIG: commit 88d97ea82cbe dmaengine: idxd: add halt interrupt support. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/163114224352.846654.14334468363464318828.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/irq.c | 5 +++++ drivers/dma/idxd/registers.h | 1 + 2 files changed, 6 insertions(+) diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c index 79fcfc4883e4..17f2f8a31b63 100644 --- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -63,6 +63,9 @@ static int process_misc_interrupts(struct idxd_device *idxd, u32 cause) int i; bool err = false; + if (cause & IDXD_INTC_HALT_STATE) + goto halt; + if (cause & IDXD_INTC_ERR) { spin_lock(&idxd->dev_lock); for (i = 0; i < 4; i++) @@ -121,6 +124,7 @@ static int process_misc_interrupts(struct idxd_device *idxd, u32 cause) if (!err) return 0; +halt: gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET); if (gensts.state == IDXD_DEVICE_STATE_HALT) { idxd->state = IDXD_DEV_HALTED; @@ -134,6 +138,7 @@ static int process_misc_interrupts(struct idxd_device *idxd, u32 cause) queue_work(idxd->wq, &idxd->work); } else { spin_lock(&idxd->dev_lock); + idxd->state = IDXD_DEV_HALTED; idxd_wqs_quiesce(idxd); idxd_wqs_unmap_portal(idxd); idxd_device_clear_state(idxd); diff --git a/drivers/dma/idxd/registers.h b/drivers/dma/idxd/registers.h index eeb11e6eb25b..262c8220adbd 100644 --- a/drivers/dma/idxd/registers.h +++ b/drivers/dma/idxd/registers.h @@ -157,6 +157,7 @@ enum idxd_device_reset_type { #define IDXD_INTC_CMD 0x02 #define IDXD_INTC_OCCUPY 0x04 #define IDXD_INTC_PERFMON_OVFL 0x08 +#define IDXD_INTC_HALT_STATE 0x10 #define IDXD_CMD_OFFSET 0xa0 union idxd_command_reg { -- Gitee From 7160e897202da29d708b81ba161f1b0490b77124 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Wed, 1 Sep 2021 17:18:05 -0700 Subject: [PATCH 82/85] dmaengine: idxd: reconfig device after device reset command ANBZ: #22 commit e530a9f3db4188d1f4e3704b0948ef69c04d5ca6 upstream. Device reset clears the MSIXPERM table and the device registers. Re-program the MSIXPERM table and re-enable the error interrupts post reset. Intel-SIG: commit e530a9f3db41 dmaengine: idxd: reconfig device after device reset command. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Fixes: 745e92a6d816 ("dmaengine: idxd: idxd: move remove() bits for idxd 'struct device' to device.c") Reported-by: Sanjay Kumar Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/163054188513.2853562.12077053294595278181.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index ef3281fc3f8b..b1407465d5c4 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -583,6 +583,8 @@ void idxd_device_reset(struct idxd_device *idxd) spin_lock(&idxd->dev_lock); idxd_device_clear_state(idxd); idxd->state = IDXD_DEV_DISABLED; + idxd_unmask_error_interrupts(idxd); + idxd_msix_perm_setup(idxd); spin_unlock(&idxd->dev_lock); } -- Gitee From 349b92a129b77d337846f58b40ce45000bd0720d Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Mon, 25 Oct 2021 07:59:49 -0700 Subject: [PATCH 83/85] dmaengine: idxd: cleanup completion record allocation ANBZ: #22 commit 2efe58cfaad4ad94eab888d287c174de5209a0c2 upstream. According to core-api/dma-api-howto.rst, the address from dma_alloc_coherent is gauranteed to align to the smallest PAGE_SIZE order. That supercedes the 64B/32B alignment requirement of the completion record. Remove alignment adjustment code. Intel-SIG: commit 2efe58cfaad4 dmaengine: idxd: cleanup completion record allocation. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Tested-by: Jacob Pan Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/163517396063.3484297.7494385225280705372.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/device.c | 22 +++++----------------- drivers/dma/idxd/idxd.h | 2 -- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index b1407465d5c4..fab412349f7f 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -135,8 +135,6 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq) struct idxd_device *idxd = wq->idxd; struct device *dev = &idxd->pdev->dev; int rc, num_descs, i; - int align; - u64 tmp; if (wq->type != IDXD_WQT_KERNEL) return 0; @@ -148,21 +146,13 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq) if (rc < 0) return rc; - align = idxd->data->align; - wq->compls_size = num_descs * idxd->data->compl_size + align; - wq->compls_raw = dma_alloc_coherent(dev, wq->compls_size, - &wq->compls_addr_raw, GFP_KERNEL); - if (!wq->compls_raw) { + wq->compls_size = num_descs * idxd->data->compl_size; + wq->compls = dma_alloc_coherent(dev, wq->compls_size, &wq->compls_addr, GFP_KERNEL); + if (!wq->compls) { rc = -ENOMEM; goto fail_alloc_compls; } - /* Adjust alignment */ - wq->compls_addr = (wq->compls_addr_raw + (align - 1)) & ~(align - 1); - tmp = (u64)wq->compls_raw; - tmp = (tmp + (align - 1)) & ~(align - 1); - wq->compls = (struct dsa_completion_record *)tmp; - rc = alloc_descs(wq, num_descs); if (rc < 0) goto fail_alloc_descs; @@ -191,8 +181,7 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq) fail_sbitmap_init: free_descs(wq); fail_alloc_descs: - dma_free_coherent(dev, wq->compls_size, wq->compls_raw, - wq->compls_addr_raw); + dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr); fail_alloc_compls: free_hw_descs(wq); return rc; @@ -207,8 +196,7 @@ void idxd_wq_free_resources(struct idxd_wq *wq) free_hw_descs(wq); free_descs(wq); - dma_free_coherent(dev, wq->compls_size, wq->compls_raw, - wq->compls_addr_raw); + dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr); sbitmap_queue_free(&wq->sbq); } diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index bfcb03329f77..0cf8d3145870 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -187,9 +187,7 @@ struct idxd_wq { struct dsa_completion_record *compls; struct iax_completion_record *iax_compls; }; - void *compls_raw; dma_addr_t compls_addr; - dma_addr_t compls_addr_raw; int compls_size; struct idxd_desc **descs; struct sbitmap_queue sbq; -- Gitee From 7520628926883b3f25891ee9b7c44e911c595ea6 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Mon, 25 Oct 2021 08:01:04 -0700 Subject: [PATCH 84/85] dmaengine: idxd: fix resource leak on dmaengine driver disable ANBZ: #22 commit a3e340c1574b6679f5b333221284d0959095da52 upstream. The wq resources needs to be released before the kernel type is reset by __drv_disable_wq(). With dma channels unregistered and wq quiesced, all the wq resources for dmaengine can be freed. There is no need to wait until wq is disabled. With the wq->type being reset to "unknown", the driver is skipping the freeing of the resources. Intel-SIG: commit a3e340c1574b dmaengine: idxd: fix resource leak on dmaengine driver disable. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Fixes: 0cda4f6986a3 ("dmaengine: idxd: create dmaengine driver for wq 'device'") Reported-by: Jacob Pan Tested-by: Jacob Pan Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/163517405099.3484556.12521975053711345244.stgit@djiang5-desk3.ch.intel.com Signed-off-by: Vinod Koul [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- drivers/dma/idxd/dma.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/dma/idxd/dma.c b/drivers/dma/idxd/dma.c index b90b085d18cf..c39e9483206a 100644 --- a/drivers/dma/idxd/dma.c +++ b/drivers/dma/idxd/dma.c @@ -329,10 +329,9 @@ static void idxd_dmaengine_drv_remove(struct idxd_dev *idxd_dev) mutex_lock(&wq->wq_lock); idxd_wq_quiesce(wq); idxd_unregister_dma_channel(wq); + idxd_wq_free_resources(wq); __drv_disable_wq(wq); percpu_ref_exit(&wq->wq_active); - idxd_wq_free_resources(wq); - wq->type = IDXD_WQT_NONE; mutex_unlock(&wq->wq_lock); } -- Gitee From 08498864b15e3984087ba9345e82f0cba244a720 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Fri, 10 Sep 2021 15:33:32 -0700 Subject: [PATCH 85/85] x86/asm: Fix SETZ size enqcmds() build failure ANBZ: #22 commit d81ff5fe14a950f53e2833cfa196e7bb3fd5d4e3 upstream. When building under GCC 4.9 and 5.5: arch/x86/include/asm/special_insns.h: Assembler messages: arch/x86/include/asm/special_insns.h:286: Error: operand size mismatch for `setz' Change the type to "bool" for condition code arguments, as documented. Intel-SIG: commit d81ff5fe14a9 x86/asm: Fix SETZ size enqcmds() build failure. Incremental backporting patches for DSA/IAX on Intel Xeon platform. Fixes: 7f5933f81bd8 ("x86/asm: Add an enqcmds() wrapper for the ENQCMDS instruction") Co-developed-by: Arnd Bergmann Signed-off-by: Arnd Bergmann Signed-off-by: Kees Cook Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20210910223332.3224851-1-keescook@chromium.org [ Xiaochen Shen: amend commit log ] Signed-off-by: Xiaochen Shen Reviewed-by: Zelin Deng --- arch/x86/include/asm/special_insns.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/include/asm/special_insns.h b/arch/x86/include/asm/special_insns.h index 1d3cbaef4bb7..1f5b1a9dd6af 100644 --- a/arch/x86/include/asm/special_insns.h +++ b/arch/x86/include/asm/special_insns.h @@ -287,7 +287,7 @@ static inline int enqcmds(void __iomem *dst, const void *src) { const struct { char _[64]; } *__src = src; struct { char _[64]; } __iomem *__dst = dst; - int zf; + bool zf; /* * ENQCMDS %(rdx), rax -- Gitee