diff --git a/drivers/misc/pci_endpoint_test.c b/drivers/misc/pci_endpoint_test.c index 3635741c3e7a7..b7a1497157008 100644 --- a/drivers/misc/pci_endpoint_test.c +++ b/drivers/misc/pci_endpoint_test.c @@ -1325,6 +1325,8 @@ static int pci_endpoint_test_probe(struct pci_dev *pdev, misc_device->parent = &pdev->dev; misc_device->fops = &pci_endpoint_test_fops; + pci_save_state(pdev); + ret = misc_register(misc_device); if (ret) { dev_err(dev, "Failed to register device\n"); @@ -1452,12 +1454,33 @@ static const struct pci_device_id pci_endpoint_test_tbl[] = { }; MODULE_DEVICE_TABLE(pci, pci_endpoint_test_tbl); +static pci_ers_result_t pci_endpoint_test_error_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + if (state == pci_channel_io_perm_failure) + return PCI_ERS_RESULT_DISCONNECT; + + return PCI_ERS_RESULT_NEED_RESET; +} + +static pci_ers_result_t pci_endpoint_test_slot_reset(struct pci_dev *pdev) +{ + pci_restore_state(pdev); + return PCI_ERS_RESULT_RECOVERED; +} + +static const struct pci_error_handlers pci_endpoint_test_err_handler = { + .error_detected = pci_endpoint_test_error_detected, + .slot_reset = pci_endpoint_test_slot_reset, +}; + static struct pci_driver pci_endpoint_test_driver = { .name = DRV_MODULE_NAME, .id_table = pci_endpoint_test_tbl, .probe = pci_endpoint_test_probe, .remove = pci_endpoint_test_remove, .sriov_configure = pci_sriov_configure_simple, + .err_handler = &pci_endpoint_test_err_handler, }; module_pci_driver(pci_endpoint_test_driver); diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c index 7d2794945704e..31402ae218c79 100644 --- a/drivers/pci/controller/dwc/pcie-designware-ep.c +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c @@ -1153,6 +1153,11 @@ void dw_pcie_ep_cleanup(struct dw_pcie_ep *ep) { struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + if (ep->msi_iatu_mapped) { + dw_pcie_ep_unmap_addr(ep->epc, 0, 0, ep->msi_mem_phys); + ep->msi_iatu_mapped = false; + } + dwc_pcie_debugfs_deinit(pci); dw_pcie_edma_remove(pci); } diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c index 19daadee65f7c..f62017c7c84ef 100644 --- a/drivers/pci/controller/dwc/pcie-qcom.c +++ b/drivers/pci/controller/dwc/pcie-qcom.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -56,6 +57,10 @@ #define PARF_AXI_MSTR_WR_ADDR_HALT_V2 0x1a8 #define PARF_Q2A_FLUSH 0x1ac #define PARF_LTSSM 0x1b0 +#define PARF_INT_ALL_STATUS 0x224 +#define PARF_INT_ALL_CLEAR 0x228 +#define PARF_INT_ALL_MASK 0x22c +#define PARF_STATUS 0x230 #define PARF_SID_OFFSET 0x234 #define PARF_BDF_TRANSLATE_CFG 0x24c #define PARF_DBI_BASE_ADDR_V2 0x350 @@ -69,6 +74,23 @@ #define PARF_BDF_TO_SID_TABLE_N 0x2000 #define PARF_BDF_TO_SID_CFG 0x2c00 +/* + * ECAM blocker range registers. The blocked range has a write pair + * (WR_BASE/WR_LIMIT) and a read pair (RD_BASE/RD_LIMIT); each address is + * split into a low (32-bit) and a HI (upper 32-bit) register. + */ +#define PARF_BLOCK_SLV_AXI_WR_BASE 0x360 +#define PARF_BLOCK_SLV_AXI_WR_BASE_HI 0x364 +#define PARF_BLOCK_SLV_AXI_WR_LIMIT 0x368 +#define PARF_BLOCK_SLV_AXI_WR_LIMIT_HI 0x36c +#define PARF_BLOCK_SLV_AXI_RD_BASE 0x370 +#define PARF_BLOCK_SLV_AXI_RD_BASE_HI 0x374 +#define PARF_BLOCK_SLV_AXI_RD_LIMIT 0x378 +#define PARF_BLOCK_SLV_AXI_RD_LIMIT_HI 0x37c + +#define PARF_ECAM_BASE 0x380 +#define PARF_ECAM_BASE_HI 0x384 + /* ELBI registers */ #define ELBI_SYS_CTRL 0x04 #define ELBI_SYS_STTS 0x08 @@ -86,6 +108,7 @@ /* PARF_SYS_CTRL register fields */ #define MAC_PHY_POWERDOWN_IN_P2_D_MUX_EN BIT(29) +#define ECAM_BLOCKER_EN BIT(26) #define MST_WAKEUP_EN BIT(13) #define SLV_WAKEUP_EN BIT(12) #define MSTR_ACLK_CGC_DIS BIT(10) @@ -133,6 +156,13 @@ /* PARF_LTSSM register fields */ #define LTSSM_EN BIT(8) #define PARF_LTSSM_STATE_MASK GENMASK(5, 0) +#define SW_CLEAR_FLUSH_MODE BIT(10) +#define FLUSH_MODE BIT(11) + +/* PARF_INT_ALL_{STATUS/CLEAR/MASK} register fields */ +#define INT_ALL_LINK_DOWN 1 +#define PARF_INT_ALL_LINK_DOWN BIT(INT_ALL_LINK_DOWN) +#define PARF_INT_MSI_DEV_0_7 GENMASK(30, 23) /* PARF_NO_SNOOP_OVERRIDE register fields */ #define WR_NO_SNOOP_OVERRIDE_EN BIT(1) @@ -144,6 +174,9 @@ /* PARF_BDF_TO_SID_CFG fields */ #define BDF_TO_SID_BYPASS BIT(0) +/* PARF_STATUS fields */ +#define FLUSH_COMPLETED BIT(8) + /* ELBI_SYS_CTRL register fields */ #define ELBI_SYS_CTRL_LT_ENABLE BIT(0) #define ELBI_SYS_CTRL_PME_TURNOFF_MSG BIT(4) @@ -172,6 +205,7 @@ PCIE_CAP_SLOT_POWER_LIMIT_SCALE) #define PERST_DELAY_US 1000 +#define FLUSH_TIMEOUT_US 100 #define QCOM_PCIE_CRC8_POLYNOMIAL (BIT(2) | BIT(1) | BIT(0)) @@ -291,10 +325,14 @@ struct qcom_pcie { struct dentry *debugfs; struct list_head ports; struct gpio_desc *reset; + int global_irq; bool use_pm_opp; + struct mutex hw_lock; }; #define to_qcom_pcie(x) dev_get_drvdata((x)->dev) +static int qcom_pcie_reset_root_port(struct pci_host_bridge *bridge, + struct pci_dev *pdev); static void __qcom_pcie_perst_assert(struct qcom_pcie *pcie, bool assert) { @@ -421,6 +459,25 @@ static void qcom_pcie_configure_dbi_atu_base(struct qcom_pcie *pcie) } } +static void qcom_pcie_init_ecam_blocker(struct qcom_pcie *pcie) +{ + struct dw_pcie *pci = pcie->pci; + + /* ECAM base must match the DBI base address */ + writel(lower_32_bits(pci->dbi_phys_addr), pcie->parf + PARF_ECAM_BASE); + writel(upper_32_bits(pci->dbi_phys_addr), pcie->parf + PARF_ECAM_BASE_HI); + + writel(0, pcie->parf + PARF_BLOCK_SLV_AXI_WR_BASE); + writel(0, pcie->parf + PARF_BLOCK_SLV_AXI_WR_BASE_HI); + writel(U32_MAX, pcie->parf + PARF_BLOCK_SLV_AXI_WR_LIMIT); + writel(U32_MAX, pcie->parf + PARF_BLOCK_SLV_AXI_WR_LIMIT_HI); + + writel(0, pcie->parf + PARF_BLOCK_SLV_AXI_RD_BASE); + writel(0, pcie->parf + PARF_BLOCK_SLV_AXI_RD_BASE_HI); + writel(U32_MAX, pcie->parf + PARF_BLOCK_SLV_AXI_RD_LIMIT); + writel(U32_MAX, pcie->parf + PARF_BLOCK_SLV_AXI_RD_LIMIT_HI); +} + static void qcom_pcie_2_1_0_ltssm_enable(struct qcom_pcie *pcie) { struct dw_pcie *pci = pcie->pci; @@ -966,6 +1023,8 @@ static int qcom_pcie_post_init_2_3_3(struct qcom_pcie *pcie) dw_pcie_dbi_ro_wr_dis(pci); + qcom_pcie_init_ecam_blocker(pcie); + return 0; } @@ -1080,6 +1139,8 @@ static int qcom_pcie_post_init_2_7_0(struct qcom_pcie *pcie) qcom_pcie_set_slot_nccs(pcie->pci); + qcom_pcie_init_ecam_blocker(pcie); + return 0; } @@ -1298,6 +1359,8 @@ static int qcom_pcie_post_init_2_9_0(struct qcom_pcie *pcie) for (i = 0; i < 256; i++) writel(0, pcie->parf + PARF_BDF_TO_SID_TABLE_N + (4 * i)); + qcom_pcie_init_ecam_blocker(pcie); + return 0; } @@ -1358,6 +1421,18 @@ static void qcom_pcie_configure_ports(struct qcom_pcie *pcie) dw_pcie_program_t_power_on(pcie->pci, port->l1ss_t_power_on); } +static void qcom_pcie_enable_ecam_blocker(struct qcom_pcie *pcie) +{ + u32 sys_ctrl; + + sys_ctrl = readl(pcie->parf + PARF_SYS_CTRL); + sys_ctrl |= ECAM_BLOCKER_EN; + writel(sys_ctrl, pcie->parf + PARF_SYS_CTRL); + + /* Flush the write so the blocker is enabled before this function returns */ + readl(pcie->parf + PARF_SYS_CTRL); +} + static int qcom_pcie_host_init(struct dw_pcie_rp *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); @@ -1406,6 +1481,8 @@ static int qcom_pcie_host_init(struct dw_pcie_rp *pp) goto err_assert_reset; } + pp->bridge->reset_root_port = qcom_pcie_reset_root_port; + return 0; err_assert_reset: @@ -1734,6 +1811,83 @@ static int qcom_pcie_set_max_opp(struct device *dev) return ret; } +/* + * Qcom PCIe controllers only support one Root Port per controller instance. So + * this function ignores the 'pci_dev' associated with the Root Port and just + * resets the host bridge, which in turn resets the Root Port also. + */ +static int qcom_pcie_reset_root_port(struct pci_host_bridge *bridge, + struct pci_dev *pdev) +{ + struct device *dev = bridge->dev.parent; + struct qcom_pcie *pcie = dev_get_drvdata(dev); + struct dw_pcie *pci = pcie->pci; + struct dw_pcie_rp *pp = &pci->pp; + u32 val; + int ret; + + mutex_lock(&pcie->hw_lock); + + /* Wait for the pending transactions to be completed */ + ret = readl_relaxed_poll_timeout(pcie->parf + PARF_STATUS, val, + val & FLUSH_COMPLETED, 10, + FLUSH_TIMEOUT_US); + if (ret) { + dev_err(dev, "Flush completion failed: %d\n", ret); + goto out_unlock; + } + + /* Clear the FLUSH_MODE to allow the core to be reset */ + val = readl(pcie->parf + PARF_LTSSM); + val |= SW_CLEAR_FLUSH_MODE; + writel(val, pcie->parf + PARF_LTSSM); + + /* Wait for the FLUSH_MODE to clear */ + ret = readl_relaxed_poll_timeout(pcie->parf + PARF_LTSSM, val, + !(val & FLUSH_MODE), 10, + FLUSH_TIMEOUT_US); + if (ret) { + dev_err(dev, "Flush mode clear failed: %d\n", ret); + goto out_unlock; + } + + qcom_pcie_host_deinit(pp); + + ret = qcom_pcie_host_init(pp); + if (ret) { + dev_err(dev, "Host init failed\n"); + goto out_unlock; + } + + ret = dw_pcie_setup_rc(pp); + if (ret) + goto out_unlock; + + /* + * Re-enable global IRQ events as the PARF_INT_ALL_MASK register is + * non-sticky. + */ + if (pcie->global_irq) + writel_relaxed(PARF_INT_ALL_LINK_DOWN | PARF_INT_MSI_DEV_0_7, + pcie->parf + PARF_INT_ALL_MASK); + + qcom_pcie_start_link(pci); + + ret = dw_pcie_wait_for_link(pci); + /* This linkdown might have been a hot-unplug */ + if (ret == -ENODEV) + ret = 0; + if (ret) + goto out_unlock; + + dev_dbg(dev, "Root Port reset completed\n"); + +out_unlock: + mutex_unlock(&pcie->hw_lock); + + return ret; +} + static int qcom_pcie_link_transition_count(struct seq_file *s, void *data) { struct qcom_pcie *pcie = (struct qcom_pcie *)dev_get_drvdata(s->private); @@ -1771,6 +1925,32 @@ static void qcom_pcie_init_debugfs(struct qcom_pcie *pcie) qcom_pcie_link_transition_count); } +static irqreturn_t qcom_pcie_global_irq_thread(int irq, void *data) +{ + struct qcom_pcie *pcie = data; + struct dw_pcie_rp *pp = &pcie->pci->pp; + struct device *dev = pcie->pci->dev; + struct pci_dev *port; + unsigned long status = readl_relaxed(pcie->parf + PARF_INT_ALL_STATUS); + + writel_relaxed(status, pcie->parf + PARF_INT_ALL_CLEAR); + + if (test_and_clear_bit(INT_ALL_LINK_DOWN, &status)) { + dev_dbg(dev, "Received Link down event\n"); + + mutex_lock(&pcie->hw_lock); + qcom_pcie_enable_ecam_blocker(pcie); + mutex_unlock(&pcie->hw_lock); + + for_each_pci_bridge(port, pp->bridge->bus) { + if (pci_pcie_type(port) == PCI_EXP_TYPE_ROOT_PORT) + pci_host_handle_link_down(port); + } + } + + return IRQ_HANDLED; +} + static void qcom_pci_free_msi(void *ptr) { struct dw_pcie_rp *pp = (struct dw_pcie_rp *)ptr; @@ -1990,7 +2170,7 @@ static int qcom_pcie_probe(struct platform_device *pdev) struct dw_pcie_rp *pp; struct resource *res; struct dw_pcie *pci; - int ret; + int ret, irq; pcie_cfg = of_device_get_match_data(dev); if (!pcie_cfg) { @@ -2051,6 +2231,10 @@ static int qcom_pcie_probe(struct platform_device *pdev) INIT_LIST_HEAD(&pcie->ports); + ret = devm_mutex_init(dev, &pcie->hw_lock); + if (ret) + goto err_pm_runtime_put; + pci->dev = dev; pci->ops = &dw_pcie_ops; pp = &pci->pp; @@ -2135,6 +2319,32 @@ static int qcom_pcie_probe(struct platform_device *pdev) goto err_phy_exit; } + irq = platform_get_irq_byname_optional(pdev, "global"); + if (irq > 0) { + const char *name; + + name = devm_kasprintf(dev, GFP_KERNEL, "qcom_pcie_global_irq%d", + pci_domain_nr(pp->bridge->bus)); + if (!name) { + ret = -ENOMEM; + goto err_host_deinit; + } + + ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, + qcom_pcie_global_irq_thread, + IRQF_ONESHOT, name, pcie); + if (ret) { + dev_err_probe(&pdev->dev, ret, + "Failed to request Global IRQ\n"); + goto err_host_deinit; + } + + writel_relaxed(PARF_INT_ALL_LINK_DOWN | PARF_INT_MSI_DEV_0_7, + pcie->parf + PARF_INT_ALL_MASK); + + pcie->global_irq = irq; + } + qcom_pcie_icc_opp_update(pcie); if (pcie->mhi) @@ -2142,6 +2352,8 @@ static int qcom_pcie_probe(struct platform_device *pdev) return 0; +err_host_deinit: + dw_pcie_host_deinit(pp); err_phy_exit: list_for_each_entry_safe(port, tmp_port, &pcie->ports, list) { list_for_each_entry_safe(perst, tmp_perst, &port->perst, list) diff --git a/drivers/pci/controller/pci-host-common.c b/drivers/pci/controller/pci-host-common.c index 2ce6f4b661334..363d3f970b10c 100644 --- a/drivers/pci/controller/pci-host-common.c +++ b/drivers/pci/controller/pci-host-common.c @@ -13,9 +13,11 @@ #include #include #include +#include #include #include +#include "../pci.h" #include "pci-host-common.h" /** @@ -342,5 +344,38 @@ bool pci_host_common_d3cold_possible(struct pci_host_bridge *bridge, } EXPORT_SYMBOL_GPL(pci_host_common_d3cold_possible); +static pci_ers_result_t pci_host_reset_root_port(struct pci_dev *dev) +{ + int ret; + + pci_lock_rescan_remove(); + ret = pci_bus_error_reset(dev); + pci_unlock_rescan_remove(); + if (ret) { + pci_err(dev, "Failed to reset Root Port: %d\n", ret); + return PCI_ERS_RESULT_DISCONNECT; + } + + pci_info(dev, "Root Port has been reset\n"); + + return PCI_ERS_RESULT_RECOVERED; +} + +static void pci_host_recover_root_port(struct pci_dev *port) +{ +#if IS_ENABLED(CONFIG_PCIEAER) + pcie_do_recovery(port, pci_channel_io_frozen, pci_host_reset_root_port); +#else + pci_host_reset_root_port(port); +#endif +} + +void pci_host_handle_link_down(struct pci_dev *port) +{ + pci_info(port, "Recovering Root Port due to Link Down\n"); + pci_host_recover_root_port(port); +} +EXPORT_SYMBOL_GPL(pci_host_handle_link_down); + MODULE_DESCRIPTION("Common library for PCI host controller drivers"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/pci-host-common.h b/drivers/pci/controller/pci-host-common.h index 9f0f36a322214..51afd65d3c4a6 100644 --- a/drivers/pci/controller/pci-host-common.h +++ b/drivers/pci/controller/pci-host-common.h @@ -48,6 +48,7 @@ int pci_host_common_init(struct platform_device *pdev, struct pci_host_bridge *bridge, const struct pci_ecam_ops *ops); void pci_host_common_remove(struct platform_device *pdev); +void pci_host_handle_link_down(struct pci_dev *port); struct pci_config_window *pci_host_common_ecam_create(struct device *dev, struct pci_host_bridge *bridge, const struct pci_ecam_ops *ops); diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index f122f48c5fec1..5bda307520671 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -4852,6 +4852,19 @@ void pci_reset_secondary_bus(struct pci_dev *dev) void __weak pcibios_reset_secondary_bus(struct pci_dev *dev) { + struct pci_host_bridge *host = pci_find_host_bridge(dev->bus); + int ret; + + if (pci_is_root_bus(dev->bus) && host->reset_root_port) { + ret = host->reset_root_port(host, dev); + if (ret) + pci_err(dev, "Failed to reset Root Port: %d\n", ret); + else + pci_restore_state(dev); + + return; + } + pci_reset_secondary_bus(dev); } @@ -5698,6 +5711,7 @@ int pci_bus_error_reset(struct pci_dev *bridge) { return pci_reset_bridge(bridge, PCI_RESET_NO_RESTORE); } +EXPORT_SYMBOL_GPL(pci_bus_error_reset); int pci_try_reset_bridge(struct pci_dev *bridge) { diff --git a/drivers/pci/pcie/err.c b/drivers/pci/pcie/err.c index bebe4bc111d75..d77403d8855bd 100644 --- a/drivers/pci/pcie/err.c +++ b/drivers/pci/pcie/err.c @@ -256,11 +256,6 @@ pci_ers_result_t pcie_do_recovery(struct pci_dev *dev, } if (status == PCI_ERS_RESULT_NEED_RESET) { - /* - * TODO: Should call platform-specific - * functions to reset slot before calling - * drivers' slot_reset callbacks? - */ status = PCI_ERS_RESULT_RECOVERED; pci_dbg(bridge, "broadcast slot_reset message\n"); pci_walk_bridge(bridge, report_slot_reset, &status); @@ -297,3 +292,4 @@ pci_ers_result_t pcie_do_recovery(struct pci_dev *dev, return status; } +EXPORT_SYMBOL_GPL(pcie_do_recovery); diff --git a/include/linux/pci.h b/include/linux/pci.h index 09134d0a559fd..c2f2e33894bbb 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -648,6 +648,7 @@ struct pci_host_bridge { void (*release_fn)(struct pci_host_bridge *); int (*enable_device)(struct pci_host_bridge *bridge, struct pci_dev *dev); void (*disable_device)(struct pci_host_bridge *bridge, struct pci_dev *dev); + int (*reset_root_port)(struct pci_host_bridge *bridge, struct pci_dev *dev); void *release_data; unsigned int ignore_reset_delay:1; /* For entire hierarchy */ unsigned int no_ext_tags:1; /* No Extended Tags */