diff --git a/arch/arm64/configs/qcom.config b/arch/arm64/configs/qcom.config index 20bc4fd5101de..a7032fade76b0 100644 --- a/arch/arm64/configs/qcom.config +++ b/arch/arm64/configs/qcom.config @@ -144,3 +144,5 @@ CONFIG_SCMI_QCOM_MEMLAT_DEVFREQ=m # GPU Virtualization support CONFIG_DRM_VIRTIO_GPU=m CONFIG_DRM_VIRTIO_GPU_KMS=m + +CONFIG_NVME_SPEED_SWITCH=y diff --git a/drivers/nvme/host/Kconfig b/drivers/nvme/host/Kconfig index 31974c7dd20c9..ef03494eb601d 100644 --- a/drivers/nvme/host/Kconfig +++ b/drivers/nvme/host/Kconfig @@ -134,3 +134,13 @@ config NVME_APPLE To compile this driver as a module, choose M here: the module will be called nvme-apple. + +config NVME_SPEED_SWITCH + bool "NVMe adaptive PCIe link rate switching" + depends on BLK_DEV_NVME + help + This provides support for dynamic PCIe link rate switching for NVMe + devices. When enabled, the NVMe driver can dynamically adjust the + PCIe link speed and width based on workload requirement. + + If unsure, say N. diff --git a/drivers/nvme/host/Makefile b/drivers/nvme/host/Makefile index 6414ec968f99a..f33e804e92881 100644 --- a/drivers/nvme/host/Makefile +++ b/drivers/nvme/host/Makefile @@ -18,6 +18,7 @@ nvme-core-$(CONFIG_BLK_DEV_ZONED) += zns.o nvme-core-$(CONFIG_FAULT_INJECTION_DEBUG_FS) += fault_inject.o nvme-core-$(CONFIG_NVME_HWMON) += hwmon.o nvme-core-$(CONFIG_NVME_HOST_AUTH) += auth.o +nvme-core-$(CONFIG_NVME_SPEED_SWITCH) += speed_switch.o nvme-y += pci.o diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 453c1f0b2dd09..f190dd05e46a5 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -4627,6 +4627,7 @@ static void nvme_scan_work(struct work_struct *work) /* Re-read the ANA log page to not miss updates */ queue_work(nvme_wq, &ctrl->ana_work); #endif + nvme_speed_switch_start(ctrl); } /* @@ -5051,6 +5052,7 @@ EXPORT_SYMBOL_GPL(nvme_remove_io_tag_set); void nvme_stop_ctrl(struct nvme_ctrl *ctrl) { + nvme_speed_switch_exit(ctrl); nvme_mpath_stop(ctrl); nvme_auth_stop(ctrl); nvme_stop_failfast_work(ctrl); @@ -5266,6 +5268,8 @@ int nvme_add_ctrl(struct nvme_ctrl *ctrl) nvme_fault_inject_init(&ctrl->fault_inject, dev_name(ctrl->device)); nvme_get_ctrl(ctrl); + nvme_speed_switch_init(ctrl); + return 0; } EXPORT_SYMBOL_GPL(nvme_add_ctrl); diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 824651cc898db..fb958ff13bbc5 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -331,6 +331,38 @@ enum nvme_ctrl_flags { NVME_CTRL_FROZEN = 6, }; +#ifdef CONFIG_NVME_SPEED_SWITCH +struct nvme_speed_switch_stats { + u64 read_bytes; + u64 write_bytes; +}; + +struct nvme_speed_switch { + bool initialized; + bool enabled; + u8 max_speed; + u8 cur_speed; + u8 target_speed; + u8 min_speed; + u32 monitor_interval; /* ms */ + u32 threshold; /* KB */ + u32 up_cnt; + u32 down_cnt; + u32 up_threshold; + u32 down_threshold; + u32 idle_cnt; + struct timer_list timer; + struct work_struct work; + struct nvme_speed_switch_stats __percpu *stats; + atomic_t timer_active; +}; + +enum nvme_speed_timer_state { + NVME_SPEED_TIMER_INACTIVE, + NVME_SPEED_TIMER_ACTIVE, +}; +#endif + struct nvme_ctrl { bool comp_seen; bool identified; @@ -472,8 +504,23 @@ struct nvme_ctrl { enum nvme_dctype dctype; u16 awupf; /* 0's based value. */ +#ifdef CONFIG_NVME_SPEED_SWITCH + struct nvme_speed_switch speed_switch; +#endif }; +#ifdef CONFIG_NVME_SPEED_SWITCH +void nvme_update_io_stats(struct nvme_ctrl *ctrl, struct request *req); +void nvme_speed_switch_start(struct nvme_ctrl *ctrl); +void nvme_speed_switch_init(struct nvme_ctrl *ctrl); +void nvme_speed_switch_exit(struct nvme_ctrl *ctrl); +#else +static inline void nvme_update_io_stats(struct nvme_ctrl *ctrl, struct request *req) {} +static inline void nvme_speed_switch_start(struct nvme_ctrl *ctrl) {} +static inline void nvme_speed_switch_init(struct nvme_ctrl *ctrl) {} +static inline void nvme_speed_switch_exit(struct nvme_ctrl *ctrl) {} +#endif + static inline enum nvme_ctrl_state nvme_ctrl_state(struct nvme_ctrl *ctrl) { return READ_ONCE(ctrl->state); @@ -1029,6 +1076,9 @@ extern const struct attribute_group nvme_dev_attrs_group; extern const struct attribute_group nvme_dev_diag_attrs_group; extern const struct attribute_group *nvme_subsys_attrs_groups[]; extern const struct attribute_group *nvme_dev_attr_groups[]; +#ifdef CONFIG_NVME_SPEED_SWITCH +extern const struct attribute_group nvme_speed_attr_group; +#endif extern const struct block_device_operations nvme_bdev_ops; void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl); diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 69932d640b537..862c38242f69c 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -1455,6 +1455,9 @@ static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx, ret = nvme_prep_rq(req); if (unlikely(ret)) return ret; + + nvme_update_io_stats(&dev->ctrl, req); + spin_lock(&nvmeq->sq_lock); nvme_sq_copy_cmd(nvmeq, &iod->cmd); nvme_write_sq_db(nvmeq, bd->last); @@ -1473,6 +1476,7 @@ static void nvme_submit_cmds(struct nvme_queue *nvmeq, struct rq_list *rqlist) while ((req = rq_list_pop(rqlist))) { struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + nvme_update_io_stats(&nvmeq->dev->ctrl, req); nvme_sq_copy_cmd(nvmeq, &iod->cmd); } nvme_write_sq_db(nvmeq, true); diff --git a/drivers/nvme/host/speed_switch.c b/drivers/nvme/host/speed_switch.c new file mode 100644 index 0000000000000..6cacac1ff6925 --- /dev/null +++ b/drivers/nvme/host/speed_switch.c @@ -0,0 +1,418 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * NVMe link speed switch driver + * + * Adjusts the PCIe link rate of NVMe devices dynamically based on the I/O + * workload: the link is downgraded to a configurable minimum rate during + * periods of low activity to save power, and upgraded to the maximum + * supported rate as soon as the workload requires it. + * + * Copyright (c) 2026 Hygon Information Technology Co., Ltd. + * Copyright (c) 2026 Kylin Software Co., Ltd. + * + */ + +#include +#include +#include +#include +#include + +#include "nvme.h" + +/* + * PCIe link bandwidth in KB/s for each generation (rows, Gen1..Gen5) and + * lane width (columns, x1..x16). The table is used to derive the I/O + * threshold exceed/below which the link can be upgraded/downgraded. + */ +static const u32 nvme_speed_table[][5] = { + { 312, 625, 1250, 2500, 5000 }, /* Gen1 */ + { 625, 1250, 2500, 5000, 10000 }, /* Gen2 */ + { 1000, 2000, 4000, 8000, 16000 }, /* Gen3 */ + { 2000, 4000, 8000, 16000, 32000 }, /* Gen4 */ + { 4000, 8000, 16000, 32000, 64000 },/* Gen5 */ +}; + +static int nvme_bandwidth_index(enum pcie_link_width width) +{ + switch (width) { + case PCIE_LNK_X1: + return 0; + case PCIE_LNK_X2: + return 1; + case PCIE_LNK_X4: + return 2; + case PCIE_LNK_X8: + return 3; + case PCIE_LNK_X16: + return 4; + default: + return -EINVAL; + } +} + +/* + * Read the maximum supported link rate of the upstream port from the Link + * Capability register. + */ +static int nvme_get_max_link_speed(struct pci_dev *pdev) +{ + u16 linkcap; + + pcie_capability_read_word(pdev, PCI_EXP_LNKCAP, &linkcap); + + return linkcap & PCI_EXP_LNKCAP_SLS; +} + +static enum pci_bus_speed nvme_speed_to_bus_speed(u8 gen) +{ + return gen + PCIE_SPEED_2_5GT - 1; +} + +/* + * Work item that actually changes the link rate. The queues are frozen + * before retraining to quiesce I/O and resumed once the switch completed. + */ +static void nvme_speed_switch_work(struct work_struct *work) +{ + struct nvme_speed_switch *sw = container_of(work, struct nvme_speed_switch, work); + struct nvme_ctrl *ctrl = container_of(sw, struct nvme_ctrl, speed_switch); + struct pci_dev *pdev = to_pci_dev(ctrl->dev); + struct pci_dev *bridge = pdev->bus->self; + u8 cur_speed = READ_ONCE(sw->cur_speed); + int ret; + + if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE) + return; + + /* Freeze I/O to avoid timeouts during link retraining. */ + nvme_start_freeze(ctrl); + nvme_wait_freeze(ctrl); + + ret = pcie_set_target_speed(bridge, nvme_speed_to_bus_speed(sw->target_speed), + true); + if (ret) { + dev_warn(ctrl->device, + "failed to set target rate Gen%u (%d), trying rollback to Gen%u\n", + sw->target_speed, ret, cur_speed); + ret = pcie_set_target_speed(bridge, nvme_speed_to_bus_speed(cur_speed), + true); + if (ret) { + dev_err(ctrl->device, + "rollback to Gen%u failed (%d), disabling speed switch\n", + cur_speed, ret); + WRITE_ONCE(sw->enabled, false); + nvme_unfreeze(ctrl); + return; + } + } else { + WRITE_ONCE(sw->cur_speed, sw->target_speed); + } + + nvme_unfreeze(ctrl); + dev_dbg(ctrl->device, "link rate changed to Gen%u\n", + READ_ONCE(sw->cur_speed)); +} + +/* + * Compute the I/O threshold (in KB) for a monitoring window. + * + * The threshold is derived from the following constraint: for the link + * speed to be upgraded, the transfer time of the data at the minimum + * speed must be greater than the sum of the transfer time at maximum + * speed and the link retraining time (100us): + * + * data / min_bw >= data / max_bw + 100us + * + * which yields data <= min_bw * max_bw * 0.1ms / (max_bw - min_bw). + * The bandwidths are taken from the link capability of the upstream port + * and the link width capability of the device. + */ +static u32 nvme_calc_speed_threshold(struct nvme_speed_switch *sw, + enum pcie_link_width max_width) +{ + struct nvme_ctrl *ctrl = container_of(sw, struct nvme_ctrl, speed_switch); + int bw_idx, min_gen = sw->min_speed, max_gen = sw->max_speed; + u64 min_bw, max_bw, x, y; + u32 threshold; + + if (min_gen < 1 || min_gen > 5 || max_gen < 1 || max_gen > 5) { + dev_warn(ctrl->device, + "invalid link rate range for threshold: min=Gen%d, max=Gen%d\n", + min_gen, max_gen); + return 0; + } + + bw_idx = nvme_bandwidth_index(max_width); + if (bw_idx < 0) { + dev_warn(ctrl->device, + "unsupported link width (%d) for threshold calculation\n", + max_width); + return 0; + } + + min_bw = nvme_speed_table[min_gen - 1][bw_idx]; + max_bw = nvme_speed_table[max_gen - 1][bw_idx]; + if (min_bw >= max_bw) { + dev_warn(ctrl->device, + "invalid bandwidth for threshold: min=%llu KB/s, max=%llu KB/s\n", + min_bw, max_bw); + return 0; + } + + x = min_bw * max_bw; + y = 10ULL * (max_bw - min_bw); + threshold = div64_u64(x + (y >> 1), y); + + dev_dbg(ctrl->device, + "I/O threshold: %u KB (min=Gen%d, max=Gen%d, width=x%d)\n", + threshold, min_gen, max_gen, max_width); + + return threshold; +} + +/* Called on the I/O submission path to accumulate the transferred bytes. */ +void nvme_update_io_stats(struct nvme_ctrl *ctrl, struct request *req) +{ + struct nvme_speed_switch *sw = &ctrl->speed_switch; + struct nvme_speed_switch_stats *stat; + enum req_op op = req_op(req); + unsigned int bytes = blk_rq_bytes(req); + + if (!READ_ONCE(sw->enabled)) + return; + + if (op != REQ_OP_READ && op != REQ_OP_WRITE) + return; + + if (!sw->stats) { + dev_dbg(ctrl->device, "I/O statistics not allocated, skip accounting\n"); + return; + } + + stat = get_cpu_ptr(sw->stats); + if (op == REQ_OP_READ) + stat->read_bytes += bytes; + else + stat->write_bytes += bytes; + put_cpu_ptr(sw->stats); + + if (atomic_cmpxchg(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE, + NVME_SPEED_TIMER_ACTIVE) != NVME_SPEED_TIMER_INACTIVE) + return; + + /* First I/O of the activity period: compute the threshold and arm the timer. */ + WRITE_ONCE(sw->threshold, nvme_calc_speed_threshold(sw, + pcie_get_width_cap(to_pci_dev(ctrl->dev)))); + if (!READ_ONCE(sw->threshold)) { + dev_warn(ctrl->device, + "failed to compute speed switch threshold, keeping current rate\n"); + atomic_set(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE); + return; + } + mod_timer(&sw->timer, jiffies + msecs_to_jiffies(READ_ONCE(sw->monitor_interval))); +} +EXPORT_SYMBOL_GPL(nvme_update_io_stats); + +static void nvme_clear_io_stats(struct nvme_speed_switch *sw) +{ + int cpu; + + for_each_possible_cpu(cpu) { + struct nvme_speed_switch_stats *stat = per_cpu_ptr(sw->stats, cpu); + + stat->read_bytes = 0; + stat->write_bytes = 0; + } +} + +/* + * Aggregate the per-CPU I/O counters of the last monitoring window and + * decide the target rate. Hysteresis counters are used to avoid + * excessive rate switching when the workload fluctuates around the + * threshold: the rate is only upgraded after the threshold was exceeded + * for a number of consecutive windows, and only downgraded after it + * stayed below the threshold for a number of consecutive windows. When + * no I/O was observed for 10 seconds the monitoring is stopped. + */ +static int nvme_check_io_and_decide_speed(struct nvme_speed_switch *sw, + bool *io_activity) +{ + struct nvme_ctrl *ctrl = container_of(sw, struct nvme_ctrl, speed_switch); + u64 read_bytes = 0, write_bytes = 0; + unsigned long read_kb, write_kb; + int cpu; + + for_each_possible_cpu(cpu) { + struct nvme_speed_switch_stats *stat = per_cpu_ptr(sw->stats, cpu); + + read_bytes += stat->read_bytes; + write_bytes += stat->write_bytes; + stat->read_bytes = 0; + stat->write_bytes = 0; + } + + if (read_bytes || write_bytes) { + *io_activity = true; + sw->idle_cnt = 0; + } else { + sw->idle_cnt++; + } + + /* Stop monitoring after 10 seconds of inactivity (100 windows of 100ms). */ + if (sw->idle_cnt >= 100) + *io_activity = false; + + read_kb = read_bytes / 1024; + write_kb = write_bytes / 1024; + dev_dbg(ctrl->device, + "I/O in window: read=%lu KB, write=%lu KB, threshold=%u KB\n", + read_kb, write_kb, READ_ONCE(sw->threshold)); + + if (read_kb >= sw->threshold || write_kb >= sw->threshold) { + sw->up_cnt++; + sw->down_cnt = 0; + } else { + sw->down_cnt++; + sw->up_cnt = 0; + } + + if (sw->up_cnt > READ_ONCE(sw->up_threshold)) { + sw->up_cnt = 0; + return sw->max_speed; + } + + if (sw->down_cnt > READ_ONCE(sw->down_threshold) || !*io_activity) { + sw->down_cnt = 0; + return sw->min_speed; + } + + return READ_ONCE(sw->cur_speed); +} + +static void nvme_speed_switch_timer_fn(struct timer_list *t) +{ + struct nvme_speed_switch *sw = container_of(t, struct nvme_speed_switch, timer); + bool io_activity = true; + + if (!READ_ONCE(sw->enabled)) { + atomic_set(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE); + return; + } + + sw->target_speed = nvme_check_io_and_decide_speed(sw, &io_activity); + if (sw->target_speed != READ_ONCE(sw->cur_speed)) + schedule_work(&sw->work); + + if (!io_activity) { + atomic_set(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE); + return; + } + mod_timer(t, jiffies + msecs_to_jiffies(READ_ONCE(sw->monitor_interval))); +} + +static void nvme_speed_switch_params_init(struct nvme_speed_switch *sw) +{ + sw->monitor_interval = 100; + sw->min_speed = PCI_EXP_LNKSTA_CLS_2_5GB; + /* Upgrade immediately once the threshold is exceeded. */ + sw->up_threshold = 0; + /* Downgrade after 10 consecutive below-threshold windows. */ + sw->down_threshold = 10; +} + +void nvme_speed_switch_init(struct nvme_ctrl *ctrl) +{ + struct nvme_speed_switch *sw = &ctrl->speed_switch; + struct pci_dev *pdev; + int max_speed, ret; + + if (!dev_is_pci(ctrl->dev)) { + dev_err(ctrl->device, "link rate switching requires a PCI device\n"); + return; + } + + pdev = to_pci_dev(ctrl->dev); + if (!pdev->bus->self) { + dev_err(ctrl->device, + "link rate switching requires a PCIe upstream port\n"); + return; + } + + nvme_speed_switch_params_init(sw); + + max_speed = nvme_get_max_link_speed(pdev->bus->self); + if (max_speed <= sw->min_speed) { + dev_err(ctrl->device, + "invalid link rate range: min=Gen%u, max=Gen%d\n", + sw->min_speed, max_speed); + return; + } + sw->max_speed = max_speed; + + sw->stats = alloc_percpu(struct nvme_speed_switch_stats); + if (!sw->stats) { + dev_err(ctrl->device, "failed to allocate per-CPU I/O statistics\n"); + return; + } + nvme_clear_io_stats(sw); + + WRITE_ONCE(sw->cur_speed, sw->max_speed); + WRITE_ONCE(sw->threshold, nvme_calc_speed_threshold(sw, pcie_get_width_cap(pdev))); + if (!READ_ONCE(sw->threshold)) { + dev_err(ctrl->device, "failed to compute speed switch threshold\n"); + free_percpu(sw->stats); + sw->stats = NULL; + return; + } + + INIT_WORK(&sw->work, nvme_speed_switch_work); + timer_setup(&sw->timer, nvme_speed_switch_timer_fn, 0); + atomic_set(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE); + WRITE_ONCE(sw->enabled, true); + + ret = sysfs_create_group(&ctrl->device->kobj, &nvme_speed_attr_group); + if (ret) { + dev_err(ctrl->device, + "failed to create link rate switching sysfs group (%d)\n", + ret); + timer_delete_sync(&sw->timer); + free_percpu(sw->stats); + sw->stats = NULL; + WRITE_ONCE(sw->enabled, false); + return; + } + sw->initialized = true; + + dev_info(ctrl->device, + "NVMe link rate switching initialized (min Gen%u, max Gen%d)\n", + sw->min_speed, sw->max_speed); +} + +void nvme_speed_switch_start(struct nvme_ctrl *ctrl) +{ + struct nvme_speed_switch *sw = &ctrl->speed_switch; + + if (!READ_ONCE(sw->enabled)) + return; + + atomic_set(&sw->timer_active, NVME_SPEED_TIMER_ACTIVE); + mod_timer(&sw->timer, jiffies + msecs_to_jiffies(READ_ONCE(sw->monitor_interval))); +} + +void nvme_speed_switch_exit(struct nvme_ctrl *ctrl) +{ + struct nvme_speed_switch *sw = &ctrl->speed_switch; + + if (!sw->initialized) + return; + + sw->initialized = false; + WRITE_ONCE(sw->enabled, false); + timer_delete_sync(&sw->timer); + cancel_work_sync(&sw->work); + sysfs_remove_group(&ctrl->device->kobj, &nvme_speed_attr_group); + + free_percpu(sw->stats); + sw->stats = NULL; +} diff --git a/drivers/nvme/host/sysfs.c b/drivers/nvme/host/sysfs.c index 75b2d69b59578..cd899b87bdb06 100644 --- a/drivers/nvme/host/sysfs.c +++ b/drivers/nvme/host/sysfs.c @@ -1308,3 +1308,146 @@ const struct attribute_group *nvme_subsys_attrs_groups[] = { &nvme_subsys_attrs_group, NULL, }; + +#ifdef CONFIG_NVME_SPEED_SWITCH +enum nvme_speed_attr_id { + NVME_SPEED_ATTR_ENABLE, + NVME_SPEED_ATTR_MONITOR_INTERVAL, + NVME_SPEED_ATTR_MIN_SPEED, + NVME_SPEED_ATTR_UP_THRESHOLD, + NVME_SPEED_ATTR_DOWN_THRESHOLD, +}; + +struct nvme_speed_attr { + struct device_attribute attr; + enum nvme_speed_attr_id id; +}; + +static ssize_t nvme_speed_attr_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct nvme_ctrl *ctrl = dev_get_drvdata(dev); + struct nvme_speed_switch *sw = &ctrl->speed_switch; + struct nvme_speed_attr *sa = + container_of(attr, struct nvme_speed_attr, attr); + u32 val; + + switch (sa->id) { + case NVME_SPEED_ATTR_ENABLE: + val = READ_ONCE(sw->enabled); + break; + case NVME_SPEED_ATTR_MONITOR_INTERVAL: + val = READ_ONCE(sw->monitor_interval); + break; + case NVME_SPEED_ATTR_MIN_SPEED: + val = READ_ONCE(sw->min_speed); + break; + case NVME_SPEED_ATTR_UP_THRESHOLD: + val = READ_ONCE(sw->up_threshold); + break; + case NVME_SPEED_ATTR_DOWN_THRESHOLD: + val = READ_ONCE(sw->down_threshold); + break; + default: + return -EINVAL; + } + + return sysfs_emit(buf, "%u\n", val); +} + +static ssize_t nvme_speed_attr_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct nvme_ctrl *ctrl = dev_get_drvdata(dev); + struct nvme_speed_switch *sw = &ctrl->speed_switch; + struct nvme_speed_attr *sa = + container_of(attr, struct nvme_speed_attr, attr); + u32 val; + int ret; + + ret = kstrtou32(buf, 10, &val); + if (ret) + return ret; + + switch (sa->id) { + case NVME_SPEED_ATTR_ENABLE: + if (val > 1) + return -EINVAL; + if (!val && READ_ONCE(sw->enabled)) { + WRITE_ONCE(sw->enabled, false); + timer_delete_sync(&sw->timer); + cancel_work_sync(&sw->work); + atomic_set(&sw->timer_active, NVME_SPEED_TIMER_INACTIVE); + } else + WRITE_ONCE(sw->enabled, val); + break; + case NVME_SPEED_ATTR_MONITOR_INTERVAL: + if (val < 100) + return -EINVAL; + WRITE_ONCE(sw->monitor_interval, val); + break; + case NVME_SPEED_ATTR_MIN_SPEED: + if (val < 1 || val > 5) + return -EINVAL; + if (val > sw->max_speed) + val = sw->max_speed; + WRITE_ONCE(sw->min_speed, val); + break; + case NVME_SPEED_ATTR_UP_THRESHOLD: + WRITE_ONCE(sw->up_threshold, val); + break; + case NVME_SPEED_ATTR_DOWN_THRESHOLD: + WRITE_ONCE(sw->down_threshold, val); + break; + default: + return -EINVAL; + } + + return count; +} + +static struct nvme_speed_attr nvme_speed_attr_enable = { + .attr = __ATTR(enable, 0644, + nvme_speed_attr_show, nvme_speed_attr_store), + .id = NVME_SPEED_ATTR_ENABLE, +}; + +static struct nvme_speed_attr nvme_speed_attr_monitor_interval = { + .attr = __ATTR(monitor_interval, 0644, + nvme_speed_attr_show, nvme_speed_attr_store), + .id = NVME_SPEED_ATTR_MONITOR_INTERVAL, +}; + +static struct nvme_speed_attr nvme_speed_attr_min_speed = { + .attr = __ATTR(min_speed, 0644, + nvme_speed_attr_show, nvme_speed_attr_store), + .id = NVME_SPEED_ATTR_MIN_SPEED, +}; + +static struct nvme_speed_attr nvme_speed_attr_up_threshold = { + .attr = __ATTR(up_threshold, 0644, + nvme_speed_attr_show, nvme_speed_attr_store), + .id = NVME_SPEED_ATTR_UP_THRESHOLD, +}; + +static struct nvme_speed_attr nvme_speed_attr_down_threshold = { + .attr = __ATTR(down_threshold, 0644, + nvme_speed_attr_show, nvme_speed_attr_store), + .id = NVME_SPEED_ATTR_DOWN_THRESHOLD, +}; + +static struct attribute *nvme_speed_attrs[] = { + &nvme_speed_attr_enable.attr.attr, + &nvme_speed_attr_monitor_interval.attr.attr, + &nvme_speed_attr_min_speed.attr.attr, + &nvme_speed_attr_up_threshold.attr.attr, + &nvme_speed_attr_down_threshold.attr.attr, + NULL, +}; + +const struct attribute_group nvme_speed_attr_group = { + .name = "speed", + .attrs = nvme_speed_attrs, +}; +#endif /* CONFIG_NVME_SPEED_SWITCH */ diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c index 19daadee65f7c..ecd8a108a8289 100644 --- a/drivers/pci/controller/dwc/pcie-qcom.c +++ b/drivers/pci/controller/dwc/pcie-qcom.c @@ -377,6 +377,26 @@ static void qcom_pcie_set_slot_nccs(struct dw_pcie *pci) dw_pcie_dbi_ro_wr_dis(pci); } +static void qcom_pcie_set_link_bw_notif_cap(struct dw_pcie *pci) +{ + u16 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + u32 val; + + dw_pcie_dbi_ro_wr_en(pci); + + /* + * The controller hardwires Link Bandwidth Notification Capability to 0, + * even though the Root Port can retrain to any of the speeds advertised + * in LNKCAP2. Set the bit so that the PCIe port driver binds bwctrl and + * registers the link speed thermal cooling device. + */ + val = readl(pci->dbi_base + offset + PCI_EXP_LNKCAP); + val |= PCI_EXP_LNKCAP_LBNC; + writel(val, pci->dbi_base + offset + PCI_EXP_LNKCAP); + + dw_pcie_dbi_ro_wr_dis(pci); +} + static void qcom_pcie_configure_dbi_base(struct qcom_pcie *pcie) { struct dw_pcie *pci = pcie->pci; @@ -1079,6 +1099,7 @@ static int qcom_pcie_post_init_2_7_0(struct qcom_pcie *pcie) pcie->parf + PARF_NO_SNOOP_OVERRIDE); qcom_pcie_set_slot_nccs(pcie->pci); + qcom_pcie_set_link_bw_notif_cap(pcie->pci); return 0; } @@ -1400,6 +1421,8 @@ static int qcom_pcie_host_init(struct dw_pcie_rp *pp) qcom_pcie_perst_deassert(pcie); + pp->bridge->disable_aspm_for_retrain = true; + if (pcie->cfg->ops->config_sid) { ret = pcie->cfg->ops->config_sid(pcie); if (ret) diff --git a/drivers/pci/pcie/bwctrl.c b/drivers/pci/pcie/bwctrl.c index c4c8d260bf969..9c4fc3947a24e 100644 --- a/drivers/pci/pcie/bwctrl.c +++ b/drivers/pci/pcie/bwctrl.c @@ -28,9 +28,11 @@ #include #include #include +#include #include #include #include +#include #include "../pci.h" #include "portdrv.h" @@ -120,6 +122,38 @@ static int pcie_bwctrl_change_speed(struct pci_dev *port, u16 target_speed, bool return pcie_retrain_link(port, use_lt); } +static int pcie_set_opp(struct pci_dev *pdev, struct pci_host_bridge *host, + enum pci_bus_speed speed) +{ + struct device *dev = host->dev.parent; + struct dev_pm_opp_key key = {}; + int ret, freq_mbps, width; + unsigned long freq_kbps; + struct dev_pm_opp *opp; + u16 lnksta; + + pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta); + width = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta); + + freq_mbps = pcie_dev_speed_mbps(speed); + if (freq_mbps < 0) + return -EINVAL; + + freq_kbps = freq_mbps * KILO; + key.freq = freq_kbps; + key.level = pci_bus_speed2lnkctl2(speed); + key.bw = 0; + opp = dev_pm_opp_find_key_exact(dev, &key, true); + if (!IS_ERR(opp)) { + ret = dev_pm_opp_set_opp(dev, opp); + if (ret) + dev_err(dev, "Failed to set OPP for freq (%lu): %d\n", + freq_kbps * width, ret); + dev_pm_opp_put(opp); + } + return 0; +} + /** * pcie_set_target_speed - Set downstream Link Speed for PCIe Port * @port: PCIe Port @@ -140,9 +174,12 @@ static int pcie_bwctrl_change_speed(struct pci_dev *port, u16 target_speed, bool int pcie_set_target_speed(struct pci_dev *port, enum pci_bus_speed speed_req, bool use_lt) { + struct pci_host_bridge *host = pci_find_host_bridge(port->bus); + bool is_rootbus = pci_is_root_bus(port->bus); struct pci_bus *bus = port->subordinate; + struct pci_dev *child = NULL; + int aspm_state = 0, ret; u16 target_speed; - int ret; if (WARN_ON_ONCE(!pcie_valid_speed(speed_req))) return -EINVAL; @@ -152,6 +189,24 @@ int pcie_set_target_speed(struct pci_dev *port, enum pci_bus_speed speed_req, target_speed = pcie_bwctrl_select_speed(port, speed_req); + /* + * The host bridge driver may need to be scaled for targeted speed + * otherwise link might not come up at requested speed. + */ + if (is_rootbus && host && bus) { + /* Get function 0 of downstream device */ + list_for_each_entry(child, &bus->devices, bus_list) + if (PCI_FUNC(child->devfn) == 0) + break; + + if (child && host->disable_aspm_for_retrain) { + aspm_state = pcie_aspm_enabled(child); + pci_disable_link_state_locked(child, PCIE_LINK_STATE_ALL); + } + if (speed_req > bus->cur_bus_speed) + pcie_set_opp(port, host, speed_req); + } + scoped_guard(rwsem_read, &pcie_bwctrl_setspeed_rwsem) { struct pcie_bwctrl_data *data = port->link_bwctrl; @@ -176,8 +231,15 @@ int pcie_set_target_speed(struct pci_dev *port, enum pci_bus_speed speed_req, !list_empty(&bus->devices)) ret = -EAGAIN; + if (bus && is_rootbus && host) { + if (child && host->disable_aspm_for_retrain) + pci_force_enable_link_state(child, aspm_state); + pcie_set_opp(port, host, bus->cur_bus_speed); + } + return ret; } +EXPORT_SYMBOL_GPL(pcie_set_target_speed); static void pcie_bwnotif_enable(struct pcie_device *srv) { diff --git a/include/linux/pci.h b/include/linux/pci.h index 09134d0a559fd..cb34eb4ba961b 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -664,6 +664,7 @@ struct pci_host_bridge { unsigned int msi_domain:1; /* Bridge wants MSI domain */ unsigned int broken_l1ss_resume:1; /* Resuming from L1SS during system suspend is broken */ + unsigned int disable_aspm_for_retrain:1; /* Disable ASPM before link retain */ /* Resource alignment requirements */ resource_size_t (*align_resource)(struct pci_dev *dev,