// SPDX-License-Identifier: GPL-2.0 /* * Driver for HiSilicon PCIe tune and trace device * * Copyright (c) 2022 HiSilicon Technologies Co., Ltd. * Author: Yicong Yang */ #include #include #include #include #include #include #include #include #include #include #include #include #include "hisi_ptt.h" /* Dynamic CPU hotplug state used by PTT */ static enum cpuhp_state hisi_ptt_pmu_online; static bool hisi_ptt_wait_tuning_finish(struct hisi_ptt *hisi_ptt) { u32 val; return !readl_poll_timeout(hisi_ptt->iobase + HISI_PTT_TUNING_INT_STAT, val, !(val & HISI_PTT_TUNING_INT_STAT_MASK), HISI_PTT_WAIT_POLL_INTERVAL_US, HISI_PTT_WAIT_TUNE_TIMEOUT_US); } static ssize_t hisi_ptt_tune_attr_show(struct device *dev, struct device_attribute *attr, char *buf) { struct hisi_ptt *hisi_ptt = to_hisi_ptt(dev_get_drvdata(dev)); struct dev_ext_attribute *ext_attr; struct hisi_ptt_tune_desc *desc; u32 reg; u16 val; ext_attr = container_of(attr, struct dev_ext_attribute, attr); desc = ext_attr->var; mutex_lock(&hisi_ptt->tune_lock); reg = readl(hisi_ptt->iobase + HISI_PTT_TUNING_CTRL); reg &= ~(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB); reg |= FIELD_PREP(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB, desc->event_code); writel(reg, hisi_ptt->iobase + HISI_PTT_TUNING_CTRL); /* Write all 1 to indicates it's the read process */ writel(~0U, hisi_ptt->iobase + HISI_PTT_TUNING_DATA); if (!hisi_ptt_wait_tuning_finish(hisi_ptt)) { mutex_unlock(&hisi_ptt->tune_lock); return -ETIMEDOUT; } reg = readl(hisi_ptt->iobase + HISI_PTT_TUNING_DATA); reg &= HISI_PTT_TUNING_DATA_VAL_MASK; val = FIELD_GET(HISI_PTT_TUNING_DATA_VAL_MASK, reg); mutex_unlock(&hisi_ptt->tune_lock); return sysfs_emit(buf, "%u\n", val); } static ssize_t hisi_ptt_tune_attr_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct hisi_ptt *hisi_ptt = to_hisi_ptt(dev_get_drvdata(dev)); struct dev_ext_attribute *ext_attr; struct hisi_ptt_tune_desc *desc; u32 reg; u16 val; ext_attr = container_of(attr, struct dev_ext_attribute, attr); desc = ext_attr->var; if (kstrtou16(buf, 10, &val)) return -EINVAL; mutex_lock(&hisi_ptt->tune_lock); reg = readl(hisi_ptt->iobase + HISI_PTT_TUNING_CTRL); reg &= ~(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB); reg |= FIELD_PREP(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB, desc->event_code); writel(reg, hisi_ptt->iobase + HISI_PTT_TUNING_CTRL); writel(FIELD_PREP(HISI_PTT_TUNING_DATA_VAL_MASK, val), hisi_ptt->iobase + HISI_PTT_TUNING_DATA); if (!hisi_ptt_wait_tuning_finish(hisi_ptt)) { mutex_unlock(&hisi_ptt->tune_lock); return -ETIMEDOUT; } mutex_unlock(&hisi_ptt->tune_lock); return count; } #define HISI_PTT_TUNE_ATTR(_name, _val, _show, _store) \ static struct hisi_ptt_tune_desc _name##_desc = { \ .name = #_name, \ .event_code = (_val), \ }; \ static struct dev_ext_attribute hisi_ptt_##_name##_attr = { \ .attr = __ATTR(_name, 0600, _show, _store), \ .var = &_name##_desc, \ } #define HISI_PTT_TUNE_ATTR_COMMON(_name, _val) \ HISI_PTT_TUNE_ATTR(_name, _val, \ hisi_ptt_tune_attr_show, \ hisi_ptt_tune_attr_store) /* * The value of the tuning event are composed of two parts: main event code * in BIT[0,15] and subevent code in BIT[16,23]. For example, qox_tx_cpl is * a subevent of 'Tx path QoS control' which for tuning the weight of Tx * completion TLPs. See hisi_ptt.rst documentation for more information. */ #define HISI_PTT_TUNE_QOS_TX_CPL (0x4 | (3 << 16)) #define HISI_PTT_TUNE_QOS_TX_NP (0x4 | (4 << 16)) #define HISI_PTT_TUNE_QOS_TX_P (0x4 | (5 << 16)) #define HISI_PTT_TUNE_RX_ALLOC_BUF_LEVEL (0x5 | (6 << 16)) #define HISI_PTT_TUNE_TX_ALLOC_BUF_LEVEL (0x5 | (7 << 16)) HISI_PTT_TUNE_ATTR_COMMON(qos_tx_cpl, HISI_PTT_TUNE_QOS_TX_CPL); HISI_PTT_TUNE_ATTR_COMMON(qos_tx_np, HISI_PTT_TUNE_QOS_TX_NP); HISI_PTT_TUNE_ATTR_COMMON(qos_tx_p, HISI_PTT_TUNE_QOS_TX_P); HISI_PTT_TUNE_ATTR_COMMON(rx_alloc_buf_level, HISI_PTT_TUNE_RX_ALLOC_BUF_LEVEL); HISI_PTT_TUNE_ATTR_COMMON(tx_alloc_buf_level, HISI_PTT_TUNE_TX_ALLOC_BUF_LEVEL); static struct attribute *hisi_ptt_tune_attrs[] = { &hisi_ptt_qos_tx_cpl_attr.attr.attr, &hisi_ptt_qos_tx_np_attr.attr.attr, &hisi_ptt_qos_tx_p_attr.attr.attr, &hisi_ptt_rx_alloc_buf_level_attr.attr.attr, &hisi_ptt_tx_alloc_buf_level_attr.attr.attr, NULL, }; static struct attribute_group hisi_ptt_tune_group = { .name = "tune", .attrs = hisi_ptt_tune_attrs, }; static u16 hisi_ptt_get_filter_val(u16 devid, bool is_port) { if (is_port) return BIT(HISI_PCIE_CORE_PORT_ID(devid & 0xff)); return devid; } static bool hisi_ptt_wait_trace_hw_idle(struct hisi_ptt *hisi_ptt) { u32 val; return !readl_poll_timeout_atomic(hisi_ptt->iobase + HISI_PTT_TRACE_STS, val, val & HISI_PTT_TRACE_IDLE, HISI_PTT_WAIT_POLL_INTERVAL_US, HISI_PTT_WAIT_TRACE_TIMEOUT_US); } static void hisi_ptt_wait_dma_reset_done(struct hisi_ptt *hisi_ptt) { u32 val; readl_poll_timeout_atomic(hisi_ptt->iobase + HISI_PTT_TRACE_WR_STS, val, !val, HISI_PTT_RESET_POLL_INTERVAL_US, HISI_PTT_RESET_TIMEOUT_US); } static void hisi_ptt_trace_end(struct hisi_ptt *hisi_ptt) { writel(0, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); hisi_ptt->trace_ctrl.started = false; } static int hisi_ptt_trace_start(struct hisi_ptt *hisi_ptt) { struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl; u32 val; int i; /* Check device idle before start trace */ if (!hisi_ptt_wait_trace_hw_idle(hisi_ptt)) { pci_err(hisi_ptt->pdev, "Failed to start trace, the device is still busy\n"); return -EBUSY; } ctrl->started = true; /* Reset the DMA before start tracing */ val = readl(hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); val |= HISI_PTT_TRACE_CTRL_RST; writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); hisi_ptt_wait_dma_reset_done(hisi_ptt); val = readl(hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); val &= ~HISI_PTT_TRACE_CTRL_RST; writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); /* Reset the index of current buffer */ hisi_ptt->trace_ctrl.buf_index = 0; /* Zero the trace buffers */ for (i = 0; i < HISI_PTT_TRACE_BUF_CNT; i++) memset(ctrl->trace_buf[i].addr, 0, HISI_PTT_TRACE_BUF_SIZE); /* Clear the interrupt status */ writel(HISI_PTT_TRACE_INT_STAT_MASK, hisi_ptt->iobase + HISI_PTT_TRACE_INT_STAT); writel(0, hisi_ptt->iobase + HISI_PTT_TRACE_INT_MASK); /* Set the trace control register */ val = FIELD_PREP(HISI_PTT_TRACE_CTRL_TYPE_SEL, ctrl->type); val |= FIELD_PREP(HISI_PTT_TRACE_CTRL_RXTX_SEL, ctrl->direction); val |= FIELD_PREP(HISI_PTT_TRACE_CTRL_DATA_FORMAT, ctrl->format); val |= FIELD_PREP(HISI_PTT_TRACE_CTRL_TARGET_SEL, hisi_ptt->trace_ctrl.filter); if (!hisi_ptt->trace_ctrl.is_port) val |= HISI_PTT_TRACE_CTRL_FILTER_MODE; /* Start the Trace */ val |= HISI_PTT_TRACE_CTRL_EN; writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); return 0; } static int hisi_ptt_update_aux(struct hisi_ptt *hisi_ptt, int index, bool stop) { struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl; struct perf_output_handle *handle = &ctrl->handle; struct perf_event *event = handle->event; struct hisi_ptt_pmu_buf *buf; size_t size; void *addr; buf = perf_get_aux(handle); if (!buf || !handle->size) return -EINVAL; addr = ctrl->trace_buf[ctrl->buf_index].addr; /* * If we're going to stop, read the size of already traced data from * HISI_PTT_TRACE_WR_STS. Otherwise we're coming from the interrupt, * the data size is always HISI_PTT_TRACE_BUF_SIZE. */ if (stop) { u32 reg; reg = readl(hisi_ptt->iobase + HISI_PTT_TRACE_WR_STS); size = FIELD_GET(HISI_PTT_TRACE_WR_STS_WRITE, reg); } else { size = HISI_PTT_TRACE_BUF_SIZE; } memcpy(buf->base + buf->pos, addr, size); buf->pos += size; /* * Just commit the traced data if we're going to stop. Otherwise if the * resident AUX buffer cannot contain the data of next trace buffer, * apply a new one. */ if (stop) { perf_aux_output_end(handle, buf->pos); } else if (buf->length - buf->pos < HISI_PTT_TRACE_BUF_SIZE) { perf_aux_output_end(handle, buf->pos); buf = perf_aux_output_begin(handle, event); if (!buf) return -EINVAL; buf->pos = handle->head % buf->length; if (buf->length - buf->pos < HISI_PTT_TRACE_BUF_SIZE) { perf_aux_output_end(handle, 0); return -EINVAL; } } return 0; } static irqreturn_t hisi_ptt_isr(int irq, void *context) { struct hisi_ptt *hisi_ptt = context; u32 status, buf_idx; status = readl(hisi_ptt->iobase + HISI_PTT_TRACE_INT_STAT); if (!(status & HISI_PTT_TRACE_INT_STAT_MASK)) return IRQ_NONE; buf_idx = ffs(status) - 1; /* Clear the interrupt status of buffer @buf_idx */ writel(status, hisi_ptt->iobase + HISI_PTT_TRACE_INT_STAT); /* * Update the AUX buffer and cache the current buffer index, * as we need to know this and save the data when the trace * is ended out of the interrupt handler. End the trace * if the updating fails. */ if (hisi_ptt_update_aux(hisi_ptt, buf_idx, false)) hisi_ptt_trace_end(hisi_ptt); else hisi_ptt->trace_ctrl.buf_index = (buf_idx + 1) % HISI_PTT_TRACE_BUF_CNT; return IRQ_HANDLED; } static void hisi_ptt_irq_free_vectors(void *pdev) { pci_free_irq_vectors(pdev); } static int hisi_ptt_register_irq(struct hisi_ptt *hisi_ptt) { struct pci_dev *pdev = hisi_ptt->pdev; int ret; ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI); if (ret < 0) { pci_err(pdev, "failed to allocate irq vector, ret = %d\n", ret); return ret; } ret = devm_add_action_or_reset(&pdev->dev, hisi_ptt_irq_free_vectors, pdev); if (ret < 0) return ret; hisi_ptt->trace_irq = pci_irq_vector(pdev, HISI_PTT_TRACE_DMA_IRQ); ret = devm_request_threaded_irq(&pdev->dev, hisi_ptt->trace_irq, NULL, hisi_ptt_isr, 0, DRV_NAME, hisi_ptt); if (ret) { pci_err(pdev, "failed to request irq %d, ret = %d\n", hisi_ptt->trace_irq, ret); return ret; } return 0; } static void hisi_ptt_del_free_filter(struct hisi_ptt *hisi_ptt, struct hisi_ptt_filter_desc *filter) { if (filter->is_port) hisi_ptt->port_mask &= ~hisi_ptt_get_filter_val(filter->devid, true); list_del(&filter->list); kfree(filter->name); kfree(filter); } static struct hisi_ptt_filter_desc * hisi_ptt_alloc_add_filter(struct hisi_ptt *hisi_ptt, u16 devid, bool is_port) { struct hisi_ptt_filter_desc *filter; u8 devfn = devid & 0xff; char *filter_name; filter_name = kasprintf(GFP_KERNEL, "%04x:%02x:%02x.%d", pci_domain_nr(hisi_ptt->pdev->bus), PCI_BUS_NUM(devid), PCI_SLOT(devfn), PCI_FUNC(devfn)); if (!filter_name) { pci_err(hisi_ptt->pdev, "failed to allocate name for filter %04x:%02x:%02x.%d\n", pci_domain_nr(hisi_ptt->pdev->bus), PCI_BUS_NUM(devid), PCI_SLOT(devfn), PCI_FUNC(devfn)); return NULL; } filter = kzalloc(sizeof(*filter), GFP_KERNEL); if (!filter) { pci_err(hisi_ptt->pdev, "failed to add filter for %s\n", filter_name); kfree(filter_name); return NULL; } filter->name = filter_name; filter->is_port = is_port; filter->devid = devid; if (filter->is_port) { list_add_tail(&filter->list, &hisi_ptt->port_filters); /* Update the available port mask */ hisi_ptt->port_mask |= hisi_ptt_get_filter_val(filter->devid, true); } else { list_add_tail(&filter->list, &hisi_ptt->req_filters); } return filter; } static ssize_t hisi_ptt_filter_show(struct device *dev, struct device_attribute *attr, char *buf) { struct hisi_ptt_filter_desc *filter; unsigned long filter_val; filter = container_of(attr, struct hisi_ptt_filter_desc, attr); filter_val = hisi_ptt_get_filter_val(filter->devid, filter->is_port) | (filter->is_port ? HISI_PTT_PMU_FILTER_IS_PORT : 0); return sysfs_emit(buf, "0x%05lx\n", filter_val); } static int hisi_ptt_create_rp_filter_attr(struct hisi_ptt *hisi_ptt, struct hisi_ptt_filter_desc *filter) { struct kobject *kobj = &hisi_ptt->hisi_ptt_pmu.dev->kobj; sysfs_attr_init(&filter->attr.attr); filter->attr.attr.name = filter->name; filter->attr.attr.mode = 0400; /* DEVICE_ATTR_ADMIN_RO */ filter->attr.show = hisi_ptt_filter_show; return sysfs_add_file_to_group(kobj, &filter->attr.attr, HISI_PTT_RP_FILTERS_GRP_NAME); } static void hisi_ptt_remove_rp_filter_attr(struct hisi_ptt *hisi_ptt, struct hisi_ptt_filter_desc *filter) { struct kobject *kobj = &hisi_ptt->hisi_ptt_pmu.dev->kobj; sysfs_remove_file_from_group(kobj, &filter->attr.attr, HISI_PTT_RP_FILTERS_GRP_NAME); } static int hisi_ptt_create_req_filter_attr(struct hisi_ptt *hisi_ptt, struct hisi_ptt_filter_desc *filter) { struct kobject *kobj = &hisi_ptt->hisi_ptt_pmu.dev->kobj; sysfs_attr_init(&filter->attr.attr); filter->attr.attr.name = filter->name; filter->attr.attr.mode = 0400; /* DEVICE_ATTR_ADMIN_RO */ filter->attr.show = hisi_ptt_filter_show; return sysfs_add_file_to_group(kobj, &filter->attr.attr, HISI_PTT_REQ_FILTERS_GRP_NAME); } static void hisi_ptt_remove_req_filter_attr(struct hisi_ptt *hisi_ptt, struct hisi_ptt_filter_desc *filter) { struct kobject *kobj = &hisi_ptt->hisi_ptt_pmu.dev->kobj; sysfs_remove_file_from_group(kobj, &filter->attr.attr, HISI_PTT_REQ_FILTERS_GRP_NAME); } static int hisi_ptt_create_filter_attr(struct hisi_ptt *hisi_ptt, struct hisi_ptt_filter_desc *filter) { int ret; if (filter->is_port) ret = hisi_ptt_create_rp_filter_attr(hisi_ptt, filter); else ret = hisi_ptt_create_req_filter_attr(hisi_ptt, filter); if (ret) pci_err(hisi_ptt->pdev, "failed to create sysfs attribute for filter %s\n", filter->name); return ret; } static void hisi_ptt_remove_filter_attr(struct hisi_ptt *hisi_ptt, struct hisi_ptt_filter_desc *filter) { if (filter->is_port) hisi_ptt_remove_rp_filter_attr(hisi_ptt, filter); else hisi_ptt_remove_req_filter_attr(hisi_ptt, filter); } static void hisi_ptt_remove_all_filter_attributes(void *data) { struct hisi_ptt_filter_desc *filter; struct hisi_ptt *hisi_ptt = data; mutex_lock(&hisi_ptt->filter_lock); list_for_each_entry(filter, &hisi_ptt->req_filters, list) hisi_ptt_remove_filter_attr(hisi_ptt, filter); list_for_each_entry(filter, &hisi_ptt->port_filters, list) hisi_ptt_remove_filter_attr(hisi_ptt, filter); hisi_ptt->sysfs_inited = false; mutex_unlock(&hisi_ptt->filter_lock); } static int hisi_ptt_init_filter_attributes(struct hisi_ptt *hisi_ptt) { struct hisi_ptt_filter_desc *filter; int ret; mutex_lock(&hisi_ptt->filter_lock); /* * Register the reset callback in the first stage. In reset we traverse * the filters list to remove the sysfs attributes so the callback can * be called safely even without below filter attributes creation. */ ret = devm_add_action(&hisi_ptt->pdev->dev, hisi_ptt_remove_all_filter_attributes, hisi_ptt); if (ret) goto out; list_for_each_entry(filter, &hisi_ptt->port_filters, list) { ret = hisi_ptt_create_filter_attr(hisi_ptt, filter); if (ret) goto out; } list_for_each_entry(filter, &hisi_ptt->req_filters, list) { ret = hisi_ptt_create_filter_attr(hisi_ptt, filter); if (ret) goto out; } hisi_ptt->sysfs_inited = true; out: mutex_unlock(&hisi_ptt->filter_lock); return ret; } static void hisi_ptt_update_filters(struct work_struct *work) { struct delayed_work *delayed_work = to_delayed_work(work); struct hisi_ptt_filter_update_info info; struct hisi_ptt_filter_desc *filter; struct hisi_ptt *hisi_ptt; hisi_ptt = container_of(delayed_work, struct hisi_ptt, work); if (!mutex_trylock(&hisi_ptt->filter_lock)) { schedule_delayed_work(&hisi_ptt->work, HISI_PTT_WORK_DELAY_MS); return; } while (kfifo_get(&hisi_ptt->filter_update_kfifo, &info)) { if (info.is_add) { /* * Notify the users if failed to add this filter, others * still work and available. See the comments in * hisi_ptt_init_filters(). */ filter = hisi_ptt_alloc_add_filter(hisi_ptt, info.devid, info.is_port); if (!filter) continue; /* * If filters' sysfs entries hasn't been initialized, * then we're still at probe stage. Add the filters to * the list and later hisi_ptt_init_filter_attributes() * will create sysfs attributes for all the filters. */ if (hisi_ptt->sysfs_inited && hisi_ptt_create_filter_attr(hisi_ptt, filter)) { hisi_ptt_del_free_filter(hisi_ptt, filter); continue; } } else { struct hisi_ptt_filter_desc *tmp; struct list_head *target_list; target_list = info.is_port ? &hisi_ptt->port_filters : &hisi_ptt->req_filters; list_for_each_entry_safe(filter, tmp, target_list, list) if (filter->devid == info.devid) { if (hisi_ptt->sysfs_inited) hisi_ptt_remove_filter_attr(hisi_ptt, filter); hisi_ptt_del_free_filter(hisi_ptt, filter); break; } } } mutex_unlock(&hisi_ptt->filter_lock); } /* * A PCI bus notifier is used here for dynamically updating the filter * list. */ static int hisi_ptt_notifier_call(struct notifier_block *nb, unsigned long action, void *data) { struct hisi_ptt *hisi_ptt = container_of(nb, struct hisi_ptt, hisi_ptt_nb); struct hisi_ptt_filter_update_info info; struct pci_dev *pdev, *root_port; struct device *dev = data; u32 port_devid; pdev = to_pci_dev(dev); root_port = pcie_find_root_port(pdev); if (!root_port) return 0; port_devid = pci_dev_id(root_port); if (port_devid < hisi_ptt->lower_bdf || port_devid > hisi_ptt->upper_bdf) return 0; info.is_port = pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT; info.devid = pci_dev_id(pdev); switch (action) { case BUS_NOTIFY_ADD_DEVICE: info.is_add = true; break; case BUS_NOTIFY_DEL_DEVICE: info.is_add = false; break; default: return 0; } /* * The FIFO size is 16 which is sufficient for almost all the cases, * since each PCIe core will have most 8 Root Ports (typically only * 1~4 Root Ports). On failure log the failed filter and let user * handle it. */ if (kfifo_in_spinlocked(&hisi_ptt->filter_update_kfifo, &info, 1, &hisi_ptt->filter_update_lock)) schedule_delayed_work(&hisi_ptt->work, 0); else pci_warn(hisi_ptt->pdev, "filter update fifo overflow for target %s\n", pci_name(pdev)); return 0; } static int hisi_ptt_init_filters(struct pci_dev *pdev, void *data) { struct pci_dev *root_port = pcie_find_root_port(pdev); struct hisi_ptt_filter_desc *filter; struct hisi_ptt *hisi_ptt = data; u32 port_devid; if (!root_port) return 0; port_devid = pci_dev_id(root_port); if (port_devid < hisi_ptt->lower_bdf || port_devid > hisi_ptt->upper_bdf) return 0; /* * We won't fail the probe if filter allocation failed here. The filters * should be partial initialized and users would know which filter fails * through the log. Other functions of PTT device are still available. */ filter = hisi_ptt_alloc_add_filter(hisi_ptt, pci_dev_id(pdev), pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT); if (!filter) return -ENOMEM; return 0; } static void hisi_ptt_release_filters(void *data) { struct hisi_ptt_filter_desc *filter, *tmp; struct hisi_ptt *hisi_ptt = data; list_for_each_entry_safe(filter, tmp, &hisi_ptt->req_filters, list) hisi_ptt_del_free_filter(hisi_ptt, filter); list_for_each_entry_safe(filter, tmp, &hisi_ptt->port_filters, list) hisi_ptt_del_free_filter(hisi_ptt, filter); } static int hisi_ptt_config_trace_buf(struct hisi_ptt *hisi_ptt) { struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl; struct device *dev = &hisi_ptt->pdev->dev; int i; ctrl->trace_buf = devm_kcalloc(dev, HISI_PTT_TRACE_BUF_CNT, sizeof(*ctrl->trace_buf), GFP_KERNEL); if (!ctrl->trace_buf) return -ENOMEM; for (i = 0; i < HISI_PTT_TRACE_BUF_CNT; ++i) { ctrl->trace_buf[i].addr = dmam_alloc_coherent(dev, HISI_PTT_TRACE_BUF_SIZE, &ctrl->trace_buf[i].dma, GFP_KERNEL); if (!ctrl->trace_buf[i].addr) return -ENOMEM; } /* Configure the trace DMA buffer */ for (i = 0; i < HISI_PTT_TRACE_BUF_CNT; i++) { writel(lower_32_bits(ctrl->trace_buf[i].dma), hisi_ptt->iobase + HISI_PTT_TRACE_ADDR_BASE_LO_0 + i * HISI_PTT_TRACE_ADDR_STRIDE); writel(upper_32_bits(ctrl->trace_buf[i].dma), hisi_ptt->iobase + HISI_PTT_TRACE_ADDR_BASE_HI_0 + i * HISI_PTT_TRACE_ADDR_STRIDE); } writel(HISI_PTT_TRACE_BUF_SIZE, hisi_ptt->iobase + HISI_PTT_TRACE_ADDR_SIZE); return 0; } static int hisi_ptt_init_ctrls(struct hisi_ptt *hisi_ptt) { struct pci_dev *pdev = hisi_ptt->pdev; struct pci_bus *bus; int ret; u32 reg; INIT_DELAYED_WORK(&hisi_ptt->work, hisi_ptt_update_filters); INIT_KFIFO(hisi_ptt->filter_update_kfifo); spin_lock_init(&hisi_ptt->filter_update_lock); INIT_LIST_HEAD(&hisi_ptt->port_filters); INIT_LIST_HEAD(&hisi_ptt->req_filters); mutex_init(&hisi_ptt->filter_lock); ret = hisi_ptt_config_trace_buf(hisi_ptt); if (ret) return ret; /* * The device range register provides the information about the root * ports which the RCiEP can control and trace. The RCiEP and the root * ports which it supports are on the same PCIe core, with same domain * number but maybe different bus number. The device range register * will tell us which root ports we can support, Bit[31:16] indicates * the upper BDF numbers of the root port, while Bit[15:0] indicates * the lower. */ reg = readl(hisi_ptt->iobase + HISI_PTT_DEVICE_RANGE); hisi_ptt->upper_bdf = FIELD_GET(HISI_PTT_DEVICE_RANGE_UPPER, reg); hisi_ptt->lower_bdf = FIELD_GET(HISI_PTT_DEVICE_RANGE_LOWER, reg); bus = pci_find_bus(pci_domain_nr(pdev->bus), PCI_BUS_NUM(hisi_ptt->upper_bdf)); if (bus) pci_walk_bus(bus, hisi_ptt_init_filters, hisi_ptt); ret = devm_add_action_or_reset(&pdev->dev, hisi_ptt_release_filters, hisi_ptt); if (ret) return ret; hisi_ptt->trace_ctrl.on_cpu = -1; return 0; } static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, char *buf) { struct hisi_ptt *hisi_ptt = to_hisi_ptt(dev_get_drvdata(dev)); const cpumask_t *cpumask = cpumask_of_node(dev_to_node(&hisi_ptt->pdev->dev)); return cpumap_print_to_pagebuf(true, buf, cpumask); } static DEVICE_ATTR_RO(cpumask); static struct attribute *hisi_ptt_cpumask_attrs[] = { &dev_attr_cpumask.attr, NULL }; static const struct attribute_group hisi_ptt_cpumask_attr_group = { .attrs = hisi_ptt_cpumask_attrs, }; /* * Bit 19 indicates the filter type, 1 for Root Port filter and 0 for Requester * filter. Bit[15:0] indicates the filter value, for Root Port filter it's * a bit mask of desired ports and for Requester filter it's the Requester ID * of the desired PCIe function. Bit[18:16] is reserved for extension. * * See hisi_ptt.rst documentation for detailed information. */ PMU_FORMAT_ATTR(filter, "config:0-19"); PMU_FORMAT_ATTR(direction, "config:20-23"); PMU_FORMAT_ATTR(type, "config:24-31"); PMU_FORMAT_ATTR(format, "config:32-35"); static struct attribute *hisi_ptt_pmu_format_attrs[] = { &format_attr_filter.attr, &format_attr_direction.attr, &format_attr_type.attr, &format_attr_format.attr, NULL }; static struct attribute_group hisi_ptt_pmu_format_group = { .name = "format", .attrs = hisi_ptt_pmu_format_attrs, }; static ssize_t hisi_ptt_filter_multiselect_show(struct device *dev, struct device_attribute *attr, char *buf) { struct dev_ext_attribute *ext_attr; ext_attr = container_of(attr, struct dev_ext_attribute, attr); return sysfs_emit(buf, "%s\n", (char *)ext_attr->var); } static struct dev_ext_attribute root_port_filters_multiselect = { .attr = { .attr = { .name = "multiselect", .mode = 0400 }, .show = hisi_ptt_filter_multiselect_show, }, .var = "1", }; static struct attribute *hisi_ptt_pmu_root_ports_attrs[] = { &root_port_filters_multiselect.attr.attr, NULL }; static struct attribute_group hisi_ptt_pmu_root_ports_group = { .name = HISI_PTT_RP_FILTERS_GRP_NAME, .attrs = hisi_ptt_pmu_root_ports_attrs, }; static struct dev_ext_attribute requester_filters_multiselect = { .attr = { .attr = { .name = "multiselect", .mode = 0400 }, .show = hisi_ptt_filter_multiselect_show, }, .var = "0", }; static struct attribute *hisi_ptt_pmu_requesters_attrs[] = { &requester_filters_multiselect.attr.attr, NULL }; static struct attribute_group hisi_ptt_pmu_requesters_group = { .name = HISI_PTT_REQ_FILTERS_GRP_NAME, .attrs = hisi_ptt_pmu_requesters_attrs, }; static const struct attribute_group *hisi_ptt_pmu_groups[] = { &hisi_ptt_cpumask_attr_group, &hisi_ptt_pmu_format_group, &hisi_ptt_tune_group, &hisi_ptt_pmu_root_ports_group, &hisi_ptt_pmu_requesters_group, NULL }; static int hisi_ptt_trace_valid_direction(u32 val) { /* * The direction values have different effects according to the data * format (specified in the parentheses). TLP set A/B means different * set of TLP types. See hisi_ptt.rst documentation for more details. */ static const u32 hisi_ptt_trace_available_direction[] = { 0, /* inbound(4DW) or reserved(8DW) */ 1, /* outbound(4DW) */ 2, /* {in, out}bound(4DW) or inbound(8DW), TLP set A */ 3, /* {in, out}bound(4DW) or inbound(8DW), TLP set B */ }; int i; for (i = 0; i < ARRAY_SIZE(hisi_ptt_trace_available_direction); i++) { if (val == hisi_ptt_trace_available_direction[i]) return 0; } return -EINVAL; } static int hisi_ptt_trace_valid_type(u32 val) { /* Different types can be set simultaneously */ static const u32 hisi_ptt_trace_available_type[] = { 1, /* posted_request */ 2, /* non-posted_request */ 4, /* completion */ }; int i; if (!val) return -EINVAL; /* * Walk the available list and clear the valid bits of * the config. If there is any resident bit after the * walk then the config is invalid. */ for (i = 0; i < ARRAY_SIZE(hisi_ptt_trace_available_type); i++) val &= ~hisi_ptt_trace_available_type[i]; if (val) return -EINVAL; return 0; } static int hisi_ptt_trace_valid_format(u32 val) { static const u32 hisi_ptt_trace_availble_format[] = { 0, /* 4DW */ 1, /* 8DW */ }; int i; for (i = 0; i < ARRAY_SIZE(hisi_ptt_trace_availble_format); i++) { if (val == hisi_ptt_trace_availble_format[i]) return 0; } return -EINVAL; } static int hisi_ptt_trace_valid_filter(struct hisi_ptt *hisi_ptt, u64 config) { unsigned long val, port_mask = hisi_ptt->port_mask; struct hisi_ptt_filter_desc *filter; int ret = 0; hisi_ptt->trace_ctrl.is_port = FIELD_GET(HISI_PTT_PMU_FILTER_IS_PORT, config); val = FIELD_GET(HISI_PTT_PMU_FILTER_VAL_MASK, config); /* * Port filters are defined as bit mask. For port filters, check * the bits in the @val are within the range of hisi_ptt->port_mask * and whether it's empty or not, otherwise user has specified * some unsupported root ports. * * For Requester ID filters, walk the available filter list to see * whether we have one matched. */ mutex_lock(&hisi_ptt->filter_lock); if (!hisi_ptt->trace_ctrl.is_port) { list_for_each_entry(filter, &hisi_ptt->req_filters, list) { if (val == hisi_ptt_get_filter_val(filter->devid, filter->is_port)) goto out; } } else if (bitmap_subset(&val, &port_mask, BITS_PER_LONG)) { goto out; } ret = -EINVAL; out: mutex_unlock(&hisi_ptt->filter_lock); return ret; } static void hisi_ptt_pmu_init_configs(struct hisi_ptt *hisi_ptt, struct perf_event *event) { struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl; u32 val; val = FIELD_GET(HISI_PTT_PMU_FILTER_VAL_MASK, event->attr.config); hisi_ptt->trace_ctrl.filter = val; val = FIELD_GET(HISI_PTT_PMU_DIRECTION_MASK, event->attr.config); ctrl->direction = val; val = FIELD_GET(HISI_PTT_PMU_TYPE_MASK, event->attr.config); ctrl->type = val; val = FIELD_GET(HISI_PTT_PMU_FORMAT_MASK, event->attr.config); ctrl->format = val; } static int hisi_ptt_pmu_event_init(struct perf_event *event) { struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu); int ret; u32 val; if (event->cpu < 0) { dev_dbg(event->pmu->dev, "Per-task mode not supported\n"); return -EOPNOTSUPP; } if (event->attr.type != hisi_ptt->hisi_ptt_pmu.type) return -ENOENT; ret = hisi_ptt_trace_valid_filter(hisi_ptt, event->attr.config); if (ret < 0) return ret; val = FIELD_GET(HISI_PTT_PMU_DIRECTION_MASK, event->attr.config); ret = hisi_ptt_trace_valid_direction(val); if (ret < 0) return ret; val = FIELD_GET(HISI_PTT_PMU_TYPE_MASK, event->attr.config); ret = hisi_ptt_trace_valid_type(val); if (ret < 0) return ret; val = FIELD_GET(HISI_PTT_PMU_FORMAT_MASK, event->attr.config); return hisi_ptt_trace_valid_format(val); } static void *hisi_ptt_pmu_setup_aux(struct perf_event *event, void **pages, int nr_pages, bool overwrite) { struct hisi_ptt_pmu_buf *buf; struct page **pagelist; int i; if (overwrite) { dev_warn(event->pmu->dev, "Overwrite mode is not supported\n"); return NULL; } /* If the pages size less than buffers, we cannot start trace */ if (nr_pages < HISI_PTT_TRACE_TOTAL_BUF_SIZE / PAGE_SIZE) return NULL; buf = kzalloc(sizeof(*buf), GFP_KERNEL); if (!buf) return NULL; pagelist = kcalloc(nr_pages, sizeof(*pagelist), GFP_KERNEL); if (!pagelist) goto err; for (i = 0; i < nr_pages; i++) pagelist[i] = virt_to_page(pages[i]); buf->base = vmap(pagelist, nr_pages, VM_MAP, PAGE_KERNEL); if (!buf->base) { kfree(pagelist); goto err; } buf->nr_pages = nr_pages; buf->length = nr_pages * PAGE_SIZE; buf->pos = 0; kfree(pagelist); return buf; err: kfree(buf); return NULL; } static void hisi_ptt_pmu_free_aux(void *aux) { struct hisi_ptt_pmu_buf *buf = aux; vunmap(buf->base); kfree(buf); } static void hisi_ptt_pmu_start(struct perf_event *event, int flags) { struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu); struct perf_output_handle *handle = &hisi_ptt->trace_ctrl.handle; struct hw_perf_event *hwc = &event->hw; struct device *dev = event->pmu->dev; struct hisi_ptt_pmu_buf *buf; int cpu = event->cpu; int ret; hwc->state = 0; /* Serialize the perf process if user specified several CPUs */ spin_lock(&hisi_ptt->pmu_lock); if (hisi_ptt->trace_ctrl.started) { dev_dbg(dev, "trace has already started\n"); goto stop; } /* * Handle the interrupt on the same cpu which starts the trace to avoid * context mismatch. Otherwise we'll trigger the WARN from the perf * core in event_function_local(). If CPU passed is offline we'll fail * here, just log it since we can do nothing here. */ ret = irq_set_affinity(hisi_ptt->trace_irq, cpumask_of(cpu)); if (ret) dev_warn(dev, "failed to set the affinity of trace interrupt\n"); hisi_ptt->trace_ctrl.on_cpu = cpu; buf = perf_aux_output_begin(handle, event); if (!buf) { dev_dbg(dev, "aux output begin failed\n"); goto stop; } buf->pos = handle->head % buf->length; hisi_ptt_pmu_init_configs(hisi_ptt, event); ret = hisi_ptt_trace_start(hisi_ptt); if (ret) { dev_dbg(dev, "trace start failed, ret = %d\n", ret); perf_aux_output_end(handle, 0); goto stop; } spin_unlock(&hisi_ptt->pmu_lock); return; stop: event->hw.state |= PERF_HES_STOPPED; spin_unlock(&hisi_ptt->pmu_lock); } static void hisi_ptt_pmu_stop(struct perf_event *event, int flags) { struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu); struct hw_perf_event *hwc = &event->hw; if (hwc->state & PERF_HES_STOPPED) return; spin_lock(&hisi_ptt->pmu_lock); if (hisi_ptt->trace_ctrl.started) { hisi_ptt_trace_end(hisi_ptt); if (!hisi_ptt_wait_trace_hw_idle(hisi_ptt)) dev_warn(event->pmu->dev, "Device is still busy\n"); hisi_ptt_update_aux(hisi_ptt, hisi_ptt->trace_ctrl.buf_index, true); } spin_unlock(&hisi_ptt->pmu_lock); hwc->state |= PERF_HES_STOPPED; perf_event_update_userpage(event); hwc->state |= PERF_HES_UPTODATE; } static int hisi_ptt_pmu_add(struct perf_event *event, int flags) { struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu); struct hw_perf_event *hwc = &event->hw; int cpu = event->cpu; /* Only allow the cpus on the device's node to add the event */ if (!cpumask_test_cpu(cpu, cpumask_of_node(dev_to_node(&hisi_ptt->pdev->dev)))) return 0; hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; if (flags & PERF_EF_START) { hisi_ptt_pmu_start(event, PERF_EF_RELOAD); if (hwc->state & PERF_HES_STOPPED) return -EINVAL; } return 0; } static void hisi_ptt_pmu_del(struct perf_event *event, int flags) { hisi_ptt_pmu_stop(event, PERF_EF_UPDATE); } static void hisi_ptt_pmu_read(struct perf_event *event) { } static void hisi_ptt_remove_cpuhp_instance(void *hotplug_node) { cpuhp_state_remove_instance_nocalls(hisi_ptt_pmu_online, hotplug_node); } static void hisi_ptt_unregister_pmu(void *pmu) { perf_pmu_unregister(pmu); } static int hisi_ptt_register_pmu(struct hisi_ptt *hisi_ptt) { u16 core_id, sicl_id; char *pmu_name; u32 reg; int ret; ret = cpuhp_state_add_instance_nocalls(hisi_ptt_pmu_online, &hisi_ptt->hotplug_node); if (ret) return ret; ret = devm_add_action_or_reset(&hisi_ptt->pdev->dev, hisi_ptt_remove_cpuhp_instance, &hisi_ptt->hotplug_node); if (ret) return ret; mutex_init(&hisi_ptt->tune_lock); spin_lock_init(&hisi_ptt->pmu_lock); hisi_ptt->hisi_ptt_pmu = (struct pmu) { .module = THIS_MODULE, .capabilities = PERF_PMU_CAP_EXCLUSIVE | PERF_PMU_CAP_NO_EXCLUDE, .task_ctx_nr = perf_sw_context, .attr_groups = hisi_ptt_pmu_groups, .event_init = hisi_ptt_pmu_event_init, .setup_aux = hisi_ptt_pmu_setup_aux, .free_aux = hisi_ptt_pmu_free_aux, .start = hisi_ptt_pmu_start, .stop = hisi_ptt_pmu_stop, .add = hisi_ptt_pmu_add, .del = hisi_ptt_pmu_del, .read = hisi_ptt_pmu_read, }; reg = readl(hisi_ptt->iobase + HISI_PTT_LOCATION); core_id = FIELD_GET(HISI_PTT_CORE_ID, reg); sicl_id = FIELD_GET(HISI_PTT_SICL_ID, reg); pmu_name = devm_kasprintf(&hisi_ptt->pdev->dev, GFP_KERNEL, "hisi_ptt%u_%u", sicl_id, core_id); if (!pmu_name) return -ENOMEM; ret = perf_pmu_register(&hisi_ptt->hisi_ptt_pmu, pmu_name, -1); if (ret) return ret; return devm_add_action_or_reset(&hisi_ptt->pdev->dev, hisi_ptt_unregister_pmu, &hisi_ptt->hisi_ptt_pmu); } static void hisi_ptt_unregister_filter_update_notifier(void *data) { struct hisi_ptt *hisi_ptt = data; bus_unregister_notifier(&pci_bus_type, &hisi_ptt->hisi_ptt_nb); /* Cancel any work that has been queued */ cancel_delayed_work_sync(&hisi_ptt->work); } /* Register the bus notifier for dynamically updating the filter list */ static int hisi_ptt_register_filter_update_notifier(struct hisi_ptt *hisi_ptt) { int ret; hisi_ptt->hisi_ptt_nb.notifier_call = hisi_ptt_notifier_call; ret = bus_register_notifier(&pci_bus_type, &hisi_ptt->hisi_ptt_nb); if (ret) return ret; return devm_add_action_or_reset(&hisi_ptt->pdev->dev, hisi_ptt_unregister_filter_update_notifier, hisi_ptt); } /* * The DMA of PTT trace can only use direct mappings due to some * hardware restriction. Check whether there is no IOMMU or the * policy of the IOMMU domain is passthrough, otherwise the trace * cannot work. * * The PTT device is supposed to behind an ARM SMMUv3, which * should have passthrough the device by a quirk. */ static int hisi_ptt_check_iommu_mapping(struct pci_dev *pdev) { struct iommu_domain *iommu_domain; iommu_domain = iommu_get_domain_for_dev(&pdev->dev); if (!iommu_domain || iommu_domain->type == IOMMU_DOMAIN_IDENTITY) return 0; return -EOPNOTSUPP; } static int hisi_ptt_probe(struct pci_dev *pdev, const struct pci_device_id *id) { struct hisi_ptt *hisi_ptt; int ret; ret = hisi_ptt_check_iommu_mapping(pdev); if (ret) { pci_err(pdev, "requires direct DMA mappings\n"); return ret; } hisi_ptt = devm_kzalloc(&pdev->dev, sizeof(*hisi_ptt), GFP_KERNEL); if (!hisi_ptt) return -ENOMEM; hisi_ptt->pdev = pdev; pci_set_drvdata(pdev, hisi_ptt); ret = pcim_enable_device(pdev); if (ret) { pci_err(pdev, "failed to enable device, ret = %d\n", ret); return ret; } ret = pcim_iomap_regions(pdev, BIT(2), DRV_NAME); if (ret) { pci_err(pdev, "failed to remap io memory, ret = %d\n", ret); return ret; } hisi_ptt->iobase = pcim_iomap_table(pdev)[2]; ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64)); if (ret) { pci_err(pdev, "failed to set 64 bit dma mask, ret = %d\n", ret); return ret; } pci_set_master(pdev); ret = hisi_ptt_register_irq(hisi_ptt); if (ret) return ret; ret = hisi_ptt_init_ctrls(hisi_ptt); if (ret) { pci_err(pdev, "failed to init controls, ret = %d\n", ret); return ret; } ret = hisi_ptt_register_filter_update_notifier(hisi_ptt); if (ret) pci_warn(pdev, "failed to register filter update notifier, ret = %d", ret); ret = hisi_ptt_register_pmu(hisi_ptt); if (ret) { pci_err(pdev, "failed to register PMU device, ret = %d", ret); return ret; } ret = hisi_ptt_init_filter_attributes(hisi_ptt); if (ret) { pci_err(pdev, "failed to init sysfs filter attributes, ret = %d", ret); return ret; } return 0; } static const struct pci_device_id hisi_ptt_id_tbl[] = { { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa12e) }, { } }; MODULE_DEVICE_TABLE(pci, hisi_ptt_id_tbl); static struct pci_driver hisi_ptt_driver = { .name = DRV_NAME, .id_table = hisi_ptt_id_tbl, .probe = hisi_ptt_probe, }; static int hisi_ptt_cpu_teardown(unsigned int cpu, struct hlist_node *node) { struct hisi_ptt *hisi_ptt; struct device *dev; int target, src; hisi_ptt = hlist_entry_safe(node, struct hisi_ptt, hotplug_node); src = hisi_ptt->trace_ctrl.on_cpu; dev = hisi_ptt->hisi_ptt_pmu.dev; if (!hisi_ptt->trace_ctrl.started || src != cpu) return 0; target = cpumask_any_but(cpumask_of_node(dev_to_node(&hisi_ptt->pdev->dev)), cpu); if (target >= nr_cpu_ids) { dev_err(dev, "no available cpu for perf context migration\n"); return 0; } perf_pmu_migrate_context(&hisi_ptt->hisi_ptt_pmu, src, target); /* * Also make sure the interrupt bind to the migrated CPU as well. Warn * the user on failure here. */ if (irq_set_affinity(hisi_ptt->trace_irq, cpumask_of(target))) dev_warn(dev, "failed to set the affinity of trace interrupt\n"); hisi_ptt->trace_ctrl.on_cpu = target; return 0; } static int __init hisi_ptt_init(void) { int ret; ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRV_NAME, NULL, hisi_ptt_cpu_teardown); if (ret < 0) return ret; hisi_ptt_pmu_online = ret; ret = pci_register_driver(&hisi_ptt_driver); if (ret) cpuhp_remove_multi_state(hisi_ptt_pmu_online); return ret; } module_init(hisi_ptt_init); static void __exit hisi_ptt_exit(void) { pci_unregister_driver(&hisi_ptt_driver); cpuhp_remove_multi_state(hisi_ptt_pmu_online); } module_exit(hisi_ptt_exit); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Yicong Yang "); MODULE_DESCRIPTION("Driver for HiSilicon PCIe tune and trace device");