1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Driver for HiSilicon PCIe tune and trace device 4 * 5 * Copyright (c) 2022 HiSilicon Technologies Co., Ltd. 6 * Author: Yicong Yang <yangyicong@hisilicon.com> 7 */ 8 9 #include <linux/bitfield.h> 10 #include <linux/bitops.h> 11 #include <linux/cpuhotplug.h> 12 #include <linux/delay.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/interrupt.h> 15 #include <linux/io.h> 16 #include <linux/iommu.h> 17 #include <linux/iopoll.h> 18 #include <linux/module.h> 19 #include <linux/sysfs.h> 20 #include <linux/vmalloc.h> 21 22 #include "hisi_ptt.h" 23 24 /* Dynamic CPU hotplug state used by PTT */ 25 static enum cpuhp_state hisi_ptt_pmu_online; 26 27 static bool hisi_ptt_wait_tuning_finish(struct hisi_ptt *hisi_ptt) 28 { 29 u32 val; 30 31 return !readl_poll_timeout(hisi_ptt->iobase + HISI_PTT_TUNING_INT_STAT, 32 val, !(val & HISI_PTT_TUNING_INT_STAT_MASK), 33 HISI_PTT_WAIT_POLL_INTERVAL_US, 34 HISI_PTT_WAIT_TUNE_TIMEOUT_US); 35 } 36 37 static ssize_t hisi_ptt_tune_attr_show(struct device *dev, 38 struct device_attribute *attr, 39 char *buf) 40 { 41 struct hisi_ptt *hisi_ptt = to_hisi_ptt(dev_get_drvdata(dev)); 42 struct dev_ext_attribute *ext_attr; 43 struct hisi_ptt_tune_desc *desc; 44 u32 reg; 45 u16 val; 46 47 ext_attr = container_of(attr, struct dev_ext_attribute, attr); 48 desc = ext_attr->var; 49 50 mutex_lock(&hisi_ptt->tune_lock); 51 52 reg = readl(hisi_ptt->iobase + HISI_PTT_TUNING_CTRL); 53 reg &= ~(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB); 54 reg |= FIELD_PREP(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB, 55 desc->event_code); 56 writel(reg, hisi_ptt->iobase + HISI_PTT_TUNING_CTRL); 57 58 /* Write all 1 to indicates it's the read process */ 59 writel(~0U, hisi_ptt->iobase + HISI_PTT_TUNING_DATA); 60 61 if (!hisi_ptt_wait_tuning_finish(hisi_ptt)) { 62 mutex_unlock(&hisi_ptt->tune_lock); 63 return -ETIMEDOUT; 64 } 65 66 reg = readl(hisi_ptt->iobase + HISI_PTT_TUNING_DATA); 67 reg &= HISI_PTT_TUNING_DATA_VAL_MASK; 68 val = FIELD_GET(HISI_PTT_TUNING_DATA_VAL_MASK, reg); 69 70 mutex_unlock(&hisi_ptt->tune_lock); 71 return sysfs_emit(buf, "%u\n", val); 72 } 73 74 static ssize_t hisi_ptt_tune_attr_store(struct device *dev, 75 struct device_attribute *attr, 76 const char *buf, size_t count) 77 { 78 struct hisi_ptt *hisi_ptt = to_hisi_ptt(dev_get_drvdata(dev)); 79 struct dev_ext_attribute *ext_attr; 80 struct hisi_ptt_tune_desc *desc; 81 u32 reg; 82 u16 val; 83 84 ext_attr = container_of(attr, struct dev_ext_attribute, attr); 85 desc = ext_attr->var; 86 87 if (kstrtou16(buf, 10, &val)) 88 return -EINVAL; 89 90 mutex_lock(&hisi_ptt->tune_lock); 91 92 reg = readl(hisi_ptt->iobase + HISI_PTT_TUNING_CTRL); 93 reg &= ~(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB); 94 reg |= FIELD_PREP(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB, 95 desc->event_code); 96 writel(reg, hisi_ptt->iobase + HISI_PTT_TUNING_CTRL); 97 writel(FIELD_PREP(HISI_PTT_TUNING_DATA_VAL_MASK, val), 98 hisi_ptt->iobase + HISI_PTT_TUNING_DATA); 99 100 if (!hisi_ptt_wait_tuning_finish(hisi_ptt)) { 101 mutex_unlock(&hisi_ptt->tune_lock); 102 return -ETIMEDOUT; 103 } 104 105 mutex_unlock(&hisi_ptt->tune_lock); 106 return count; 107 } 108 109 #define HISI_PTT_TUNE_ATTR(_name, _val, _show, _store) \ 110 static struct hisi_ptt_tune_desc _name##_desc = { \ 111 .name = #_name, \ 112 .event_code = (_val), \ 113 }; \ 114 static struct dev_ext_attribute hisi_ptt_##_name##_attr = { \ 115 .attr = __ATTR(_name, 0600, _show, _store), \ 116 .var = &_name##_desc, \ 117 } 118 119 #define HISI_PTT_TUNE_ATTR_COMMON(_name, _val) \ 120 HISI_PTT_TUNE_ATTR(_name, _val, \ 121 hisi_ptt_tune_attr_show, \ 122 hisi_ptt_tune_attr_store) 123 124 /* 125 * The value of the tuning event are composed of two parts: main event code 126 * in BIT[0,15] and subevent code in BIT[16,23]. For example, qox_tx_cpl is 127 * a subevent of 'Tx path QoS control' which for tuning the weight of Tx 128 * completion TLPs. See hisi_ptt.rst documentation for more information. 129 */ 130 #define HISI_PTT_TUNE_QOS_TX_CPL (0x4 | (3 << 16)) 131 #define HISI_PTT_TUNE_QOS_TX_NP (0x4 | (4 << 16)) 132 #define HISI_PTT_TUNE_QOS_TX_P (0x4 | (5 << 16)) 133 #define HISI_PTT_TUNE_RX_ALLOC_BUF_LEVEL (0x5 | (6 << 16)) 134 #define HISI_PTT_TUNE_TX_ALLOC_BUF_LEVEL (0x5 | (7 << 16)) 135 136 HISI_PTT_TUNE_ATTR_COMMON(qos_tx_cpl, HISI_PTT_TUNE_QOS_TX_CPL); 137 HISI_PTT_TUNE_ATTR_COMMON(qos_tx_np, HISI_PTT_TUNE_QOS_TX_NP); 138 HISI_PTT_TUNE_ATTR_COMMON(qos_tx_p, HISI_PTT_TUNE_QOS_TX_P); 139 HISI_PTT_TUNE_ATTR_COMMON(rx_alloc_buf_level, HISI_PTT_TUNE_RX_ALLOC_BUF_LEVEL); 140 HISI_PTT_TUNE_ATTR_COMMON(tx_alloc_buf_level, HISI_PTT_TUNE_TX_ALLOC_BUF_LEVEL); 141 142 static struct attribute *hisi_ptt_tune_attrs[] = { 143 &hisi_ptt_qos_tx_cpl_attr.attr.attr, 144 &hisi_ptt_qos_tx_np_attr.attr.attr, 145 &hisi_ptt_qos_tx_p_attr.attr.attr, 146 &hisi_ptt_rx_alloc_buf_level_attr.attr.attr, 147 &hisi_ptt_tx_alloc_buf_level_attr.attr.attr, 148 NULL, 149 }; 150 151 static struct attribute_group hisi_ptt_tune_group = { 152 .name = "tune", 153 .attrs = hisi_ptt_tune_attrs, 154 }; 155 156 static u16 hisi_ptt_get_filter_val(u16 devid, bool is_port) 157 { 158 if (is_port) 159 return BIT(HISI_PCIE_CORE_PORT_ID(devid & 0xff)); 160 161 return devid; 162 } 163 164 static bool hisi_ptt_wait_trace_hw_idle(struct hisi_ptt *hisi_ptt) 165 { 166 u32 val; 167 168 return !readl_poll_timeout_atomic(hisi_ptt->iobase + HISI_PTT_TRACE_STS, 169 val, val & HISI_PTT_TRACE_IDLE, 170 HISI_PTT_WAIT_POLL_INTERVAL_US, 171 HISI_PTT_WAIT_TRACE_TIMEOUT_US); 172 } 173 174 static void hisi_ptt_wait_dma_reset_done(struct hisi_ptt *hisi_ptt) 175 { 176 u32 val; 177 178 readl_poll_timeout_atomic(hisi_ptt->iobase + HISI_PTT_TRACE_WR_STS, 179 val, !val, HISI_PTT_RESET_POLL_INTERVAL_US, 180 HISI_PTT_RESET_TIMEOUT_US); 181 } 182 183 static void hisi_ptt_trace_end(struct hisi_ptt *hisi_ptt) 184 { 185 writel(0, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); 186 hisi_ptt->trace_ctrl.started = false; 187 } 188 189 static int hisi_ptt_trace_start(struct hisi_ptt *hisi_ptt) 190 { 191 struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl; 192 u32 val; 193 int i; 194 195 /* Check device idle before start trace */ 196 if (!hisi_ptt_wait_trace_hw_idle(hisi_ptt)) { 197 pci_err(hisi_ptt->pdev, "Failed to start trace, the device is still busy\n"); 198 return -EBUSY; 199 } 200 201 ctrl->started = true; 202 203 /* Reset the DMA before start tracing */ 204 val = readl(hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); 205 val |= HISI_PTT_TRACE_CTRL_RST; 206 writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); 207 208 hisi_ptt_wait_dma_reset_done(hisi_ptt); 209 210 val = readl(hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); 211 val &= ~HISI_PTT_TRACE_CTRL_RST; 212 writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); 213 214 /* Reset the index of current buffer */ 215 hisi_ptt->trace_ctrl.buf_index = 0; 216 217 /* Zero the trace buffers */ 218 for (i = 0; i < HISI_PTT_TRACE_BUF_CNT; i++) 219 memset(ctrl->trace_buf[i].addr, 0, HISI_PTT_TRACE_BUF_SIZE); 220 221 /* Clear the interrupt status */ 222 writel(HISI_PTT_TRACE_INT_STAT_MASK, hisi_ptt->iobase + HISI_PTT_TRACE_INT_STAT); 223 writel(0, hisi_ptt->iobase + HISI_PTT_TRACE_INT_MASK); 224 225 /* Set the trace control register */ 226 val = FIELD_PREP(HISI_PTT_TRACE_CTRL_TYPE_SEL, ctrl->type); 227 val |= FIELD_PREP(HISI_PTT_TRACE_CTRL_RXTX_SEL, ctrl->direction); 228 val |= FIELD_PREP(HISI_PTT_TRACE_CTRL_DATA_FORMAT, ctrl->format); 229 val |= FIELD_PREP(HISI_PTT_TRACE_CTRL_TARGET_SEL, hisi_ptt->trace_ctrl.filter); 230 if (!hisi_ptt->trace_ctrl.is_port) 231 val |= HISI_PTT_TRACE_CTRL_FILTER_MODE; 232 233 /* Start the Trace */ 234 val |= HISI_PTT_TRACE_CTRL_EN; 235 writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); 236 237 return 0; 238 } 239 240 static int hisi_ptt_update_aux(struct hisi_ptt *hisi_ptt, int index, bool stop) 241 { 242 struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl; 243 struct perf_output_handle *handle = &ctrl->handle; 244 struct perf_event *event = handle->event; 245 struct hisi_ptt_pmu_buf *buf; 246 size_t size; 247 void *addr; 248 249 buf = perf_get_aux(handle); 250 if (!buf || !handle->size) 251 return -EINVAL; 252 253 addr = ctrl->trace_buf[ctrl->buf_index].addr; 254 255 /* 256 * If we're going to stop, read the size of already traced data from 257 * HISI_PTT_TRACE_WR_STS. Otherwise we're coming from the interrupt, 258 * the data size is always HISI_PTT_TRACE_BUF_SIZE. 259 */ 260 if (stop) { 261 u32 reg; 262 263 reg = readl(hisi_ptt->iobase + HISI_PTT_TRACE_WR_STS); 264 size = FIELD_GET(HISI_PTT_TRACE_WR_STS_WRITE, reg); 265 } else { 266 size = HISI_PTT_TRACE_BUF_SIZE; 267 } 268 269 memcpy(buf->base + buf->pos, addr, size); 270 buf->pos += size; 271 272 /* 273 * Just commit the traced data if we're going to stop. Otherwise if the 274 * resident AUX buffer cannot contain the data of next trace buffer, 275 * apply a new one. 276 */ 277 if (stop) { 278 perf_aux_output_end(handle, buf->pos); 279 } else if (buf->length - buf->pos < HISI_PTT_TRACE_BUF_SIZE) { 280 perf_aux_output_end(handle, buf->pos); 281 282 buf = perf_aux_output_begin(handle, event); 283 if (!buf) 284 return -EINVAL; 285 286 buf->pos = handle->head % buf->length; 287 if (buf->length - buf->pos < HISI_PTT_TRACE_BUF_SIZE) { 288 perf_aux_output_end(handle, 0); 289 return -EINVAL; 290 } 291 } 292 293 return 0; 294 } 295 296 static irqreturn_t hisi_ptt_isr(int irq, void *context) 297 { 298 struct hisi_ptt *hisi_ptt = context; 299 u32 status, buf_idx; 300 301 status = readl(hisi_ptt->iobase + HISI_PTT_TRACE_INT_STAT); 302 if (!(status & HISI_PTT_TRACE_INT_STAT_MASK)) 303 return IRQ_NONE; 304 305 buf_idx = ffs(status) - 1; 306 307 /* Clear the interrupt status of buffer @buf_idx */ 308 writel(status, hisi_ptt->iobase + HISI_PTT_TRACE_INT_STAT); 309 310 /* 311 * Update the AUX buffer and cache the current buffer index, 312 * as we need to know this and save the data when the trace 313 * is ended out of the interrupt handler. End the trace 314 * if the updating fails. 315 */ 316 if (hisi_ptt_update_aux(hisi_ptt, buf_idx, false)) 317 hisi_ptt_trace_end(hisi_ptt); 318 else 319 hisi_ptt->trace_ctrl.buf_index = (buf_idx + 1) % HISI_PTT_TRACE_BUF_CNT; 320 321 return IRQ_HANDLED; 322 } 323 324 static void hisi_ptt_irq_free_vectors(void *pdev) 325 { 326 pci_free_irq_vectors(pdev); 327 } 328 329 static int hisi_ptt_register_irq(struct hisi_ptt *hisi_ptt) 330 { 331 struct pci_dev *pdev = hisi_ptt->pdev; 332 int ret; 333 334 ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI); 335 if (ret < 0) { 336 pci_err(pdev, "failed to allocate irq vector, ret = %d\n", ret); 337 return ret; 338 } 339 340 ret = devm_add_action_or_reset(&pdev->dev, hisi_ptt_irq_free_vectors, pdev); 341 if (ret < 0) 342 return ret; 343 344 hisi_ptt->trace_irq = pci_irq_vector(pdev, HISI_PTT_TRACE_DMA_IRQ); 345 ret = devm_request_threaded_irq(&pdev->dev, hisi_ptt->trace_irq, 346 NULL, hisi_ptt_isr, 0, 347 DRV_NAME, hisi_ptt); 348 if (ret) { 349 pci_err(pdev, "failed to request irq %d, ret = %d\n", 350 hisi_ptt->trace_irq, ret); 351 return ret; 352 } 353 354 return 0; 355 } 356 357 static void hisi_ptt_del_free_filter(struct hisi_ptt *hisi_ptt, 358 struct hisi_ptt_filter_desc *filter) 359 { 360 if (filter->is_port) 361 hisi_ptt->port_mask &= ~hisi_ptt_get_filter_val(filter->devid, true); 362 363 list_del(&filter->list); 364 kfree(filter->name); 365 kfree(filter); 366 } 367 368 static struct hisi_ptt_filter_desc * 369 hisi_ptt_alloc_add_filter(struct hisi_ptt *hisi_ptt, u16 devid, bool is_port) 370 { 371 struct hisi_ptt_filter_desc *filter; 372 u8 devfn = devid & 0xff; 373 char *filter_name; 374 375 filter_name = kasprintf(GFP_KERNEL, "%04x:%02x:%02x.%d", pci_domain_nr(hisi_ptt->pdev->bus), 376 PCI_BUS_NUM(devid), PCI_SLOT(devfn), PCI_FUNC(devfn)); 377 if (!filter_name) { 378 pci_err(hisi_ptt->pdev, "failed to allocate name for filter %04x:%02x:%02x.%d\n", 379 pci_domain_nr(hisi_ptt->pdev->bus), PCI_BUS_NUM(devid), 380 PCI_SLOT(devfn), PCI_FUNC(devfn)); 381 return NULL; 382 } 383 384 filter = kzalloc(sizeof(*filter), GFP_KERNEL); 385 if (!filter) { 386 pci_err(hisi_ptt->pdev, "failed to add filter for %s\n", 387 filter_name); 388 kfree(filter_name); 389 return NULL; 390 } 391 392 filter->name = filter_name; 393 filter->is_port = is_port; 394 filter->devid = devid; 395 396 if (filter->is_port) { 397 list_add_tail(&filter->list, &hisi_ptt->port_filters); 398 399 /* Update the available port mask */ 400 hisi_ptt->port_mask |= hisi_ptt_get_filter_val(filter->devid, true); 401 } else { 402 list_add_tail(&filter->list, &hisi_ptt->req_filters); 403 } 404 405 return filter; 406 } 407 408 static ssize_t hisi_ptt_filter_show(struct device *dev, struct device_attribute *attr, 409 char *buf) 410 { 411 struct hisi_ptt_filter_desc *filter; 412 unsigned long filter_val; 413 414 filter = container_of(attr, struct hisi_ptt_filter_desc, attr); 415 filter_val = hisi_ptt_get_filter_val(filter->devid, filter->is_port) | 416 (filter->is_port ? HISI_PTT_PMU_FILTER_IS_PORT : 0); 417 418 return sysfs_emit(buf, "0x%05lx\n", filter_val); 419 } 420 421 static int hisi_ptt_create_rp_filter_attr(struct hisi_ptt *hisi_ptt, 422 struct hisi_ptt_filter_desc *filter) 423 { 424 struct kobject *kobj = &hisi_ptt->hisi_ptt_pmu.dev->kobj; 425 426 sysfs_attr_init(&filter->attr.attr); 427 filter->attr.attr.name = filter->name; 428 filter->attr.attr.mode = 0400; /* DEVICE_ATTR_ADMIN_RO */ 429 filter->attr.show = hisi_ptt_filter_show; 430 431 return sysfs_add_file_to_group(kobj, &filter->attr.attr, 432 HISI_PTT_RP_FILTERS_GRP_NAME); 433 } 434 435 static void hisi_ptt_remove_rp_filter_attr(struct hisi_ptt *hisi_ptt, 436 struct hisi_ptt_filter_desc *filter) 437 { 438 struct kobject *kobj = &hisi_ptt->hisi_ptt_pmu.dev->kobj; 439 440 sysfs_remove_file_from_group(kobj, &filter->attr.attr, 441 HISI_PTT_RP_FILTERS_GRP_NAME); 442 } 443 444 static int hisi_ptt_create_req_filter_attr(struct hisi_ptt *hisi_ptt, 445 struct hisi_ptt_filter_desc *filter) 446 { 447 struct kobject *kobj = &hisi_ptt->hisi_ptt_pmu.dev->kobj; 448 449 sysfs_attr_init(&filter->attr.attr); 450 filter->attr.attr.name = filter->name; 451 filter->attr.attr.mode = 0400; /* DEVICE_ATTR_ADMIN_RO */ 452 filter->attr.show = hisi_ptt_filter_show; 453 454 return sysfs_add_file_to_group(kobj, &filter->attr.attr, 455 HISI_PTT_REQ_FILTERS_GRP_NAME); 456 } 457 458 static void hisi_ptt_remove_req_filter_attr(struct hisi_ptt *hisi_ptt, 459 struct hisi_ptt_filter_desc *filter) 460 { 461 struct kobject *kobj = &hisi_ptt->hisi_ptt_pmu.dev->kobj; 462 463 sysfs_remove_file_from_group(kobj, &filter->attr.attr, 464 HISI_PTT_REQ_FILTERS_GRP_NAME); 465 } 466 467 static int hisi_ptt_create_filter_attr(struct hisi_ptt *hisi_ptt, 468 struct hisi_ptt_filter_desc *filter) 469 { 470 int ret; 471 472 if (filter->is_port) 473 ret = hisi_ptt_create_rp_filter_attr(hisi_ptt, filter); 474 else 475 ret = hisi_ptt_create_req_filter_attr(hisi_ptt, filter); 476 477 if (ret) 478 pci_err(hisi_ptt->pdev, "failed to create sysfs attribute for filter %s\n", 479 filter->name); 480 481 return ret; 482 } 483 484 static void hisi_ptt_remove_filter_attr(struct hisi_ptt *hisi_ptt, 485 struct hisi_ptt_filter_desc *filter) 486 { 487 if (filter->is_port) 488 hisi_ptt_remove_rp_filter_attr(hisi_ptt, filter); 489 else 490 hisi_ptt_remove_req_filter_attr(hisi_ptt, filter); 491 } 492 493 static void hisi_ptt_remove_all_filter_attributes(void *data) 494 { 495 struct hisi_ptt_filter_desc *filter; 496 struct hisi_ptt *hisi_ptt = data; 497 498 mutex_lock(&hisi_ptt->filter_lock); 499 500 list_for_each_entry(filter, &hisi_ptt->req_filters, list) 501 hisi_ptt_remove_filter_attr(hisi_ptt, filter); 502 503 list_for_each_entry(filter, &hisi_ptt->port_filters, list) 504 hisi_ptt_remove_filter_attr(hisi_ptt, filter); 505 506 hisi_ptt->sysfs_inited = false; 507 mutex_unlock(&hisi_ptt->filter_lock); 508 } 509 510 static int hisi_ptt_init_filter_attributes(struct hisi_ptt *hisi_ptt) 511 { 512 struct hisi_ptt_filter_desc *filter; 513 int ret; 514 515 mutex_lock(&hisi_ptt->filter_lock); 516 517 /* 518 * Register the reset callback in the first stage. In reset we traverse 519 * the filters list to remove the sysfs attributes so the callback can 520 * be called safely even without below filter attributes creation. 521 */ 522 ret = devm_add_action(&hisi_ptt->pdev->dev, 523 hisi_ptt_remove_all_filter_attributes, 524 hisi_ptt); 525 if (ret) 526 goto out; 527 528 list_for_each_entry(filter, &hisi_ptt->port_filters, list) { 529 ret = hisi_ptt_create_filter_attr(hisi_ptt, filter); 530 if (ret) 531 goto out; 532 } 533 534 list_for_each_entry(filter, &hisi_ptt->req_filters, list) { 535 ret = hisi_ptt_create_filter_attr(hisi_ptt, filter); 536 if (ret) 537 goto out; 538 } 539 540 hisi_ptt->sysfs_inited = true; 541 out: 542 mutex_unlock(&hisi_ptt->filter_lock); 543 return ret; 544 } 545 546 static void hisi_ptt_update_filters(struct work_struct *work) 547 { 548 struct delayed_work *delayed_work = to_delayed_work(work); 549 struct hisi_ptt_filter_update_info info; 550 struct hisi_ptt_filter_desc *filter; 551 struct hisi_ptt *hisi_ptt; 552 553 hisi_ptt = container_of(delayed_work, struct hisi_ptt, work); 554 555 if (!mutex_trylock(&hisi_ptt->filter_lock)) { 556 schedule_delayed_work(&hisi_ptt->work, HISI_PTT_WORK_DELAY_MS); 557 return; 558 } 559 560 while (kfifo_get(&hisi_ptt->filter_update_kfifo, &info)) { 561 if (info.is_add) { 562 /* 563 * Notify the users if failed to add this filter, others 564 * still work and available. See the comments in 565 * hisi_ptt_init_filters(). 566 */ 567 filter = hisi_ptt_alloc_add_filter(hisi_ptt, info.devid, info.is_port); 568 if (!filter) 569 continue; 570 571 /* 572 * If filters' sysfs entries hasn't been initialized, 573 * then we're still at probe stage. Add the filters to 574 * the list and later hisi_ptt_init_filter_attributes() 575 * will create sysfs attributes for all the filters. 576 */ 577 if (hisi_ptt->sysfs_inited && 578 hisi_ptt_create_filter_attr(hisi_ptt, filter)) { 579 hisi_ptt_del_free_filter(hisi_ptt, filter); 580 continue; 581 } 582 } else { 583 struct hisi_ptt_filter_desc *tmp; 584 struct list_head *target_list; 585 586 target_list = info.is_port ? &hisi_ptt->port_filters : 587 &hisi_ptt->req_filters; 588 589 list_for_each_entry_safe(filter, tmp, target_list, list) 590 if (filter->devid == info.devid) { 591 if (hisi_ptt->sysfs_inited) 592 hisi_ptt_remove_filter_attr(hisi_ptt, filter); 593 594 hisi_ptt_del_free_filter(hisi_ptt, filter); 595 break; 596 } 597 } 598 } 599 600 mutex_unlock(&hisi_ptt->filter_lock); 601 } 602 603 /* 604 * A PCI bus notifier is used here for dynamically updating the filter 605 * list. 606 */ 607 static int hisi_ptt_notifier_call(struct notifier_block *nb, unsigned long action, 608 void *data) 609 { 610 struct hisi_ptt *hisi_ptt = container_of(nb, struct hisi_ptt, hisi_ptt_nb); 611 struct hisi_ptt_filter_update_info info; 612 struct pci_dev *pdev, *root_port; 613 struct device *dev = data; 614 u32 port_devid; 615 616 pdev = to_pci_dev(dev); 617 root_port = pcie_find_root_port(pdev); 618 if (!root_port) 619 return 0; 620 621 port_devid = PCI_DEVID(root_port->bus->number, root_port->devfn); 622 if (port_devid < hisi_ptt->lower_bdf || 623 port_devid > hisi_ptt->upper_bdf) 624 return 0; 625 626 info.is_port = pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT; 627 info.devid = PCI_DEVID(pdev->bus->number, pdev->devfn); 628 629 switch (action) { 630 case BUS_NOTIFY_ADD_DEVICE: 631 info.is_add = true; 632 break; 633 case BUS_NOTIFY_DEL_DEVICE: 634 info.is_add = false; 635 break; 636 default: 637 return 0; 638 } 639 640 /* 641 * The FIFO size is 16 which is sufficient for almost all the cases, 642 * since each PCIe core will have most 8 Root Ports (typically only 643 * 1~4 Root Ports). On failure log the failed filter and let user 644 * handle it. 645 */ 646 if (kfifo_in_spinlocked(&hisi_ptt->filter_update_kfifo, &info, 1, 647 &hisi_ptt->filter_update_lock)) 648 schedule_delayed_work(&hisi_ptt->work, 0); 649 else 650 pci_warn(hisi_ptt->pdev, 651 "filter update fifo overflow for target %s\n", 652 pci_name(pdev)); 653 654 return 0; 655 } 656 657 static int hisi_ptt_init_filters(struct pci_dev *pdev, void *data) 658 { 659 struct pci_dev *root_port = pcie_find_root_port(pdev); 660 struct hisi_ptt_filter_desc *filter; 661 struct hisi_ptt *hisi_ptt = data; 662 u32 port_devid; 663 664 if (!root_port) 665 return 0; 666 667 port_devid = PCI_DEVID(root_port->bus->number, root_port->devfn); 668 if (port_devid < hisi_ptt->lower_bdf || 669 port_devid > hisi_ptt->upper_bdf) 670 return 0; 671 672 /* 673 * We won't fail the probe if filter allocation failed here. The filters 674 * should be partial initialized and users would know which filter fails 675 * through the log. Other functions of PTT device are still available. 676 */ 677 filter = hisi_ptt_alloc_add_filter(hisi_ptt, PCI_DEVID(pdev->bus->number, pdev->devfn), 678 pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT); 679 if (!filter) 680 return -ENOMEM; 681 682 return 0; 683 } 684 685 static void hisi_ptt_release_filters(void *data) 686 { 687 struct hisi_ptt_filter_desc *filter, *tmp; 688 struct hisi_ptt *hisi_ptt = data; 689 690 list_for_each_entry_safe(filter, tmp, &hisi_ptt->req_filters, list) 691 hisi_ptt_del_free_filter(hisi_ptt, filter); 692 693 list_for_each_entry_safe(filter, tmp, &hisi_ptt->port_filters, list) 694 hisi_ptt_del_free_filter(hisi_ptt, filter); 695 } 696 697 static int hisi_ptt_config_trace_buf(struct hisi_ptt *hisi_ptt) 698 { 699 struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl; 700 struct device *dev = &hisi_ptt->pdev->dev; 701 int i; 702 703 ctrl->trace_buf = devm_kcalloc(dev, HISI_PTT_TRACE_BUF_CNT, 704 sizeof(*ctrl->trace_buf), GFP_KERNEL); 705 if (!ctrl->trace_buf) 706 return -ENOMEM; 707 708 for (i = 0; i < HISI_PTT_TRACE_BUF_CNT; ++i) { 709 ctrl->trace_buf[i].addr = dmam_alloc_coherent(dev, HISI_PTT_TRACE_BUF_SIZE, 710 &ctrl->trace_buf[i].dma, 711 GFP_KERNEL); 712 if (!ctrl->trace_buf[i].addr) 713 return -ENOMEM; 714 } 715 716 /* Configure the trace DMA buffer */ 717 for (i = 0; i < HISI_PTT_TRACE_BUF_CNT; i++) { 718 writel(lower_32_bits(ctrl->trace_buf[i].dma), 719 hisi_ptt->iobase + HISI_PTT_TRACE_ADDR_BASE_LO_0 + 720 i * HISI_PTT_TRACE_ADDR_STRIDE); 721 writel(upper_32_bits(ctrl->trace_buf[i].dma), 722 hisi_ptt->iobase + HISI_PTT_TRACE_ADDR_BASE_HI_0 + 723 i * HISI_PTT_TRACE_ADDR_STRIDE); 724 } 725 writel(HISI_PTT_TRACE_BUF_SIZE, hisi_ptt->iobase + HISI_PTT_TRACE_ADDR_SIZE); 726 727 return 0; 728 } 729 730 static int hisi_ptt_init_ctrls(struct hisi_ptt *hisi_ptt) 731 { 732 struct pci_dev *pdev = hisi_ptt->pdev; 733 struct pci_bus *bus; 734 int ret; 735 u32 reg; 736 737 INIT_DELAYED_WORK(&hisi_ptt->work, hisi_ptt_update_filters); 738 INIT_KFIFO(hisi_ptt->filter_update_kfifo); 739 spin_lock_init(&hisi_ptt->filter_update_lock); 740 741 INIT_LIST_HEAD(&hisi_ptt->port_filters); 742 INIT_LIST_HEAD(&hisi_ptt->req_filters); 743 mutex_init(&hisi_ptt->filter_lock); 744 745 ret = hisi_ptt_config_trace_buf(hisi_ptt); 746 if (ret) 747 return ret; 748 749 /* 750 * The device range register provides the information about the root 751 * ports which the RCiEP can control and trace. The RCiEP and the root 752 * ports which it supports are on the same PCIe core, with same domain 753 * number but maybe different bus number. The device range register 754 * will tell us which root ports we can support, Bit[31:16] indicates 755 * the upper BDF numbers of the root port, while Bit[15:0] indicates 756 * the lower. 757 */ 758 reg = readl(hisi_ptt->iobase + HISI_PTT_DEVICE_RANGE); 759 hisi_ptt->upper_bdf = FIELD_GET(HISI_PTT_DEVICE_RANGE_UPPER, reg); 760 hisi_ptt->lower_bdf = FIELD_GET(HISI_PTT_DEVICE_RANGE_LOWER, reg); 761 762 bus = pci_find_bus(pci_domain_nr(pdev->bus), PCI_BUS_NUM(hisi_ptt->upper_bdf)); 763 if (bus) 764 pci_walk_bus(bus, hisi_ptt_init_filters, hisi_ptt); 765 766 ret = devm_add_action_or_reset(&pdev->dev, hisi_ptt_release_filters, hisi_ptt); 767 if (ret) 768 return ret; 769 770 hisi_ptt->trace_ctrl.on_cpu = -1; 771 return 0; 772 } 773 774 static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, 775 char *buf) 776 { 777 struct hisi_ptt *hisi_ptt = to_hisi_ptt(dev_get_drvdata(dev)); 778 const cpumask_t *cpumask = cpumask_of_node(dev_to_node(&hisi_ptt->pdev->dev)); 779 780 return cpumap_print_to_pagebuf(true, buf, cpumask); 781 } 782 static DEVICE_ATTR_RO(cpumask); 783 784 static struct attribute *hisi_ptt_cpumask_attrs[] = { 785 &dev_attr_cpumask.attr, 786 NULL 787 }; 788 789 static const struct attribute_group hisi_ptt_cpumask_attr_group = { 790 .attrs = hisi_ptt_cpumask_attrs, 791 }; 792 793 /* 794 * Bit 19 indicates the filter type, 1 for Root Port filter and 0 for Requester 795 * filter. Bit[15:0] indicates the filter value, for Root Port filter it's 796 * a bit mask of desired ports and for Requester filter it's the Requester ID 797 * of the desired PCIe function. Bit[18:16] is reserved for extension. 798 * 799 * See hisi_ptt.rst documentation for detailed information. 800 */ 801 PMU_FORMAT_ATTR(filter, "config:0-19"); 802 PMU_FORMAT_ATTR(direction, "config:20-23"); 803 PMU_FORMAT_ATTR(type, "config:24-31"); 804 PMU_FORMAT_ATTR(format, "config:32-35"); 805 806 static struct attribute *hisi_ptt_pmu_format_attrs[] = { 807 &format_attr_filter.attr, 808 &format_attr_direction.attr, 809 &format_attr_type.attr, 810 &format_attr_format.attr, 811 NULL 812 }; 813 814 static struct attribute_group hisi_ptt_pmu_format_group = { 815 .name = "format", 816 .attrs = hisi_ptt_pmu_format_attrs, 817 }; 818 819 static ssize_t hisi_ptt_filter_multiselect_show(struct device *dev, 820 struct device_attribute *attr, 821 char *buf) 822 { 823 struct dev_ext_attribute *ext_attr; 824 825 ext_attr = container_of(attr, struct dev_ext_attribute, attr); 826 return sysfs_emit(buf, "%s\n", (char *)ext_attr->var); 827 } 828 829 static struct dev_ext_attribute root_port_filters_multiselect = { 830 .attr = { 831 .attr = { .name = "multiselect", .mode = 0400 }, 832 .show = hisi_ptt_filter_multiselect_show, 833 }, 834 .var = "1", 835 }; 836 837 static struct attribute *hisi_ptt_pmu_root_ports_attrs[] = { 838 &root_port_filters_multiselect.attr.attr, 839 NULL 840 }; 841 842 static struct attribute_group hisi_ptt_pmu_root_ports_group = { 843 .name = HISI_PTT_RP_FILTERS_GRP_NAME, 844 .attrs = hisi_ptt_pmu_root_ports_attrs, 845 }; 846 847 static struct dev_ext_attribute requester_filters_multiselect = { 848 .attr = { 849 .attr = { .name = "multiselect", .mode = 0400 }, 850 .show = hisi_ptt_filter_multiselect_show, 851 }, 852 .var = "0", 853 }; 854 855 static struct attribute *hisi_ptt_pmu_requesters_attrs[] = { 856 &requester_filters_multiselect.attr.attr, 857 NULL 858 }; 859 860 static struct attribute_group hisi_ptt_pmu_requesters_group = { 861 .name = HISI_PTT_REQ_FILTERS_GRP_NAME, 862 .attrs = hisi_ptt_pmu_requesters_attrs, 863 }; 864 865 static const struct attribute_group *hisi_ptt_pmu_groups[] = { 866 &hisi_ptt_cpumask_attr_group, 867 &hisi_ptt_pmu_format_group, 868 &hisi_ptt_tune_group, 869 &hisi_ptt_pmu_root_ports_group, 870 &hisi_ptt_pmu_requesters_group, 871 NULL 872 }; 873 874 static int hisi_ptt_trace_valid_direction(u32 val) 875 { 876 /* 877 * The direction values have different effects according to the data 878 * format (specified in the parentheses). TLP set A/B means different 879 * set of TLP types. See hisi_ptt.rst documentation for more details. 880 */ 881 static const u32 hisi_ptt_trace_available_direction[] = { 882 0, /* inbound(4DW) or reserved(8DW) */ 883 1, /* outbound(4DW) */ 884 2, /* {in, out}bound(4DW) or inbound(8DW), TLP set A */ 885 3, /* {in, out}bound(4DW) or inbound(8DW), TLP set B */ 886 }; 887 int i; 888 889 for (i = 0; i < ARRAY_SIZE(hisi_ptt_trace_available_direction); i++) { 890 if (val == hisi_ptt_trace_available_direction[i]) 891 return 0; 892 } 893 894 return -EINVAL; 895 } 896 897 static int hisi_ptt_trace_valid_type(u32 val) 898 { 899 /* Different types can be set simultaneously */ 900 static const u32 hisi_ptt_trace_available_type[] = { 901 1, /* posted_request */ 902 2, /* non-posted_request */ 903 4, /* completion */ 904 }; 905 int i; 906 907 if (!val) 908 return -EINVAL; 909 910 /* 911 * Walk the available list and clear the valid bits of 912 * the config. If there is any resident bit after the 913 * walk then the config is invalid. 914 */ 915 for (i = 0; i < ARRAY_SIZE(hisi_ptt_trace_available_type); i++) 916 val &= ~hisi_ptt_trace_available_type[i]; 917 918 if (val) 919 return -EINVAL; 920 921 return 0; 922 } 923 924 static int hisi_ptt_trace_valid_format(u32 val) 925 { 926 static const u32 hisi_ptt_trace_availble_format[] = { 927 0, /* 4DW */ 928 1, /* 8DW */ 929 }; 930 int i; 931 932 for (i = 0; i < ARRAY_SIZE(hisi_ptt_trace_availble_format); i++) { 933 if (val == hisi_ptt_trace_availble_format[i]) 934 return 0; 935 } 936 937 return -EINVAL; 938 } 939 940 static int hisi_ptt_trace_valid_filter(struct hisi_ptt *hisi_ptt, u64 config) 941 { 942 unsigned long val, port_mask = hisi_ptt->port_mask; 943 struct hisi_ptt_filter_desc *filter; 944 int ret = 0; 945 946 hisi_ptt->trace_ctrl.is_port = FIELD_GET(HISI_PTT_PMU_FILTER_IS_PORT, config); 947 val = FIELD_GET(HISI_PTT_PMU_FILTER_VAL_MASK, config); 948 949 /* 950 * Port filters are defined as bit mask. For port filters, check 951 * the bits in the @val are within the range of hisi_ptt->port_mask 952 * and whether it's empty or not, otherwise user has specified 953 * some unsupported root ports. 954 * 955 * For Requester ID filters, walk the available filter list to see 956 * whether we have one matched. 957 */ 958 mutex_lock(&hisi_ptt->filter_lock); 959 if (!hisi_ptt->trace_ctrl.is_port) { 960 list_for_each_entry(filter, &hisi_ptt->req_filters, list) { 961 if (val == hisi_ptt_get_filter_val(filter->devid, filter->is_port)) 962 goto out; 963 } 964 } else if (bitmap_subset(&val, &port_mask, BITS_PER_LONG)) { 965 goto out; 966 } 967 968 ret = -EINVAL; 969 out: 970 mutex_unlock(&hisi_ptt->filter_lock); 971 return ret; 972 } 973 974 static void hisi_ptt_pmu_init_configs(struct hisi_ptt *hisi_ptt, struct perf_event *event) 975 { 976 struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl; 977 u32 val; 978 979 val = FIELD_GET(HISI_PTT_PMU_FILTER_VAL_MASK, event->attr.config); 980 hisi_ptt->trace_ctrl.filter = val; 981 982 val = FIELD_GET(HISI_PTT_PMU_DIRECTION_MASK, event->attr.config); 983 ctrl->direction = val; 984 985 val = FIELD_GET(HISI_PTT_PMU_TYPE_MASK, event->attr.config); 986 ctrl->type = val; 987 988 val = FIELD_GET(HISI_PTT_PMU_FORMAT_MASK, event->attr.config); 989 ctrl->format = val; 990 } 991 992 static int hisi_ptt_pmu_event_init(struct perf_event *event) 993 { 994 struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu); 995 int ret; 996 u32 val; 997 998 if (event->cpu < 0) { 999 dev_dbg(event->pmu->dev, "Per-task mode not supported\n"); 1000 return -EOPNOTSUPP; 1001 } 1002 1003 if (event->attr.type != hisi_ptt->hisi_ptt_pmu.type) 1004 return -ENOENT; 1005 1006 ret = hisi_ptt_trace_valid_filter(hisi_ptt, event->attr.config); 1007 if (ret < 0) 1008 return ret; 1009 1010 val = FIELD_GET(HISI_PTT_PMU_DIRECTION_MASK, event->attr.config); 1011 ret = hisi_ptt_trace_valid_direction(val); 1012 if (ret < 0) 1013 return ret; 1014 1015 val = FIELD_GET(HISI_PTT_PMU_TYPE_MASK, event->attr.config); 1016 ret = hisi_ptt_trace_valid_type(val); 1017 if (ret < 0) 1018 return ret; 1019 1020 val = FIELD_GET(HISI_PTT_PMU_FORMAT_MASK, event->attr.config); 1021 return hisi_ptt_trace_valid_format(val); 1022 } 1023 1024 static void *hisi_ptt_pmu_setup_aux(struct perf_event *event, void **pages, 1025 int nr_pages, bool overwrite) 1026 { 1027 struct hisi_ptt_pmu_buf *buf; 1028 struct page **pagelist; 1029 int i; 1030 1031 if (overwrite) { 1032 dev_warn(event->pmu->dev, "Overwrite mode is not supported\n"); 1033 return NULL; 1034 } 1035 1036 /* If the pages size less than buffers, we cannot start trace */ 1037 if (nr_pages < HISI_PTT_TRACE_TOTAL_BUF_SIZE / PAGE_SIZE) 1038 return NULL; 1039 1040 buf = kzalloc(sizeof(*buf), GFP_KERNEL); 1041 if (!buf) 1042 return NULL; 1043 1044 pagelist = kcalloc(nr_pages, sizeof(*pagelist), GFP_KERNEL); 1045 if (!pagelist) 1046 goto err; 1047 1048 for (i = 0; i < nr_pages; i++) 1049 pagelist[i] = virt_to_page(pages[i]); 1050 1051 buf->base = vmap(pagelist, nr_pages, VM_MAP, PAGE_KERNEL); 1052 if (!buf->base) { 1053 kfree(pagelist); 1054 goto err; 1055 } 1056 1057 buf->nr_pages = nr_pages; 1058 buf->length = nr_pages * PAGE_SIZE; 1059 buf->pos = 0; 1060 1061 kfree(pagelist); 1062 return buf; 1063 err: 1064 kfree(buf); 1065 return NULL; 1066 } 1067 1068 static void hisi_ptt_pmu_free_aux(void *aux) 1069 { 1070 struct hisi_ptt_pmu_buf *buf = aux; 1071 1072 vunmap(buf->base); 1073 kfree(buf); 1074 } 1075 1076 static void hisi_ptt_pmu_start(struct perf_event *event, int flags) 1077 { 1078 struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu); 1079 struct perf_output_handle *handle = &hisi_ptt->trace_ctrl.handle; 1080 struct hw_perf_event *hwc = &event->hw; 1081 struct device *dev = event->pmu->dev; 1082 struct hisi_ptt_pmu_buf *buf; 1083 int cpu = event->cpu; 1084 int ret; 1085 1086 hwc->state = 0; 1087 1088 /* Serialize the perf process if user specified several CPUs */ 1089 spin_lock(&hisi_ptt->pmu_lock); 1090 if (hisi_ptt->trace_ctrl.started) { 1091 dev_dbg(dev, "trace has already started\n"); 1092 goto stop; 1093 } 1094 1095 /* 1096 * Handle the interrupt on the same cpu which starts the trace to avoid 1097 * context mismatch. Otherwise we'll trigger the WARN from the perf 1098 * core in event_function_local(). If CPU passed is offline we'll fail 1099 * here, just log it since we can do nothing here. 1100 */ 1101 ret = irq_set_affinity(hisi_ptt->trace_irq, cpumask_of(cpu)); 1102 if (ret) 1103 dev_warn(dev, "failed to set the affinity of trace interrupt\n"); 1104 1105 hisi_ptt->trace_ctrl.on_cpu = cpu; 1106 1107 buf = perf_aux_output_begin(handle, event); 1108 if (!buf) { 1109 dev_dbg(dev, "aux output begin failed\n"); 1110 goto stop; 1111 } 1112 1113 buf->pos = handle->head % buf->length; 1114 1115 hisi_ptt_pmu_init_configs(hisi_ptt, event); 1116 1117 ret = hisi_ptt_trace_start(hisi_ptt); 1118 if (ret) { 1119 dev_dbg(dev, "trace start failed, ret = %d\n", ret); 1120 perf_aux_output_end(handle, 0); 1121 goto stop; 1122 } 1123 1124 spin_unlock(&hisi_ptt->pmu_lock); 1125 return; 1126 stop: 1127 event->hw.state |= PERF_HES_STOPPED; 1128 spin_unlock(&hisi_ptt->pmu_lock); 1129 } 1130 1131 static void hisi_ptt_pmu_stop(struct perf_event *event, int flags) 1132 { 1133 struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu); 1134 struct hw_perf_event *hwc = &event->hw; 1135 1136 if (hwc->state & PERF_HES_STOPPED) 1137 return; 1138 1139 spin_lock(&hisi_ptt->pmu_lock); 1140 if (hisi_ptt->trace_ctrl.started) { 1141 hisi_ptt_trace_end(hisi_ptt); 1142 1143 if (!hisi_ptt_wait_trace_hw_idle(hisi_ptt)) 1144 dev_warn(event->pmu->dev, "Device is still busy\n"); 1145 1146 hisi_ptt_update_aux(hisi_ptt, hisi_ptt->trace_ctrl.buf_index, true); 1147 } 1148 spin_unlock(&hisi_ptt->pmu_lock); 1149 1150 hwc->state |= PERF_HES_STOPPED; 1151 perf_event_update_userpage(event); 1152 hwc->state |= PERF_HES_UPTODATE; 1153 } 1154 1155 static int hisi_ptt_pmu_add(struct perf_event *event, int flags) 1156 { 1157 struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu); 1158 struct hw_perf_event *hwc = &event->hw; 1159 int cpu = event->cpu; 1160 1161 /* Only allow the cpus on the device's node to add the event */ 1162 if (!cpumask_test_cpu(cpu, cpumask_of_node(dev_to_node(&hisi_ptt->pdev->dev)))) 1163 return 0; 1164 1165 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 1166 1167 if (flags & PERF_EF_START) { 1168 hisi_ptt_pmu_start(event, PERF_EF_RELOAD); 1169 if (hwc->state & PERF_HES_STOPPED) 1170 return -EINVAL; 1171 } 1172 1173 return 0; 1174 } 1175 1176 static void hisi_ptt_pmu_del(struct perf_event *event, int flags) 1177 { 1178 hisi_ptt_pmu_stop(event, PERF_EF_UPDATE); 1179 } 1180 1181 static void hisi_ptt_remove_cpuhp_instance(void *hotplug_node) 1182 { 1183 cpuhp_state_remove_instance_nocalls(hisi_ptt_pmu_online, hotplug_node); 1184 } 1185 1186 static void hisi_ptt_unregister_pmu(void *pmu) 1187 { 1188 perf_pmu_unregister(pmu); 1189 } 1190 1191 static int hisi_ptt_register_pmu(struct hisi_ptt *hisi_ptt) 1192 { 1193 u16 core_id, sicl_id; 1194 char *pmu_name; 1195 u32 reg; 1196 int ret; 1197 1198 ret = cpuhp_state_add_instance_nocalls(hisi_ptt_pmu_online, 1199 &hisi_ptt->hotplug_node); 1200 if (ret) 1201 return ret; 1202 1203 ret = devm_add_action_or_reset(&hisi_ptt->pdev->dev, 1204 hisi_ptt_remove_cpuhp_instance, 1205 &hisi_ptt->hotplug_node); 1206 if (ret) 1207 return ret; 1208 1209 mutex_init(&hisi_ptt->tune_lock); 1210 spin_lock_init(&hisi_ptt->pmu_lock); 1211 1212 hisi_ptt->hisi_ptt_pmu = (struct pmu) { 1213 .module = THIS_MODULE, 1214 .capabilities = PERF_PMU_CAP_EXCLUSIVE | PERF_PMU_CAP_NO_EXCLUDE, 1215 .task_ctx_nr = perf_sw_context, 1216 .attr_groups = hisi_ptt_pmu_groups, 1217 .event_init = hisi_ptt_pmu_event_init, 1218 .setup_aux = hisi_ptt_pmu_setup_aux, 1219 .free_aux = hisi_ptt_pmu_free_aux, 1220 .start = hisi_ptt_pmu_start, 1221 .stop = hisi_ptt_pmu_stop, 1222 .add = hisi_ptt_pmu_add, 1223 .del = hisi_ptt_pmu_del, 1224 }; 1225 1226 reg = readl(hisi_ptt->iobase + HISI_PTT_LOCATION); 1227 core_id = FIELD_GET(HISI_PTT_CORE_ID, reg); 1228 sicl_id = FIELD_GET(HISI_PTT_SICL_ID, reg); 1229 1230 pmu_name = devm_kasprintf(&hisi_ptt->pdev->dev, GFP_KERNEL, "hisi_ptt%u_%u", 1231 sicl_id, core_id); 1232 if (!pmu_name) 1233 return -ENOMEM; 1234 1235 ret = perf_pmu_register(&hisi_ptt->hisi_ptt_pmu, pmu_name, -1); 1236 if (ret) 1237 return ret; 1238 1239 return devm_add_action_or_reset(&hisi_ptt->pdev->dev, 1240 hisi_ptt_unregister_pmu, 1241 &hisi_ptt->hisi_ptt_pmu); 1242 } 1243 1244 static void hisi_ptt_unregister_filter_update_notifier(void *data) 1245 { 1246 struct hisi_ptt *hisi_ptt = data; 1247 1248 bus_unregister_notifier(&pci_bus_type, &hisi_ptt->hisi_ptt_nb); 1249 1250 /* Cancel any work that has been queued */ 1251 cancel_delayed_work_sync(&hisi_ptt->work); 1252 } 1253 1254 /* Register the bus notifier for dynamically updating the filter list */ 1255 static int hisi_ptt_register_filter_update_notifier(struct hisi_ptt *hisi_ptt) 1256 { 1257 int ret; 1258 1259 hisi_ptt->hisi_ptt_nb.notifier_call = hisi_ptt_notifier_call; 1260 ret = bus_register_notifier(&pci_bus_type, &hisi_ptt->hisi_ptt_nb); 1261 if (ret) 1262 return ret; 1263 1264 return devm_add_action_or_reset(&hisi_ptt->pdev->dev, 1265 hisi_ptt_unregister_filter_update_notifier, 1266 hisi_ptt); 1267 } 1268 1269 /* 1270 * The DMA of PTT trace can only use direct mappings due to some 1271 * hardware restriction. Check whether there is no IOMMU or the 1272 * policy of the IOMMU domain is passthrough, otherwise the trace 1273 * cannot work. 1274 * 1275 * The PTT device is supposed to behind an ARM SMMUv3, which 1276 * should have passthrough the device by a quirk. 1277 */ 1278 static int hisi_ptt_check_iommu_mapping(struct pci_dev *pdev) 1279 { 1280 struct iommu_domain *iommu_domain; 1281 1282 iommu_domain = iommu_get_domain_for_dev(&pdev->dev); 1283 if (!iommu_domain || iommu_domain->type == IOMMU_DOMAIN_IDENTITY) 1284 return 0; 1285 1286 return -EOPNOTSUPP; 1287 } 1288 1289 static int hisi_ptt_probe(struct pci_dev *pdev, 1290 const struct pci_device_id *id) 1291 { 1292 struct hisi_ptt *hisi_ptt; 1293 int ret; 1294 1295 ret = hisi_ptt_check_iommu_mapping(pdev); 1296 if (ret) { 1297 pci_err(pdev, "requires direct DMA mappings\n"); 1298 return ret; 1299 } 1300 1301 hisi_ptt = devm_kzalloc(&pdev->dev, sizeof(*hisi_ptt), GFP_KERNEL); 1302 if (!hisi_ptt) 1303 return -ENOMEM; 1304 1305 hisi_ptt->pdev = pdev; 1306 pci_set_drvdata(pdev, hisi_ptt); 1307 1308 ret = pcim_enable_device(pdev); 1309 if (ret) { 1310 pci_err(pdev, "failed to enable device, ret = %d\n", ret); 1311 return ret; 1312 } 1313 1314 ret = pcim_iomap_regions(pdev, BIT(2), DRV_NAME); 1315 if (ret) { 1316 pci_err(pdev, "failed to remap io memory, ret = %d\n", ret); 1317 return ret; 1318 } 1319 1320 hisi_ptt->iobase = pcim_iomap_table(pdev)[2]; 1321 1322 ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64)); 1323 if (ret) { 1324 pci_err(pdev, "failed to set 64 bit dma mask, ret = %d\n", ret); 1325 return ret; 1326 } 1327 1328 pci_set_master(pdev); 1329 1330 ret = hisi_ptt_register_irq(hisi_ptt); 1331 if (ret) 1332 return ret; 1333 1334 ret = hisi_ptt_init_ctrls(hisi_ptt); 1335 if (ret) { 1336 pci_err(pdev, "failed to init controls, ret = %d\n", ret); 1337 return ret; 1338 } 1339 1340 ret = hisi_ptt_register_filter_update_notifier(hisi_ptt); 1341 if (ret) 1342 pci_warn(pdev, "failed to register filter update notifier, ret = %d", ret); 1343 1344 ret = hisi_ptt_register_pmu(hisi_ptt); 1345 if (ret) { 1346 pci_err(pdev, "failed to register PMU device, ret = %d", ret); 1347 return ret; 1348 } 1349 1350 ret = hisi_ptt_init_filter_attributes(hisi_ptt); 1351 if (ret) { 1352 pci_err(pdev, "failed to init sysfs filter attributes, ret = %d", ret); 1353 return ret; 1354 } 1355 1356 return 0; 1357 } 1358 1359 static const struct pci_device_id hisi_ptt_id_tbl[] = { 1360 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa12e) }, 1361 { } 1362 }; 1363 MODULE_DEVICE_TABLE(pci, hisi_ptt_id_tbl); 1364 1365 static struct pci_driver hisi_ptt_driver = { 1366 .name = DRV_NAME, 1367 .id_table = hisi_ptt_id_tbl, 1368 .probe = hisi_ptt_probe, 1369 }; 1370 1371 static int hisi_ptt_cpu_teardown(unsigned int cpu, struct hlist_node *node) 1372 { 1373 struct hisi_ptt *hisi_ptt; 1374 struct device *dev; 1375 int target, src; 1376 1377 hisi_ptt = hlist_entry_safe(node, struct hisi_ptt, hotplug_node); 1378 src = hisi_ptt->trace_ctrl.on_cpu; 1379 dev = hisi_ptt->hisi_ptt_pmu.dev; 1380 1381 if (!hisi_ptt->trace_ctrl.started || src != cpu) 1382 return 0; 1383 1384 target = cpumask_any_but(cpumask_of_node(dev_to_node(&hisi_ptt->pdev->dev)), cpu); 1385 if (target >= nr_cpu_ids) { 1386 dev_err(dev, "no available cpu for perf context migration\n"); 1387 return 0; 1388 } 1389 1390 perf_pmu_migrate_context(&hisi_ptt->hisi_ptt_pmu, src, target); 1391 1392 /* 1393 * Also make sure the interrupt bind to the migrated CPU as well. Warn 1394 * the user on failure here. 1395 */ 1396 if (irq_set_affinity(hisi_ptt->trace_irq, cpumask_of(target))) 1397 dev_warn(dev, "failed to set the affinity of trace interrupt\n"); 1398 1399 hisi_ptt->trace_ctrl.on_cpu = target; 1400 return 0; 1401 } 1402 1403 static int __init hisi_ptt_init(void) 1404 { 1405 int ret; 1406 1407 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRV_NAME, NULL, 1408 hisi_ptt_cpu_teardown); 1409 if (ret < 0) 1410 return ret; 1411 hisi_ptt_pmu_online = ret; 1412 1413 ret = pci_register_driver(&hisi_ptt_driver); 1414 if (ret) 1415 cpuhp_remove_multi_state(hisi_ptt_pmu_online); 1416 1417 return ret; 1418 } 1419 module_init(hisi_ptt_init); 1420 1421 static void __exit hisi_ptt_exit(void) 1422 { 1423 pci_unregister_driver(&hisi_ptt_driver); 1424 cpuhp_remove_multi_state(hisi_ptt_pmu_online); 1425 } 1426 module_exit(hisi_ptt_exit); 1427 1428 MODULE_LICENSE("GPL"); 1429 MODULE_AUTHOR("Yicong Yang <yangyicong@hisilicon.com>"); 1430 MODULE_DESCRIPTION("Driver for HiSilicon PCIe tune and trace device"); 1431