1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright IBM Corp. 2012 4 * 5 * Author(s): 6 * Jan Glauber <jang@linux.vnet.ibm.com> 7 */ 8 9 #define KMSG_COMPONENT "zpci" 10 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 11 12 #include <linux/compat.h> 13 #include <linux/kernel.h> 14 #include <linux/miscdevice.h> 15 #include <linux/slab.h> 16 #include <linux/err.h> 17 #include <linux/delay.h> 18 #include <linux/pci.h> 19 #include <linux/uaccess.h> 20 #include <asm/asm-extable.h> 21 #include <asm/pci_debug.h> 22 #include <asm/pci_clp.h> 23 #include <asm/clp.h> 24 #include <uapi/asm/clp.h> 25 26 #include "pci_bus.h" 27 28 bool zpci_unique_uid; 29 30 void update_uid_checking(bool new) 31 { 32 if (zpci_unique_uid != new) 33 zpci_dbg(3, "uid checking:%d\n", new); 34 35 zpci_unique_uid = new; 36 } 37 38 static inline void zpci_err_clp(unsigned int rsp, int rc) 39 { 40 struct { 41 unsigned int rsp; 42 int rc; 43 } __packed data = {rsp, rc}; 44 45 zpci_err_hex(&data, sizeof(data)); 46 } 47 48 /* 49 * Call Logical Processor with c=1, lps=0 and command 1 50 * to get the bit mask of installed logical processors 51 */ 52 static inline int clp_get_ilp(unsigned long *ilp) 53 { 54 unsigned long mask; 55 int cc = 3; 56 57 asm volatile ( 58 " .insn rrf,0xb9a00000,%[mask],%[cmd],8,0\n" 59 "0: ipm %[cc]\n" 60 " srl %[cc],28\n" 61 "1:\n" 62 EX_TABLE(0b, 1b) 63 : [cc] "+d" (cc), [mask] "=d" (mask) : [cmd] "a" (1) 64 : "cc"); 65 *ilp = mask; 66 return cc; 67 } 68 69 /* 70 * Call Logical Processor with c=0, the give constant lps and an lpcb request. 71 */ 72 static __always_inline int clp_req(void *data, unsigned int lps) 73 { 74 struct { u8 _[CLP_BLK_SIZE]; } *req = data; 75 u64 ignored; 76 int cc = 3; 77 78 asm volatile ( 79 " .insn rrf,0xb9a00000,%[ign],%[req],0,%[lps]\n" 80 "0: ipm %[cc]\n" 81 " srl %[cc],28\n" 82 "1:\n" 83 EX_TABLE(0b, 1b) 84 : [cc] "+d" (cc), [ign] "=d" (ignored), "+m" (*req) 85 : [req] "a" (req), [lps] "i" (lps) 86 : "cc"); 87 return cc; 88 } 89 90 static void *clp_alloc_block(gfp_t gfp_mask) 91 { 92 return (void *) __get_free_pages(gfp_mask, get_order(CLP_BLK_SIZE)); 93 } 94 95 static void clp_free_block(void *ptr) 96 { 97 free_pages((unsigned long) ptr, get_order(CLP_BLK_SIZE)); 98 } 99 100 static void clp_store_query_pci_fngrp(struct zpci_dev *zdev, 101 struct clp_rsp_query_pci_grp *response) 102 { 103 zdev->tlb_refresh = response->refresh; 104 zdev->dma_mask = response->dasm; 105 zdev->msi_addr = response->msia; 106 zdev->max_msi = response->noi; 107 zdev->fmb_update = response->mui; 108 zdev->version = response->version; 109 zdev->maxstbl = response->maxstbl; 110 zdev->dtsm = response->dtsm; 111 112 switch (response->version) { 113 case 1: 114 zdev->max_bus_speed = PCIE_SPEED_5_0GT; 115 break; 116 default: 117 zdev->max_bus_speed = PCI_SPEED_UNKNOWN; 118 break; 119 } 120 } 121 122 static int clp_query_pci_fngrp(struct zpci_dev *zdev, u8 pfgid) 123 { 124 struct clp_req_rsp_query_pci_grp *rrb; 125 int rc; 126 127 rrb = clp_alloc_block(GFP_KERNEL); 128 if (!rrb) 129 return -ENOMEM; 130 131 memset(rrb, 0, sizeof(*rrb)); 132 rrb->request.hdr.len = sizeof(rrb->request); 133 rrb->request.hdr.cmd = CLP_QUERY_PCI_FNGRP; 134 rrb->response.hdr.len = sizeof(rrb->response); 135 rrb->request.pfgid = pfgid; 136 137 rc = clp_req(rrb, CLP_LPS_PCI); 138 if (!rc && rrb->response.hdr.rsp == CLP_RC_OK) 139 clp_store_query_pci_fngrp(zdev, &rrb->response); 140 else { 141 zpci_err("Q PCI FGRP:\n"); 142 zpci_err_clp(rrb->response.hdr.rsp, rc); 143 rc = -EIO; 144 } 145 clp_free_block(rrb); 146 return rc; 147 } 148 149 static int clp_store_query_pci_fn(struct zpci_dev *zdev, 150 struct clp_rsp_query_pci *response) 151 { 152 int i; 153 154 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 155 zdev->bars[i].val = le32_to_cpu(response->bar[i]); 156 zdev->bars[i].size = response->bar_size[i]; 157 } 158 zdev->start_dma = response->sdma; 159 zdev->end_dma = response->edma; 160 zdev->pchid = response->pchid; 161 zdev->pfgid = response->pfgid; 162 zdev->pft = response->pft; 163 zdev->vfn = response->vfn; 164 zdev->port = response->port; 165 zdev->uid = response->uid; 166 zdev->fmb_length = sizeof(u32) * response->fmb_len; 167 zdev->rid_available = response->rid_avail; 168 zdev->is_physfn = response->is_physfn; 169 if (!s390_pci_no_rid && zdev->rid_available) 170 zdev->devfn = response->rid & ZPCI_RID_MASK_DEVFN; 171 172 memcpy(zdev->pfip, response->pfip, sizeof(zdev->pfip)); 173 if (response->util_str_avail) { 174 memcpy(zdev->util_str, response->util_str, 175 sizeof(zdev->util_str)); 176 zdev->util_str_avail = 1; 177 } 178 zdev->mio_capable = response->mio_addr_avail; 179 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 180 if (!(response->mio.valid & (1 << (PCI_STD_NUM_BARS - i - 1)))) 181 continue; 182 183 zdev->bars[i].mio_wb = (void __iomem *) response->mio.addr[i].wb; 184 zdev->bars[i].mio_wt = (void __iomem *) response->mio.addr[i].wt; 185 } 186 return 0; 187 } 188 189 int clp_query_pci_fn(struct zpci_dev *zdev) 190 { 191 struct clp_req_rsp_query_pci *rrb; 192 int rc; 193 194 rrb = clp_alloc_block(GFP_KERNEL); 195 if (!rrb) 196 return -ENOMEM; 197 198 memset(rrb, 0, sizeof(*rrb)); 199 rrb->request.hdr.len = sizeof(rrb->request); 200 rrb->request.hdr.cmd = CLP_QUERY_PCI_FN; 201 rrb->response.hdr.len = sizeof(rrb->response); 202 rrb->request.fh = zdev->fh; 203 204 rc = clp_req(rrb, CLP_LPS_PCI); 205 if (!rc && rrb->response.hdr.rsp == CLP_RC_OK) { 206 rc = clp_store_query_pci_fn(zdev, &rrb->response); 207 if (rc) 208 goto out; 209 rc = clp_query_pci_fngrp(zdev, rrb->response.pfgid); 210 } else { 211 zpci_err("Q PCI FN:\n"); 212 zpci_err_clp(rrb->response.hdr.rsp, rc); 213 rc = -EIO; 214 } 215 out: 216 clp_free_block(rrb); 217 return rc; 218 } 219 220 /** 221 * clp_set_pci_fn() - Execute a command on a PCI function 222 * @zdev: Function that will be affected 223 * @fh: Out parameter for updated function handle 224 * @nr_dma_as: DMA address space number 225 * @command: The command code to execute 226 * 227 * Returns: 0 on success, < 0 for Linux errors (e.g. -ENOMEM), and 228 * > 0 for non-success platform responses 229 */ 230 static int clp_set_pci_fn(struct zpci_dev *zdev, u32 *fh, u8 nr_dma_as, u8 command) 231 { 232 struct clp_req_rsp_set_pci *rrb; 233 int rc, retries = 100; 234 u32 gisa = 0; 235 236 *fh = 0; 237 rrb = clp_alloc_block(GFP_KERNEL); 238 if (!rrb) 239 return -ENOMEM; 240 241 if (command != CLP_SET_DISABLE_PCI_FN) 242 gisa = zdev->gisa; 243 244 do { 245 memset(rrb, 0, sizeof(*rrb)); 246 rrb->request.hdr.len = sizeof(rrb->request); 247 rrb->request.hdr.cmd = CLP_SET_PCI_FN; 248 rrb->response.hdr.len = sizeof(rrb->response); 249 rrb->request.fh = zdev->fh; 250 rrb->request.oc = command; 251 rrb->request.ndas = nr_dma_as; 252 rrb->request.gisa = gisa; 253 254 rc = clp_req(rrb, CLP_LPS_PCI); 255 if (rrb->response.hdr.rsp == CLP_RC_SETPCIFN_BUSY) { 256 retries--; 257 if (retries < 0) 258 break; 259 msleep(20); 260 } 261 } while (rrb->response.hdr.rsp == CLP_RC_SETPCIFN_BUSY); 262 263 if (!rc && rrb->response.hdr.rsp == CLP_RC_OK) { 264 *fh = rrb->response.fh; 265 } else { 266 zpci_err("Set PCI FN:\n"); 267 zpci_err_clp(rrb->response.hdr.rsp, rc); 268 if (!rc) 269 rc = rrb->response.hdr.rsp; 270 } 271 clp_free_block(rrb); 272 return rc; 273 } 274 275 int clp_setup_writeback_mio(void) 276 { 277 struct clp_req_rsp_slpc_pci *rrb; 278 u8 wb_bit_pos; 279 int rc; 280 281 rrb = clp_alloc_block(GFP_KERNEL); 282 if (!rrb) 283 return -ENOMEM; 284 285 memset(rrb, 0, sizeof(*rrb)); 286 rrb->request.hdr.len = sizeof(rrb->request); 287 rrb->request.hdr.cmd = CLP_SLPC; 288 rrb->response.hdr.len = sizeof(rrb->response); 289 290 rc = clp_req(rrb, CLP_LPS_PCI); 291 if (!rc && rrb->response.hdr.rsp == CLP_RC_OK) { 292 if (rrb->response.vwb) { 293 wb_bit_pos = rrb->response.mio_wb; 294 set_bit_inv(wb_bit_pos, &mio_wb_bit_mask); 295 zpci_dbg(3, "wb bit: %d\n", wb_bit_pos); 296 } else { 297 zpci_dbg(3, "wb bit: n.a.\n"); 298 } 299 300 } else { 301 zpci_err("SLPC PCI:\n"); 302 zpci_err_clp(rrb->response.hdr.rsp, rc); 303 rc = -EIO; 304 } 305 clp_free_block(rrb); 306 return rc; 307 } 308 309 int clp_enable_fh(struct zpci_dev *zdev, u32 *fh, u8 nr_dma_as) 310 { 311 int rc; 312 313 rc = clp_set_pci_fn(zdev, fh, nr_dma_as, CLP_SET_ENABLE_PCI_FN); 314 zpci_dbg(3, "ena fid:%x, fh:%x, rc:%d\n", zdev->fid, *fh, rc); 315 if (!rc && zpci_use_mio(zdev)) { 316 rc = clp_set_pci_fn(zdev, fh, nr_dma_as, CLP_SET_ENABLE_MIO); 317 zpci_dbg(3, "ena mio fid:%x, fh:%x, rc:%d\n", 318 zdev->fid, *fh, rc); 319 if (rc) 320 clp_disable_fh(zdev, fh); 321 } 322 return rc; 323 } 324 325 int clp_disable_fh(struct zpci_dev *zdev, u32 *fh) 326 { 327 int rc; 328 329 if (!zdev_enabled(zdev)) 330 return 0; 331 332 rc = clp_set_pci_fn(zdev, fh, 0, CLP_SET_DISABLE_PCI_FN); 333 zpci_dbg(3, "dis fid:%x, fh:%x, rc:%d\n", zdev->fid, *fh, rc); 334 return rc; 335 } 336 337 static int clp_list_pci_req(struct clp_req_rsp_list_pci *rrb, 338 u64 *resume_token, int *nentries) 339 { 340 int rc; 341 342 memset(rrb, 0, sizeof(*rrb)); 343 rrb->request.hdr.len = sizeof(rrb->request); 344 rrb->request.hdr.cmd = CLP_LIST_PCI; 345 /* store as many entries as possible */ 346 rrb->response.hdr.len = CLP_BLK_SIZE - LIST_PCI_HDR_LEN; 347 rrb->request.resume_token = *resume_token; 348 349 /* Get PCI function handle list */ 350 rc = clp_req(rrb, CLP_LPS_PCI); 351 if (rc || rrb->response.hdr.rsp != CLP_RC_OK) { 352 zpci_err("List PCI FN:\n"); 353 zpci_err_clp(rrb->response.hdr.rsp, rc); 354 return -EIO; 355 } 356 357 update_uid_checking(rrb->response.uid_checking); 358 WARN_ON_ONCE(rrb->response.entry_size != 359 sizeof(struct clp_fh_list_entry)); 360 361 *nentries = (rrb->response.hdr.len - LIST_PCI_HDR_LEN) / 362 rrb->response.entry_size; 363 *resume_token = rrb->response.resume_token; 364 365 return rc; 366 } 367 368 static int clp_list_pci(struct clp_req_rsp_list_pci *rrb, void *data, 369 void (*cb)(struct clp_fh_list_entry *, void *)) 370 { 371 u64 resume_token = 0; 372 int nentries, i, rc; 373 374 do { 375 rc = clp_list_pci_req(rrb, &resume_token, &nentries); 376 if (rc) 377 return rc; 378 for (i = 0; i < nentries; i++) 379 cb(&rrb->response.fh_list[i], data); 380 } while (resume_token); 381 382 return rc; 383 } 384 385 static int clp_find_pci(struct clp_req_rsp_list_pci *rrb, u32 fid, 386 struct clp_fh_list_entry *entry) 387 { 388 struct clp_fh_list_entry *fh_list; 389 u64 resume_token = 0; 390 int nentries, i, rc; 391 392 do { 393 rc = clp_list_pci_req(rrb, &resume_token, &nentries); 394 if (rc) 395 return rc; 396 fh_list = rrb->response.fh_list; 397 for (i = 0; i < nentries; i++) { 398 if (fh_list[i].fid == fid) { 399 *entry = fh_list[i]; 400 return 0; 401 } 402 } 403 } while (resume_token); 404 405 return -ENODEV; 406 } 407 408 static void __clp_add(struct clp_fh_list_entry *entry, void *data) 409 { 410 struct zpci_dev *zdev; 411 412 if (!entry->vendor_id) 413 return; 414 415 zdev = get_zdev_by_fid(entry->fid); 416 if (zdev) { 417 zpci_zdev_put(zdev); 418 return; 419 } 420 zpci_create_device(entry->fid, entry->fh, entry->config_state); 421 } 422 423 int clp_scan_pci_devices(void) 424 { 425 struct clp_req_rsp_list_pci *rrb; 426 int rc; 427 428 rrb = clp_alloc_block(GFP_KERNEL); 429 if (!rrb) 430 return -ENOMEM; 431 432 rc = clp_list_pci(rrb, NULL, __clp_add); 433 434 clp_free_block(rrb); 435 return rc; 436 } 437 438 /* 439 * Get the current function handle of the function matching @fid 440 */ 441 int clp_refresh_fh(u32 fid, u32 *fh) 442 { 443 struct clp_req_rsp_list_pci *rrb; 444 struct clp_fh_list_entry entry; 445 int rc; 446 447 rrb = clp_alloc_block(GFP_NOWAIT); 448 if (!rrb) 449 return -ENOMEM; 450 451 rc = clp_find_pci(rrb, fid, &entry); 452 if (!rc) 453 *fh = entry.fh; 454 455 clp_free_block(rrb); 456 return rc; 457 } 458 459 int clp_get_state(u32 fid, enum zpci_state *state) 460 { 461 struct clp_req_rsp_list_pci *rrb; 462 struct clp_fh_list_entry entry; 463 int rc; 464 465 rrb = clp_alloc_block(GFP_ATOMIC); 466 if (!rrb) 467 return -ENOMEM; 468 469 rc = clp_find_pci(rrb, fid, &entry); 470 if (!rc) { 471 *state = entry.config_state; 472 } else if (rc == -ENODEV) { 473 *state = ZPCI_FN_STATE_RESERVED; 474 rc = 0; 475 } 476 477 clp_free_block(rrb); 478 return rc; 479 } 480 481 static int clp_base_slpc(struct clp_req *req, struct clp_req_rsp_slpc *lpcb) 482 { 483 unsigned long limit = PAGE_SIZE - sizeof(lpcb->request); 484 485 if (lpcb->request.hdr.len != sizeof(lpcb->request) || 486 lpcb->response.hdr.len > limit) 487 return -EINVAL; 488 return clp_req(lpcb, CLP_LPS_BASE) ? -EOPNOTSUPP : 0; 489 } 490 491 static int clp_base_command(struct clp_req *req, struct clp_req_hdr *lpcb) 492 { 493 switch (lpcb->cmd) { 494 case 0x0001: /* store logical-processor characteristics */ 495 return clp_base_slpc(req, (void *) lpcb); 496 default: 497 return -EINVAL; 498 } 499 } 500 501 static int clp_pci_slpc(struct clp_req *req, struct clp_req_rsp_slpc_pci *lpcb) 502 { 503 unsigned long limit = PAGE_SIZE - sizeof(lpcb->request); 504 505 if (lpcb->request.hdr.len != sizeof(lpcb->request) || 506 lpcb->response.hdr.len > limit) 507 return -EINVAL; 508 return clp_req(lpcb, CLP_LPS_PCI) ? -EOPNOTSUPP : 0; 509 } 510 511 static int clp_pci_list(struct clp_req *req, struct clp_req_rsp_list_pci *lpcb) 512 { 513 unsigned long limit = PAGE_SIZE - sizeof(lpcb->request); 514 515 if (lpcb->request.hdr.len != sizeof(lpcb->request) || 516 lpcb->response.hdr.len > limit) 517 return -EINVAL; 518 if (lpcb->request.reserved2 != 0) 519 return -EINVAL; 520 return clp_req(lpcb, CLP_LPS_PCI) ? -EOPNOTSUPP : 0; 521 } 522 523 static int clp_pci_query(struct clp_req *req, 524 struct clp_req_rsp_query_pci *lpcb) 525 { 526 unsigned long limit = PAGE_SIZE - sizeof(lpcb->request); 527 528 if (lpcb->request.hdr.len != sizeof(lpcb->request) || 529 lpcb->response.hdr.len > limit) 530 return -EINVAL; 531 if (lpcb->request.reserved2 != 0 || lpcb->request.reserved3 != 0) 532 return -EINVAL; 533 return clp_req(lpcb, CLP_LPS_PCI) ? -EOPNOTSUPP : 0; 534 } 535 536 static int clp_pci_query_grp(struct clp_req *req, 537 struct clp_req_rsp_query_pci_grp *lpcb) 538 { 539 unsigned long limit = PAGE_SIZE - sizeof(lpcb->request); 540 541 if (lpcb->request.hdr.len != sizeof(lpcb->request) || 542 lpcb->response.hdr.len > limit) 543 return -EINVAL; 544 if (lpcb->request.reserved2 != 0 || lpcb->request.reserved3 != 0 || 545 lpcb->request.reserved4 != 0) 546 return -EINVAL; 547 return clp_req(lpcb, CLP_LPS_PCI) ? -EOPNOTSUPP : 0; 548 } 549 550 static int clp_pci_command(struct clp_req *req, struct clp_req_hdr *lpcb) 551 { 552 switch (lpcb->cmd) { 553 case 0x0001: /* store logical-processor characteristics */ 554 return clp_pci_slpc(req, (void *) lpcb); 555 case 0x0002: /* list PCI functions */ 556 return clp_pci_list(req, (void *) lpcb); 557 case 0x0003: /* query PCI function */ 558 return clp_pci_query(req, (void *) lpcb); 559 case 0x0004: /* query PCI function group */ 560 return clp_pci_query_grp(req, (void *) lpcb); 561 default: 562 return -EINVAL; 563 } 564 } 565 566 static int clp_normal_command(struct clp_req *req) 567 { 568 struct clp_req_hdr *lpcb; 569 void __user *uptr; 570 int rc; 571 572 rc = -EINVAL; 573 if (req->lps != 0 && req->lps != 2) 574 goto out; 575 576 rc = -ENOMEM; 577 lpcb = clp_alloc_block(GFP_KERNEL); 578 if (!lpcb) 579 goto out; 580 581 rc = -EFAULT; 582 uptr = (void __force __user *)(unsigned long) req->data_p; 583 if (copy_from_user(lpcb, uptr, PAGE_SIZE) != 0) 584 goto out_free; 585 586 rc = -EINVAL; 587 if (lpcb->fmt != 0 || lpcb->reserved1 != 0 || lpcb->reserved2 != 0) 588 goto out_free; 589 590 switch (req->lps) { 591 case 0: 592 rc = clp_base_command(req, lpcb); 593 break; 594 case 2: 595 rc = clp_pci_command(req, lpcb); 596 break; 597 } 598 if (rc) 599 goto out_free; 600 601 rc = -EFAULT; 602 if (copy_to_user(uptr, lpcb, PAGE_SIZE) != 0) 603 goto out_free; 604 605 rc = 0; 606 607 out_free: 608 clp_free_block(lpcb); 609 out: 610 return rc; 611 } 612 613 static int clp_immediate_command(struct clp_req *req) 614 { 615 void __user *uptr; 616 unsigned long ilp; 617 int exists; 618 619 if (req->cmd > 1 || clp_get_ilp(&ilp) != 0) 620 return -EINVAL; 621 622 uptr = (void __force __user *)(unsigned long) req->data_p; 623 if (req->cmd == 0) { 624 /* Command code 0: test for a specific processor */ 625 exists = test_bit_inv(req->lps, &ilp); 626 return put_user(exists, (int __user *) uptr); 627 } 628 /* Command code 1: return bit mask of installed processors */ 629 return put_user(ilp, (unsigned long __user *) uptr); 630 } 631 632 static long clp_misc_ioctl(struct file *filp, unsigned int cmd, 633 unsigned long arg) 634 { 635 struct clp_req req; 636 void __user *argp; 637 638 if (cmd != CLP_SYNC) 639 return -EINVAL; 640 641 argp = is_compat_task() ? compat_ptr(arg) : (void __user *) arg; 642 if (copy_from_user(&req, argp, sizeof(req))) 643 return -EFAULT; 644 if (req.r != 0) 645 return -EINVAL; 646 return req.c ? clp_immediate_command(&req) : clp_normal_command(&req); 647 } 648 649 static int clp_misc_release(struct inode *inode, struct file *filp) 650 { 651 return 0; 652 } 653 654 static const struct file_operations clp_misc_fops = { 655 .owner = THIS_MODULE, 656 .open = nonseekable_open, 657 .release = clp_misc_release, 658 .unlocked_ioctl = clp_misc_ioctl, 659 .compat_ioctl = clp_misc_ioctl, 660 .llseek = no_llseek, 661 }; 662 663 static struct miscdevice clp_misc_device = { 664 .minor = MISC_DYNAMIC_MINOR, 665 .name = "clp", 666 .fops = &clp_misc_fops, 667 }; 668 669 static int __init clp_misc_init(void) 670 { 671 return misc_register(&clp_misc_device); 672 } 673 674 device_initcall(clp_misc_init); 675