1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. 4 */ 5 6 /* 7 * Oracle Data Analytics Accelerator (DAX) 8 * 9 * DAX is a coprocessor which resides on the SPARC M7 (DAX1) and M8 10 * (DAX2) processor chips, and has direct access to the CPU's L3 11 * caches as well as physical memory. It can perform several 12 * operations on data streams with various input and output formats. 13 * The driver provides a transport mechanism only and has limited 14 * knowledge of the various opcodes and data formats. A user space 15 * library provides high level services and translates these into low 16 * level commands which are then passed into the driver and 17 * subsequently the hypervisor and the coprocessor. The library is 18 * the recommended way for applications to use the coprocessor, and 19 * the driver interface is not intended for general use. 20 * 21 * See Documentation/sparc/oradax/oracle-dax.rst for more details. 22 */ 23 24 #include <linux/uaccess.h> 25 #include <linux/module.h> 26 #include <linux/delay.h> 27 #include <linux/cdev.h> 28 #include <linux/slab.h> 29 #include <linux/mm.h> 30 31 #include <asm/hypervisor.h> 32 #include <asm/mdesc.h> 33 #include <asm/oradax.h> 34 35 MODULE_LICENSE("GPL"); 36 MODULE_DESCRIPTION("Driver for Oracle Data Analytics Accelerator"); 37 38 #define DAX_DBG_FLG_BASIC 0x01 39 #define DAX_DBG_FLG_STAT 0x02 40 #define DAX_DBG_FLG_INFO 0x04 41 #define DAX_DBG_FLG_ALL 0xff 42 43 #define dax_err(fmt, ...) pr_err("%s: " fmt "\n", __func__, ##__VA_ARGS__) 44 #define dax_info(fmt, ...) pr_info("%s: " fmt "\n", __func__, ##__VA_ARGS__) 45 46 #define dax_dbg(fmt, ...) do { \ 47 if (dax_debug & DAX_DBG_FLG_BASIC)\ 48 dax_info(fmt, ##__VA_ARGS__); \ 49 } while (0) 50 #define dax_stat_dbg(fmt, ...) do { \ 51 if (dax_debug & DAX_DBG_FLG_STAT) \ 52 dax_info(fmt, ##__VA_ARGS__); \ 53 } while (0) 54 #define dax_info_dbg(fmt, ...) do { \ 55 if (dax_debug & DAX_DBG_FLG_INFO) \ 56 dax_info(fmt, ##__VA_ARGS__); \ 57 } while (0) 58 59 #define DAX1_MINOR 1 60 #define DAX1_MAJOR 1 61 #define DAX2_MINOR 0 62 #define DAX2_MAJOR 2 63 64 #define DAX1_STR "ORCL,sun4v-dax" 65 #define DAX2_STR "ORCL,sun4v-dax2" 66 67 #define DAX_CA_ELEMS (DAX_MMAP_LEN / sizeof(struct dax_cca)) 68 69 #define DAX_CCB_USEC 100 70 #define DAX_CCB_RETRIES 10000 71 72 /* stream types */ 73 enum { 74 OUT, 75 PRI, 76 SEC, 77 TBL, 78 NUM_STREAM_TYPES 79 }; 80 81 /* completion status */ 82 #define CCA_STAT_NOT_COMPLETED 0 83 #define CCA_STAT_COMPLETED 1 84 #define CCA_STAT_FAILED 2 85 #define CCA_STAT_KILLED 3 86 #define CCA_STAT_NOT_RUN 4 87 #define CCA_STAT_PIPE_OUT 5 88 #define CCA_STAT_PIPE_SRC 6 89 #define CCA_STAT_PIPE_DST 7 90 91 /* completion err */ 92 #define CCA_ERR_SUCCESS 0x0 /* no error */ 93 #define CCA_ERR_OVERFLOW 0x1 /* buffer overflow */ 94 #define CCA_ERR_DECODE 0x2 /* CCB decode error */ 95 #define CCA_ERR_PAGE_OVERFLOW 0x3 /* page overflow */ 96 #define CCA_ERR_KILLED 0x7 /* command was killed */ 97 #define CCA_ERR_TIMEOUT 0x8 /* Timeout */ 98 #define CCA_ERR_ADI 0x9 /* ADI error */ 99 #define CCA_ERR_DATA_FMT 0xA /* data format error */ 100 #define CCA_ERR_OTHER_NO_RETRY 0xE /* Other error, do not retry */ 101 #define CCA_ERR_OTHER_RETRY 0xF /* Other error, retry */ 102 #define CCA_ERR_PARTIAL_SYMBOL 0x80 /* QP partial symbol warning */ 103 104 /* CCB address types */ 105 #define DAX_ADDR_TYPE_NONE 0 106 #define DAX_ADDR_TYPE_VA_ALT 1 /* secondary context */ 107 #define DAX_ADDR_TYPE_RA 2 /* real address */ 108 #define DAX_ADDR_TYPE_VA 3 /* virtual address */ 109 110 /* dax_header_t opcode */ 111 #define DAX_OP_SYNC_NOP 0x0 112 #define DAX_OP_EXTRACT 0x1 113 #define DAX_OP_SCAN_VALUE 0x2 114 #define DAX_OP_SCAN_RANGE 0x3 115 #define DAX_OP_TRANSLATE 0x4 116 #define DAX_OP_SELECT 0x5 117 #define DAX_OP_INVERT 0x10 /* OR with translate, scan opcodes */ 118 119 struct dax_header { 120 u32 ccb_version:4; /* 31:28 CCB Version */ 121 /* 27:24 Sync Flags */ 122 u32 pipe:1; /* Pipeline */ 123 u32 longccb:1; /* Longccb. Set for scan with lu2, lu3, lu4. */ 124 u32 cond:1; /* Conditional */ 125 u32 serial:1; /* Serial */ 126 u32 opcode:8; /* 23:16 Opcode */ 127 /* 15:0 Address Type. */ 128 u32 reserved:3; /* 15:13 reserved */ 129 u32 table_addr_type:2; /* 12:11 Huffman Table Address Type */ 130 u32 out_addr_type:3; /* 10:8 Destination Address Type */ 131 u32 sec_addr_type:3; /* 7:5 Secondary Source Address Type */ 132 u32 pri_addr_type:3; /* 4:2 Primary Source Address Type */ 133 u32 cca_addr_type:2; /* 1:0 Completion Address Type */ 134 }; 135 136 struct dax_control { 137 u32 pri_fmt:4; /* 31:28 Primary Input Format */ 138 u32 pri_elem_size:5; /* 27:23 Primary Input Element Size(less1) */ 139 u32 pri_offset:3; /* 22:20 Primary Input Starting Offset */ 140 u32 sec_encoding:1; /* 19 Secondary Input Encoding */ 141 /* (must be 0 for Select) */ 142 u32 sec_offset:3; /* 18:16 Secondary Input Starting Offset */ 143 u32 sec_elem_size:2; /* 15:14 Secondary Input Element Size */ 144 /* (must be 0 for Select) */ 145 u32 out_fmt:2; /* 13:12 Output Format */ 146 u32 out_elem_size:2; /* 11:10 Output Element Size */ 147 u32 misc:10; /* 9:0 Opcode specific info */ 148 }; 149 150 struct dax_data_access { 151 u64 flow_ctrl:2; /* 63:62 Flow Control Type */ 152 u64 pipe_target:2; /* 61:60 Pipeline Target */ 153 u64 out_buf_size:20; /* 59:40 Output Buffer Size */ 154 /* (cachelines less 1) */ 155 u64 unused1:8; /* 39:32 Reserved, Set to 0 */ 156 u64 out_alloc:5; /* 31:27 Output Allocation */ 157 u64 unused2:1; /* 26 Reserved */ 158 u64 pri_len_fmt:2; /* 25:24 Input Length Format */ 159 u64 pri_len:24; /* 23:0 Input Element/Byte/Bit Count */ 160 /* (less 1) */ 161 }; 162 163 struct dax_ccb { 164 struct dax_header hdr; /* CCB Header */ 165 struct dax_control ctrl;/* Control Word */ 166 void *ca; /* Completion Address */ 167 void *pri; /* Primary Input Address */ 168 struct dax_data_access dac; /* Data Access Control */ 169 void *sec; /* Secondary Input Address */ 170 u64 dword5; /* depends on opcode */ 171 void *out; /* Output Address */ 172 void *tbl; /* Table Address or bitmap */ 173 }; 174 175 struct dax_cca { 176 u8 status; /* user may mwait on this address */ 177 u8 err; /* user visible error notification */ 178 u8 rsvd[2]; /* reserved */ 179 u32 n_remaining; /* for QP partial symbol warning */ 180 u32 output_sz; /* output in bytes */ 181 u32 rsvd2; /* reserved */ 182 u64 run_cycles; /* run time in OCND2 cycles */ 183 u64 run_stats; /* nothing reported in version 1.0 */ 184 u32 n_processed; /* number input elements */ 185 u32 rsvd3[5]; /* reserved */ 186 u64 retval; /* command return value */ 187 u64 rsvd4[8]; /* reserved */ 188 }; 189 190 /* per thread CCB context */ 191 struct dax_ctx { 192 struct dax_ccb *ccb_buf; 193 u64 ccb_buf_ra; /* cached RA of ccb_buf */ 194 struct dax_cca *ca_buf; 195 u64 ca_buf_ra; /* cached RA of ca_buf */ 196 struct page *pages[DAX_CA_ELEMS][NUM_STREAM_TYPES]; 197 /* array of locked pages */ 198 struct task_struct *owner; /* thread that owns ctx */ 199 struct task_struct *client; /* requesting thread */ 200 union ccb_result result; 201 u32 ccb_count; 202 u32 fail_count; 203 }; 204 205 /* driver public entry points */ 206 static int dax_open(struct inode *inode, struct file *file); 207 static ssize_t dax_read(struct file *filp, char __user *buf, 208 size_t count, loff_t *ppos); 209 static ssize_t dax_write(struct file *filp, const char __user *buf, 210 size_t count, loff_t *ppos); 211 static int dax_devmap(struct file *f, struct vm_area_struct *vma); 212 static int dax_close(struct inode *i, struct file *f); 213 214 static const struct file_operations dax_fops = { 215 .owner = THIS_MODULE, 216 .open = dax_open, 217 .read = dax_read, 218 .write = dax_write, 219 .mmap = dax_devmap, 220 .release = dax_close, 221 }; 222 223 static int dax_ccb_exec(struct dax_ctx *ctx, const char __user *buf, 224 size_t count, loff_t *ppos); 225 static int dax_ccb_info(u64 ca, struct ccb_info_result *info); 226 static int dax_ccb_kill(u64 ca, u16 *kill_res); 227 228 static struct cdev c_dev; 229 static struct class *cl; 230 static dev_t first; 231 232 static int max_ccb_version; 233 static int dax_debug; 234 module_param(dax_debug, int, 0644); 235 MODULE_PARM_DESC(dax_debug, "Debug flags"); 236 237 static int __init dax_attach(void) 238 { 239 unsigned long dummy, hv_rv, major, minor, minor_requested, max_ccbs; 240 struct mdesc_handle *hp = mdesc_grab(); 241 char *prop, *dax_name; 242 bool found = false; 243 int len, ret = 0; 244 u64 pn; 245 246 if (hp == NULL) { 247 dax_err("Unable to grab mdesc"); 248 return -ENODEV; 249 } 250 251 mdesc_for_each_node_by_name(hp, pn, "virtual-device") { 252 prop = (char *)mdesc_get_property(hp, pn, "name", &len); 253 if (prop == NULL) 254 continue; 255 if (strncmp(prop, "dax", strlen("dax"))) 256 continue; 257 dax_dbg("Found node 0x%llx = %s", pn, prop); 258 259 prop = (char *)mdesc_get_property(hp, pn, "compatible", &len); 260 if (prop == NULL) 261 continue; 262 dax_dbg("Found node 0x%llx = %s", pn, prop); 263 found = true; 264 break; 265 } 266 267 if (!found) { 268 dax_err("No DAX device found"); 269 ret = -ENODEV; 270 goto done; 271 } 272 273 if (strncmp(prop, DAX2_STR, strlen(DAX2_STR)) == 0) { 274 dax_name = DAX_NAME "2"; 275 major = DAX2_MAJOR; 276 minor_requested = DAX2_MINOR; 277 max_ccb_version = 1; 278 dax_dbg("MD indicates DAX2 coprocessor"); 279 } else if (strncmp(prop, DAX1_STR, strlen(DAX1_STR)) == 0) { 280 dax_name = DAX_NAME "1"; 281 major = DAX1_MAJOR; 282 minor_requested = DAX1_MINOR; 283 max_ccb_version = 0; 284 dax_dbg("MD indicates DAX1 coprocessor"); 285 } else { 286 dax_err("Unknown dax type: %s", prop); 287 ret = -ENODEV; 288 goto done; 289 } 290 291 minor = minor_requested; 292 dax_dbg("Registering DAX HV api with major %ld minor %ld", major, 293 minor); 294 if (sun4v_hvapi_register(HV_GRP_DAX, major, &minor)) { 295 dax_err("hvapi_register failed"); 296 ret = -ENODEV; 297 goto done; 298 } else { 299 dax_dbg("Max minor supported by HV = %ld (major %ld)", minor, 300 major); 301 minor = min(minor, minor_requested); 302 dax_dbg("registered DAX major %ld minor %ld", major, minor); 303 } 304 305 /* submit a zero length ccb array to query coprocessor queue size */ 306 hv_rv = sun4v_ccb_submit(0, 0, HV_CCB_QUERY_CMD, 0, &max_ccbs, &dummy); 307 if (hv_rv != 0) { 308 dax_err("get_hwqueue_size failed with status=%ld and max_ccbs=%ld", 309 hv_rv, max_ccbs); 310 ret = -ENODEV; 311 goto done; 312 } 313 314 if (max_ccbs != DAX_MAX_CCBS) { 315 dax_err("HV reports unsupported max_ccbs=%ld", max_ccbs); 316 ret = -ENODEV; 317 goto done; 318 } 319 320 if (alloc_chrdev_region(&first, 0, 1, DAX_NAME) < 0) { 321 dax_err("alloc_chrdev_region failed"); 322 ret = -ENXIO; 323 goto done; 324 } 325 326 cl = class_create(THIS_MODULE, DAX_NAME); 327 if (IS_ERR(cl)) { 328 dax_err("class_create failed"); 329 ret = PTR_ERR(cl); 330 goto class_error; 331 } 332 333 if (device_create(cl, NULL, first, NULL, dax_name) == NULL) { 334 dax_err("device_create failed"); 335 ret = -ENXIO; 336 goto device_error; 337 } 338 339 cdev_init(&c_dev, &dax_fops); 340 if (cdev_add(&c_dev, first, 1) == -1) { 341 dax_err("cdev_add failed"); 342 ret = -ENXIO; 343 goto cdev_error; 344 } 345 346 pr_info("Attached DAX module\n"); 347 goto done; 348 349 cdev_error: 350 device_destroy(cl, first); 351 device_error: 352 class_destroy(cl); 353 class_error: 354 unregister_chrdev_region(first, 1); 355 done: 356 mdesc_release(hp); 357 return ret; 358 } 359 module_init(dax_attach); 360 361 static void __exit dax_detach(void) 362 { 363 pr_info("Cleaning up DAX module\n"); 364 cdev_del(&c_dev); 365 device_destroy(cl, first); 366 class_destroy(cl); 367 unregister_chrdev_region(first, 1); 368 } 369 module_exit(dax_detach); 370 371 /* map completion area */ 372 static int dax_devmap(struct file *f, struct vm_area_struct *vma) 373 { 374 struct dax_ctx *ctx = (struct dax_ctx *)f->private_data; 375 size_t len = vma->vm_end - vma->vm_start; 376 377 dax_dbg("len=0x%lx, flags=0x%lx", len, vma->vm_flags); 378 379 if (ctx->owner != current) { 380 dax_dbg("devmap called from wrong thread"); 381 return -EINVAL; 382 } 383 384 if (len != DAX_MMAP_LEN) { 385 dax_dbg("len(%lu) != DAX_MMAP_LEN(%d)", len, DAX_MMAP_LEN); 386 return -EINVAL; 387 } 388 389 /* completion area is mapped read-only for user */ 390 if (vma->vm_flags & VM_WRITE) 391 return -EPERM; 392 vma->vm_flags &= ~VM_MAYWRITE; 393 394 if (remap_pfn_range(vma, vma->vm_start, ctx->ca_buf_ra >> PAGE_SHIFT, 395 len, vma->vm_page_prot)) 396 return -EAGAIN; 397 398 dax_dbg("mmapped completion area at uva 0x%lx", vma->vm_start); 399 return 0; 400 } 401 402 /* Unlock user pages. Called during dequeue or device close */ 403 static void dax_unlock_pages(struct dax_ctx *ctx, int ccb_index, int nelem) 404 { 405 int i, j; 406 407 for (i = ccb_index; i < ccb_index + nelem; i++) { 408 for (j = 0; j < NUM_STREAM_TYPES; j++) { 409 struct page *p = ctx->pages[i][j]; 410 411 if (p) { 412 dax_dbg("freeing page %p", p); 413 if (j == OUT) 414 set_page_dirty(p); 415 put_page(p); 416 ctx->pages[i][j] = NULL; 417 } 418 } 419 } 420 } 421 422 static int dax_lock_page(void *va, struct page **p) 423 { 424 int ret; 425 426 dax_dbg("uva %p", va); 427 428 ret = get_user_pages_fast((unsigned long)va, 1, FOLL_WRITE, p); 429 if (ret == 1) { 430 dax_dbg("locked page %p, for VA %p", *p, va); 431 return 0; 432 } 433 434 dax_dbg("get_user_pages failed, va=%p, ret=%d", va, ret); 435 return -1; 436 } 437 438 static int dax_lock_pages(struct dax_ctx *ctx, int idx, 439 int nelem, u64 *err_va) 440 { 441 int i; 442 443 for (i = 0; i < nelem; i++) { 444 struct dax_ccb *ccbp = &ctx->ccb_buf[i]; 445 446 /* 447 * For each address in the CCB whose type is virtual, 448 * lock the page and change the type to virtual alternate 449 * context. On error, return the offending address in 450 * err_va. 451 */ 452 if (ccbp->hdr.out_addr_type == DAX_ADDR_TYPE_VA) { 453 dax_dbg("output"); 454 if (dax_lock_page(ccbp->out, 455 &ctx->pages[i + idx][OUT]) != 0) { 456 *err_va = (u64)ccbp->out; 457 goto error; 458 } 459 ccbp->hdr.out_addr_type = DAX_ADDR_TYPE_VA_ALT; 460 } 461 462 if (ccbp->hdr.pri_addr_type == DAX_ADDR_TYPE_VA) { 463 dax_dbg("input"); 464 if (dax_lock_page(ccbp->pri, 465 &ctx->pages[i + idx][PRI]) != 0) { 466 *err_va = (u64)ccbp->pri; 467 goto error; 468 } 469 ccbp->hdr.pri_addr_type = DAX_ADDR_TYPE_VA_ALT; 470 } 471 472 if (ccbp->hdr.sec_addr_type == DAX_ADDR_TYPE_VA) { 473 dax_dbg("sec input"); 474 if (dax_lock_page(ccbp->sec, 475 &ctx->pages[i + idx][SEC]) != 0) { 476 *err_va = (u64)ccbp->sec; 477 goto error; 478 } 479 ccbp->hdr.sec_addr_type = DAX_ADDR_TYPE_VA_ALT; 480 } 481 482 if (ccbp->hdr.table_addr_type == DAX_ADDR_TYPE_VA) { 483 dax_dbg("tbl"); 484 if (dax_lock_page(ccbp->tbl, 485 &ctx->pages[i + idx][TBL]) != 0) { 486 *err_va = (u64)ccbp->tbl; 487 goto error; 488 } 489 ccbp->hdr.table_addr_type = DAX_ADDR_TYPE_VA_ALT; 490 } 491 492 /* skip over 2nd 64 bytes of long CCB */ 493 if (ccbp->hdr.longccb) 494 i++; 495 } 496 return DAX_SUBMIT_OK; 497 498 error: 499 dax_unlock_pages(ctx, idx, nelem); 500 return DAX_SUBMIT_ERR_NOACCESS; 501 } 502 503 static void dax_ccb_wait(struct dax_ctx *ctx, int idx) 504 { 505 int ret, nretries; 506 u16 kill_res; 507 508 dax_dbg("idx=%d", idx); 509 510 for (nretries = 0; nretries < DAX_CCB_RETRIES; nretries++) { 511 if (ctx->ca_buf[idx].status == CCA_STAT_NOT_COMPLETED) 512 udelay(DAX_CCB_USEC); 513 else 514 return; 515 } 516 dax_dbg("ctx (%p): CCB[%d] timed out, wait usec=%d, retries=%d. Killing ccb", 517 (void *)ctx, idx, DAX_CCB_USEC, DAX_CCB_RETRIES); 518 519 ret = dax_ccb_kill(ctx->ca_buf_ra + idx * sizeof(struct dax_cca), 520 &kill_res); 521 dax_dbg("Kill CCB[%d] %s", idx, ret ? "failed" : "succeeded"); 522 } 523 524 static int dax_close(struct inode *ino, struct file *f) 525 { 526 struct dax_ctx *ctx = (struct dax_ctx *)f->private_data; 527 int i; 528 529 f->private_data = NULL; 530 531 for (i = 0; i < DAX_CA_ELEMS; i++) { 532 if (ctx->ca_buf[i].status == CCA_STAT_NOT_COMPLETED) { 533 dax_dbg("CCB[%d] not completed", i); 534 dax_ccb_wait(ctx, i); 535 } 536 dax_unlock_pages(ctx, i, 1); 537 } 538 539 kfree(ctx->ccb_buf); 540 kfree(ctx->ca_buf); 541 dax_stat_dbg("CCBs: %d good, %d bad", ctx->ccb_count, ctx->fail_count); 542 kfree(ctx); 543 544 return 0; 545 } 546 547 static ssize_t dax_read(struct file *f, char __user *buf, 548 size_t count, loff_t *ppos) 549 { 550 struct dax_ctx *ctx = f->private_data; 551 552 if (ctx->client != current) 553 return -EUSERS; 554 555 ctx->client = NULL; 556 557 if (count != sizeof(union ccb_result)) 558 return -EINVAL; 559 if (copy_to_user(buf, &ctx->result, sizeof(union ccb_result))) 560 return -EFAULT; 561 return count; 562 } 563 564 static ssize_t dax_write(struct file *f, const char __user *buf, 565 size_t count, loff_t *ppos) 566 { 567 struct dax_ctx *ctx = f->private_data; 568 struct dax_command hdr; 569 unsigned long ca; 570 int i, idx, ret; 571 572 if (ctx->client != NULL) 573 return -EINVAL; 574 575 if (count == 0 || count > DAX_MAX_CCBS * sizeof(struct dax_ccb)) 576 return -EINVAL; 577 578 if (count % sizeof(struct dax_ccb) == 0) 579 return dax_ccb_exec(ctx, buf, count, ppos); /* CCB EXEC */ 580 581 if (count != sizeof(struct dax_command)) 582 return -EINVAL; 583 584 /* immediate command */ 585 if (ctx->owner != current) 586 return -EUSERS; 587 588 if (copy_from_user(&hdr, buf, sizeof(hdr))) 589 return -EFAULT; 590 591 ca = ctx->ca_buf_ra + hdr.ca_offset; 592 593 switch (hdr.command) { 594 case CCB_KILL: 595 if (hdr.ca_offset >= DAX_MMAP_LEN) { 596 dax_dbg("invalid ca_offset (%d) >= ca_buflen (%d)", 597 hdr.ca_offset, DAX_MMAP_LEN); 598 return -EINVAL; 599 } 600 601 ret = dax_ccb_kill(ca, &ctx->result.kill.action); 602 if (ret != 0) { 603 dax_dbg("dax_ccb_kill failed (ret=%d)", ret); 604 return ret; 605 } 606 607 dax_info_dbg("killed (ca_offset %d)", hdr.ca_offset); 608 idx = hdr.ca_offset / sizeof(struct dax_cca); 609 ctx->ca_buf[idx].status = CCA_STAT_KILLED; 610 ctx->ca_buf[idx].err = CCA_ERR_KILLED; 611 ctx->client = current; 612 return count; 613 614 case CCB_INFO: 615 if (hdr.ca_offset >= DAX_MMAP_LEN) { 616 dax_dbg("invalid ca_offset (%d) >= ca_buflen (%d)", 617 hdr.ca_offset, DAX_MMAP_LEN); 618 return -EINVAL; 619 } 620 621 ret = dax_ccb_info(ca, &ctx->result.info); 622 if (ret != 0) { 623 dax_dbg("dax_ccb_info failed (ret=%d)", ret); 624 return ret; 625 } 626 627 dax_info_dbg("info succeeded on ca_offset %d", hdr.ca_offset); 628 ctx->client = current; 629 return count; 630 631 case CCB_DEQUEUE: 632 for (i = 0; i < DAX_CA_ELEMS; i++) { 633 if (ctx->ca_buf[i].status != 634 CCA_STAT_NOT_COMPLETED) 635 dax_unlock_pages(ctx, i, 1); 636 } 637 return count; 638 639 default: 640 return -EINVAL; 641 } 642 } 643 644 static int dax_open(struct inode *inode, struct file *f) 645 { 646 struct dax_ctx *ctx = NULL; 647 int i; 648 649 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 650 if (ctx == NULL) 651 goto done; 652 653 ctx->ccb_buf = kcalloc(DAX_MAX_CCBS, sizeof(struct dax_ccb), 654 GFP_KERNEL); 655 if (ctx->ccb_buf == NULL) 656 goto done; 657 658 ctx->ccb_buf_ra = virt_to_phys(ctx->ccb_buf); 659 dax_dbg("ctx->ccb_buf=0x%p, ccb_buf_ra=0x%llx", 660 (void *)ctx->ccb_buf, ctx->ccb_buf_ra); 661 662 /* allocate CCB completion area buffer */ 663 ctx->ca_buf = kzalloc(DAX_MMAP_LEN, GFP_KERNEL); 664 if (ctx->ca_buf == NULL) 665 goto alloc_error; 666 for (i = 0; i < DAX_CA_ELEMS; i++) 667 ctx->ca_buf[i].status = CCA_STAT_COMPLETED; 668 669 ctx->ca_buf_ra = virt_to_phys(ctx->ca_buf); 670 dax_dbg("ctx=0x%p, ctx->ca_buf=0x%p, ca_buf_ra=0x%llx", 671 (void *)ctx, (void *)ctx->ca_buf, ctx->ca_buf_ra); 672 673 ctx->owner = current; 674 f->private_data = ctx; 675 return 0; 676 677 alloc_error: 678 kfree(ctx->ccb_buf); 679 done: 680 kfree(ctx); 681 return -ENOMEM; 682 } 683 684 static char *dax_hv_errno(unsigned long hv_ret, int *ret) 685 { 686 switch (hv_ret) { 687 case HV_EBADALIGN: 688 *ret = -EFAULT; 689 return "HV_EBADALIGN"; 690 case HV_ENORADDR: 691 *ret = -EFAULT; 692 return "HV_ENORADDR"; 693 case HV_EINVAL: 694 *ret = -EINVAL; 695 return "HV_EINVAL"; 696 case HV_EWOULDBLOCK: 697 *ret = -EAGAIN; 698 return "HV_EWOULDBLOCK"; 699 case HV_ENOACCESS: 700 *ret = -EPERM; 701 return "HV_ENOACCESS"; 702 default: 703 break; 704 } 705 706 *ret = -EIO; 707 return "UNKNOWN"; 708 } 709 710 static int dax_ccb_kill(u64 ca, u16 *kill_res) 711 { 712 unsigned long hv_ret; 713 int count, ret = 0; 714 char *err_str; 715 716 for (count = 0; count < DAX_CCB_RETRIES; count++) { 717 dax_dbg("attempting kill on ca_ra 0x%llx", ca); 718 hv_ret = sun4v_ccb_kill(ca, kill_res); 719 720 if (hv_ret == HV_EOK) { 721 dax_info_dbg("HV_EOK (ca_ra 0x%llx): %d", ca, 722 *kill_res); 723 } else { 724 err_str = dax_hv_errno(hv_ret, &ret); 725 dax_dbg("%s (ca_ra 0x%llx)", err_str, ca); 726 } 727 728 if (ret != -EAGAIN) 729 return ret; 730 dax_info_dbg("ccb_kill count = %d", count); 731 udelay(DAX_CCB_USEC); 732 } 733 734 return -EAGAIN; 735 } 736 737 static int dax_ccb_info(u64 ca, struct ccb_info_result *info) 738 { 739 unsigned long hv_ret; 740 char *err_str; 741 int ret = 0; 742 743 dax_dbg("attempting info on ca_ra 0x%llx", ca); 744 hv_ret = sun4v_ccb_info(ca, info); 745 746 if (hv_ret == HV_EOK) { 747 dax_info_dbg("HV_EOK (ca_ra 0x%llx): %d", ca, info->state); 748 if (info->state == DAX_CCB_ENQUEUED) { 749 dax_info_dbg("dax_unit %d, queue_num %d, queue_pos %d", 750 info->inst_num, info->q_num, info->q_pos); 751 } 752 } else { 753 err_str = dax_hv_errno(hv_ret, &ret); 754 dax_dbg("%s (ca_ra 0x%llx)", err_str, ca); 755 } 756 757 return ret; 758 } 759 760 static void dax_prt_ccbs(struct dax_ccb *ccb, int nelem) 761 { 762 int i, j; 763 u64 *ccbp; 764 765 dax_dbg("ccb buffer:"); 766 for (i = 0; i < nelem; i++) { 767 ccbp = (u64 *)&ccb[i]; 768 dax_dbg(" %sccb[%d]", ccb[i].hdr.longccb ? "long " : "", i); 769 for (j = 0; j < 8; j++) 770 dax_dbg("\tccb[%d].dwords[%d]=0x%llx", 771 i, j, *(ccbp + j)); 772 } 773 } 774 775 /* 776 * Validates user CCB content. Also sets completion address and address types 777 * for all addresses contained in CCB. 778 */ 779 static int dax_preprocess_usr_ccbs(struct dax_ctx *ctx, int idx, int nelem) 780 { 781 int i; 782 783 /* 784 * The user is not allowed to specify real address types in 785 * the CCB header. This must be enforced by the kernel before 786 * submitting the CCBs to HV. The only allowed values for all 787 * address fields are VA or IMM 788 */ 789 for (i = 0; i < nelem; i++) { 790 struct dax_ccb *ccbp = &ctx->ccb_buf[i]; 791 unsigned long ca_offset; 792 793 if (ccbp->hdr.ccb_version > max_ccb_version) 794 return DAX_SUBMIT_ERR_CCB_INVAL; 795 796 switch (ccbp->hdr.opcode) { 797 case DAX_OP_SYNC_NOP: 798 case DAX_OP_EXTRACT: 799 case DAX_OP_SCAN_VALUE: 800 case DAX_OP_SCAN_RANGE: 801 case DAX_OP_TRANSLATE: 802 case DAX_OP_SCAN_VALUE | DAX_OP_INVERT: 803 case DAX_OP_SCAN_RANGE | DAX_OP_INVERT: 804 case DAX_OP_TRANSLATE | DAX_OP_INVERT: 805 case DAX_OP_SELECT: 806 break; 807 default: 808 return DAX_SUBMIT_ERR_CCB_INVAL; 809 } 810 811 if (ccbp->hdr.out_addr_type != DAX_ADDR_TYPE_VA && 812 ccbp->hdr.out_addr_type != DAX_ADDR_TYPE_NONE) { 813 dax_dbg("invalid out_addr_type in user CCB[%d]", i); 814 return DAX_SUBMIT_ERR_CCB_INVAL; 815 } 816 817 if (ccbp->hdr.pri_addr_type != DAX_ADDR_TYPE_VA && 818 ccbp->hdr.pri_addr_type != DAX_ADDR_TYPE_NONE) { 819 dax_dbg("invalid pri_addr_type in user CCB[%d]", i); 820 return DAX_SUBMIT_ERR_CCB_INVAL; 821 } 822 823 if (ccbp->hdr.sec_addr_type != DAX_ADDR_TYPE_VA && 824 ccbp->hdr.sec_addr_type != DAX_ADDR_TYPE_NONE) { 825 dax_dbg("invalid sec_addr_type in user CCB[%d]", i); 826 return DAX_SUBMIT_ERR_CCB_INVAL; 827 } 828 829 if (ccbp->hdr.table_addr_type != DAX_ADDR_TYPE_VA && 830 ccbp->hdr.table_addr_type != DAX_ADDR_TYPE_NONE) { 831 dax_dbg("invalid table_addr_type in user CCB[%d]", i); 832 return DAX_SUBMIT_ERR_CCB_INVAL; 833 } 834 835 /* set completion (real) address and address type */ 836 ccbp->hdr.cca_addr_type = DAX_ADDR_TYPE_RA; 837 ca_offset = (idx + i) * sizeof(struct dax_cca); 838 ccbp->ca = (void *)ctx->ca_buf_ra + ca_offset; 839 memset(&ctx->ca_buf[idx + i], 0, sizeof(struct dax_cca)); 840 841 dax_dbg("ccb[%d]=%p, ca_offset=0x%lx, compl RA=0x%llx", 842 i, ccbp, ca_offset, ctx->ca_buf_ra + ca_offset); 843 844 /* skip over 2nd 64 bytes of long CCB */ 845 if (ccbp->hdr.longccb) 846 i++; 847 } 848 849 return DAX_SUBMIT_OK; 850 } 851 852 static int dax_ccb_exec(struct dax_ctx *ctx, const char __user *buf, 853 size_t count, loff_t *ppos) 854 { 855 unsigned long accepted_len, hv_rv; 856 int i, idx, nccbs, naccepted; 857 858 ctx->client = current; 859 idx = *ppos; 860 nccbs = count / sizeof(struct dax_ccb); 861 862 if (ctx->owner != current) { 863 dax_dbg("wrong thread"); 864 ctx->result.exec.status = DAX_SUBMIT_ERR_THR_INIT; 865 return 0; 866 } 867 dax_dbg("args: ccb_buf_len=%ld, idx=%d", count, idx); 868 869 /* for given index and length, verify ca_buf range exists */ 870 if (idx < 0 || idx > (DAX_CA_ELEMS - nccbs)) { 871 ctx->result.exec.status = DAX_SUBMIT_ERR_NO_CA_AVAIL; 872 return 0; 873 } 874 875 /* 876 * Copy CCBs into kernel buffer to prevent modification by the 877 * user in between validation and submission. 878 */ 879 if (copy_from_user(ctx->ccb_buf, buf, count)) { 880 dax_dbg("copyin of user CCB buffer failed"); 881 ctx->result.exec.status = DAX_SUBMIT_ERR_CCB_ARR_MMU_MISS; 882 return 0; 883 } 884 885 /* check to see if ca_buf[idx] .. ca_buf[idx + nccbs] are available */ 886 for (i = idx; i < idx + nccbs; i++) { 887 if (ctx->ca_buf[i].status == CCA_STAT_NOT_COMPLETED) { 888 dax_dbg("CA range not available, dequeue needed"); 889 ctx->result.exec.status = DAX_SUBMIT_ERR_NO_CA_AVAIL; 890 return 0; 891 } 892 } 893 dax_unlock_pages(ctx, idx, nccbs); 894 895 ctx->result.exec.status = dax_preprocess_usr_ccbs(ctx, idx, nccbs); 896 if (ctx->result.exec.status != DAX_SUBMIT_OK) 897 return 0; 898 899 ctx->result.exec.status = dax_lock_pages(ctx, idx, nccbs, 900 &ctx->result.exec.status_data); 901 if (ctx->result.exec.status != DAX_SUBMIT_OK) 902 return 0; 903 904 if (dax_debug & DAX_DBG_FLG_BASIC) 905 dax_prt_ccbs(ctx->ccb_buf, nccbs); 906 907 hv_rv = sun4v_ccb_submit(ctx->ccb_buf_ra, count, 908 HV_CCB_QUERY_CMD | HV_CCB_VA_SECONDARY, 0, 909 &accepted_len, &ctx->result.exec.status_data); 910 911 switch (hv_rv) { 912 case HV_EOK: 913 /* 914 * Hcall succeeded with no errors but the accepted 915 * length may be less than the requested length. The 916 * only way the driver can resubmit the remainder is 917 * to wait for completion of the submitted CCBs since 918 * there is no way to guarantee the ordering semantics 919 * required by the client applications. Therefore we 920 * let the user library deal with resubmissions. 921 */ 922 ctx->result.exec.status = DAX_SUBMIT_OK; 923 break; 924 case HV_EWOULDBLOCK: 925 /* 926 * This is a transient HV API error. The user library 927 * can retry. 928 */ 929 dax_dbg("hcall returned HV_EWOULDBLOCK"); 930 ctx->result.exec.status = DAX_SUBMIT_ERR_WOULDBLOCK; 931 break; 932 case HV_ENOMAP: 933 /* 934 * HV was unable to translate a VA. The VA it could 935 * not translate is returned in the status_data param. 936 */ 937 dax_dbg("hcall returned HV_ENOMAP"); 938 ctx->result.exec.status = DAX_SUBMIT_ERR_NOMAP; 939 break; 940 case HV_EINVAL: 941 /* 942 * This is the result of an invalid user CCB as HV is 943 * validating some of the user CCB fields. Pass this 944 * error back to the user. There is no supporting info 945 * to isolate the invalid field. 946 */ 947 dax_dbg("hcall returned HV_EINVAL"); 948 ctx->result.exec.status = DAX_SUBMIT_ERR_CCB_INVAL; 949 break; 950 case HV_ENOACCESS: 951 /* 952 * HV found a VA that did not have the appropriate 953 * permissions (such as the w bit). The VA in question 954 * is returned in status_data param. 955 */ 956 dax_dbg("hcall returned HV_ENOACCESS"); 957 ctx->result.exec.status = DAX_SUBMIT_ERR_NOACCESS; 958 break; 959 case HV_EUNAVAILABLE: 960 /* 961 * The requested CCB operation could not be performed 962 * at this time. Return the specific unavailable code 963 * in the status_data field. 964 */ 965 dax_dbg("hcall returned HV_EUNAVAILABLE"); 966 ctx->result.exec.status = DAX_SUBMIT_ERR_UNAVAIL; 967 break; 968 default: 969 ctx->result.exec.status = DAX_SUBMIT_ERR_INTERNAL; 970 dax_dbg("unknown hcall return value (%ld)", hv_rv); 971 break; 972 } 973 974 /* unlock pages associated with the unaccepted CCBs */ 975 naccepted = accepted_len / sizeof(struct dax_ccb); 976 dax_unlock_pages(ctx, idx + naccepted, nccbs - naccepted); 977 978 /* mark unaccepted CCBs as not completed */ 979 for (i = idx + naccepted; i < idx + nccbs; i++) 980 ctx->ca_buf[i].status = CCA_STAT_COMPLETED; 981 982 ctx->ccb_count += naccepted; 983 ctx->fail_count += nccbs - naccepted; 984 985 dax_dbg("hcall rv=%ld, accepted_len=%ld, status_data=0x%llx, ret status=%d", 986 hv_rv, accepted_len, ctx->result.exec.status_data, 987 ctx->result.exec.status); 988 989 if (count == accepted_len) 990 ctx->client = NULL; /* no read needed to complete protocol */ 991 return accepted_len; 992 } 993