1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2021 Western Digital Corporation or its affiliates. 4 * Copyright (C) 2022 Ventana Micro Systems Inc. 5 * 6 * Authors: 7 * Anup Patel <apatel@ventanamicro.com> 8 */ 9 10 #include <linux/atomic.h> 11 #include <linux/bitmap.h> 12 #include <linux/kvm_host.h> 13 #include <linux/math.h> 14 #include <linux/spinlock.h> 15 #include <linux/swab.h> 16 #include <kvm/iodev.h> 17 #include <asm/csr.h> 18 #include <asm/kvm_aia_imsic.h> 19 20 #define IMSIC_MAX_EIX (IMSIC_MAX_ID / BITS_PER_TYPE(u64)) 21 22 struct imsic_mrif_eix { 23 unsigned long eip[BITS_PER_TYPE(u64) / BITS_PER_LONG]; 24 unsigned long eie[BITS_PER_TYPE(u64) / BITS_PER_LONG]; 25 }; 26 27 struct imsic_mrif { 28 struct imsic_mrif_eix eix[IMSIC_MAX_EIX]; 29 unsigned long eithreshold; 30 unsigned long eidelivery; 31 }; 32 33 struct imsic { 34 struct kvm_io_device iodev; 35 36 u32 nr_msis; 37 u32 nr_eix; 38 u32 nr_hw_eix; 39 40 /* 41 * At any point in time, the register state is in 42 * one of the following places: 43 * 44 * 1) Hardware: IMSIC VS-file (vsfile_cpu >= 0) 45 * 2) Software: IMSIC SW-file (vsfile_cpu < 0) 46 */ 47 48 /* IMSIC VS-file */ 49 rwlock_t vsfile_lock; 50 int vsfile_cpu; 51 int vsfile_hgei; 52 void __iomem *vsfile_va; 53 phys_addr_t vsfile_pa; 54 55 /* IMSIC SW-file */ 56 struct imsic_mrif *swfile; 57 phys_addr_t swfile_pa; 58 }; 59 60 #define imsic_vs_csr_read(__c) \ 61 ({ \ 62 unsigned long __r; \ 63 csr_write(CSR_VSISELECT, __c); \ 64 __r = csr_read(CSR_VSIREG); \ 65 __r; \ 66 }) 67 68 #define imsic_read_switchcase(__ireg) \ 69 case __ireg: \ 70 return imsic_vs_csr_read(__ireg); 71 #define imsic_read_switchcase_2(__ireg) \ 72 imsic_read_switchcase(__ireg + 0) \ 73 imsic_read_switchcase(__ireg + 1) 74 #define imsic_read_switchcase_4(__ireg) \ 75 imsic_read_switchcase_2(__ireg + 0) \ 76 imsic_read_switchcase_2(__ireg + 2) 77 #define imsic_read_switchcase_8(__ireg) \ 78 imsic_read_switchcase_4(__ireg + 0) \ 79 imsic_read_switchcase_4(__ireg + 4) 80 #define imsic_read_switchcase_16(__ireg) \ 81 imsic_read_switchcase_8(__ireg + 0) \ 82 imsic_read_switchcase_8(__ireg + 8) 83 #define imsic_read_switchcase_32(__ireg) \ 84 imsic_read_switchcase_16(__ireg + 0) \ 85 imsic_read_switchcase_16(__ireg + 16) 86 #define imsic_read_switchcase_64(__ireg) \ 87 imsic_read_switchcase_32(__ireg + 0) \ 88 imsic_read_switchcase_32(__ireg + 32) 89 90 static unsigned long imsic_eix_read(int ireg) 91 { 92 switch (ireg) { 93 imsic_read_switchcase_64(IMSIC_EIP0) 94 imsic_read_switchcase_64(IMSIC_EIE0) 95 } 96 97 return 0; 98 } 99 100 #define imsic_vs_csr_swap(__c, __v) \ 101 ({ \ 102 unsigned long __r; \ 103 csr_write(CSR_VSISELECT, __c); \ 104 __r = csr_swap(CSR_VSIREG, __v); \ 105 __r; \ 106 }) 107 108 #define imsic_swap_switchcase(__ireg, __v) \ 109 case __ireg: \ 110 return imsic_vs_csr_swap(__ireg, __v); 111 #define imsic_swap_switchcase_2(__ireg, __v) \ 112 imsic_swap_switchcase(__ireg + 0, __v) \ 113 imsic_swap_switchcase(__ireg + 1, __v) 114 #define imsic_swap_switchcase_4(__ireg, __v) \ 115 imsic_swap_switchcase_2(__ireg + 0, __v) \ 116 imsic_swap_switchcase_2(__ireg + 2, __v) 117 #define imsic_swap_switchcase_8(__ireg, __v) \ 118 imsic_swap_switchcase_4(__ireg + 0, __v) \ 119 imsic_swap_switchcase_4(__ireg + 4, __v) 120 #define imsic_swap_switchcase_16(__ireg, __v) \ 121 imsic_swap_switchcase_8(__ireg + 0, __v) \ 122 imsic_swap_switchcase_8(__ireg + 8, __v) 123 #define imsic_swap_switchcase_32(__ireg, __v) \ 124 imsic_swap_switchcase_16(__ireg + 0, __v) \ 125 imsic_swap_switchcase_16(__ireg + 16, __v) 126 #define imsic_swap_switchcase_64(__ireg, __v) \ 127 imsic_swap_switchcase_32(__ireg + 0, __v) \ 128 imsic_swap_switchcase_32(__ireg + 32, __v) 129 130 static unsigned long imsic_eix_swap(int ireg, unsigned long val) 131 { 132 switch (ireg) { 133 imsic_swap_switchcase_64(IMSIC_EIP0, val) 134 imsic_swap_switchcase_64(IMSIC_EIE0, val) 135 } 136 137 return 0; 138 } 139 140 #define imsic_vs_csr_write(__c, __v) \ 141 do { \ 142 csr_write(CSR_VSISELECT, __c); \ 143 csr_write(CSR_VSIREG, __v); \ 144 } while (0) 145 146 #define imsic_write_switchcase(__ireg, __v) \ 147 case __ireg: \ 148 imsic_vs_csr_write(__ireg, __v); \ 149 break; 150 #define imsic_write_switchcase_2(__ireg, __v) \ 151 imsic_write_switchcase(__ireg + 0, __v) \ 152 imsic_write_switchcase(__ireg + 1, __v) 153 #define imsic_write_switchcase_4(__ireg, __v) \ 154 imsic_write_switchcase_2(__ireg + 0, __v) \ 155 imsic_write_switchcase_2(__ireg + 2, __v) 156 #define imsic_write_switchcase_8(__ireg, __v) \ 157 imsic_write_switchcase_4(__ireg + 0, __v) \ 158 imsic_write_switchcase_4(__ireg + 4, __v) 159 #define imsic_write_switchcase_16(__ireg, __v) \ 160 imsic_write_switchcase_8(__ireg + 0, __v) \ 161 imsic_write_switchcase_8(__ireg + 8, __v) 162 #define imsic_write_switchcase_32(__ireg, __v) \ 163 imsic_write_switchcase_16(__ireg + 0, __v) \ 164 imsic_write_switchcase_16(__ireg + 16, __v) 165 #define imsic_write_switchcase_64(__ireg, __v) \ 166 imsic_write_switchcase_32(__ireg + 0, __v) \ 167 imsic_write_switchcase_32(__ireg + 32, __v) 168 169 static void imsic_eix_write(int ireg, unsigned long val) 170 { 171 switch (ireg) { 172 imsic_write_switchcase_64(IMSIC_EIP0, val) 173 imsic_write_switchcase_64(IMSIC_EIE0, val) 174 } 175 } 176 177 #define imsic_vs_csr_set(__c, __v) \ 178 do { \ 179 csr_write(CSR_VSISELECT, __c); \ 180 csr_set(CSR_VSIREG, __v); \ 181 } while (0) 182 183 #define imsic_set_switchcase(__ireg, __v) \ 184 case __ireg: \ 185 imsic_vs_csr_set(__ireg, __v); \ 186 break; 187 #define imsic_set_switchcase_2(__ireg, __v) \ 188 imsic_set_switchcase(__ireg + 0, __v) \ 189 imsic_set_switchcase(__ireg + 1, __v) 190 #define imsic_set_switchcase_4(__ireg, __v) \ 191 imsic_set_switchcase_2(__ireg + 0, __v) \ 192 imsic_set_switchcase_2(__ireg + 2, __v) 193 #define imsic_set_switchcase_8(__ireg, __v) \ 194 imsic_set_switchcase_4(__ireg + 0, __v) \ 195 imsic_set_switchcase_4(__ireg + 4, __v) 196 #define imsic_set_switchcase_16(__ireg, __v) \ 197 imsic_set_switchcase_8(__ireg + 0, __v) \ 198 imsic_set_switchcase_8(__ireg + 8, __v) 199 #define imsic_set_switchcase_32(__ireg, __v) \ 200 imsic_set_switchcase_16(__ireg + 0, __v) \ 201 imsic_set_switchcase_16(__ireg + 16, __v) 202 #define imsic_set_switchcase_64(__ireg, __v) \ 203 imsic_set_switchcase_32(__ireg + 0, __v) \ 204 imsic_set_switchcase_32(__ireg + 32, __v) 205 206 static void imsic_eix_set(int ireg, unsigned long val) 207 { 208 switch (ireg) { 209 imsic_set_switchcase_64(IMSIC_EIP0, val) 210 imsic_set_switchcase_64(IMSIC_EIE0, val) 211 } 212 } 213 214 static unsigned long imsic_mrif_atomic_rmw(struct imsic_mrif *mrif, 215 unsigned long *ptr, 216 unsigned long new_val, 217 unsigned long wr_mask) 218 { 219 unsigned long old_val = 0, tmp = 0; 220 221 __asm__ __volatile__ ( 222 "0: lr.w.aq %1, %0\n" 223 " and %2, %1, %3\n" 224 " or %2, %2, %4\n" 225 " sc.w.rl %2, %2, %0\n" 226 " bnez %2, 0b" 227 : "+A" (*ptr), "+r" (old_val), "+r" (tmp) 228 : "r" (~wr_mask), "r" (new_val & wr_mask) 229 : "memory"); 230 231 return old_val; 232 } 233 234 static unsigned long imsic_mrif_atomic_or(struct imsic_mrif *mrif, 235 unsigned long *ptr, 236 unsigned long val) 237 { 238 return atomic_long_fetch_or(val, (atomic_long_t *)ptr); 239 } 240 241 #define imsic_mrif_atomic_write(__mrif, __ptr, __new_val) \ 242 imsic_mrif_atomic_rmw(__mrif, __ptr, __new_val, -1UL) 243 #define imsic_mrif_atomic_read(__mrif, __ptr) \ 244 imsic_mrif_atomic_or(__mrif, __ptr, 0) 245 246 static u32 imsic_mrif_topei(struct imsic_mrif *mrif, u32 nr_eix, u32 nr_msis) 247 { 248 struct imsic_mrif_eix *eix; 249 u32 i, imin, imax, ei, max_msi; 250 unsigned long eipend[BITS_PER_TYPE(u64) / BITS_PER_LONG]; 251 unsigned long eithreshold = imsic_mrif_atomic_read(mrif, 252 &mrif->eithreshold); 253 254 max_msi = (eithreshold && (eithreshold <= nr_msis)) ? 255 eithreshold : nr_msis; 256 for (ei = 0; ei < nr_eix; ei++) { 257 eix = &mrif->eix[ei]; 258 eipend[0] = imsic_mrif_atomic_read(mrif, &eix->eie[0]) & 259 imsic_mrif_atomic_read(mrif, &eix->eip[0]); 260 #ifdef CONFIG_32BIT 261 eipend[1] = imsic_mrif_atomic_read(mrif, &eix->eie[1]) & 262 imsic_mrif_atomic_read(mrif, &eix->eip[1]); 263 if (!eipend[0] && !eipend[1]) 264 #else 265 if (!eipend[0]) 266 #endif 267 continue; 268 269 imin = ei * BITS_PER_TYPE(u64); 270 imax = ((imin + BITS_PER_TYPE(u64)) < max_msi) ? 271 imin + BITS_PER_TYPE(u64) : max_msi; 272 for (i = (!imin) ? 1 : imin; i < imax; i++) { 273 if (test_bit(i - imin, eipend)) 274 return (i << TOPEI_ID_SHIFT) | i; 275 } 276 } 277 278 return 0; 279 } 280 281 static int imsic_mrif_isel_check(u32 nr_eix, unsigned long isel) 282 { 283 u32 num = 0; 284 285 switch (isel) { 286 case IMSIC_EIDELIVERY: 287 case IMSIC_EITHRESHOLD: 288 break; 289 case IMSIC_EIP0 ... IMSIC_EIP63: 290 num = isel - IMSIC_EIP0; 291 break; 292 case IMSIC_EIE0 ... IMSIC_EIE63: 293 num = isel - IMSIC_EIE0; 294 break; 295 default: 296 return -ENOENT; 297 } 298 #ifndef CONFIG_32BIT 299 if (num & 0x1) 300 return -EINVAL; 301 #endif 302 if ((num / 2) >= nr_eix) 303 return -EINVAL; 304 305 return 0; 306 } 307 308 static int imsic_mrif_rmw(struct imsic_mrif *mrif, u32 nr_eix, 309 unsigned long isel, unsigned long *val, 310 unsigned long new_val, unsigned long wr_mask) 311 { 312 bool pend; 313 struct imsic_mrif_eix *eix; 314 unsigned long *ei, num, old_val = 0; 315 316 switch (isel) { 317 case IMSIC_EIDELIVERY: 318 old_val = imsic_mrif_atomic_rmw(mrif, &mrif->eidelivery, 319 new_val, wr_mask & 0x1); 320 break; 321 case IMSIC_EITHRESHOLD: 322 old_val = imsic_mrif_atomic_rmw(mrif, &mrif->eithreshold, 323 new_val, wr_mask & (IMSIC_MAX_ID - 1)); 324 break; 325 case IMSIC_EIP0 ... IMSIC_EIP63: 326 case IMSIC_EIE0 ... IMSIC_EIE63: 327 if (isel >= IMSIC_EIP0 && isel <= IMSIC_EIP63) { 328 pend = true; 329 num = isel - IMSIC_EIP0; 330 } else { 331 pend = false; 332 num = isel - IMSIC_EIE0; 333 } 334 335 if ((num / 2) >= nr_eix) 336 return -EINVAL; 337 eix = &mrif->eix[num / 2]; 338 339 #ifndef CONFIG_32BIT 340 if (num & 0x1) 341 return -EINVAL; 342 ei = (pend) ? &eix->eip[0] : &eix->eie[0]; 343 #else 344 ei = (pend) ? &eix->eip[num & 0x1] : &eix->eie[num & 0x1]; 345 #endif 346 347 /* Bit0 of EIP0 or EIE0 is read-only */ 348 if (!num) 349 wr_mask &= ~BIT(0); 350 351 old_val = imsic_mrif_atomic_rmw(mrif, ei, new_val, wr_mask); 352 break; 353 default: 354 return -ENOENT; 355 } 356 357 if (val) 358 *val = old_val; 359 360 return 0; 361 } 362 363 struct imsic_vsfile_read_data { 364 int hgei; 365 u32 nr_eix; 366 bool clear; 367 struct imsic_mrif *mrif; 368 }; 369 370 static void imsic_vsfile_local_read(void *data) 371 { 372 u32 i; 373 struct imsic_mrif_eix *eix; 374 struct imsic_vsfile_read_data *idata = data; 375 struct imsic_mrif *mrif = idata->mrif; 376 unsigned long new_hstatus, old_hstatus, old_vsiselect; 377 378 old_vsiselect = csr_read(CSR_VSISELECT); 379 old_hstatus = csr_read(CSR_HSTATUS); 380 new_hstatus = old_hstatus & ~HSTATUS_VGEIN; 381 new_hstatus |= ((unsigned long)idata->hgei) << HSTATUS_VGEIN_SHIFT; 382 csr_write(CSR_HSTATUS, new_hstatus); 383 384 /* 385 * We don't use imsic_mrif_atomic_xyz() functions to store 386 * values in MRIF because imsic_vsfile_read() is always called 387 * with pointer to temporary MRIF on stack. 388 */ 389 390 if (idata->clear) { 391 mrif->eidelivery = imsic_vs_csr_swap(IMSIC_EIDELIVERY, 0); 392 mrif->eithreshold = imsic_vs_csr_swap(IMSIC_EITHRESHOLD, 0); 393 for (i = 0; i < idata->nr_eix; i++) { 394 eix = &mrif->eix[i]; 395 eix->eip[0] = imsic_eix_swap(IMSIC_EIP0 + i * 2, 0); 396 eix->eie[0] = imsic_eix_swap(IMSIC_EIE0 + i * 2, 0); 397 #ifdef CONFIG_32BIT 398 eix->eip[1] = imsic_eix_swap(IMSIC_EIP0 + i * 2 + 1, 0); 399 eix->eie[1] = imsic_eix_swap(IMSIC_EIE0 + i * 2 + 1, 0); 400 #endif 401 } 402 } else { 403 mrif->eidelivery = imsic_vs_csr_read(IMSIC_EIDELIVERY); 404 mrif->eithreshold = imsic_vs_csr_read(IMSIC_EITHRESHOLD); 405 for (i = 0; i < idata->nr_eix; i++) { 406 eix = &mrif->eix[i]; 407 eix->eip[0] = imsic_eix_read(IMSIC_EIP0 + i * 2); 408 eix->eie[0] = imsic_eix_read(IMSIC_EIE0 + i * 2); 409 #ifdef CONFIG_32BIT 410 eix->eip[1] = imsic_eix_read(IMSIC_EIP0 + i * 2 + 1); 411 eix->eie[1] = imsic_eix_read(IMSIC_EIE0 + i * 2 + 1); 412 #endif 413 } 414 } 415 416 csr_write(CSR_HSTATUS, old_hstatus); 417 csr_write(CSR_VSISELECT, old_vsiselect); 418 } 419 420 static void imsic_vsfile_read(int vsfile_hgei, int vsfile_cpu, u32 nr_eix, 421 bool clear, struct imsic_mrif *mrif) 422 { 423 struct imsic_vsfile_read_data idata; 424 425 /* We can only read clear if we have a IMSIC VS-file */ 426 if (vsfile_cpu < 0 || vsfile_hgei <= 0) 427 return; 428 429 /* We can only read clear on local CPU */ 430 idata.hgei = vsfile_hgei; 431 idata.nr_eix = nr_eix; 432 idata.clear = clear; 433 idata.mrif = mrif; 434 on_each_cpu_mask(cpumask_of(vsfile_cpu), 435 imsic_vsfile_local_read, &idata, 1); 436 } 437 438 struct imsic_vsfile_rw_data { 439 int hgei; 440 int isel; 441 bool write; 442 unsigned long val; 443 }; 444 445 static void imsic_vsfile_local_rw(void *data) 446 { 447 struct imsic_vsfile_rw_data *idata = data; 448 unsigned long new_hstatus, old_hstatus, old_vsiselect; 449 450 old_vsiselect = csr_read(CSR_VSISELECT); 451 old_hstatus = csr_read(CSR_HSTATUS); 452 new_hstatus = old_hstatus & ~HSTATUS_VGEIN; 453 new_hstatus |= ((unsigned long)idata->hgei) << HSTATUS_VGEIN_SHIFT; 454 csr_write(CSR_HSTATUS, new_hstatus); 455 456 switch (idata->isel) { 457 case IMSIC_EIDELIVERY: 458 if (idata->write) 459 imsic_vs_csr_write(IMSIC_EIDELIVERY, idata->val); 460 else 461 idata->val = imsic_vs_csr_read(IMSIC_EIDELIVERY); 462 break; 463 case IMSIC_EITHRESHOLD: 464 if (idata->write) 465 imsic_vs_csr_write(IMSIC_EITHRESHOLD, idata->val); 466 else 467 idata->val = imsic_vs_csr_read(IMSIC_EITHRESHOLD); 468 break; 469 case IMSIC_EIP0 ... IMSIC_EIP63: 470 case IMSIC_EIE0 ... IMSIC_EIE63: 471 #ifndef CONFIG_32BIT 472 if (idata->isel & 0x1) 473 break; 474 #endif 475 if (idata->write) 476 imsic_eix_write(idata->isel, idata->val); 477 else 478 idata->val = imsic_eix_read(idata->isel); 479 break; 480 default: 481 break; 482 } 483 484 csr_write(CSR_HSTATUS, old_hstatus); 485 csr_write(CSR_VSISELECT, old_vsiselect); 486 } 487 488 static int imsic_vsfile_rw(int vsfile_hgei, int vsfile_cpu, u32 nr_eix, 489 unsigned long isel, bool write, 490 unsigned long *val) 491 { 492 int rc; 493 struct imsic_vsfile_rw_data rdata; 494 495 /* We can only access register if we have a IMSIC VS-file */ 496 if (vsfile_cpu < 0 || vsfile_hgei <= 0) 497 return -EINVAL; 498 499 /* Check IMSIC register iselect */ 500 rc = imsic_mrif_isel_check(nr_eix, isel); 501 if (rc) 502 return rc; 503 504 /* We can only access register on local CPU */ 505 rdata.hgei = vsfile_hgei; 506 rdata.isel = isel; 507 rdata.write = write; 508 rdata.val = (write) ? *val : 0; 509 on_each_cpu_mask(cpumask_of(vsfile_cpu), 510 imsic_vsfile_local_rw, &rdata, 1); 511 512 if (!write) 513 *val = rdata.val; 514 515 return 0; 516 } 517 518 static void imsic_vsfile_local_clear(int vsfile_hgei, u32 nr_eix) 519 { 520 u32 i; 521 unsigned long new_hstatus, old_hstatus, old_vsiselect; 522 523 /* We can only zero-out if we have a IMSIC VS-file */ 524 if (vsfile_hgei <= 0) 525 return; 526 527 old_vsiselect = csr_read(CSR_VSISELECT); 528 old_hstatus = csr_read(CSR_HSTATUS); 529 new_hstatus = old_hstatus & ~HSTATUS_VGEIN; 530 new_hstatus |= ((unsigned long)vsfile_hgei) << HSTATUS_VGEIN_SHIFT; 531 csr_write(CSR_HSTATUS, new_hstatus); 532 533 imsic_vs_csr_write(IMSIC_EIDELIVERY, 0); 534 imsic_vs_csr_write(IMSIC_EITHRESHOLD, 0); 535 for (i = 0; i < nr_eix; i++) { 536 imsic_eix_write(IMSIC_EIP0 + i * 2, 0); 537 imsic_eix_write(IMSIC_EIE0 + i * 2, 0); 538 #ifdef CONFIG_32BIT 539 imsic_eix_write(IMSIC_EIP0 + i * 2 + 1, 0); 540 imsic_eix_write(IMSIC_EIE0 + i * 2 + 1, 0); 541 #endif 542 } 543 544 csr_write(CSR_HSTATUS, old_hstatus); 545 csr_write(CSR_VSISELECT, old_vsiselect); 546 } 547 548 static void imsic_vsfile_local_update(int vsfile_hgei, u32 nr_eix, 549 struct imsic_mrif *mrif) 550 { 551 u32 i; 552 struct imsic_mrif_eix *eix; 553 unsigned long new_hstatus, old_hstatus, old_vsiselect; 554 555 /* We can only update if we have a HW IMSIC context */ 556 if (vsfile_hgei <= 0) 557 return; 558 559 /* 560 * We don't use imsic_mrif_atomic_xyz() functions to read values 561 * from MRIF in this function because it is always called with 562 * pointer to temporary MRIF on stack. 563 */ 564 565 old_vsiselect = csr_read(CSR_VSISELECT); 566 old_hstatus = csr_read(CSR_HSTATUS); 567 new_hstatus = old_hstatus & ~HSTATUS_VGEIN; 568 new_hstatus |= ((unsigned long)vsfile_hgei) << HSTATUS_VGEIN_SHIFT; 569 csr_write(CSR_HSTATUS, new_hstatus); 570 571 for (i = 0; i < nr_eix; i++) { 572 eix = &mrif->eix[i]; 573 imsic_eix_set(IMSIC_EIP0 + i * 2, eix->eip[0]); 574 imsic_eix_set(IMSIC_EIE0 + i * 2, eix->eie[0]); 575 #ifdef CONFIG_32BIT 576 imsic_eix_set(IMSIC_EIP0 + i * 2 + 1, eix->eip[1]); 577 imsic_eix_set(IMSIC_EIE0 + i * 2 + 1, eix->eie[1]); 578 #endif 579 } 580 imsic_vs_csr_write(IMSIC_EITHRESHOLD, mrif->eithreshold); 581 imsic_vs_csr_write(IMSIC_EIDELIVERY, mrif->eidelivery); 582 583 csr_write(CSR_HSTATUS, old_hstatus); 584 csr_write(CSR_VSISELECT, old_vsiselect); 585 } 586 587 static void imsic_vsfile_cleanup(struct imsic *imsic) 588 { 589 int old_vsfile_hgei, old_vsfile_cpu; 590 unsigned long flags; 591 592 /* 593 * We don't use imsic_mrif_atomic_xyz() functions to clear the 594 * SW-file in this function because it is always called when the 595 * VCPU is being destroyed. 596 */ 597 598 write_lock_irqsave(&imsic->vsfile_lock, flags); 599 old_vsfile_hgei = imsic->vsfile_hgei; 600 old_vsfile_cpu = imsic->vsfile_cpu; 601 imsic->vsfile_cpu = imsic->vsfile_hgei = -1; 602 imsic->vsfile_va = NULL; 603 imsic->vsfile_pa = 0; 604 write_unlock_irqrestore(&imsic->vsfile_lock, flags); 605 606 memset(imsic->swfile, 0, sizeof(*imsic->swfile)); 607 608 if (old_vsfile_cpu >= 0) 609 kvm_riscv_aia_free_hgei(old_vsfile_cpu, old_vsfile_hgei); 610 } 611 612 static void imsic_swfile_extirq_update(struct kvm_vcpu *vcpu) 613 { 614 struct imsic *imsic = vcpu->arch.aia_context.imsic_state; 615 struct imsic_mrif *mrif = imsic->swfile; 616 617 if (imsic_mrif_atomic_read(mrif, &mrif->eidelivery) && 618 imsic_mrif_topei(mrif, imsic->nr_eix, imsic->nr_msis)) 619 kvm_riscv_vcpu_set_interrupt(vcpu, IRQ_VS_EXT); 620 else 621 kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_EXT); 622 } 623 624 static void imsic_swfile_read(struct kvm_vcpu *vcpu, bool clear, 625 struct imsic_mrif *mrif) 626 { 627 struct imsic *imsic = vcpu->arch.aia_context.imsic_state; 628 629 /* 630 * We don't use imsic_mrif_atomic_xyz() functions to read and 631 * write SW-file and MRIF in this function because it is always 632 * called when VCPU is not using SW-file and the MRIF points to 633 * a temporary MRIF on stack. 634 */ 635 636 memcpy(mrif, imsic->swfile, sizeof(*mrif)); 637 if (clear) { 638 memset(imsic->swfile, 0, sizeof(*imsic->swfile)); 639 kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_EXT); 640 } 641 } 642 643 static void imsic_swfile_update(struct kvm_vcpu *vcpu, 644 struct imsic_mrif *mrif) 645 { 646 u32 i; 647 struct imsic_mrif_eix *seix, *eix; 648 struct imsic *imsic = vcpu->arch.aia_context.imsic_state; 649 struct imsic_mrif *smrif = imsic->swfile; 650 651 imsic_mrif_atomic_write(smrif, &smrif->eidelivery, mrif->eidelivery); 652 imsic_mrif_atomic_write(smrif, &smrif->eithreshold, mrif->eithreshold); 653 for (i = 0; i < imsic->nr_eix; i++) { 654 seix = &smrif->eix[i]; 655 eix = &mrif->eix[i]; 656 imsic_mrif_atomic_or(smrif, &seix->eip[0], eix->eip[0]); 657 imsic_mrif_atomic_or(smrif, &seix->eie[0], eix->eie[0]); 658 #ifdef CONFIG_32BIT 659 imsic_mrif_atomic_or(smrif, &seix->eip[1], eix->eip[1]); 660 imsic_mrif_atomic_or(smrif, &seix->eie[1], eix->eie[1]); 661 #endif 662 } 663 664 imsic_swfile_extirq_update(vcpu); 665 } 666 667 void kvm_riscv_vcpu_aia_imsic_release(struct kvm_vcpu *vcpu) 668 { 669 unsigned long flags; 670 struct imsic_mrif tmrif; 671 int old_vsfile_hgei, old_vsfile_cpu; 672 struct imsic *imsic = vcpu->arch.aia_context.imsic_state; 673 674 /* Read and clear IMSIC VS-file details */ 675 write_lock_irqsave(&imsic->vsfile_lock, flags); 676 old_vsfile_hgei = imsic->vsfile_hgei; 677 old_vsfile_cpu = imsic->vsfile_cpu; 678 imsic->vsfile_cpu = imsic->vsfile_hgei = -1; 679 imsic->vsfile_va = NULL; 680 imsic->vsfile_pa = 0; 681 write_unlock_irqrestore(&imsic->vsfile_lock, flags); 682 683 /* Do nothing, if no IMSIC VS-file to release */ 684 if (old_vsfile_cpu < 0) 685 return; 686 687 /* 688 * At this point, all interrupt producers are still using 689 * the old IMSIC VS-file so we first re-direct all interrupt 690 * producers. 691 */ 692 693 /* Purge the G-stage mapping */ 694 kvm_riscv_gstage_iounmap(vcpu->kvm, 695 vcpu->arch.aia_context.imsic_addr, 696 IMSIC_MMIO_PAGE_SZ); 697 698 /* TODO: Purge the IOMMU mapping ??? */ 699 700 /* 701 * At this point, all interrupt producers have been re-directed 702 * to somewhere else so we move register state from the old IMSIC 703 * VS-file to the IMSIC SW-file. 704 */ 705 706 /* Read and clear register state from old IMSIC VS-file */ 707 memset(&tmrif, 0, sizeof(tmrif)); 708 imsic_vsfile_read(old_vsfile_hgei, old_vsfile_cpu, imsic->nr_hw_eix, 709 true, &tmrif); 710 711 /* Update register state in IMSIC SW-file */ 712 imsic_swfile_update(vcpu, &tmrif); 713 714 /* Free-up old IMSIC VS-file */ 715 kvm_riscv_aia_free_hgei(old_vsfile_cpu, old_vsfile_hgei); 716 } 717 718 int kvm_riscv_vcpu_aia_imsic_update(struct kvm_vcpu *vcpu) 719 { 720 unsigned long flags; 721 phys_addr_t new_vsfile_pa; 722 struct imsic_mrif tmrif; 723 void __iomem *new_vsfile_va; 724 struct kvm *kvm = vcpu->kvm; 725 struct kvm_run *run = vcpu->run; 726 struct kvm_vcpu_aia *vaia = &vcpu->arch.aia_context; 727 struct imsic *imsic = vaia->imsic_state; 728 int ret = 0, new_vsfile_hgei = -1, old_vsfile_hgei, old_vsfile_cpu; 729 730 /* Do nothing for emulation mode */ 731 if (kvm->arch.aia.mode == KVM_DEV_RISCV_AIA_MODE_EMUL) 732 return 1; 733 734 /* Read old IMSIC VS-file details */ 735 read_lock_irqsave(&imsic->vsfile_lock, flags); 736 old_vsfile_hgei = imsic->vsfile_hgei; 737 old_vsfile_cpu = imsic->vsfile_cpu; 738 read_unlock_irqrestore(&imsic->vsfile_lock, flags); 739 740 /* Do nothing if we are continuing on same CPU */ 741 if (old_vsfile_cpu == vcpu->cpu) 742 return 1; 743 744 /* Allocate new IMSIC VS-file */ 745 ret = kvm_riscv_aia_alloc_hgei(vcpu->cpu, vcpu, 746 &new_vsfile_va, &new_vsfile_pa); 747 if (ret <= 0) { 748 /* For HW acceleration mode, we can't continue */ 749 if (kvm->arch.aia.mode == KVM_DEV_RISCV_AIA_MODE_HWACCEL) { 750 run->fail_entry.hardware_entry_failure_reason = 751 CSR_HSTATUS; 752 run->fail_entry.cpu = vcpu->cpu; 753 run->exit_reason = KVM_EXIT_FAIL_ENTRY; 754 return 0; 755 } 756 757 /* Release old IMSIC VS-file */ 758 if (old_vsfile_cpu >= 0) 759 kvm_riscv_vcpu_aia_imsic_release(vcpu); 760 761 /* For automatic mode, we continue */ 762 goto done; 763 } 764 new_vsfile_hgei = ret; 765 766 /* 767 * At this point, all interrupt producers are still using 768 * to the old IMSIC VS-file so we first move all interrupt 769 * producers to the new IMSIC VS-file. 770 */ 771 772 /* Zero-out new IMSIC VS-file */ 773 imsic_vsfile_local_clear(new_vsfile_hgei, imsic->nr_hw_eix); 774 775 /* Update G-stage mapping for the new IMSIC VS-file */ 776 ret = kvm_riscv_gstage_ioremap(kvm, vcpu->arch.aia_context.imsic_addr, 777 new_vsfile_pa, IMSIC_MMIO_PAGE_SZ, 778 true, true); 779 if (ret) 780 goto fail_free_vsfile_hgei; 781 782 /* TODO: Update the IOMMU mapping ??? */ 783 784 /* Update new IMSIC VS-file details in IMSIC context */ 785 write_lock_irqsave(&imsic->vsfile_lock, flags); 786 imsic->vsfile_hgei = new_vsfile_hgei; 787 imsic->vsfile_cpu = vcpu->cpu; 788 imsic->vsfile_va = new_vsfile_va; 789 imsic->vsfile_pa = new_vsfile_pa; 790 write_unlock_irqrestore(&imsic->vsfile_lock, flags); 791 792 /* 793 * At this point, all interrupt producers have been moved 794 * to the new IMSIC VS-file so we move register state from 795 * the old IMSIC VS/SW-file to the new IMSIC VS-file. 796 */ 797 798 memset(&tmrif, 0, sizeof(tmrif)); 799 if (old_vsfile_cpu >= 0) { 800 /* Read and clear register state from old IMSIC VS-file */ 801 imsic_vsfile_read(old_vsfile_hgei, old_vsfile_cpu, 802 imsic->nr_hw_eix, true, &tmrif); 803 804 /* Free-up old IMSIC VS-file */ 805 kvm_riscv_aia_free_hgei(old_vsfile_cpu, old_vsfile_hgei); 806 } else { 807 /* Read and clear register state from IMSIC SW-file */ 808 imsic_swfile_read(vcpu, true, &tmrif); 809 } 810 811 /* Restore register state in the new IMSIC VS-file */ 812 imsic_vsfile_local_update(new_vsfile_hgei, imsic->nr_hw_eix, &tmrif); 813 814 done: 815 /* Set VCPU HSTATUS.VGEIN to new IMSIC VS-file */ 816 vcpu->arch.guest_context.hstatus &= ~HSTATUS_VGEIN; 817 if (new_vsfile_hgei > 0) 818 vcpu->arch.guest_context.hstatus |= 819 ((unsigned long)new_vsfile_hgei) << HSTATUS_VGEIN_SHIFT; 820 821 /* Continue run-loop */ 822 return 1; 823 824 fail_free_vsfile_hgei: 825 kvm_riscv_aia_free_hgei(vcpu->cpu, new_vsfile_hgei); 826 return ret; 827 } 828 829 int kvm_riscv_vcpu_aia_imsic_rmw(struct kvm_vcpu *vcpu, unsigned long isel, 830 unsigned long *val, unsigned long new_val, 831 unsigned long wr_mask) 832 { 833 u32 topei; 834 struct imsic_mrif_eix *eix; 835 int r, rc = KVM_INSN_CONTINUE_NEXT_SEPC; 836 struct imsic *imsic = vcpu->arch.aia_context.imsic_state; 837 838 if (isel == KVM_RISCV_AIA_IMSIC_TOPEI) { 839 /* Read pending and enabled interrupt with highest priority */ 840 topei = imsic_mrif_topei(imsic->swfile, imsic->nr_eix, 841 imsic->nr_msis); 842 if (val) 843 *val = topei; 844 845 /* Writes ignore value and clear top pending interrupt */ 846 if (topei && wr_mask) { 847 topei >>= TOPEI_ID_SHIFT; 848 if (topei) { 849 eix = &imsic->swfile->eix[topei / 850 BITS_PER_TYPE(u64)]; 851 clear_bit(topei & (BITS_PER_TYPE(u64) - 1), 852 eix->eip); 853 } 854 } 855 } else { 856 r = imsic_mrif_rmw(imsic->swfile, imsic->nr_eix, isel, 857 val, new_val, wr_mask); 858 /* Forward unknown IMSIC register to user-space */ 859 if (r) 860 rc = (r == -ENOENT) ? 0 : KVM_INSN_ILLEGAL_TRAP; 861 } 862 863 if (wr_mask) 864 imsic_swfile_extirq_update(vcpu); 865 866 return rc; 867 } 868 869 int kvm_riscv_aia_imsic_rw_attr(struct kvm *kvm, unsigned long type, 870 bool write, unsigned long *val) 871 { 872 u32 isel, vcpu_id; 873 unsigned long flags; 874 struct imsic *imsic; 875 struct kvm_vcpu *vcpu; 876 int rc, vsfile_hgei, vsfile_cpu; 877 878 if (!kvm_riscv_aia_initialized(kvm)) 879 return -ENODEV; 880 881 vcpu_id = KVM_DEV_RISCV_AIA_IMSIC_GET_VCPU(type); 882 vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id); 883 if (!vcpu) 884 return -ENODEV; 885 886 isel = KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(type); 887 imsic = vcpu->arch.aia_context.imsic_state; 888 889 read_lock_irqsave(&imsic->vsfile_lock, flags); 890 891 rc = 0; 892 vsfile_hgei = imsic->vsfile_hgei; 893 vsfile_cpu = imsic->vsfile_cpu; 894 if (vsfile_cpu < 0) { 895 if (write) { 896 rc = imsic_mrif_rmw(imsic->swfile, imsic->nr_eix, 897 isel, NULL, *val, -1UL); 898 imsic_swfile_extirq_update(vcpu); 899 } else 900 rc = imsic_mrif_rmw(imsic->swfile, imsic->nr_eix, 901 isel, val, 0, 0); 902 } 903 904 read_unlock_irqrestore(&imsic->vsfile_lock, flags); 905 906 if (!rc && vsfile_cpu >= 0) 907 rc = imsic_vsfile_rw(vsfile_hgei, vsfile_cpu, imsic->nr_eix, 908 isel, write, val); 909 910 return rc; 911 } 912 913 int kvm_riscv_aia_imsic_has_attr(struct kvm *kvm, unsigned long type) 914 { 915 u32 isel, vcpu_id; 916 struct imsic *imsic; 917 struct kvm_vcpu *vcpu; 918 919 if (!kvm_riscv_aia_initialized(kvm)) 920 return -ENODEV; 921 922 vcpu_id = KVM_DEV_RISCV_AIA_IMSIC_GET_VCPU(type); 923 vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id); 924 if (!vcpu) 925 return -ENODEV; 926 927 isel = KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(type); 928 imsic = vcpu->arch.aia_context.imsic_state; 929 return imsic_mrif_isel_check(imsic->nr_eix, isel); 930 } 931 932 void kvm_riscv_vcpu_aia_imsic_reset(struct kvm_vcpu *vcpu) 933 { 934 struct imsic *imsic = vcpu->arch.aia_context.imsic_state; 935 936 if (!imsic) 937 return; 938 939 kvm_riscv_vcpu_aia_imsic_release(vcpu); 940 941 memset(imsic->swfile, 0, sizeof(*imsic->swfile)); 942 } 943 944 int kvm_riscv_vcpu_aia_imsic_inject(struct kvm_vcpu *vcpu, 945 u32 guest_index, u32 offset, u32 iid) 946 { 947 unsigned long flags; 948 struct imsic_mrif_eix *eix; 949 struct imsic *imsic = vcpu->arch.aia_context.imsic_state; 950 951 /* We only emulate one IMSIC MMIO page for each Guest VCPU */ 952 if (!imsic || !iid || guest_index || 953 (offset != IMSIC_MMIO_SETIPNUM_LE && 954 offset != IMSIC_MMIO_SETIPNUM_BE)) 955 return -ENODEV; 956 957 iid = (offset == IMSIC_MMIO_SETIPNUM_BE) ? __swab32(iid) : iid; 958 if (imsic->nr_msis <= iid) 959 return -EINVAL; 960 961 read_lock_irqsave(&imsic->vsfile_lock, flags); 962 963 if (imsic->vsfile_cpu >= 0) { 964 writel(iid, imsic->vsfile_va + IMSIC_MMIO_SETIPNUM_LE); 965 kvm_vcpu_kick(vcpu); 966 } else { 967 eix = &imsic->swfile->eix[iid / BITS_PER_TYPE(u64)]; 968 set_bit(iid & (BITS_PER_TYPE(u64) - 1), eix->eip); 969 imsic_swfile_extirq_update(vcpu); 970 } 971 972 read_unlock_irqrestore(&imsic->vsfile_lock, flags); 973 974 return 0; 975 } 976 977 static int imsic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev, 978 gpa_t addr, int len, void *val) 979 { 980 if (len != 4 || (addr & 0x3) != 0) 981 return -EOPNOTSUPP; 982 983 *((u32 *)val) = 0; 984 985 return 0; 986 } 987 988 static int imsic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev, 989 gpa_t addr, int len, const void *val) 990 { 991 struct kvm_msi msi = { 0 }; 992 993 if (len != 4 || (addr & 0x3) != 0) 994 return -EOPNOTSUPP; 995 996 msi.address_hi = addr >> 32; 997 msi.address_lo = (u32)addr; 998 msi.data = *((const u32 *)val); 999 kvm_riscv_aia_inject_msi(vcpu->kvm, &msi); 1000 1001 return 0; 1002 }; 1003 1004 static struct kvm_io_device_ops imsic_iodoev_ops = { 1005 .read = imsic_mmio_read, 1006 .write = imsic_mmio_write, 1007 }; 1008 1009 int kvm_riscv_vcpu_aia_imsic_init(struct kvm_vcpu *vcpu) 1010 { 1011 int ret = 0; 1012 struct imsic *imsic; 1013 struct page *swfile_page; 1014 struct kvm *kvm = vcpu->kvm; 1015 1016 /* Fail if we have zero IDs */ 1017 if (!kvm->arch.aia.nr_ids) 1018 return -EINVAL; 1019 1020 /* Allocate IMSIC context */ 1021 imsic = kzalloc(sizeof(*imsic), GFP_KERNEL); 1022 if (!imsic) 1023 return -ENOMEM; 1024 vcpu->arch.aia_context.imsic_state = imsic; 1025 1026 /* Setup IMSIC context */ 1027 imsic->nr_msis = kvm->arch.aia.nr_ids + 1; 1028 rwlock_init(&imsic->vsfile_lock); 1029 imsic->nr_eix = BITS_TO_U64(imsic->nr_msis); 1030 imsic->nr_hw_eix = BITS_TO_U64(kvm_riscv_aia_max_ids); 1031 imsic->vsfile_hgei = imsic->vsfile_cpu = -1; 1032 1033 /* Setup IMSIC SW-file */ 1034 swfile_page = alloc_pages(GFP_KERNEL | __GFP_ZERO, 1035 get_order(sizeof(*imsic->swfile))); 1036 if (!swfile_page) { 1037 ret = -ENOMEM; 1038 goto fail_free_imsic; 1039 } 1040 imsic->swfile = page_to_virt(swfile_page); 1041 imsic->swfile_pa = page_to_phys(swfile_page); 1042 1043 /* Setup IO device */ 1044 kvm_iodevice_init(&imsic->iodev, &imsic_iodoev_ops); 1045 mutex_lock(&kvm->slots_lock); 1046 ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, 1047 vcpu->arch.aia_context.imsic_addr, 1048 KVM_DEV_RISCV_IMSIC_SIZE, 1049 &imsic->iodev); 1050 mutex_unlock(&kvm->slots_lock); 1051 if (ret) 1052 goto fail_free_swfile; 1053 1054 return 0; 1055 1056 fail_free_swfile: 1057 free_pages((unsigned long)imsic->swfile, 1058 get_order(sizeof(*imsic->swfile))); 1059 fail_free_imsic: 1060 vcpu->arch.aia_context.imsic_state = NULL; 1061 kfree(imsic); 1062 return ret; 1063 } 1064 1065 void kvm_riscv_vcpu_aia_imsic_cleanup(struct kvm_vcpu *vcpu) 1066 { 1067 struct kvm *kvm = vcpu->kvm; 1068 struct imsic *imsic = vcpu->arch.aia_context.imsic_state; 1069 1070 if (!imsic) 1071 return; 1072 1073 imsic_vsfile_cleanup(imsic); 1074 1075 mutex_lock(&kvm->slots_lock); 1076 kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &imsic->iodev); 1077 mutex_unlock(&kvm->slots_lock); 1078 1079 free_pages((unsigned long)imsic->swfile, 1080 get_order(sizeof(*imsic->swfile))); 1081 1082 vcpu->arch.aia_context.imsic_state = NULL; 1083 kfree(imsic); 1084 } 1085