1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2021 Western Digital Corporation or its affiliates. 4 * Copyright (C) 2022 Ventana Micro Systems Inc. 5 * 6 * Authors: 7 * Anup Patel <apatel@ventanamicro.com> 8 */ 9 10 #include <linux/atomic.h> 11 #include <linux/bitmap.h> 12 #include <linux/kvm_host.h> 13 #include <linux/math.h> 14 #include <linux/spinlock.h> 15 #include <linux/swab.h> 16 #include <kvm/iodev.h> 17 #include <asm/csr.h> 18 #include <asm/kvm_aia_imsic.h> 19 20 #define IMSIC_MAX_EIX (IMSIC_MAX_ID / BITS_PER_TYPE(u64)) 21 22 struct imsic_mrif_eix { 23 unsigned long eip[BITS_PER_TYPE(u64) / BITS_PER_LONG]; 24 unsigned long eie[BITS_PER_TYPE(u64) / BITS_PER_LONG]; 25 }; 26 27 struct imsic_mrif { 28 struct imsic_mrif_eix eix[IMSIC_MAX_EIX]; 29 unsigned long eithreshold; 30 unsigned long eidelivery; 31 }; 32 33 struct imsic { 34 struct kvm_io_device iodev; 35 36 u32 nr_msis; 37 u32 nr_eix; 38 u32 nr_hw_eix; 39 40 /* 41 * At any point in time, the register state is in 42 * one of the following places: 43 * 44 * 1) Hardware: IMSIC VS-file (vsfile_cpu >= 0) 45 * 2) Software: IMSIC SW-file (vsfile_cpu < 0) 46 */ 47 48 /* IMSIC VS-file */ 49 rwlock_t vsfile_lock; 50 int vsfile_cpu; 51 int vsfile_hgei; 52 void __iomem *vsfile_va; 53 phys_addr_t vsfile_pa; 54 55 /* IMSIC SW-file */ 56 struct imsic_mrif *swfile; 57 phys_addr_t swfile_pa; 58 spinlock_t swfile_extirq_lock; 59 }; 60 61 #define imsic_vs_csr_read(__c) \ 62 ({ \ 63 unsigned long __r; \ 64 csr_write(CSR_VSISELECT, __c); \ 65 __r = csr_read(CSR_VSIREG); \ 66 __r; \ 67 }) 68 69 #define imsic_read_switchcase(__ireg) \ 70 case __ireg: \ 71 return imsic_vs_csr_read(__ireg); 72 #define imsic_read_switchcase_2(__ireg) \ 73 imsic_read_switchcase(__ireg + 0) \ 74 imsic_read_switchcase(__ireg + 1) 75 #define imsic_read_switchcase_4(__ireg) \ 76 imsic_read_switchcase_2(__ireg + 0) \ 77 imsic_read_switchcase_2(__ireg + 2) 78 #define imsic_read_switchcase_8(__ireg) \ 79 imsic_read_switchcase_4(__ireg + 0) \ 80 imsic_read_switchcase_4(__ireg + 4) 81 #define imsic_read_switchcase_16(__ireg) \ 82 imsic_read_switchcase_8(__ireg + 0) \ 83 imsic_read_switchcase_8(__ireg + 8) 84 #define imsic_read_switchcase_32(__ireg) \ 85 imsic_read_switchcase_16(__ireg + 0) \ 86 imsic_read_switchcase_16(__ireg + 16) 87 #define imsic_read_switchcase_64(__ireg) \ 88 imsic_read_switchcase_32(__ireg + 0) \ 89 imsic_read_switchcase_32(__ireg + 32) 90 91 static unsigned long imsic_eix_read(int ireg) 92 { 93 switch (ireg) { 94 imsic_read_switchcase_64(IMSIC_EIP0) 95 imsic_read_switchcase_64(IMSIC_EIE0) 96 } 97 98 return 0; 99 } 100 101 #define imsic_vs_csr_swap(__c, __v) \ 102 ({ \ 103 unsigned long __r; \ 104 csr_write(CSR_VSISELECT, __c); \ 105 __r = csr_swap(CSR_VSIREG, __v); \ 106 __r; \ 107 }) 108 109 #define imsic_swap_switchcase(__ireg, __v) \ 110 case __ireg: \ 111 return imsic_vs_csr_swap(__ireg, __v); 112 #define imsic_swap_switchcase_2(__ireg, __v) \ 113 imsic_swap_switchcase(__ireg + 0, __v) \ 114 imsic_swap_switchcase(__ireg + 1, __v) 115 #define imsic_swap_switchcase_4(__ireg, __v) \ 116 imsic_swap_switchcase_2(__ireg + 0, __v) \ 117 imsic_swap_switchcase_2(__ireg + 2, __v) 118 #define imsic_swap_switchcase_8(__ireg, __v) \ 119 imsic_swap_switchcase_4(__ireg + 0, __v) \ 120 imsic_swap_switchcase_4(__ireg + 4, __v) 121 #define imsic_swap_switchcase_16(__ireg, __v) \ 122 imsic_swap_switchcase_8(__ireg + 0, __v) \ 123 imsic_swap_switchcase_8(__ireg + 8, __v) 124 #define imsic_swap_switchcase_32(__ireg, __v) \ 125 imsic_swap_switchcase_16(__ireg + 0, __v) \ 126 imsic_swap_switchcase_16(__ireg + 16, __v) 127 #define imsic_swap_switchcase_64(__ireg, __v) \ 128 imsic_swap_switchcase_32(__ireg + 0, __v) \ 129 imsic_swap_switchcase_32(__ireg + 32, __v) 130 131 static unsigned long imsic_eix_swap(int ireg, unsigned long val) 132 { 133 switch (ireg) { 134 imsic_swap_switchcase_64(IMSIC_EIP0, val) 135 imsic_swap_switchcase_64(IMSIC_EIE0, val) 136 } 137 138 return 0; 139 } 140 141 #define imsic_vs_csr_write(__c, __v) \ 142 do { \ 143 csr_write(CSR_VSISELECT, __c); \ 144 csr_write(CSR_VSIREG, __v); \ 145 } while (0) 146 147 #define imsic_write_switchcase(__ireg, __v) \ 148 case __ireg: \ 149 imsic_vs_csr_write(__ireg, __v); \ 150 break; 151 #define imsic_write_switchcase_2(__ireg, __v) \ 152 imsic_write_switchcase(__ireg + 0, __v) \ 153 imsic_write_switchcase(__ireg + 1, __v) 154 #define imsic_write_switchcase_4(__ireg, __v) \ 155 imsic_write_switchcase_2(__ireg + 0, __v) \ 156 imsic_write_switchcase_2(__ireg + 2, __v) 157 #define imsic_write_switchcase_8(__ireg, __v) \ 158 imsic_write_switchcase_4(__ireg + 0, __v) \ 159 imsic_write_switchcase_4(__ireg + 4, __v) 160 #define imsic_write_switchcase_16(__ireg, __v) \ 161 imsic_write_switchcase_8(__ireg + 0, __v) \ 162 imsic_write_switchcase_8(__ireg + 8, __v) 163 #define imsic_write_switchcase_32(__ireg, __v) \ 164 imsic_write_switchcase_16(__ireg + 0, __v) \ 165 imsic_write_switchcase_16(__ireg + 16, __v) 166 #define imsic_write_switchcase_64(__ireg, __v) \ 167 imsic_write_switchcase_32(__ireg + 0, __v) \ 168 imsic_write_switchcase_32(__ireg + 32, __v) 169 170 static void imsic_eix_write(int ireg, unsigned long val) 171 { 172 switch (ireg) { 173 imsic_write_switchcase_64(IMSIC_EIP0, val) 174 imsic_write_switchcase_64(IMSIC_EIE0, val) 175 } 176 } 177 178 #define imsic_vs_csr_set(__c, __v) \ 179 do { \ 180 csr_write(CSR_VSISELECT, __c); \ 181 csr_set(CSR_VSIREG, __v); \ 182 } while (0) 183 184 #define imsic_set_switchcase(__ireg, __v) \ 185 case __ireg: \ 186 imsic_vs_csr_set(__ireg, __v); \ 187 break; 188 #define imsic_set_switchcase_2(__ireg, __v) \ 189 imsic_set_switchcase(__ireg + 0, __v) \ 190 imsic_set_switchcase(__ireg + 1, __v) 191 #define imsic_set_switchcase_4(__ireg, __v) \ 192 imsic_set_switchcase_2(__ireg + 0, __v) \ 193 imsic_set_switchcase_2(__ireg + 2, __v) 194 #define imsic_set_switchcase_8(__ireg, __v) \ 195 imsic_set_switchcase_4(__ireg + 0, __v) \ 196 imsic_set_switchcase_4(__ireg + 4, __v) 197 #define imsic_set_switchcase_16(__ireg, __v) \ 198 imsic_set_switchcase_8(__ireg + 0, __v) \ 199 imsic_set_switchcase_8(__ireg + 8, __v) 200 #define imsic_set_switchcase_32(__ireg, __v) \ 201 imsic_set_switchcase_16(__ireg + 0, __v) \ 202 imsic_set_switchcase_16(__ireg + 16, __v) 203 #define imsic_set_switchcase_64(__ireg, __v) \ 204 imsic_set_switchcase_32(__ireg + 0, __v) \ 205 imsic_set_switchcase_32(__ireg + 32, __v) 206 207 static void imsic_eix_set(int ireg, unsigned long val) 208 { 209 switch (ireg) { 210 imsic_set_switchcase_64(IMSIC_EIP0, val) 211 imsic_set_switchcase_64(IMSIC_EIE0, val) 212 } 213 } 214 215 static unsigned long imsic_mrif_atomic_rmw(struct imsic_mrif *mrif, 216 unsigned long *ptr, 217 unsigned long new_val, 218 unsigned long wr_mask) 219 { 220 unsigned long old_val = 0, tmp = 0; 221 222 __asm__ __volatile__ ( 223 "0: lr.w.aq %1, %0\n" 224 " and %2, %1, %3\n" 225 " or %2, %2, %4\n" 226 " sc.w.rl %2, %2, %0\n" 227 " bnez %2, 0b" 228 : "+A" (*ptr), "+r" (old_val), "+r" (tmp) 229 : "r" (~wr_mask), "r" (new_val & wr_mask) 230 : "memory"); 231 232 return old_val; 233 } 234 235 static unsigned long imsic_mrif_atomic_or(struct imsic_mrif *mrif, 236 unsigned long *ptr, 237 unsigned long val) 238 { 239 return atomic_long_fetch_or(val, (atomic_long_t *)ptr); 240 } 241 242 #define imsic_mrif_atomic_write(__mrif, __ptr, __new_val) \ 243 imsic_mrif_atomic_rmw(__mrif, __ptr, __new_val, -1UL) 244 #define imsic_mrif_atomic_read(__mrif, __ptr) \ 245 imsic_mrif_atomic_or(__mrif, __ptr, 0) 246 247 static u32 imsic_mrif_topei(struct imsic_mrif *mrif, u32 nr_eix, u32 nr_msis) 248 { 249 struct imsic_mrif_eix *eix; 250 u32 i, imin, imax, ei, max_msi; 251 unsigned long eipend[BITS_PER_TYPE(u64) / BITS_PER_LONG]; 252 unsigned long eithreshold = imsic_mrif_atomic_read(mrif, 253 &mrif->eithreshold); 254 255 max_msi = (eithreshold && (eithreshold <= nr_msis)) ? 256 eithreshold : nr_msis; 257 for (ei = 0; ei < nr_eix; ei++) { 258 eix = &mrif->eix[ei]; 259 eipend[0] = imsic_mrif_atomic_read(mrif, &eix->eie[0]) & 260 imsic_mrif_atomic_read(mrif, &eix->eip[0]); 261 #ifdef CONFIG_32BIT 262 eipend[1] = imsic_mrif_atomic_read(mrif, &eix->eie[1]) & 263 imsic_mrif_atomic_read(mrif, &eix->eip[1]); 264 if (!eipend[0] && !eipend[1]) 265 #else 266 if (!eipend[0]) 267 #endif 268 continue; 269 270 imin = ei * BITS_PER_TYPE(u64); 271 imax = ((imin + BITS_PER_TYPE(u64)) < max_msi) ? 272 imin + BITS_PER_TYPE(u64) : max_msi; 273 for (i = (!imin) ? 1 : imin; i < imax; i++) { 274 if (test_bit(i - imin, eipend)) 275 return (i << TOPEI_ID_SHIFT) | i; 276 } 277 } 278 279 return 0; 280 } 281 282 static int imsic_mrif_isel_check(u32 nr_eix, unsigned long isel) 283 { 284 u32 num = 0; 285 286 switch (isel) { 287 case IMSIC_EIDELIVERY: 288 case IMSIC_EITHRESHOLD: 289 break; 290 case IMSIC_EIP0 ... IMSIC_EIP63: 291 num = isel - IMSIC_EIP0; 292 break; 293 case IMSIC_EIE0 ... IMSIC_EIE63: 294 num = isel - IMSIC_EIE0; 295 break; 296 default: 297 return -ENOENT; 298 } 299 #ifndef CONFIG_32BIT 300 if (num & 0x1) 301 return -EINVAL; 302 #endif 303 if ((num / 2) >= nr_eix) 304 return -EINVAL; 305 306 return 0; 307 } 308 309 static int imsic_mrif_rmw(struct imsic_mrif *mrif, u32 nr_eix, 310 unsigned long isel, unsigned long *val, 311 unsigned long new_val, unsigned long wr_mask) 312 { 313 bool pend; 314 struct imsic_mrif_eix *eix; 315 unsigned long *ei, num, old_val = 0; 316 317 switch (isel) { 318 case IMSIC_EIDELIVERY: 319 old_val = imsic_mrif_atomic_rmw(mrif, &mrif->eidelivery, 320 new_val, wr_mask & 0x1); 321 break; 322 case IMSIC_EITHRESHOLD: 323 old_val = imsic_mrif_atomic_rmw(mrif, &mrif->eithreshold, 324 new_val, wr_mask & (IMSIC_MAX_ID - 1)); 325 break; 326 case IMSIC_EIP0 ... IMSIC_EIP63: 327 case IMSIC_EIE0 ... IMSIC_EIE63: 328 if (isel >= IMSIC_EIP0 && isel <= IMSIC_EIP63) { 329 pend = true; 330 num = isel - IMSIC_EIP0; 331 } else { 332 pend = false; 333 num = isel - IMSIC_EIE0; 334 } 335 336 if ((num / 2) >= nr_eix) 337 return -EINVAL; 338 eix = &mrif->eix[num / 2]; 339 340 #ifndef CONFIG_32BIT 341 if (num & 0x1) 342 return -EINVAL; 343 ei = (pend) ? &eix->eip[0] : &eix->eie[0]; 344 #else 345 ei = (pend) ? &eix->eip[num & 0x1] : &eix->eie[num & 0x1]; 346 #endif 347 348 /* Bit0 of EIP0 or EIE0 is read-only */ 349 if (!num) 350 wr_mask &= ~BIT(0); 351 352 old_val = imsic_mrif_atomic_rmw(mrif, ei, new_val, wr_mask); 353 break; 354 default: 355 return -ENOENT; 356 } 357 358 if (val) 359 *val = old_val; 360 361 return 0; 362 } 363 364 struct imsic_vsfile_read_data { 365 int hgei; 366 u32 nr_eix; 367 bool clear; 368 struct imsic_mrif *mrif; 369 }; 370 371 static void imsic_vsfile_local_read(void *data) 372 { 373 u32 i; 374 struct imsic_mrif_eix *eix; 375 struct imsic_vsfile_read_data *idata = data; 376 struct imsic_mrif *mrif = idata->mrif; 377 unsigned long new_hstatus, old_hstatus, old_vsiselect; 378 379 old_vsiselect = csr_read(CSR_VSISELECT); 380 old_hstatus = csr_read(CSR_HSTATUS); 381 new_hstatus = old_hstatus & ~HSTATUS_VGEIN; 382 new_hstatus |= ((unsigned long)idata->hgei) << HSTATUS_VGEIN_SHIFT; 383 csr_write(CSR_HSTATUS, new_hstatus); 384 385 /* 386 * We don't use imsic_mrif_atomic_xyz() functions to store 387 * values in MRIF because imsic_vsfile_read() is always called 388 * with pointer to temporary MRIF on stack. 389 */ 390 391 if (idata->clear) { 392 mrif->eidelivery = imsic_vs_csr_swap(IMSIC_EIDELIVERY, 0); 393 mrif->eithreshold = imsic_vs_csr_swap(IMSIC_EITHRESHOLD, 0); 394 for (i = 0; i < idata->nr_eix; i++) { 395 eix = &mrif->eix[i]; 396 eix->eip[0] = imsic_eix_swap(IMSIC_EIP0 + i * 2, 0); 397 eix->eie[0] = imsic_eix_swap(IMSIC_EIE0 + i * 2, 0); 398 #ifdef CONFIG_32BIT 399 eix->eip[1] = imsic_eix_swap(IMSIC_EIP0 + i * 2 + 1, 0); 400 eix->eie[1] = imsic_eix_swap(IMSIC_EIE0 + i * 2 + 1, 0); 401 #endif 402 } 403 } else { 404 mrif->eidelivery = imsic_vs_csr_read(IMSIC_EIDELIVERY); 405 mrif->eithreshold = imsic_vs_csr_read(IMSIC_EITHRESHOLD); 406 for (i = 0; i < idata->nr_eix; i++) { 407 eix = &mrif->eix[i]; 408 eix->eip[0] = imsic_eix_read(IMSIC_EIP0 + i * 2); 409 eix->eie[0] = imsic_eix_read(IMSIC_EIE0 + i * 2); 410 #ifdef CONFIG_32BIT 411 eix->eip[1] = imsic_eix_read(IMSIC_EIP0 + i * 2 + 1); 412 eix->eie[1] = imsic_eix_read(IMSIC_EIE0 + i * 2 + 1); 413 #endif 414 } 415 } 416 417 csr_write(CSR_HSTATUS, old_hstatus); 418 csr_write(CSR_VSISELECT, old_vsiselect); 419 } 420 421 static void imsic_vsfile_read(int vsfile_hgei, int vsfile_cpu, u32 nr_eix, 422 bool clear, struct imsic_mrif *mrif) 423 { 424 struct imsic_vsfile_read_data idata; 425 426 /* We can only read clear if we have a IMSIC VS-file */ 427 if (vsfile_cpu < 0 || vsfile_hgei <= 0) 428 return; 429 430 /* We can only read clear on local CPU */ 431 idata.hgei = vsfile_hgei; 432 idata.nr_eix = nr_eix; 433 idata.clear = clear; 434 idata.mrif = mrif; 435 on_each_cpu_mask(cpumask_of(vsfile_cpu), 436 imsic_vsfile_local_read, &idata, 1); 437 } 438 439 struct imsic_vsfile_rw_data { 440 int hgei; 441 int isel; 442 bool write; 443 unsigned long val; 444 }; 445 446 static void imsic_vsfile_local_rw(void *data) 447 { 448 struct imsic_vsfile_rw_data *idata = data; 449 unsigned long new_hstatus, old_hstatus, old_vsiselect; 450 451 old_vsiselect = csr_read(CSR_VSISELECT); 452 old_hstatus = csr_read(CSR_HSTATUS); 453 new_hstatus = old_hstatus & ~HSTATUS_VGEIN; 454 new_hstatus |= ((unsigned long)idata->hgei) << HSTATUS_VGEIN_SHIFT; 455 csr_write(CSR_HSTATUS, new_hstatus); 456 457 switch (idata->isel) { 458 case IMSIC_EIDELIVERY: 459 if (idata->write) 460 imsic_vs_csr_write(IMSIC_EIDELIVERY, idata->val); 461 else 462 idata->val = imsic_vs_csr_read(IMSIC_EIDELIVERY); 463 break; 464 case IMSIC_EITHRESHOLD: 465 if (idata->write) 466 imsic_vs_csr_write(IMSIC_EITHRESHOLD, idata->val); 467 else 468 idata->val = imsic_vs_csr_read(IMSIC_EITHRESHOLD); 469 break; 470 case IMSIC_EIP0 ... IMSIC_EIP63: 471 case IMSIC_EIE0 ... IMSIC_EIE63: 472 #ifndef CONFIG_32BIT 473 if (idata->isel & 0x1) 474 break; 475 #endif 476 if (idata->write) 477 imsic_eix_write(idata->isel, idata->val); 478 else 479 idata->val = imsic_eix_read(idata->isel); 480 break; 481 default: 482 break; 483 } 484 485 csr_write(CSR_HSTATUS, old_hstatus); 486 csr_write(CSR_VSISELECT, old_vsiselect); 487 } 488 489 static int imsic_vsfile_rw(int vsfile_hgei, int vsfile_cpu, u32 nr_eix, 490 unsigned long isel, bool write, 491 unsigned long *val) 492 { 493 int rc; 494 struct imsic_vsfile_rw_data rdata; 495 496 /* We can only access register if we have a IMSIC VS-file */ 497 if (vsfile_cpu < 0 || vsfile_hgei <= 0) 498 return -EINVAL; 499 500 /* Check IMSIC register iselect */ 501 rc = imsic_mrif_isel_check(nr_eix, isel); 502 if (rc) 503 return rc; 504 505 /* We can only access register on local CPU */ 506 rdata.hgei = vsfile_hgei; 507 rdata.isel = isel; 508 rdata.write = write; 509 rdata.val = (write) ? *val : 0; 510 on_each_cpu_mask(cpumask_of(vsfile_cpu), 511 imsic_vsfile_local_rw, &rdata, 1); 512 513 if (!write) 514 *val = rdata.val; 515 516 return 0; 517 } 518 519 static void imsic_vsfile_local_clear(int vsfile_hgei, u32 nr_eix) 520 { 521 u32 i; 522 unsigned long new_hstatus, old_hstatus, old_vsiselect; 523 524 /* We can only zero-out if we have a IMSIC VS-file */ 525 if (vsfile_hgei <= 0) 526 return; 527 528 old_vsiselect = csr_read(CSR_VSISELECT); 529 old_hstatus = csr_read(CSR_HSTATUS); 530 new_hstatus = old_hstatus & ~HSTATUS_VGEIN; 531 new_hstatus |= ((unsigned long)vsfile_hgei) << HSTATUS_VGEIN_SHIFT; 532 csr_write(CSR_HSTATUS, new_hstatus); 533 534 imsic_vs_csr_write(IMSIC_EIDELIVERY, 0); 535 imsic_vs_csr_write(IMSIC_EITHRESHOLD, 0); 536 for (i = 0; i < nr_eix; i++) { 537 imsic_eix_write(IMSIC_EIP0 + i * 2, 0); 538 imsic_eix_write(IMSIC_EIE0 + i * 2, 0); 539 #ifdef CONFIG_32BIT 540 imsic_eix_write(IMSIC_EIP0 + i * 2 + 1, 0); 541 imsic_eix_write(IMSIC_EIE0 + i * 2 + 1, 0); 542 #endif 543 } 544 545 csr_write(CSR_HSTATUS, old_hstatus); 546 csr_write(CSR_VSISELECT, old_vsiselect); 547 } 548 549 static void imsic_vsfile_local_update(int vsfile_hgei, u32 nr_eix, 550 struct imsic_mrif *mrif) 551 { 552 u32 i; 553 struct imsic_mrif_eix *eix; 554 unsigned long new_hstatus, old_hstatus, old_vsiselect; 555 556 /* We can only update if we have a HW IMSIC context */ 557 if (vsfile_hgei <= 0) 558 return; 559 560 /* 561 * We don't use imsic_mrif_atomic_xyz() functions to read values 562 * from MRIF in this function because it is always called with 563 * pointer to temporary MRIF on stack. 564 */ 565 566 old_vsiselect = csr_read(CSR_VSISELECT); 567 old_hstatus = csr_read(CSR_HSTATUS); 568 new_hstatus = old_hstatus & ~HSTATUS_VGEIN; 569 new_hstatus |= ((unsigned long)vsfile_hgei) << HSTATUS_VGEIN_SHIFT; 570 csr_write(CSR_HSTATUS, new_hstatus); 571 572 for (i = 0; i < nr_eix; i++) { 573 eix = &mrif->eix[i]; 574 imsic_eix_set(IMSIC_EIP0 + i * 2, eix->eip[0]); 575 imsic_eix_set(IMSIC_EIE0 + i * 2, eix->eie[0]); 576 #ifdef CONFIG_32BIT 577 imsic_eix_set(IMSIC_EIP0 + i * 2 + 1, eix->eip[1]); 578 imsic_eix_set(IMSIC_EIE0 + i * 2 + 1, eix->eie[1]); 579 #endif 580 } 581 imsic_vs_csr_write(IMSIC_EITHRESHOLD, mrif->eithreshold); 582 imsic_vs_csr_write(IMSIC_EIDELIVERY, mrif->eidelivery); 583 584 csr_write(CSR_HSTATUS, old_hstatus); 585 csr_write(CSR_VSISELECT, old_vsiselect); 586 } 587 588 static void imsic_vsfile_cleanup(struct imsic *imsic) 589 { 590 int old_vsfile_hgei, old_vsfile_cpu; 591 unsigned long flags; 592 593 /* 594 * We don't use imsic_mrif_atomic_xyz() functions to clear the 595 * SW-file in this function because it is always called when the 596 * VCPU is being destroyed. 597 */ 598 599 write_lock_irqsave(&imsic->vsfile_lock, flags); 600 old_vsfile_hgei = imsic->vsfile_hgei; 601 old_vsfile_cpu = imsic->vsfile_cpu; 602 imsic->vsfile_cpu = imsic->vsfile_hgei = -1; 603 imsic->vsfile_va = NULL; 604 imsic->vsfile_pa = 0; 605 write_unlock_irqrestore(&imsic->vsfile_lock, flags); 606 607 memset(imsic->swfile, 0, sizeof(*imsic->swfile)); 608 609 if (old_vsfile_cpu >= 0) 610 kvm_riscv_aia_free_hgei(old_vsfile_cpu, old_vsfile_hgei); 611 } 612 613 static void imsic_swfile_extirq_update(struct kvm_vcpu *vcpu) 614 { 615 struct imsic *imsic = vcpu->arch.aia_context.imsic_state; 616 struct imsic_mrif *mrif = imsic->swfile; 617 unsigned long flags; 618 619 /* 620 * The critical section is necessary during external interrupt 621 * updates to avoid the risk of losing interrupts due to potential 622 * interruptions between reading topei and updating pending status. 623 */ 624 625 spin_lock_irqsave(&imsic->swfile_extirq_lock, flags); 626 627 if (imsic_mrif_atomic_read(mrif, &mrif->eidelivery) && 628 imsic_mrif_topei(mrif, imsic->nr_eix, imsic->nr_msis)) 629 kvm_riscv_vcpu_set_interrupt(vcpu, IRQ_VS_EXT); 630 else 631 kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_EXT); 632 633 spin_unlock_irqrestore(&imsic->swfile_extirq_lock, flags); 634 } 635 636 static void imsic_swfile_read(struct kvm_vcpu *vcpu, bool clear, 637 struct imsic_mrif *mrif) 638 { 639 struct imsic *imsic = vcpu->arch.aia_context.imsic_state; 640 641 /* 642 * We don't use imsic_mrif_atomic_xyz() functions to read and 643 * write SW-file and MRIF in this function because it is always 644 * called when VCPU is not using SW-file and the MRIF points to 645 * a temporary MRIF on stack. 646 */ 647 648 memcpy(mrif, imsic->swfile, sizeof(*mrif)); 649 if (clear) { 650 memset(imsic->swfile, 0, sizeof(*imsic->swfile)); 651 kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_EXT); 652 } 653 } 654 655 static void imsic_swfile_update(struct kvm_vcpu *vcpu, 656 struct imsic_mrif *mrif) 657 { 658 u32 i; 659 struct imsic_mrif_eix *seix, *eix; 660 struct imsic *imsic = vcpu->arch.aia_context.imsic_state; 661 struct imsic_mrif *smrif = imsic->swfile; 662 663 imsic_mrif_atomic_write(smrif, &smrif->eidelivery, mrif->eidelivery); 664 imsic_mrif_atomic_write(smrif, &smrif->eithreshold, mrif->eithreshold); 665 for (i = 0; i < imsic->nr_eix; i++) { 666 seix = &smrif->eix[i]; 667 eix = &mrif->eix[i]; 668 imsic_mrif_atomic_or(smrif, &seix->eip[0], eix->eip[0]); 669 imsic_mrif_atomic_or(smrif, &seix->eie[0], eix->eie[0]); 670 #ifdef CONFIG_32BIT 671 imsic_mrif_atomic_or(smrif, &seix->eip[1], eix->eip[1]); 672 imsic_mrif_atomic_or(smrif, &seix->eie[1], eix->eie[1]); 673 #endif 674 } 675 676 imsic_swfile_extirq_update(vcpu); 677 } 678 679 void kvm_riscv_vcpu_aia_imsic_release(struct kvm_vcpu *vcpu) 680 { 681 unsigned long flags; 682 struct imsic_mrif tmrif; 683 int old_vsfile_hgei, old_vsfile_cpu; 684 struct imsic *imsic = vcpu->arch.aia_context.imsic_state; 685 686 /* Read and clear IMSIC VS-file details */ 687 write_lock_irqsave(&imsic->vsfile_lock, flags); 688 old_vsfile_hgei = imsic->vsfile_hgei; 689 old_vsfile_cpu = imsic->vsfile_cpu; 690 imsic->vsfile_cpu = imsic->vsfile_hgei = -1; 691 imsic->vsfile_va = NULL; 692 imsic->vsfile_pa = 0; 693 write_unlock_irqrestore(&imsic->vsfile_lock, flags); 694 695 /* Do nothing, if no IMSIC VS-file to release */ 696 if (old_vsfile_cpu < 0) 697 return; 698 699 /* 700 * At this point, all interrupt producers are still using 701 * the old IMSIC VS-file so we first re-direct all interrupt 702 * producers. 703 */ 704 705 /* Purge the G-stage mapping */ 706 kvm_riscv_gstage_iounmap(vcpu->kvm, 707 vcpu->arch.aia_context.imsic_addr, 708 IMSIC_MMIO_PAGE_SZ); 709 710 /* TODO: Purge the IOMMU mapping ??? */ 711 712 /* 713 * At this point, all interrupt producers have been re-directed 714 * to somewhere else so we move register state from the old IMSIC 715 * VS-file to the IMSIC SW-file. 716 */ 717 718 /* Read and clear register state from old IMSIC VS-file */ 719 memset(&tmrif, 0, sizeof(tmrif)); 720 imsic_vsfile_read(old_vsfile_hgei, old_vsfile_cpu, imsic->nr_hw_eix, 721 true, &tmrif); 722 723 /* Update register state in IMSIC SW-file */ 724 imsic_swfile_update(vcpu, &tmrif); 725 726 /* Free-up old IMSIC VS-file */ 727 kvm_riscv_aia_free_hgei(old_vsfile_cpu, old_vsfile_hgei); 728 } 729 730 int kvm_riscv_vcpu_aia_imsic_update(struct kvm_vcpu *vcpu) 731 { 732 unsigned long flags; 733 phys_addr_t new_vsfile_pa; 734 struct imsic_mrif tmrif; 735 void __iomem *new_vsfile_va; 736 struct kvm *kvm = vcpu->kvm; 737 struct kvm_run *run = vcpu->run; 738 struct kvm_vcpu_aia *vaia = &vcpu->arch.aia_context; 739 struct imsic *imsic = vaia->imsic_state; 740 int ret = 0, new_vsfile_hgei = -1, old_vsfile_hgei, old_vsfile_cpu; 741 742 /* Do nothing for emulation mode */ 743 if (kvm->arch.aia.mode == KVM_DEV_RISCV_AIA_MODE_EMUL) 744 return 1; 745 746 /* Read old IMSIC VS-file details */ 747 read_lock_irqsave(&imsic->vsfile_lock, flags); 748 old_vsfile_hgei = imsic->vsfile_hgei; 749 old_vsfile_cpu = imsic->vsfile_cpu; 750 read_unlock_irqrestore(&imsic->vsfile_lock, flags); 751 752 /* Do nothing if we are continuing on same CPU */ 753 if (old_vsfile_cpu == vcpu->cpu) 754 return 1; 755 756 /* Allocate new IMSIC VS-file */ 757 ret = kvm_riscv_aia_alloc_hgei(vcpu->cpu, vcpu, 758 &new_vsfile_va, &new_vsfile_pa); 759 if (ret <= 0) { 760 /* For HW acceleration mode, we can't continue */ 761 if (kvm->arch.aia.mode == KVM_DEV_RISCV_AIA_MODE_HWACCEL) { 762 run->fail_entry.hardware_entry_failure_reason = 763 CSR_HSTATUS; 764 run->fail_entry.cpu = vcpu->cpu; 765 run->exit_reason = KVM_EXIT_FAIL_ENTRY; 766 return 0; 767 } 768 769 /* Release old IMSIC VS-file */ 770 if (old_vsfile_cpu >= 0) 771 kvm_riscv_vcpu_aia_imsic_release(vcpu); 772 773 /* For automatic mode, we continue */ 774 goto done; 775 } 776 new_vsfile_hgei = ret; 777 778 /* 779 * At this point, all interrupt producers are still using 780 * to the old IMSIC VS-file so we first move all interrupt 781 * producers to the new IMSIC VS-file. 782 */ 783 784 /* Zero-out new IMSIC VS-file */ 785 imsic_vsfile_local_clear(new_vsfile_hgei, imsic->nr_hw_eix); 786 787 /* Update G-stage mapping for the new IMSIC VS-file */ 788 ret = kvm_riscv_gstage_ioremap(kvm, vcpu->arch.aia_context.imsic_addr, 789 new_vsfile_pa, IMSIC_MMIO_PAGE_SZ, 790 true, true); 791 if (ret) 792 goto fail_free_vsfile_hgei; 793 794 /* TODO: Update the IOMMU mapping ??? */ 795 796 /* Update new IMSIC VS-file details in IMSIC context */ 797 write_lock_irqsave(&imsic->vsfile_lock, flags); 798 imsic->vsfile_hgei = new_vsfile_hgei; 799 imsic->vsfile_cpu = vcpu->cpu; 800 imsic->vsfile_va = new_vsfile_va; 801 imsic->vsfile_pa = new_vsfile_pa; 802 write_unlock_irqrestore(&imsic->vsfile_lock, flags); 803 804 /* 805 * At this point, all interrupt producers have been moved 806 * to the new IMSIC VS-file so we move register state from 807 * the old IMSIC VS/SW-file to the new IMSIC VS-file. 808 */ 809 810 memset(&tmrif, 0, sizeof(tmrif)); 811 if (old_vsfile_cpu >= 0) { 812 /* Read and clear register state from old IMSIC VS-file */ 813 imsic_vsfile_read(old_vsfile_hgei, old_vsfile_cpu, 814 imsic->nr_hw_eix, true, &tmrif); 815 816 /* Free-up old IMSIC VS-file */ 817 kvm_riscv_aia_free_hgei(old_vsfile_cpu, old_vsfile_hgei); 818 } else { 819 /* Read and clear register state from IMSIC SW-file */ 820 imsic_swfile_read(vcpu, true, &tmrif); 821 } 822 823 /* Restore register state in the new IMSIC VS-file */ 824 imsic_vsfile_local_update(new_vsfile_hgei, imsic->nr_hw_eix, &tmrif); 825 826 done: 827 /* Set VCPU HSTATUS.VGEIN to new IMSIC VS-file */ 828 vcpu->arch.guest_context.hstatus &= ~HSTATUS_VGEIN; 829 if (new_vsfile_hgei > 0) 830 vcpu->arch.guest_context.hstatus |= 831 ((unsigned long)new_vsfile_hgei) << HSTATUS_VGEIN_SHIFT; 832 833 /* Continue run-loop */ 834 return 1; 835 836 fail_free_vsfile_hgei: 837 kvm_riscv_aia_free_hgei(vcpu->cpu, new_vsfile_hgei); 838 return ret; 839 } 840 841 int kvm_riscv_vcpu_aia_imsic_rmw(struct kvm_vcpu *vcpu, unsigned long isel, 842 unsigned long *val, unsigned long new_val, 843 unsigned long wr_mask) 844 { 845 u32 topei; 846 struct imsic_mrif_eix *eix; 847 int r, rc = KVM_INSN_CONTINUE_NEXT_SEPC; 848 struct imsic *imsic = vcpu->arch.aia_context.imsic_state; 849 850 if (isel == KVM_RISCV_AIA_IMSIC_TOPEI) { 851 /* Read pending and enabled interrupt with highest priority */ 852 topei = imsic_mrif_topei(imsic->swfile, imsic->nr_eix, 853 imsic->nr_msis); 854 if (val) 855 *val = topei; 856 857 /* Writes ignore value and clear top pending interrupt */ 858 if (topei && wr_mask) { 859 topei >>= TOPEI_ID_SHIFT; 860 if (topei) { 861 eix = &imsic->swfile->eix[topei / 862 BITS_PER_TYPE(u64)]; 863 clear_bit(topei & (BITS_PER_TYPE(u64) - 1), 864 eix->eip); 865 } 866 } 867 } else { 868 r = imsic_mrif_rmw(imsic->swfile, imsic->nr_eix, isel, 869 val, new_val, wr_mask); 870 /* Forward unknown IMSIC register to user-space */ 871 if (r) 872 rc = (r == -ENOENT) ? 0 : KVM_INSN_ILLEGAL_TRAP; 873 } 874 875 if (wr_mask) 876 imsic_swfile_extirq_update(vcpu); 877 878 return rc; 879 } 880 881 int kvm_riscv_aia_imsic_rw_attr(struct kvm *kvm, unsigned long type, 882 bool write, unsigned long *val) 883 { 884 u32 isel, vcpu_id; 885 unsigned long flags; 886 struct imsic *imsic; 887 struct kvm_vcpu *vcpu; 888 int rc, vsfile_hgei, vsfile_cpu; 889 890 if (!kvm_riscv_aia_initialized(kvm)) 891 return -ENODEV; 892 893 vcpu_id = KVM_DEV_RISCV_AIA_IMSIC_GET_VCPU(type); 894 vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id); 895 if (!vcpu) 896 return -ENODEV; 897 898 isel = KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(type); 899 imsic = vcpu->arch.aia_context.imsic_state; 900 901 read_lock_irqsave(&imsic->vsfile_lock, flags); 902 903 rc = 0; 904 vsfile_hgei = imsic->vsfile_hgei; 905 vsfile_cpu = imsic->vsfile_cpu; 906 if (vsfile_cpu < 0) { 907 if (write) { 908 rc = imsic_mrif_rmw(imsic->swfile, imsic->nr_eix, 909 isel, NULL, *val, -1UL); 910 imsic_swfile_extirq_update(vcpu); 911 } else 912 rc = imsic_mrif_rmw(imsic->swfile, imsic->nr_eix, 913 isel, val, 0, 0); 914 } 915 916 read_unlock_irqrestore(&imsic->vsfile_lock, flags); 917 918 if (!rc && vsfile_cpu >= 0) 919 rc = imsic_vsfile_rw(vsfile_hgei, vsfile_cpu, imsic->nr_eix, 920 isel, write, val); 921 922 return rc; 923 } 924 925 int kvm_riscv_aia_imsic_has_attr(struct kvm *kvm, unsigned long type) 926 { 927 u32 isel, vcpu_id; 928 struct imsic *imsic; 929 struct kvm_vcpu *vcpu; 930 931 if (!kvm_riscv_aia_initialized(kvm)) 932 return -ENODEV; 933 934 vcpu_id = KVM_DEV_RISCV_AIA_IMSIC_GET_VCPU(type); 935 vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id); 936 if (!vcpu) 937 return -ENODEV; 938 939 isel = KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(type); 940 imsic = vcpu->arch.aia_context.imsic_state; 941 return imsic_mrif_isel_check(imsic->nr_eix, isel); 942 } 943 944 void kvm_riscv_vcpu_aia_imsic_reset(struct kvm_vcpu *vcpu) 945 { 946 struct imsic *imsic = vcpu->arch.aia_context.imsic_state; 947 948 if (!imsic) 949 return; 950 951 kvm_riscv_vcpu_aia_imsic_release(vcpu); 952 953 memset(imsic->swfile, 0, sizeof(*imsic->swfile)); 954 } 955 956 int kvm_riscv_vcpu_aia_imsic_inject(struct kvm_vcpu *vcpu, 957 u32 guest_index, u32 offset, u32 iid) 958 { 959 unsigned long flags; 960 struct imsic_mrif_eix *eix; 961 struct imsic *imsic = vcpu->arch.aia_context.imsic_state; 962 963 /* We only emulate one IMSIC MMIO page for each Guest VCPU */ 964 if (!imsic || !iid || guest_index || 965 (offset != IMSIC_MMIO_SETIPNUM_LE && 966 offset != IMSIC_MMIO_SETIPNUM_BE)) 967 return -ENODEV; 968 969 iid = (offset == IMSIC_MMIO_SETIPNUM_BE) ? __swab32(iid) : iid; 970 if (imsic->nr_msis <= iid) 971 return -EINVAL; 972 973 read_lock_irqsave(&imsic->vsfile_lock, flags); 974 975 if (imsic->vsfile_cpu >= 0) { 976 writel(iid, imsic->vsfile_va + IMSIC_MMIO_SETIPNUM_LE); 977 kvm_vcpu_kick(vcpu); 978 } else { 979 eix = &imsic->swfile->eix[iid / BITS_PER_TYPE(u64)]; 980 set_bit(iid & (BITS_PER_TYPE(u64) - 1), eix->eip); 981 imsic_swfile_extirq_update(vcpu); 982 } 983 984 read_unlock_irqrestore(&imsic->vsfile_lock, flags); 985 986 return 0; 987 } 988 989 static int imsic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev, 990 gpa_t addr, int len, void *val) 991 { 992 if (len != 4 || (addr & 0x3) != 0) 993 return -EOPNOTSUPP; 994 995 *((u32 *)val) = 0; 996 997 return 0; 998 } 999 1000 static int imsic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev, 1001 gpa_t addr, int len, const void *val) 1002 { 1003 struct kvm_msi msi = { 0 }; 1004 1005 if (len != 4 || (addr & 0x3) != 0) 1006 return -EOPNOTSUPP; 1007 1008 msi.address_hi = addr >> 32; 1009 msi.address_lo = (u32)addr; 1010 msi.data = *((const u32 *)val); 1011 kvm_riscv_aia_inject_msi(vcpu->kvm, &msi); 1012 1013 return 0; 1014 }; 1015 1016 static struct kvm_io_device_ops imsic_iodoev_ops = { 1017 .read = imsic_mmio_read, 1018 .write = imsic_mmio_write, 1019 }; 1020 1021 int kvm_riscv_vcpu_aia_imsic_init(struct kvm_vcpu *vcpu) 1022 { 1023 int ret = 0; 1024 struct imsic *imsic; 1025 struct page *swfile_page; 1026 struct kvm *kvm = vcpu->kvm; 1027 1028 /* Fail if we have zero IDs */ 1029 if (!kvm->arch.aia.nr_ids) 1030 return -EINVAL; 1031 1032 /* Allocate IMSIC context */ 1033 imsic = kzalloc(sizeof(*imsic), GFP_KERNEL); 1034 if (!imsic) 1035 return -ENOMEM; 1036 vcpu->arch.aia_context.imsic_state = imsic; 1037 1038 /* Setup IMSIC context */ 1039 imsic->nr_msis = kvm->arch.aia.nr_ids + 1; 1040 rwlock_init(&imsic->vsfile_lock); 1041 imsic->nr_eix = BITS_TO_U64(imsic->nr_msis); 1042 imsic->nr_hw_eix = BITS_TO_U64(kvm_riscv_aia_max_ids); 1043 imsic->vsfile_hgei = imsic->vsfile_cpu = -1; 1044 1045 /* Setup IMSIC SW-file */ 1046 swfile_page = alloc_pages(GFP_KERNEL | __GFP_ZERO, 1047 get_order(sizeof(*imsic->swfile))); 1048 if (!swfile_page) { 1049 ret = -ENOMEM; 1050 goto fail_free_imsic; 1051 } 1052 imsic->swfile = page_to_virt(swfile_page); 1053 imsic->swfile_pa = page_to_phys(swfile_page); 1054 spin_lock_init(&imsic->swfile_extirq_lock); 1055 1056 /* Setup IO device */ 1057 kvm_iodevice_init(&imsic->iodev, &imsic_iodoev_ops); 1058 mutex_lock(&kvm->slots_lock); 1059 ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, 1060 vcpu->arch.aia_context.imsic_addr, 1061 KVM_DEV_RISCV_IMSIC_SIZE, 1062 &imsic->iodev); 1063 mutex_unlock(&kvm->slots_lock); 1064 if (ret) 1065 goto fail_free_swfile; 1066 1067 return 0; 1068 1069 fail_free_swfile: 1070 free_pages((unsigned long)imsic->swfile, 1071 get_order(sizeof(*imsic->swfile))); 1072 fail_free_imsic: 1073 vcpu->arch.aia_context.imsic_state = NULL; 1074 kfree(imsic); 1075 return ret; 1076 } 1077 1078 void kvm_riscv_vcpu_aia_imsic_cleanup(struct kvm_vcpu *vcpu) 1079 { 1080 struct kvm *kvm = vcpu->kvm; 1081 struct imsic *imsic = vcpu->arch.aia_context.imsic_state; 1082 1083 if (!imsic) 1084 return; 1085 1086 imsic_vsfile_cleanup(imsic); 1087 1088 mutex_lock(&kvm->slots_lock); 1089 kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &imsic->iodev); 1090 mutex_unlock(&kvm->slots_lock); 1091 1092 free_pages((unsigned long)imsic->swfile, 1093 get_order(sizeof(*imsic->swfile))); 1094 1095 vcpu->arch.aia_context.imsic_state = NULL; 1096 kfree(imsic); 1097 } 1098