1 /* 2 * spu_switch.c 3 * 4 * (C) Copyright IBM Corp. 2005 5 * 6 * Author: Mark Nutter <mnutter@us.ibm.com> 7 * 8 * Host-side part of SPU context switch sequence outlined in 9 * Synergistic Processor Element, Book IV. 10 * 11 * A fully premptive switch of an SPE is very expensive in terms 12 * of time and system resources. SPE Book IV indicates that SPE 13 * allocation should follow a "serially reusable device" model, 14 * in which the SPE is assigned a task until it completes. When 15 * this is not possible, this sequence may be used to premptively 16 * save, and then later (optionally) restore the context of a 17 * program executing on an SPE. 18 * 19 * 20 * This program is free software; you can redistribute it and/or modify 21 * it under the terms of the GNU General Public License as published by 22 * the Free Software Foundation; either version 2, or (at your option) 23 * any later version. 24 * 25 * This program is distributed in the hope that it will be useful, 26 * but WITHOUT ANY WARRANTY; without even the implied warranty of 27 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 28 * GNU General Public License for more details. 29 * 30 * You should have received a copy of the GNU General Public License 31 * along with this program; if not, write to the Free Software 32 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 33 */ 34 35 #include <linux/module.h> 36 #include <linux/errno.h> 37 #include <linux/hardirq.h> 38 #include <linux/sched.h> 39 #include <linux/kernel.h> 40 #include <linux/mm.h> 41 #include <linux/vmalloc.h> 42 #include <linux/smp.h> 43 #include <linux/stddef.h> 44 #include <linux/unistd.h> 45 46 #include <asm/io.h> 47 #include <asm/spu.h> 48 #include <asm/spu_priv1.h> 49 #include <asm/spu_csa.h> 50 #include <asm/mmu_context.h> 51 52 #include "spufs.h" 53 54 #include "spu_save_dump.h" 55 #include "spu_restore_dump.h" 56 57 #if 0 58 #define POLL_WHILE_TRUE(_c) { \ 59 do { \ 60 } while (_c); \ 61 } 62 #else 63 #define RELAX_SPIN_COUNT 1000 64 #define POLL_WHILE_TRUE(_c) { \ 65 do { \ 66 int _i; \ 67 for (_i=0; _i<RELAX_SPIN_COUNT && (_c); _i++) { \ 68 cpu_relax(); \ 69 } \ 70 if (unlikely(_c)) yield(); \ 71 else break; \ 72 } while (_c); \ 73 } 74 #endif /* debug */ 75 76 #define POLL_WHILE_FALSE(_c) POLL_WHILE_TRUE(!(_c)) 77 78 static inline void acquire_spu_lock(struct spu *spu) 79 { 80 /* Save, Step 1: 81 * Restore, Step 1: 82 * Acquire SPU-specific mutual exclusion lock. 83 * TBD. 84 */ 85 } 86 87 static inline void release_spu_lock(struct spu *spu) 88 { 89 /* Restore, Step 76: 90 * Release SPU-specific mutual exclusion lock. 91 * TBD. 92 */ 93 } 94 95 static inline int check_spu_isolate(struct spu_state *csa, struct spu *spu) 96 { 97 struct spu_problem __iomem *prob = spu->problem; 98 u32 isolate_state; 99 100 /* Save, Step 2: 101 * Save, Step 6: 102 * If SPU_Status[E,L,IS] any field is '1', this 103 * SPU is in isolate state and cannot be context 104 * saved at this time. 105 */ 106 isolate_state = SPU_STATUS_ISOLATED_STATE | 107 SPU_STATUS_ISOLATED_LOAD_STATUS | SPU_STATUS_ISOLATED_EXIT_STATUS; 108 return (in_be32(&prob->spu_status_R) & isolate_state) ? 1 : 0; 109 } 110 111 static inline void disable_interrupts(struct spu_state *csa, struct spu *spu) 112 { 113 /* Save, Step 3: 114 * Restore, Step 2: 115 * Save INT_Mask_class0 in CSA. 116 * Write INT_MASK_class0 with value of 0. 117 * Save INT_Mask_class1 in CSA. 118 * Write INT_MASK_class1 with value of 0. 119 * Save INT_Mask_class2 in CSA. 120 * Write INT_MASK_class2 with value of 0. 121 * Synchronize all three interrupts to be sure 122 * we no longer execute a handler on another CPU. 123 */ 124 spin_lock_irq(&spu->register_lock); 125 if (csa) { 126 csa->priv1.int_mask_class0_RW = spu_int_mask_get(spu, 0); 127 csa->priv1.int_mask_class1_RW = spu_int_mask_get(spu, 1); 128 csa->priv1.int_mask_class2_RW = spu_int_mask_get(spu, 2); 129 } 130 spu_int_mask_set(spu, 0, 0ul); 131 spu_int_mask_set(spu, 1, 0ul); 132 spu_int_mask_set(spu, 2, 0ul); 133 eieio(); 134 spin_unlock_irq(&spu->register_lock); 135 synchronize_irq(spu->irqs[0]); 136 synchronize_irq(spu->irqs[1]); 137 synchronize_irq(spu->irqs[2]); 138 } 139 140 static inline void set_watchdog_timer(struct spu_state *csa, struct spu *spu) 141 { 142 /* Save, Step 4: 143 * Restore, Step 25. 144 * Set a software watchdog timer, which specifies the 145 * maximum allowable time for a context save sequence. 146 * 147 * For present, this implementation will not set a global 148 * watchdog timer, as virtualization & variable system load 149 * may cause unpredictable execution times. 150 */ 151 } 152 153 static inline void inhibit_user_access(struct spu_state *csa, struct spu *spu) 154 { 155 /* Save, Step 5: 156 * Restore, Step 3: 157 * Inhibit user-space access (if provided) to this 158 * SPU by unmapping the virtual pages assigned to 159 * the SPU memory-mapped I/O (MMIO) for problem 160 * state. TBD. 161 */ 162 } 163 164 static inline void set_switch_pending(struct spu_state *csa, struct spu *spu) 165 { 166 /* Save, Step 7: 167 * Restore, Step 5: 168 * Set a software context switch pending flag. 169 */ 170 set_bit(SPU_CONTEXT_SWITCH_PENDING, &spu->flags); 171 mb(); 172 } 173 174 static inline void save_mfc_cntl(struct spu_state *csa, struct spu *spu) 175 { 176 struct spu_priv2 __iomem *priv2 = spu->priv2; 177 178 /* Save, Step 8: 179 * Suspend DMA and save MFC_CNTL. 180 */ 181 switch (in_be64(&priv2->mfc_control_RW) & 182 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) { 183 case MFC_CNTL_SUSPEND_IN_PROGRESS: 184 POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) & 185 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) == 186 MFC_CNTL_SUSPEND_COMPLETE); 187 /* fall through */ 188 case MFC_CNTL_SUSPEND_COMPLETE: 189 if (csa) { 190 csa->priv2.mfc_control_RW = 191 MFC_CNTL_SUSPEND_MASK | 192 MFC_CNTL_SUSPEND_DMA_QUEUE; 193 } 194 break; 195 case MFC_CNTL_NORMAL_DMA_QUEUE_OPERATION: 196 out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE); 197 POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) & 198 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) == 199 MFC_CNTL_SUSPEND_COMPLETE); 200 if (csa) { 201 csa->priv2.mfc_control_RW = 0; 202 } 203 break; 204 } 205 } 206 207 static inline void save_spu_runcntl(struct spu_state *csa, struct spu *spu) 208 { 209 struct spu_problem __iomem *prob = spu->problem; 210 211 /* Save, Step 9: 212 * Save SPU_Runcntl in the CSA. This value contains 213 * the "Application Desired State". 214 */ 215 csa->prob.spu_runcntl_RW = in_be32(&prob->spu_runcntl_RW); 216 } 217 218 static inline void save_mfc_sr1(struct spu_state *csa, struct spu *spu) 219 { 220 /* Save, Step 10: 221 * Save MFC_SR1 in the CSA. 222 */ 223 csa->priv1.mfc_sr1_RW = spu_mfc_sr1_get(spu); 224 } 225 226 static inline void save_spu_status(struct spu_state *csa, struct spu *spu) 227 { 228 struct spu_problem __iomem *prob = spu->problem; 229 230 /* Save, Step 11: 231 * Read SPU_Status[R], and save to CSA. 232 */ 233 if ((in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) == 0) { 234 csa->prob.spu_status_R = in_be32(&prob->spu_status_R); 235 } else { 236 u32 stopped; 237 238 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 239 eieio(); 240 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 241 SPU_STATUS_RUNNING); 242 stopped = 243 SPU_STATUS_INVALID_INSTR | SPU_STATUS_SINGLE_STEP | 244 SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP; 245 if ((in_be32(&prob->spu_status_R) & stopped) == 0) 246 csa->prob.spu_status_R = SPU_STATUS_RUNNING; 247 else 248 csa->prob.spu_status_R = in_be32(&prob->spu_status_R); 249 } 250 } 251 252 static inline void save_mfc_decr(struct spu_state *csa, struct spu *spu) 253 { 254 struct spu_priv2 __iomem *priv2 = spu->priv2; 255 256 /* Save, Step 12: 257 * Read MFC_CNTL[Ds]. Update saved copy of 258 * CSA.MFC_CNTL[Ds]. 259 */ 260 csa->priv2.mfc_control_RW |= 261 in_be64(&priv2->mfc_control_RW) & MFC_CNTL_DECREMENTER_RUNNING; 262 } 263 264 static inline void halt_mfc_decr(struct spu_state *csa, struct spu *spu) 265 { 266 struct spu_priv2 __iomem *priv2 = spu->priv2; 267 268 /* Save, Step 13: 269 * Write MFC_CNTL[Dh] set to a '1' to halt 270 * the decrementer. 271 */ 272 out_be64(&priv2->mfc_control_RW, 273 MFC_CNTL_DECREMENTER_HALTED | MFC_CNTL_SUSPEND_MASK); 274 eieio(); 275 } 276 277 static inline void save_timebase(struct spu_state *csa, struct spu *spu) 278 { 279 /* Save, Step 14: 280 * Read PPE Timebase High and Timebase low registers 281 * and save in CSA. TBD. 282 */ 283 csa->suspend_time = get_cycles(); 284 } 285 286 static inline void remove_other_spu_access(struct spu_state *csa, 287 struct spu *spu) 288 { 289 /* Save, Step 15: 290 * Remove other SPU access to this SPU by unmapping 291 * this SPU's pages from their address space. TBD. 292 */ 293 } 294 295 static inline void do_mfc_mssync(struct spu_state *csa, struct spu *spu) 296 { 297 struct spu_problem __iomem *prob = spu->problem; 298 299 /* Save, Step 16: 300 * Restore, Step 11. 301 * Write SPU_MSSync register. Poll SPU_MSSync[P] 302 * for a value of 0. 303 */ 304 out_be64(&prob->spc_mssync_RW, 1UL); 305 POLL_WHILE_TRUE(in_be64(&prob->spc_mssync_RW) & MS_SYNC_PENDING); 306 } 307 308 static inline void issue_mfc_tlbie(struct spu_state *csa, struct spu *spu) 309 { 310 /* Save, Step 17: 311 * Restore, Step 12. 312 * Restore, Step 48. 313 * Write TLB_Invalidate_Entry[IS,VPN,L,Lp]=0 register. 314 * Then issue a PPE sync instruction. 315 */ 316 spu_tlb_invalidate(spu); 317 mb(); 318 } 319 320 static inline void handle_pending_interrupts(struct spu_state *csa, 321 struct spu *spu) 322 { 323 /* Save, Step 18: 324 * Handle any pending interrupts from this SPU 325 * here. This is OS or hypervisor specific. One 326 * option is to re-enable interrupts to handle any 327 * pending interrupts, with the interrupt handlers 328 * recognizing the software Context Switch Pending 329 * flag, to ensure the SPU execution or MFC command 330 * queue is not restarted. TBD. 331 */ 332 } 333 334 static inline void save_mfc_queues(struct spu_state *csa, struct spu *spu) 335 { 336 struct spu_priv2 __iomem *priv2 = spu->priv2; 337 int i; 338 339 /* Save, Step 19: 340 * If MFC_Cntl[Se]=0 then save 341 * MFC command queues. 342 */ 343 if ((in_be64(&priv2->mfc_control_RW) & MFC_CNTL_DMA_QUEUES_EMPTY) == 0) { 344 for (i = 0; i < 8; i++) { 345 csa->priv2.puq[i].mfc_cq_data0_RW = 346 in_be64(&priv2->puq[i].mfc_cq_data0_RW); 347 csa->priv2.puq[i].mfc_cq_data1_RW = 348 in_be64(&priv2->puq[i].mfc_cq_data1_RW); 349 csa->priv2.puq[i].mfc_cq_data2_RW = 350 in_be64(&priv2->puq[i].mfc_cq_data2_RW); 351 csa->priv2.puq[i].mfc_cq_data3_RW = 352 in_be64(&priv2->puq[i].mfc_cq_data3_RW); 353 } 354 for (i = 0; i < 16; i++) { 355 csa->priv2.spuq[i].mfc_cq_data0_RW = 356 in_be64(&priv2->spuq[i].mfc_cq_data0_RW); 357 csa->priv2.spuq[i].mfc_cq_data1_RW = 358 in_be64(&priv2->spuq[i].mfc_cq_data1_RW); 359 csa->priv2.spuq[i].mfc_cq_data2_RW = 360 in_be64(&priv2->spuq[i].mfc_cq_data2_RW); 361 csa->priv2.spuq[i].mfc_cq_data3_RW = 362 in_be64(&priv2->spuq[i].mfc_cq_data3_RW); 363 } 364 } 365 } 366 367 static inline void save_ppu_querymask(struct spu_state *csa, struct spu *spu) 368 { 369 struct spu_problem __iomem *prob = spu->problem; 370 371 /* Save, Step 20: 372 * Save the PPU_QueryMask register 373 * in the CSA. 374 */ 375 csa->prob.dma_querymask_RW = in_be32(&prob->dma_querymask_RW); 376 } 377 378 static inline void save_ppu_querytype(struct spu_state *csa, struct spu *spu) 379 { 380 struct spu_problem __iomem *prob = spu->problem; 381 382 /* Save, Step 21: 383 * Save the PPU_QueryType register 384 * in the CSA. 385 */ 386 csa->prob.dma_querytype_RW = in_be32(&prob->dma_querytype_RW); 387 } 388 389 static inline void save_ppu_tagstatus(struct spu_state *csa, struct spu *spu) 390 { 391 struct spu_problem __iomem *prob = spu->problem; 392 393 /* Save the Prxy_TagStatus register in the CSA. 394 * 395 * It is unnecessary to restore dma_tagstatus_R, however, 396 * dma_tagstatus_R in the CSA is accessed via backing_ops, so 397 * we must save it. 398 */ 399 csa->prob.dma_tagstatus_R = in_be32(&prob->dma_tagstatus_R); 400 } 401 402 static inline void save_mfc_csr_tsq(struct spu_state *csa, struct spu *spu) 403 { 404 struct spu_priv2 __iomem *priv2 = spu->priv2; 405 406 /* Save, Step 22: 407 * Save the MFC_CSR_TSQ register 408 * in the LSCSA. 409 */ 410 csa->priv2.spu_tag_status_query_RW = 411 in_be64(&priv2->spu_tag_status_query_RW); 412 } 413 414 static inline void save_mfc_csr_cmd(struct spu_state *csa, struct spu *spu) 415 { 416 struct spu_priv2 __iomem *priv2 = spu->priv2; 417 418 /* Save, Step 23: 419 * Save the MFC_CSR_CMD1 and MFC_CSR_CMD2 420 * registers in the CSA. 421 */ 422 csa->priv2.spu_cmd_buf1_RW = in_be64(&priv2->spu_cmd_buf1_RW); 423 csa->priv2.spu_cmd_buf2_RW = in_be64(&priv2->spu_cmd_buf2_RW); 424 } 425 426 static inline void save_mfc_csr_ato(struct spu_state *csa, struct spu *spu) 427 { 428 struct spu_priv2 __iomem *priv2 = spu->priv2; 429 430 /* Save, Step 24: 431 * Save the MFC_CSR_ATO register in 432 * the CSA. 433 */ 434 csa->priv2.spu_atomic_status_RW = in_be64(&priv2->spu_atomic_status_RW); 435 } 436 437 static inline void save_mfc_tclass_id(struct spu_state *csa, struct spu *spu) 438 { 439 /* Save, Step 25: 440 * Save the MFC_TCLASS_ID register in 441 * the CSA. 442 */ 443 csa->priv1.mfc_tclass_id_RW = spu_mfc_tclass_id_get(spu); 444 } 445 446 static inline void set_mfc_tclass_id(struct spu_state *csa, struct spu *spu) 447 { 448 /* Save, Step 26: 449 * Restore, Step 23. 450 * Write the MFC_TCLASS_ID register with 451 * the value 0x10000000. 452 */ 453 spu_mfc_tclass_id_set(spu, 0x10000000); 454 eieio(); 455 } 456 457 static inline void purge_mfc_queue(struct spu_state *csa, struct spu *spu) 458 { 459 struct spu_priv2 __iomem *priv2 = spu->priv2; 460 461 /* Save, Step 27: 462 * Restore, Step 14. 463 * Write MFC_CNTL[Pc]=1 (purge queue). 464 */ 465 out_be64(&priv2->mfc_control_RW, MFC_CNTL_PURGE_DMA_REQUEST); 466 eieio(); 467 } 468 469 static inline void wait_purge_complete(struct spu_state *csa, struct spu *spu) 470 { 471 struct spu_priv2 __iomem *priv2 = spu->priv2; 472 473 /* Save, Step 28: 474 * Poll MFC_CNTL[Ps] until value '11' is read 475 * (purge complete). 476 */ 477 POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) & 478 MFC_CNTL_PURGE_DMA_STATUS_MASK) == 479 MFC_CNTL_PURGE_DMA_COMPLETE); 480 } 481 482 static inline void setup_mfc_sr1(struct spu_state *csa, struct spu *spu) 483 { 484 /* Save, Step 30: 485 * Restore, Step 18: 486 * Write MFC_SR1 with MFC_SR1[D=0,S=1] and 487 * MFC_SR1[TL,R,Pr,T] set correctly for the 488 * OS specific environment. 489 * 490 * Implementation note: The SPU-side code 491 * for save/restore is privileged, so the 492 * MFC_SR1[Pr] bit is not set. 493 * 494 */ 495 spu_mfc_sr1_set(spu, (MFC_STATE1_MASTER_RUN_CONTROL_MASK | 496 MFC_STATE1_RELOCATE_MASK | 497 MFC_STATE1_BUS_TLBIE_MASK)); 498 } 499 500 static inline void save_spu_npc(struct spu_state *csa, struct spu *spu) 501 { 502 struct spu_problem __iomem *prob = spu->problem; 503 504 /* Save, Step 31: 505 * Save SPU_NPC in the CSA. 506 */ 507 csa->prob.spu_npc_RW = in_be32(&prob->spu_npc_RW); 508 } 509 510 static inline void save_spu_privcntl(struct spu_state *csa, struct spu *spu) 511 { 512 struct spu_priv2 __iomem *priv2 = spu->priv2; 513 514 /* Save, Step 32: 515 * Save SPU_PrivCntl in the CSA. 516 */ 517 csa->priv2.spu_privcntl_RW = in_be64(&priv2->spu_privcntl_RW); 518 } 519 520 static inline void reset_spu_privcntl(struct spu_state *csa, struct spu *spu) 521 { 522 struct spu_priv2 __iomem *priv2 = spu->priv2; 523 524 /* Save, Step 33: 525 * Restore, Step 16: 526 * Write SPU_PrivCntl[S,Le,A] fields reset to 0. 527 */ 528 out_be64(&priv2->spu_privcntl_RW, 0UL); 529 eieio(); 530 } 531 532 static inline void save_spu_lslr(struct spu_state *csa, struct spu *spu) 533 { 534 struct spu_priv2 __iomem *priv2 = spu->priv2; 535 536 /* Save, Step 34: 537 * Save SPU_LSLR in the CSA. 538 */ 539 csa->priv2.spu_lslr_RW = in_be64(&priv2->spu_lslr_RW); 540 } 541 542 static inline void reset_spu_lslr(struct spu_state *csa, struct spu *spu) 543 { 544 struct spu_priv2 __iomem *priv2 = spu->priv2; 545 546 /* Save, Step 35: 547 * Restore, Step 17. 548 * Reset SPU_LSLR. 549 */ 550 out_be64(&priv2->spu_lslr_RW, LS_ADDR_MASK); 551 eieio(); 552 } 553 554 static inline void save_spu_cfg(struct spu_state *csa, struct spu *spu) 555 { 556 struct spu_priv2 __iomem *priv2 = spu->priv2; 557 558 /* Save, Step 36: 559 * Save SPU_Cfg in the CSA. 560 */ 561 csa->priv2.spu_cfg_RW = in_be64(&priv2->spu_cfg_RW); 562 } 563 564 static inline void save_pm_trace(struct spu_state *csa, struct spu *spu) 565 { 566 /* Save, Step 37: 567 * Save PM_Trace_Tag_Wait_Mask in the CSA. 568 * Not performed by this implementation. 569 */ 570 } 571 572 static inline void save_mfc_rag(struct spu_state *csa, struct spu *spu) 573 { 574 /* Save, Step 38: 575 * Save RA_GROUP_ID register and the 576 * RA_ENABLE reigster in the CSA. 577 */ 578 csa->priv1.resource_allocation_groupID_RW = 579 spu_resource_allocation_groupID_get(spu); 580 csa->priv1.resource_allocation_enable_RW = 581 spu_resource_allocation_enable_get(spu); 582 } 583 584 static inline void save_ppu_mb_stat(struct spu_state *csa, struct spu *spu) 585 { 586 struct spu_problem __iomem *prob = spu->problem; 587 588 /* Save, Step 39: 589 * Save MB_Stat register in the CSA. 590 */ 591 csa->prob.mb_stat_R = in_be32(&prob->mb_stat_R); 592 } 593 594 static inline void save_ppu_mb(struct spu_state *csa, struct spu *spu) 595 { 596 struct spu_problem __iomem *prob = spu->problem; 597 598 /* Save, Step 40: 599 * Save the PPU_MB register in the CSA. 600 */ 601 csa->prob.pu_mb_R = in_be32(&prob->pu_mb_R); 602 } 603 604 static inline void save_ppuint_mb(struct spu_state *csa, struct spu *spu) 605 { 606 struct spu_priv2 __iomem *priv2 = spu->priv2; 607 608 /* Save, Step 41: 609 * Save the PPUINT_MB register in the CSA. 610 */ 611 csa->priv2.puint_mb_R = in_be64(&priv2->puint_mb_R); 612 } 613 614 static inline void save_ch_part1(struct spu_state *csa, struct spu *spu) 615 { 616 struct spu_priv2 __iomem *priv2 = spu->priv2; 617 u64 idx, ch_indices[] = { 0UL, 3UL, 4UL, 24UL, 25UL, 27UL }; 618 int i; 619 620 /* Save, Step 42: 621 */ 622 623 /* Save CH 1, without channel count */ 624 out_be64(&priv2->spu_chnlcntptr_RW, 1); 625 csa->spu_chnldata_RW[1] = in_be64(&priv2->spu_chnldata_RW); 626 627 /* Save the following CH: [0,3,4,24,25,27] */ 628 for (i = 0; i < ARRAY_SIZE(ch_indices); i++) { 629 idx = ch_indices[i]; 630 out_be64(&priv2->spu_chnlcntptr_RW, idx); 631 eieio(); 632 csa->spu_chnldata_RW[idx] = in_be64(&priv2->spu_chnldata_RW); 633 csa->spu_chnlcnt_RW[idx] = in_be64(&priv2->spu_chnlcnt_RW); 634 out_be64(&priv2->spu_chnldata_RW, 0UL); 635 out_be64(&priv2->spu_chnlcnt_RW, 0UL); 636 eieio(); 637 } 638 } 639 640 static inline void save_spu_mb(struct spu_state *csa, struct spu *spu) 641 { 642 struct spu_priv2 __iomem *priv2 = spu->priv2; 643 int i; 644 645 /* Save, Step 43: 646 * Save SPU Read Mailbox Channel. 647 */ 648 out_be64(&priv2->spu_chnlcntptr_RW, 29UL); 649 eieio(); 650 csa->spu_chnlcnt_RW[29] = in_be64(&priv2->spu_chnlcnt_RW); 651 for (i = 0; i < 4; i++) { 652 csa->spu_mailbox_data[i] = in_be64(&priv2->spu_chnldata_RW); 653 } 654 out_be64(&priv2->spu_chnlcnt_RW, 0UL); 655 eieio(); 656 } 657 658 static inline void save_mfc_cmd(struct spu_state *csa, struct spu *spu) 659 { 660 struct spu_priv2 __iomem *priv2 = spu->priv2; 661 662 /* Save, Step 44: 663 * Save MFC_CMD Channel. 664 */ 665 out_be64(&priv2->spu_chnlcntptr_RW, 21UL); 666 eieio(); 667 csa->spu_chnlcnt_RW[21] = in_be64(&priv2->spu_chnlcnt_RW); 668 eieio(); 669 } 670 671 static inline void reset_ch(struct spu_state *csa, struct spu *spu) 672 { 673 struct spu_priv2 __iomem *priv2 = spu->priv2; 674 u64 ch_indices[4] = { 21UL, 23UL, 28UL, 30UL }; 675 u64 ch_counts[4] = { 16UL, 1UL, 1UL, 1UL }; 676 u64 idx; 677 int i; 678 679 /* Save, Step 45: 680 * Reset the following CH: [21, 23, 28, 30] 681 */ 682 for (i = 0; i < 4; i++) { 683 idx = ch_indices[i]; 684 out_be64(&priv2->spu_chnlcntptr_RW, idx); 685 eieio(); 686 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]); 687 eieio(); 688 } 689 } 690 691 static inline void resume_mfc_queue(struct spu_state *csa, struct spu *spu) 692 { 693 struct spu_priv2 __iomem *priv2 = spu->priv2; 694 695 /* Save, Step 46: 696 * Restore, Step 25. 697 * Write MFC_CNTL[Sc]=0 (resume queue processing). 698 */ 699 out_be64(&priv2->mfc_control_RW, MFC_CNTL_RESUME_DMA_QUEUE); 700 } 701 702 static inline void setup_mfc_slbs(struct spu_state *csa, struct spu *spu, 703 unsigned int *code, int code_size) 704 { 705 /* Save, Step 47: 706 * Restore, Step 30. 707 * If MFC_SR1[R]=1, write 0 to SLB_Invalidate_All 708 * register, then initialize SLB_VSID and SLB_ESID 709 * to provide access to SPU context save code and 710 * LSCSA. 711 * 712 * This implementation places both the context 713 * switch code and LSCSA in kernel address space. 714 * 715 * Further this implementation assumes that the 716 * MFC_SR1[R]=1 (in other words, assume that 717 * translation is desired by OS environment). 718 */ 719 spu_invalidate_slbs(spu); 720 spu_setup_kernel_slbs(spu, csa->lscsa, code, code_size); 721 } 722 723 static inline void set_switch_active(struct spu_state *csa, struct spu *spu) 724 { 725 /* Save, Step 48: 726 * Restore, Step 23. 727 * Change the software context switch pending flag 728 * to context switch active. 729 * 730 * This implementation does not uses a switch active flag. 731 */ 732 clear_bit(SPU_CONTEXT_SWITCH_PENDING, &spu->flags); 733 mb(); 734 } 735 736 static inline void enable_interrupts(struct spu_state *csa, struct spu *spu) 737 { 738 unsigned long class1_mask = CLASS1_ENABLE_SEGMENT_FAULT_INTR | 739 CLASS1_ENABLE_STORAGE_FAULT_INTR; 740 741 /* Save, Step 49: 742 * Restore, Step 22: 743 * Reset and then enable interrupts, as 744 * needed by OS. 745 * 746 * This implementation enables only class1 747 * (translation) interrupts. 748 */ 749 spin_lock_irq(&spu->register_lock); 750 spu_int_stat_clear(spu, 0, CLASS0_INTR_MASK); 751 spu_int_stat_clear(spu, 1, CLASS1_INTR_MASK); 752 spu_int_stat_clear(spu, 2, CLASS2_INTR_MASK); 753 spu_int_mask_set(spu, 0, 0ul); 754 spu_int_mask_set(spu, 1, class1_mask); 755 spu_int_mask_set(spu, 2, 0ul); 756 spin_unlock_irq(&spu->register_lock); 757 } 758 759 static inline int send_mfc_dma(struct spu *spu, unsigned long ea, 760 unsigned int ls_offset, unsigned int size, 761 unsigned int tag, unsigned int rclass, 762 unsigned int cmd) 763 { 764 struct spu_problem __iomem *prob = spu->problem; 765 union mfc_tag_size_class_cmd command; 766 unsigned int transfer_size; 767 volatile unsigned int status = 0x0; 768 769 while (size > 0) { 770 transfer_size = 771 (size > MFC_MAX_DMA_SIZE) ? MFC_MAX_DMA_SIZE : size; 772 command.u.mfc_size = transfer_size; 773 command.u.mfc_tag = tag; 774 command.u.mfc_rclassid = rclass; 775 command.u.mfc_cmd = cmd; 776 do { 777 out_be32(&prob->mfc_lsa_W, ls_offset); 778 out_be64(&prob->mfc_ea_W, ea); 779 out_be64(&prob->mfc_union_W.all64, command.all64); 780 status = 781 in_be32(&prob->mfc_union_W.by32.mfc_class_cmd32); 782 if (unlikely(status & 0x2)) { 783 cpu_relax(); 784 } 785 } while (status & 0x3); 786 size -= transfer_size; 787 ea += transfer_size; 788 ls_offset += transfer_size; 789 } 790 return 0; 791 } 792 793 static inline void save_ls_16kb(struct spu_state *csa, struct spu *spu) 794 { 795 unsigned long addr = (unsigned long)&csa->lscsa->ls[0]; 796 unsigned int ls_offset = 0x0; 797 unsigned int size = 16384; 798 unsigned int tag = 0; 799 unsigned int rclass = 0; 800 unsigned int cmd = MFC_PUT_CMD; 801 802 /* Save, Step 50: 803 * Issue a DMA command to copy the first 16K bytes 804 * of local storage to the CSA. 805 */ 806 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 807 } 808 809 static inline void set_spu_npc(struct spu_state *csa, struct spu *spu) 810 { 811 struct spu_problem __iomem *prob = spu->problem; 812 813 /* Save, Step 51: 814 * Restore, Step 31. 815 * Write SPU_NPC[IE]=0 and SPU_NPC[LSA] to entry 816 * point address of context save code in local 817 * storage. 818 * 819 * This implementation uses SPU-side save/restore 820 * programs with entry points at LSA of 0. 821 */ 822 out_be32(&prob->spu_npc_RW, 0); 823 eieio(); 824 } 825 826 static inline void set_signot1(struct spu_state *csa, struct spu *spu) 827 { 828 struct spu_problem __iomem *prob = spu->problem; 829 union { 830 u64 ull; 831 u32 ui[2]; 832 } addr64; 833 834 /* Save, Step 52: 835 * Restore, Step 32: 836 * Write SPU_Sig_Notify_1 register with upper 32-bits 837 * of the CSA.LSCSA effective address. 838 */ 839 addr64.ull = (u64) csa->lscsa; 840 out_be32(&prob->signal_notify1, addr64.ui[0]); 841 eieio(); 842 } 843 844 static inline void set_signot2(struct spu_state *csa, struct spu *spu) 845 { 846 struct spu_problem __iomem *prob = spu->problem; 847 union { 848 u64 ull; 849 u32 ui[2]; 850 } addr64; 851 852 /* Save, Step 53: 853 * Restore, Step 33: 854 * Write SPU_Sig_Notify_2 register with lower 32-bits 855 * of the CSA.LSCSA effective address. 856 */ 857 addr64.ull = (u64) csa->lscsa; 858 out_be32(&prob->signal_notify2, addr64.ui[1]); 859 eieio(); 860 } 861 862 static inline void send_save_code(struct spu_state *csa, struct spu *spu) 863 { 864 unsigned long addr = (unsigned long)&spu_save_code[0]; 865 unsigned int ls_offset = 0x0; 866 unsigned int size = sizeof(spu_save_code); 867 unsigned int tag = 0; 868 unsigned int rclass = 0; 869 unsigned int cmd = MFC_GETFS_CMD; 870 871 /* Save, Step 54: 872 * Issue a DMA command to copy context save code 873 * to local storage and start SPU. 874 */ 875 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 876 } 877 878 static inline void set_ppu_querymask(struct spu_state *csa, struct spu *spu) 879 { 880 struct spu_problem __iomem *prob = spu->problem; 881 882 /* Save, Step 55: 883 * Restore, Step 38. 884 * Write PPU_QueryMask=1 (enable Tag Group 0) 885 * and issue eieio instruction. 886 */ 887 out_be32(&prob->dma_querymask_RW, MFC_TAGID_TO_TAGMASK(0)); 888 eieio(); 889 } 890 891 static inline void wait_tag_complete(struct spu_state *csa, struct spu *spu) 892 { 893 struct spu_problem __iomem *prob = spu->problem; 894 u32 mask = MFC_TAGID_TO_TAGMASK(0); 895 unsigned long flags; 896 897 /* Save, Step 56: 898 * Restore, Step 39. 899 * Restore, Step 39. 900 * Restore, Step 46. 901 * Poll PPU_TagStatus[gn] until 01 (Tag group 0 complete) 902 * or write PPU_QueryType[TS]=01 and wait for Tag Group 903 * Complete Interrupt. Write INT_Stat_Class0 or 904 * INT_Stat_Class2 with value of 'handled'. 905 */ 906 POLL_WHILE_FALSE(in_be32(&prob->dma_tagstatus_R) & mask); 907 908 local_irq_save(flags); 909 spu_int_stat_clear(spu, 0, CLASS0_INTR_MASK); 910 spu_int_stat_clear(spu, 2, CLASS2_INTR_MASK); 911 local_irq_restore(flags); 912 } 913 914 static inline void wait_spu_stopped(struct spu_state *csa, struct spu *spu) 915 { 916 struct spu_problem __iomem *prob = spu->problem; 917 unsigned long flags; 918 919 /* Save, Step 57: 920 * Restore, Step 40. 921 * Poll until SPU_Status[R]=0 or wait for SPU Class 0 922 * or SPU Class 2 interrupt. Write INT_Stat_class0 923 * or INT_Stat_class2 with value of handled. 924 */ 925 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING); 926 927 local_irq_save(flags); 928 spu_int_stat_clear(spu, 0, CLASS0_INTR_MASK); 929 spu_int_stat_clear(spu, 2, CLASS2_INTR_MASK); 930 local_irq_restore(flags); 931 } 932 933 static inline int check_save_status(struct spu_state *csa, struct spu *spu) 934 { 935 struct spu_problem __iomem *prob = spu->problem; 936 u32 complete; 937 938 /* Save, Step 54: 939 * If SPU_Status[P]=1 and SPU_Status[SC] = "success", 940 * context save succeeded, otherwise context save 941 * failed. 942 */ 943 complete = ((SPU_SAVE_COMPLETE << SPU_STOP_STATUS_SHIFT) | 944 SPU_STATUS_STOPPED_BY_STOP); 945 return (in_be32(&prob->spu_status_R) != complete) ? 1 : 0; 946 } 947 948 static inline void terminate_spu_app(struct spu_state *csa, struct spu *spu) 949 { 950 /* Restore, Step 4: 951 * If required, notify the "using application" that 952 * the SPU task has been terminated. TBD. 953 */ 954 } 955 956 static inline void suspend_mfc_and_halt_decr(struct spu_state *csa, 957 struct spu *spu) 958 { 959 struct spu_priv2 __iomem *priv2 = spu->priv2; 960 961 /* Restore, Step 7: 962 * Write MFC_Cntl[Dh,Sc,Sm]='1','1','0' to suspend 963 * the queue and halt the decrementer. 964 */ 965 out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE | 966 MFC_CNTL_DECREMENTER_HALTED); 967 eieio(); 968 } 969 970 static inline void wait_suspend_mfc_complete(struct spu_state *csa, 971 struct spu *spu) 972 { 973 struct spu_priv2 __iomem *priv2 = spu->priv2; 974 975 /* Restore, Step 8: 976 * Restore, Step 47. 977 * Poll MFC_CNTL[Ss] until 11 is returned. 978 */ 979 POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) & 980 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) == 981 MFC_CNTL_SUSPEND_COMPLETE); 982 } 983 984 static inline int suspend_spe(struct spu_state *csa, struct spu *spu) 985 { 986 struct spu_problem __iomem *prob = spu->problem; 987 988 /* Restore, Step 9: 989 * If SPU_Status[R]=1, stop SPU execution 990 * and wait for stop to complete. 991 * 992 * Returns 1 if SPU_Status[R]=1 on entry. 993 * 0 otherwise 994 */ 995 if (in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) { 996 if (in_be32(&prob->spu_status_R) & 997 SPU_STATUS_ISOLATED_EXIT_STATUS) { 998 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 999 SPU_STATUS_RUNNING); 1000 } 1001 if ((in_be32(&prob->spu_status_R) & 1002 SPU_STATUS_ISOLATED_LOAD_STATUS) 1003 || (in_be32(&prob->spu_status_R) & 1004 SPU_STATUS_ISOLATED_STATE)) { 1005 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 1006 eieio(); 1007 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1008 SPU_STATUS_RUNNING); 1009 out_be32(&prob->spu_runcntl_RW, 0x2); 1010 eieio(); 1011 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1012 SPU_STATUS_RUNNING); 1013 } 1014 if (in_be32(&prob->spu_status_R) & 1015 SPU_STATUS_WAITING_FOR_CHANNEL) { 1016 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 1017 eieio(); 1018 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1019 SPU_STATUS_RUNNING); 1020 } 1021 return 1; 1022 } 1023 return 0; 1024 } 1025 1026 static inline void clear_spu_status(struct spu_state *csa, struct spu *spu) 1027 { 1028 struct spu_problem __iomem *prob = spu->problem; 1029 1030 /* Restore, Step 10: 1031 * If SPU_Status[R]=0 and SPU_Status[E,L,IS]=1, 1032 * release SPU from isolate state. 1033 */ 1034 if (!(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING)) { 1035 if (in_be32(&prob->spu_status_R) & 1036 SPU_STATUS_ISOLATED_EXIT_STATUS) { 1037 spu_mfc_sr1_set(spu, 1038 MFC_STATE1_MASTER_RUN_CONTROL_MASK); 1039 eieio(); 1040 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1041 eieio(); 1042 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1043 SPU_STATUS_RUNNING); 1044 } 1045 if ((in_be32(&prob->spu_status_R) & 1046 SPU_STATUS_ISOLATED_LOAD_STATUS) 1047 || (in_be32(&prob->spu_status_R) & 1048 SPU_STATUS_ISOLATED_STATE)) { 1049 spu_mfc_sr1_set(spu, 1050 MFC_STATE1_MASTER_RUN_CONTROL_MASK); 1051 eieio(); 1052 out_be32(&prob->spu_runcntl_RW, 0x2); 1053 eieio(); 1054 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1055 SPU_STATUS_RUNNING); 1056 } 1057 } 1058 } 1059 1060 static inline void reset_ch_part1(struct spu_state *csa, struct spu *spu) 1061 { 1062 struct spu_priv2 __iomem *priv2 = spu->priv2; 1063 u64 ch_indices[] = { 0UL, 3UL, 4UL, 24UL, 25UL, 27UL }; 1064 u64 idx; 1065 int i; 1066 1067 /* Restore, Step 20: 1068 */ 1069 1070 /* Reset CH 1 */ 1071 out_be64(&priv2->spu_chnlcntptr_RW, 1); 1072 out_be64(&priv2->spu_chnldata_RW, 0UL); 1073 1074 /* Reset the following CH: [0,3,4,24,25,27] */ 1075 for (i = 0; i < ARRAY_SIZE(ch_indices); i++) { 1076 idx = ch_indices[i]; 1077 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1078 eieio(); 1079 out_be64(&priv2->spu_chnldata_RW, 0UL); 1080 out_be64(&priv2->spu_chnlcnt_RW, 0UL); 1081 eieio(); 1082 } 1083 } 1084 1085 static inline void reset_ch_part2(struct spu_state *csa, struct spu *spu) 1086 { 1087 struct spu_priv2 __iomem *priv2 = spu->priv2; 1088 u64 ch_indices[5] = { 21UL, 23UL, 28UL, 29UL, 30UL }; 1089 u64 ch_counts[5] = { 16UL, 1UL, 1UL, 0UL, 1UL }; 1090 u64 idx; 1091 int i; 1092 1093 /* Restore, Step 21: 1094 * Reset the following CH: [21, 23, 28, 29, 30] 1095 */ 1096 for (i = 0; i < 5; i++) { 1097 idx = ch_indices[i]; 1098 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1099 eieio(); 1100 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]); 1101 eieio(); 1102 } 1103 } 1104 1105 static inline void setup_spu_status_part1(struct spu_state *csa, 1106 struct spu *spu) 1107 { 1108 u32 status_P = SPU_STATUS_STOPPED_BY_STOP; 1109 u32 status_I = SPU_STATUS_INVALID_INSTR; 1110 u32 status_H = SPU_STATUS_STOPPED_BY_HALT; 1111 u32 status_S = SPU_STATUS_SINGLE_STEP; 1112 u32 status_S_I = SPU_STATUS_SINGLE_STEP | SPU_STATUS_INVALID_INSTR; 1113 u32 status_S_P = SPU_STATUS_SINGLE_STEP | SPU_STATUS_STOPPED_BY_STOP; 1114 u32 status_P_H = SPU_STATUS_STOPPED_BY_HALT |SPU_STATUS_STOPPED_BY_STOP; 1115 u32 status_P_I = SPU_STATUS_STOPPED_BY_STOP |SPU_STATUS_INVALID_INSTR; 1116 u32 status_code; 1117 1118 /* Restore, Step 27: 1119 * If the CSA.SPU_Status[I,S,H,P]=1 then add the correct 1120 * instruction sequence to the end of the SPU based restore 1121 * code (after the "context restored" stop and signal) to 1122 * restore the correct SPU status. 1123 * 1124 * NOTE: Rather than modifying the SPU executable, we 1125 * instead add a new 'stopped_status' field to the 1126 * LSCSA. The SPU-side restore reads this field and 1127 * takes the appropriate action when exiting. 1128 */ 1129 1130 status_code = 1131 (csa->prob.spu_status_R >> SPU_STOP_STATUS_SHIFT) & 0xFFFF; 1132 if ((csa->prob.spu_status_R & status_P_I) == status_P_I) { 1133 1134 /* SPU_Status[P,I]=1 - Illegal Instruction followed 1135 * by Stop and Signal instruction, followed by 'br -4'. 1136 * 1137 */ 1138 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P_I; 1139 csa->lscsa->stopped_status.slot[1] = status_code; 1140 1141 } else if ((csa->prob.spu_status_R & status_P_H) == status_P_H) { 1142 1143 /* SPU_Status[P,H]=1 - Halt Conditional, followed 1144 * by Stop and Signal instruction, followed by 1145 * 'br -4'. 1146 */ 1147 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P_H; 1148 csa->lscsa->stopped_status.slot[1] = status_code; 1149 1150 } else if ((csa->prob.spu_status_R & status_S_P) == status_S_P) { 1151 1152 /* SPU_Status[S,P]=1 - Stop and Signal instruction 1153 * followed by 'br -4'. 1154 */ 1155 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S_P; 1156 csa->lscsa->stopped_status.slot[1] = status_code; 1157 1158 } else if ((csa->prob.spu_status_R & status_S_I) == status_S_I) { 1159 1160 /* SPU_Status[S,I]=1 - Illegal instruction followed 1161 * by 'br -4'. 1162 */ 1163 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S_I; 1164 csa->lscsa->stopped_status.slot[1] = status_code; 1165 1166 } else if ((csa->prob.spu_status_R & status_P) == status_P) { 1167 1168 /* SPU_Status[P]=1 - Stop and Signal instruction 1169 * followed by 'br -4'. 1170 */ 1171 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P; 1172 csa->lscsa->stopped_status.slot[1] = status_code; 1173 1174 } else if ((csa->prob.spu_status_R & status_H) == status_H) { 1175 1176 /* SPU_Status[H]=1 - Halt Conditional, followed 1177 * by 'br -4'. 1178 */ 1179 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_H; 1180 1181 } else if ((csa->prob.spu_status_R & status_S) == status_S) { 1182 1183 /* SPU_Status[S]=1 - Two nop instructions. 1184 */ 1185 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S; 1186 1187 } else if ((csa->prob.spu_status_R & status_I) == status_I) { 1188 1189 /* SPU_Status[I]=1 - Illegal instruction followed 1190 * by 'br -4'. 1191 */ 1192 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_I; 1193 1194 } 1195 } 1196 1197 static inline void setup_spu_status_part2(struct spu_state *csa, 1198 struct spu *spu) 1199 { 1200 u32 mask; 1201 1202 /* Restore, Step 28: 1203 * If the CSA.SPU_Status[I,S,H,P,R]=0 then 1204 * add a 'br *' instruction to the end of 1205 * the SPU based restore code. 1206 * 1207 * NOTE: Rather than modifying the SPU executable, we 1208 * instead add a new 'stopped_status' field to the 1209 * LSCSA. The SPU-side restore reads this field and 1210 * takes the appropriate action when exiting. 1211 */ 1212 mask = SPU_STATUS_INVALID_INSTR | 1213 SPU_STATUS_SINGLE_STEP | 1214 SPU_STATUS_STOPPED_BY_HALT | 1215 SPU_STATUS_STOPPED_BY_STOP | SPU_STATUS_RUNNING; 1216 if (!(csa->prob.spu_status_R & mask)) { 1217 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_R; 1218 } 1219 } 1220 1221 static inline void restore_mfc_rag(struct spu_state *csa, struct spu *spu) 1222 { 1223 /* Restore, Step 29: 1224 * Restore RA_GROUP_ID register and the 1225 * RA_ENABLE reigster from the CSA. 1226 */ 1227 spu_resource_allocation_groupID_set(spu, 1228 csa->priv1.resource_allocation_groupID_RW); 1229 spu_resource_allocation_enable_set(spu, 1230 csa->priv1.resource_allocation_enable_RW); 1231 } 1232 1233 static inline void send_restore_code(struct spu_state *csa, struct spu *spu) 1234 { 1235 unsigned long addr = (unsigned long)&spu_restore_code[0]; 1236 unsigned int ls_offset = 0x0; 1237 unsigned int size = sizeof(spu_restore_code); 1238 unsigned int tag = 0; 1239 unsigned int rclass = 0; 1240 unsigned int cmd = MFC_GETFS_CMD; 1241 1242 /* Restore, Step 37: 1243 * Issue MFC DMA command to copy context 1244 * restore code to local storage. 1245 */ 1246 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 1247 } 1248 1249 static inline void setup_decr(struct spu_state *csa, struct spu *spu) 1250 { 1251 /* Restore, Step 34: 1252 * If CSA.MFC_CNTL[Ds]=1 (decrementer was 1253 * running) then adjust decrementer, set 1254 * decrementer running status in LSCSA, 1255 * and set decrementer "wrapped" status 1256 * in LSCSA. 1257 */ 1258 if (csa->priv2.mfc_control_RW & MFC_CNTL_DECREMENTER_RUNNING) { 1259 cycles_t resume_time = get_cycles(); 1260 cycles_t delta_time = resume_time - csa->suspend_time; 1261 1262 csa->lscsa->decr_status.slot[0] = SPU_DECR_STATUS_RUNNING; 1263 if (csa->lscsa->decr.slot[0] < delta_time) { 1264 csa->lscsa->decr_status.slot[0] |= 1265 SPU_DECR_STATUS_WRAPPED; 1266 } 1267 1268 csa->lscsa->decr.slot[0] -= delta_time; 1269 } else { 1270 csa->lscsa->decr_status.slot[0] = 0; 1271 } 1272 } 1273 1274 static inline void setup_ppu_mb(struct spu_state *csa, struct spu *spu) 1275 { 1276 /* Restore, Step 35: 1277 * Copy the CSA.PU_MB data into the LSCSA. 1278 */ 1279 csa->lscsa->ppu_mb.slot[0] = csa->prob.pu_mb_R; 1280 } 1281 1282 static inline void setup_ppuint_mb(struct spu_state *csa, struct spu *spu) 1283 { 1284 /* Restore, Step 36: 1285 * Copy the CSA.PUINT_MB data into the LSCSA. 1286 */ 1287 csa->lscsa->ppuint_mb.slot[0] = csa->priv2.puint_mb_R; 1288 } 1289 1290 static inline int check_restore_status(struct spu_state *csa, struct spu *spu) 1291 { 1292 struct spu_problem __iomem *prob = spu->problem; 1293 u32 complete; 1294 1295 /* Restore, Step 40: 1296 * If SPU_Status[P]=1 and SPU_Status[SC] = "success", 1297 * context restore succeeded, otherwise context restore 1298 * failed. 1299 */ 1300 complete = ((SPU_RESTORE_COMPLETE << SPU_STOP_STATUS_SHIFT) | 1301 SPU_STATUS_STOPPED_BY_STOP); 1302 return (in_be32(&prob->spu_status_R) != complete) ? 1 : 0; 1303 } 1304 1305 static inline void restore_spu_privcntl(struct spu_state *csa, struct spu *spu) 1306 { 1307 struct spu_priv2 __iomem *priv2 = spu->priv2; 1308 1309 /* Restore, Step 41: 1310 * Restore SPU_PrivCntl from the CSA. 1311 */ 1312 out_be64(&priv2->spu_privcntl_RW, csa->priv2.spu_privcntl_RW); 1313 eieio(); 1314 } 1315 1316 static inline void restore_status_part1(struct spu_state *csa, struct spu *spu) 1317 { 1318 struct spu_problem __iomem *prob = spu->problem; 1319 u32 mask; 1320 1321 /* Restore, Step 42: 1322 * If any CSA.SPU_Status[I,S,H,P]=1, then 1323 * restore the error or single step state. 1324 */ 1325 mask = SPU_STATUS_INVALID_INSTR | 1326 SPU_STATUS_SINGLE_STEP | 1327 SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP; 1328 if (csa->prob.spu_status_R & mask) { 1329 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1330 eieio(); 1331 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1332 SPU_STATUS_RUNNING); 1333 } 1334 } 1335 1336 static inline void restore_status_part2(struct spu_state *csa, struct spu *spu) 1337 { 1338 struct spu_problem __iomem *prob = spu->problem; 1339 u32 mask; 1340 1341 /* Restore, Step 43: 1342 * If all CSA.SPU_Status[I,S,H,P,R]=0 then write 1343 * SPU_RunCntl[R0R1]='01', wait for SPU_Status[R]=1, 1344 * then write '00' to SPU_RunCntl[R0R1] and wait 1345 * for SPU_Status[R]=0. 1346 */ 1347 mask = SPU_STATUS_INVALID_INSTR | 1348 SPU_STATUS_SINGLE_STEP | 1349 SPU_STATUS_STOPPED_BY_HALT | 1350 SPU_STATUS_STOPPED_BY_STOP | SPU_STATUS_RUNNING; 1351 if (!(csa->prob.spu_status_R & mask)) { 1352 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1353 eieio(); 1354 POLL_WHILE_FALSE(in_be32(&prob->spu_status_R) & 1355 SPU_STATUS_RUNNING); 1356 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 1357 eieio(); 1358 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1359 SPU_STATUS_RUNNING); 1360 } 1361 } 1362 1363 static inline void restore_ls_16kb(struct spu_state *csa, struct spu *spu) 1364 { 1365 unsigned long addr = (unsigned long)&csa->lscsa->ls[0]; 1366 unsigned int ls_offset = 0x0; 1367 unsigned int size = 16384; 1368 unsigned int tag = 0; 1369 unsigned int rclass = 0; 1370 unsigned int cmd = MFC_GET_CMD; 1371 1372 /* Restore, Step 44: 1373 * Issue a DMA command to restore the first 1374 * 16kb of local storage from CSA. 1375 */ 1376 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 1377 } 1378 1379 static inline void suspend_mfc(struct spu_state *csa, struct spu *spu) 1380 { 1381 struct spu_priv2 __iomem *priv2 = spu->priv2; 1382 1383 /* Restore, Step 47. 1384 * Write MFC_Cntl[Sc,Sm]='1','0' to suspend 1385 * the queue. 1386 */ 1387 out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE); 1388 eieio(); 1389 } 1390 1391 static inline void clear_interrupts(struct spu_state *csa, struct spu *spu) 1392 { 1393 /* Restore, Step 49: 1394 * Write INT_MASK_class0 with value of 0. 1395 * Write INT_MASK_class1 with value of 0. 1396 * Write INT_MASK_class2 with value of 0. 1397 * Write INT_STAT_class0 with value of -1. 1398 * Write INT_STAT_class1 with value of -1. 1399 * Write INT_STAT_class2 with value of -1. 1400 */ 1401 spin_lock_irq(&spu->register_lock); 1402 spu_int_mask_set(spu, 0, 0ul); 1403 spu_int_mask_set(spu, 1, 0ul); 1404 spu_int_mask_set(spu, 2, 0ul); 1405 spu_int_stat_clear(spu, 0, CLASS0_INTR_MASK); 1406 spu_int_stat_clear(spu, 1, CLASS1_INTR_MASK); 1407 spu_int_stat_clear(spu, 2, CLASS2_INTR_MASK); 1408 spin_unlock_irq(&spu->register_lock); 1409 } 1410 1411 static inline void restore_mfc_queues(struct spu_state *csa, struct spu *spu) 1412 { 1413 struct spu_priv2 __iomem *priv2 = spu->priv2; 1414 int i; 1415 1416 /* Restore, Step 50: 1417 * If MFC_Cntl[Se]!=0 then restore 1418 * MFC command queues. 1419 */ 1420 if ((csa->priv2.mfc_control_RW & MFC_CNTL_DMA_QUEUES_EMPTY_MASK) == 0) { 1421 for (i = 0; i < 8; i++) { 1422 out_be64(&priv2->puq[i].mfc_cq_data0_RW, 1423 csa->priv2.puq[i].mfc_cq_data0_RW); 1424 out_be64(&priv2->puq[i].mfc_cq_data1_RW, 1425 csa->priv2.puq[i].mfc_cq_data1_RW); 1426 out_be64(&priv2->puq[i].mfc_cq_data2_RW, 1427 csa->priv2.puq[i].mfc_cq_data2_RW); 1428 out_be64(&priv2->puq[i].mfc_cq_data3_RW, 1429 csa->priv2.puq[i].mfc_cq_data3_RW); 1430 } 1431 for (i = 0; i < 16; i++) { 1432 out_be64(&priv2->spuq[i].mfc_cq_data0_RW, 1433 csa->priv2.spuq[i].mfc_cq_data0_RW); 1434 out_be64(&priv2->spuq[i].mfc_cq_data1_RW, 1435 csa->priv2.spuq[i].mfc_cq_data1_RW); 1436 out_be64(&priv2->spuq[i].mfc_cq_data2_RW, 1437 csa->priv2.spuq[i].mfc_cq_data2_RW); 1438 out_be64(&priv2->spuq[i].mfc_cq_data3_RW, 1439 csa->priv2.spuq[i].mfc_cq_data3_RW); 1440 } 1441 } 1442 eieio(); 1443 } 1444 1445 static inline void restore_ppu_querymask(struct spu_state *csa, struct spu *spu) 1446 { 1447 struct spu_problem __iomem *prob = spu->problem; 1448 1449 /* Restore, Step 51: 1450 * Restore the PPU_QueryMask register from CSA. 1451 */ 1452 out_be32(&prob->dma_querymask_RW, csa->prob.dma_querymask_RW); 1453 eieio(); 1454 } 1455 1456 static inline void restore_ppu_querytype(struct spu_state *csa, struct spu *spu) 1457 { 1458 struct spu_problem __iomem *prob = spu->problem; 1459 1460 /* Restore, Step 52: 1461 * Restore the PPU_QueryType register from CSA. 1462 */ 1463 out_be32(&prob->dma_querytype_RW, csa->prob.dma_querytype_RW); 1464 eieio(); 1465 } 1466 1467 static inline void restore_mfc_csr_tsq(struct spu_state *csa, struct spu *spu) 1468 { 1469 struct spu_priv2 __iomem *priv2 = spu->priv2; 1470 1471 /* Restore, Step 53: 1472 * Restore the MFC_CSR_TSQ register from CSA. 1473 */ 1474 out_be64(&priv2->spu_tag_status_query_RW, 1475 csa->priv2.spu_tag_status_query_RW); 1476 eieio(); 1477 } 1478 1479 static inline void restore_mfc_csr_cmd(struct spu_state *csa, struct spu *spu) 1480 { 1481 struct spu_priv2 __iomem *priv2 = spu->priv2; 1482 1483 /* Restore, Step 54: 1484 * Restore the MFC_CSR_CMD1 and MFC_CSR_CMD2 1485 * registers from CSA. 1486 */ 1487 out_be64(&priv2->spu_cmd_buf1_RW, csa->priv2.spu_cmd_buf1_RW); 1488 out_be64(&priv2->spu_cmd_buf2_RW, csa->priv2.spu_cmd_buf2_RW); 1489 eieio(); 1490 } 1491 1492 static inline void restore_mfc_csr_ato(struct spu_state *csa, struct spu *spu) 1493 { 1494 struct spu_priv2 __iomem *priv2 = spu->priv2; 1495 1496 /* Restore, Step 55: 1497 * Restore the MFC_CSR_ATO register from CSA. 1498 */ 1499 out_be64(&priv2->spu_atomic_status_RW, csa->priv2.spu_atomic_status_RW); 1500 } 1501 1502 static inline void restore_mfc_tclass_id(struct spu_state *csa, struct spu *spu) 1503 { 1504 /* Restore, Step 56: 1505 * Restore the MFC_TCLASS_ID register from CSA. 1506 */ 1507 spu_mfc_tclass_id_set(spu, csa->priv1.mfc_tclass_id_RW); 1508 eieio(); 1509 } 1510 1511 static inline void set_llr_event(struct spu_state *csa, struct spu *spu) 1512 { 1513 u64 ch0_cnt, ch0_data; 1514 u64 ch1_data; 1515 1516 /* Restore, Step 57: 1517 * Set the Lock Line Reservation Lost Event by: 1518 * 1. OR CSA.SPU_Event_Status with bit 21 (Lr) set to 1. 1519 * 2. If CSA.SPU_Channel_0_Count=0 and 1520 * CSA.SPU_Wr_Event_Mask[Lr]=1 and 1521 * CSA.SPU_Event_Status[Lr]=0 then set 1522 * CSA.SPU_Event_Status_Count=1. 1523 */ 1524 ch0_cnt = csa->spu_chnlcnt_RW[0]; 1525 ch0_data = csa->spu_chnldata_RW[0]; 1526 ch1_data = csa->spu_chnldata_RW[1]; 1527 csa->spu_chnldata_RW[0] |= MFC_LLR_LOST_EVENT; 1528 if ((ch0_cnt == 0) && !(ch0_data & MFC_LLR_LOST_EVENT) && 1529 (ch1_data & MFC_LLR_LOST_EVENT)) { 1530 csa->spu_chnlcnt_RW[0] = 1; 1531 } 1532 } 1533 1534 static inline void restore_decr_wrapped(struct spu_state *csa, struct spu *spu) 1535 { 1536 /* Restore, Step 58: 1537 * If the status of the CSA software decrementer 1538 * "wrapped" flag is set, OR in a '1' to 1539 * CSA.SPU_Event_Status[Tm]. 1540 */ 1541 if (!(csa->lscsa->decr_status.slot[0] & SPU_DECR_STATUS_WRAPPED)) 1542 return; 1543 1544 if ((csa->spu_chnlcnt_RW[0] == 0) && 1545 (csa->spu_chnldata_RW[1] & 0x20) && 1546 !(csa->spu_chnldata_RW[0] & 0x20)) 1547 csa->spu_chnlcnt_RW[0] = 1; 1548 1549 csa->spu_chnldata_RW[0] |= 0x20; 1550 } 1551 1552 static inline void restore_ch_part1(struct spu_state *csa, struct spu *spu) 1553 { 1554 struct spu_priv2 __iomem *priv2 = spu->priv2; 1555 u64 idx, ch_indices[] = { 0UL, 3UL, 4UL, 24UL, 25UL, 27UL }; 1556 int i; 1557 1558 /* Restore, Step 59: 1559 * Restore the following CH: [0,3,4,24,25,27] 1560 */ 1561 for (i = 0; i < ARRAY_SIZE(ch_indices); i++) { 1562 idx = ch_indices[i]; 1563 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1564 eieio(); 1565 out_be64(&priv2->spu_chnldata_RW, csa->spu_chnldata_RW[idx]); 1566 out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[idx]); 1567 eieio(); 1568 } 1569 } 1570 1571 static inline void restore_ch_part2(struct spu_state *csa, struct spu *spu) 1572 { 1573 struct spu_priv2 __iomem *priv2 = spu->priv2; 1574 u64 ch_indices[3] = { 9UL, 21UL, 23UL }; 1575 u64 ch_counts[3] = { 1UL, 16UL, 1UL }; 1576 u64 idx; 1577 int i; 1578 1579 /* Restore, Step 60: 1580 * Restore the following CH: [9,21,23]. 1581 */ 1582 ch_counts[0] = 1UL; 1583 ch_counts[1] = csa->spu_chnlcnt_RW[21]; 1584 ch_counts[2] = 1UL; 1585 for (i = 0; i < 3; i++) { 1586 idx = ch_indices[i]; 1587 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1588 eieio(); 1589 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]); 1590 eieio(); 1591 } 1592 } 1593 1594 static inline void restore_spu_lslr(struct spu_state *csa, struct spu *spu) 1595 { 1596 struct spu_priv2 __iomem *priv2 = spu->priv2; 1597 1598 /* Restore, Step 61: 1599 * Restore the SPU_LSLR register from CSA. 1600 */ 1601 out_be64(&priv2->spu_lslr_RW, csa->priv2.spu_lslr_RW); 1602 eieio(); 1603 } 1604 1605 static inline void restore_spu_cfg(struct spu_state *csa, struct spu *spu) 1606 { 1607 struct spu_priv2 __iomem *priv2 = spu->priv2; 1608 1609 /* Restore, Step 62: 1610 * Restore the SPU_Cfg register from CSA. 1611 */ 1612 out_be64(&priv2->spu_cfg_RW, csa->priv2.spu_cfg_RW); 1613 eieio(); 1614 } 1615 1616 static inline void restore_pm_trace(struct spu_state *csa, struct spu *spu) 1617 { 1618 /* Restore, Step 63: 1619 * Restore PM_Trace_Tag_Wait_Mask from CSA. 1620 * Not performed by this implementation. 1621 */ 1622 } 1623 1624 static inline void restore_spu_npc(struct spu_state *csa, struct spu *spu) 1625 { 1626 struct spu_problem __iomem *prob = spu->problem; 1627 1628 /* Restore, Step 64: 1629 * Restore SPU_NPC from CSA. 1630 */ 1631 out_be32(&prob->spu_npc_RW, csa->prob.spu_npc_RW); 1632 eieio(); 1633 } 1634 1635 static inline void restore_spu_mb(struct spu_state *csa, struct spu *spu) 1636 { 1637 struct spu_priv2 __iomem *priv2 = spu->priv2; 1638 int i; 1639 1640 /* Restore, Step 65: 1641 * Restore MFC_RdSPU_MB from CSA. 1642 */ 1643 out_be64(&priv2->spu_chnlcntptr_RW, 29UL); 1644 eieio(); 1645 out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[29]); 1646 for (i = 0; i < 4; i++) { 1647 out_be64(&priv2->spu_chnldata_RW, csa->spu_mailbox_data[i]); 1648 } 1649 eieio(); 1650 } 1651 1652 static inline void check_ppu_mb_stat(struct spu_state *csa, struct spu *spu) 1653 { 1654 struct spu_problem __iomem *prob = spu->problem; 1655 u32 dummy = 0; 1656 1657 /* Restore, Step 66: 1658 * If CSA.MB_Stat[P]=0 (mailbox empty) then 1659 * read from the PPU_MB register. 1660 */ 1661 if ((csa->prob.mb_stat_R & 0xFF) == 0) { 1662 dummy = in_be32(&prob->pu_mb_R); 1663 eieio(); 1664 } 1665 } 1666 1667 static inline void check_ppuint_mb_stat(struct spu_state *csa, struct spu *spu) 1668 { 1669 struct spu_priv2 __iomem *priv2 = spu->priv2; 1670 u64 dummy = 0UL; 1671 1672 /* Restore, Step 66: 1673 * If CSA.MB_Stat[I]=0 (mailbox empty) then 1674 * read from the PPUINT_MB register. 1675 */ 1676 if ((csa->prob.mb_stat_R & 0xFF0000) == 0) { 1677 dummy = in_be64(&priv2->puint_mb_R); 1678 eieio(); 1679 spu_int_stat_clear(spu, 2, CLASS2_ENABLE_MAILBOX_INTR); 1680 eieio(); 1681 } 1682 } 1683 1684 static inline void restore_mfc_sr1(struct spu_state *csa, struct spu *spu) 1685 { 1686 /* Restore, Step 69: 1687 * Restore the MFC_SR1 register from CSA. 1688 */ 1689 spu_mfc_sr1_set(spu, csa->priv1.mfc_sr1_RW); 1690 eieio(); 1691 } 1692 1693 static inline void restore_other_spu_access(struct spu_state *csa, 1694 struct spu *spu) 1695 { 1696 /* Restore, Step 70: 1697 * Restore other SPU mappings to this SPU. TBD. 1698 */ 1699 } 1700 1701 static inline void restore_spu_runcntl(struct spu_state *csa, struct spu *spu) 1702 { 1703 struct spu_problem __iomem *prob = spu->problem; 1704 1705 /* Restore, Step 71: 1706 * If CSA.SPU_Status[R]=1 then write 1707 * SPU_RunCntl[R0R1]='01'. 1708 */ 1709 if (csa->prob.spu_status_R & SPU_STATUS_RUNNING) { 1710 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1711 eieio(); 1712 } 1713 } 1714 1715 static inline void restore_mfc_cntl(struct spu_state *csa, struct spu *spu) 1716 { 1717 struct spu_priv2 __iomem *priv2 = spu->priv2; 1718 1719 /* Restore, Step 72: 1720 * Restore the MFC_CNTL register for the CSA. 1721 */ 1722 out_be64(&priv2->mfc_control_RW, csa->priv2.mfc_control_RW); 1723 eieio(); 1724 /* 1725 * FIXME: this is to restart a DMA that we were processing 1726 * before the save. better remember the fault information 1727 * in the csa instead. 1728 */ 1729 if ((csa->priv2.mfc_control_RW & MFC_CNTL_SUSPEND_DMA_QUEUE_MASK)) { 1730 out_be64(&priv2->mfc_control_RW, MFC_CNTL_RESTART_DMA_COMMAND); 1731 eieio(); 1732 } 1733 } 1734 1735 static inline void enable_user_access(struct spu_state *csa, struct spu *spu) 1736 { 1737 /* Restore, Step 73: 1738 * Enable user-space access (if provided) to this 1739 * SPU by mapping the virtual pages assigned to 1740 * the SPU memory-mapped I/O (MMIO) for problem 1741 * state. TBD. 1742 */ 1743 } 1744 1745 static inline void reset_switch_active(struct spu_state *csa, struct spu *spu) 1746 { 1747 /* Restore, Step 74: 1748 * Reset the "context switch active" flag. 1749 * Not performed by this implementation. 1750 */ 1751 } 1752 1753 static inline void reenable_interrupts(struct spu_state *csa, struct spu *spu) 1754 { 1755 /* Restore, Step 75: 1756 * Re-enable SPU interrupts. 1757 */ 1758 spin_lock_irq(&spu->register_lock); 1759 spu_int_mask_set(spu, 0, csa->priv1.int_mask_class0_RW); 1760 spu_int_mask_set(spu, 1, csa->priv1.int_mask_class1_RW); 1761 spu_int_mask_set(spu, 2, csa->priv1.int_mask_class2_RW); 1762 spin_unlock_irq(&spu->register_lock); 1763 } 1764 1765 static int quiece_spu(struct spu_state *prev, struct spu *spu) 1766 { 1767 /* 1768 * Combined steps 2-18 of SPU context save sequence, which 1769 * quiesce the SPU state (disable SPU execution, MFC command 1770 * queues, decrementer, SPU interrupts, etc.). 1771 * 1772 * Returns 0 on success. 1773 * 2 if failed step 2. 1774 * 6 if failed step 6. 1775 */ 1776 1777 if (check_spu_isolate(prev, spu)) { /* Step 2. */ 1778 return 2; 1779 } 1780 disable_interrupts(prev, spu); /* Step 3. */ 1781 set_watchdog_timer(prev, spu); /* Step 4. */ 1782 inhibit_user_access(prev, spu); /* Step 5. */ 1783 if (check_spu_isolate(prev, spu)) { /* Step 6. */ 1784 return 6; 1785 } 1786 set_switch_pending(prev, spu); /* Step 7. */ 1787 save_mfc_cntl(prev, spu); /* Step 8. */ 1788 save_spu_runcntl(prev, spu); /* Step 9. */ 1789 save_mfc_sr1(prev, spu); /* Step 10. */ 1790 save_spu_status(prev, spu); /* Step 11. */ 1791 save_mfc_decr(prev, spu); /* Step 12. */ 1792 halt_mfc_decr(prev, spu); /* Step 13. */ 1793 save_timebase(prev, spu); /* Step 14. */ 1794 remove_other_spu_access(prev, spu); /* Step 15. */ 1795 do_mfc_mssync(prev, spu); /* Step 16. */ 1796 issue_mfc_tlbie(prev, spu); /* Step 17. */ 1797 handle_pending_interrupts(prev, spu); /* Step 18. */ 1798 1799 return 0; 1800 } 1801 1802 static void save_csa(struct spu_state *prev, struct spu *spu) 1803 { 1804 /* 1805 * Combine steps 19-44 of SPU context save sequence, which 1806 * save regions of the privileged & problem state areas. 1807 */ 1808 1809 save_mfc_queues(prev, spu); /* Step 19. */ 1810 save_ppu_querymask(prev, spu); /* Step 20. */ 1811 save_ppu_querytype(prev, spu); /* Step 21. */ 1812 save_ppu_tagstatus(prev, spu); /* NEW. */ 1813 save_mfc_csr_tsq(prev, spu); /* Step 22. */ 1814 save_mfc_csr_cmd(prev, spu); /* Step 23. */ 1815 save_mfc_csr_ato(prev, spu); /* Step 24. */ 1816 save_mfc_tclass_id(prev, spu); /* Step 25. */ 1817 set_mfc_tclass_id(prev, spu); /* Step 26. */ 1818 purge_mfc_queue(prev, spu); /* Step 27. */ 1819 wait_purge_complete(prev, spu); /* Step 28. */ 1820 setup_mfc_sr1(prev, spu); /* Step 30. */ 1821 save_spu_npc(prev, spu); /* Step 31. */ 1822 save_spu_privcntl(prev, spu); /* Step 32. */ 1823 reset_spu_privcntl(prev, spu); /* Step 33. */ 1824 save_spu_lslr(prev, spu); /* Step 34. */ 1825 reset_spu_lslr(prev, spu); /* Step 35. */ 1826 save_spu_cfg(prev, spu); /* Step 36. */ 1827 save_pm_trace(prev, spu); /* Step 37. */ 1828 save_mfc_rag(prev, spu); /* Step 38. */ 1829 save_ppu_mb_stat(prev, spu); /* Step 39. */ 1830 save_ppu_mb(prev, spu); /* Step 40. */ 1831 save_ppuint_mb(prev, spu); /* Step 41. */ 1832 save_ch_part1(prev, spu); /* Step 42. */ 1833 save_spu_mb(prev, spu); /* Step 43. */ 1834 save_mfc_cmd(prev, spu); /* Step 44. */ 1835 reset_ch(prev, spu); /* Step 45. */ 1836 } 1837 1838 static void save_lscsa(struct spu_state *prev, struct spu *spu) 1839 { 1840 /* 1841 * Perform steps 46-57 of SPU context save sequence, 1842 * which save regions of the local store and register 1843 * file. 1844 */ 1845 1846 resume_mfc_queue(prev, spu); /* Step 46. */ 1847 /* Step 47. */ 1848 setup_mfc_slbs(prev, spu, spu_save_code, sizeof(spu_save_code)); 1849 set_switch_active(prev, spu); /* Step 48. */ 1850 enable_interrupts(prev, spu); /* Step 49. */ 1851 save_ls_16kb(prev, spu); /* Step 50. */ 1852 set_spu_npc(prev, spu); /* Step 51. */ 1853 set_signot1(prev, spu); /* Step 52. */ 1854 set_signot2(prev, spu); /* Step 53. */ 1855 send_save_code(prev, spu); /* Step 54. */ 1856 set_ppu_querymask(prev, spu); /* Step 55. */ 1857 wait_tag_complete(prev, spu); /* Step 56. */ 1858 wait_spu_stopped(prev, spu); /* Step 57. */ 1859 } 1860 1861 static void force_spu_isolate_exit(struct spu *spu) 1862 { 1863 struct spu_problem __iomem *prob = spu->problem; 1864 struct spu_priv2 __iomem *priv2 = spu->priv2; 1865 1866 /* Stop SPE execution and wait for completion. */ 1867 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 1868 iobarrier_rw(); 1869 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING); 1870 1871 /* Restart SPE master runcntl. */ 1872 spu_mfc_sr1_set(spu, MFC_STATE1_MASTER_RUN_CONTROL_MASK); 1873 iobarrier_w(); 1874 1875 /* Initiate isolate exit request and wait for completion. */ 1876 out_be64(&priv2->spu_privcntl_RW, 4LL); 1877 iobarrier_w(); 1878 out_be32(&prob->spu_runcntl_RW, 2); 1879 iobarrier_rw(); 1880 POLL_WHILE_FALSE((in_be32(&prob->spu_status_R) 1881 & SPU_STATUS_STOPPED_BY_STOP)); 1882 1883 /* Reset load request to normal. */ 1884 out_be64(&priv2->spu_privcntl_RW, SPU_PRIVCNT_LOAD_REQUEST_NORMAL); 1885 iobarrier_w(); 1886 } 1887 1888 /** 1889 * stop_spu_isolate 1890 * Check SPU run-control state and force isolated 1891 * exit function as necessary. 1892 */ 1893 static void stop_spu_isolate(struct spu *spu) 1894 { 1895 struct spu_problem __iomem *prob = spu->problem; 1896 1897 if (in_be32(&prob->spu_status_R) & SPU_STATUS_ISOLATED_STATE) { 1898 /* The SPU is in isolated state; the only way 1899 * to get it out is to perform an isolated 1900 * exit (clean) operation. 1901 */ 1902 force_spu_isolate_exit(spu); 1903 } 1904 } 1905 1906 static void harvest(struct spu_state *prev, struct spu *spu) 1907 { 1908 /* 1909 * Perform steps 2-25 of SPU context restore sequence, 1910 * which resets an SPU either after a failed save, or 1911 * when using SPU for first time. 1912 */ 1913 1914 disable_interrupts(prev, spu); /* Step 2. */ 1915 inhibit_user_access(prev, spu); /* Step 3. */ 1916 terminate_spu_app(prev, spu); /* Step 4. */ 1917 set_switch_pending(prev, spu); /* Step 5. */ 1918 stop_spu_isolate(spu); /* NEW. */ 1919 remove_other_spu_access(prev, spu); /* Step 6. */ 1920 suspend_mfc_and_halt_decr(prev, spu); /* Step 7. */ 1921 wait_suspend_mfc_complete(prev, spu); /* Step 8. */ 1922 if (!suspend_spe(prev, spu)) /* Step 9. */ 1923 clear_spu_status(prev, spu); /* Step 10. */ 1924 do_mfc_mssync(prev, spu); /* Step 11. */ 1925 issue_mfc_tlbie(prev, spu); /* Step 12. */ 1926 handle_pending_interrupts(prev, spu); /* Step 13. */ 1927 purge_mfc_queue(prev, spu); /* Step 14. */ 1928 wait_purge_complete(prev, spu); /* Step 15. */ 1929 reset_spu_privcntl(prev, spu); /* Step 16. */ 1930 reset_spu_lslr(prev, spu); /* Step 17. */ 1931 setup_mfc_sr1(prev, spu); /* Step 18. */ 1932 spu_invalidate_slbs(spu); /* Step 19. */ 1933 reset_ch_part1(prev, spu); /* Step 20. */ 1934 reset_ch_part2(prev, spu); /* Step 21. */ 1935 enable_interrupts(prev, spu); /* Step 22. */ 1936 set_switch_active(prev, spu); /* Step 23. */ 1937 set_mfc_tclass_id(prev, spu); /* Step 24. */ 1938 resume_mfc_queue(prev, spu); /* Step 25. */ 1939 } 1940 1941 static void restore_lscsa(struct spu_state *next, struct spu *spu) 1942 { 1943 /* 1944 * Perform steps 26-40 of SPU context restore sequence, 1945 * which restores regions of the local store and register 1946 * file. 1947 */ 1948 1949 set_watchdog_timer(next, spu); /* Step 26. */ 1950 setup_spu_status_part1(next, spu); /* Step 27. */ 1951 setup_spu_status_part2(next, spu); /* Step 28. */ 1952 restore_mfc_rag(next, spu); /* Step 29. */ 1953 /* Step 30. */ 1954 setup_mfc_slbs(next, spu, spu_restore_code, sizeof(spu_restore_code)); 1955 set_spu_npc(next, spu); /* Step 31. */ 1956 set_signot1(next, spu); /* Step 32. */ 1957 set_signot2(next, spu); /* Step 33. */ 1958 setup_decr(next, spu); /* Step 34. */ 1959 setup_ppu_mb(next, spu); /* Step 35. */ 1960 setup_ppuint_mb(next, spu); /* Step 36. */ 1961 send_restore_code(next, spu); /* Step 37. */ 1962 set_ppu_querymask(next, spu); /* Step 38. */ 1963 wait_tag_complete(next, spu); /* Step 39. */ 1964 wait_spu_stopped(next, spu); /* Step 40. */ 1965 } 1966 1967 static void restore_csa(struct spu_state *next, struct spu *spu) 1968 { 1969 /* 1970 * Combine steps 41-76 of SPU context restore sequence, which 1971 * restore regions of the privileged & problem state areas. 1972 */ 1973 1974 restore_spu_privcntl(next, spu); /* Step 41. */ 1975 restore_status_part1(next, spu); /* Step 42. */ 1976 restore_status_part2(next, spu); /* Step 43. */ 1977 restore_ls_16kb(next, spu); /* Step 44. */ 1978 wait_tag_complete(next, spu); /* Step 45. */ 1979 suspend_mfc(next, spu); /* Step 46. */ 1980 wait_suspend_mfc_complete(next, spu); /* Step 47. */ 1981 issue_mfc_tlbie(next, spu); /* Step 48. */ 1982 clear_interrupts(next, spu); /* Step 49. */ 1983 restore_mfc_queues(next, spu); /* Step 50. */ 1984 restore_ppu_querymask(next, spu); /* Step 51. */ 1985 restore_ppu_querytype(next, spu); /* Step 52. */ 1986 restore_mfc_csr_tsq(next, spu); /* Step 53. */ 1987 restore_mfc_csr_cmd(next, spu); /* Step 54. */ 1988 restore_mfc_csr_ato(next, spu); /* Step 55. */ 1989 restore_mfc_tclass_id(next, spu); /* Step 56. */ 1990 set_llr_event(next, spu); /* Step 57. */ 1991 restore_decr_wrapped(next, spu); /* Step 58. */ 1992 restore_ch_part1(next, spu); /* Step 59. */ 1993 restore_ch_part2(next, spu); /* Step 60. */ 1994 restore_spu_lslr(next, spu); /* Step 61. */ 1995 restore_spu_cfg(next, spu); /* Step 62. */ 1996 restore_pm_trace(next, spu); /* Step 63. */ 1997 restore_spu_npc(next, spu); /* Step 64. */ 1998 restore_spu_mb(next, spu); /* Step 65. */ 1999 check_ppu_mb_stat(next, spu); /* Step 66. */ 2000 check_ppuint_mb_stat(next, spu); /* Step 67. */ 2001 spu_invalidate_slbs(spu); /* Modified Step 68. */ 2002 restore_mfc_sr1(next, spu); /* Step 69. */ 2003 restore_other_spu_access(next, spu); /* Step 70. */ 2004 restore_spu_runcntl(next, spu); /* Step 71. */ 2005 restore_mfc_cntl(next, spu); /* Step 72. */ 2006 enable_user_access(next, spu); /* Step 73. */ 2007 reset_switch_active(next, spu); /* Step 74. */ 2008 reenable_interrupts(next, spu); /* Step 75. */ 2009 } 2010 2011 static int __do_spu_save(struct spu_state *prev, struct spu *spu) 2012 { 2013 int rc; 2014 2015 /* 2016 * SPU context save can be broken into three phases: 2017 * 2018 * (a) quiesce [steps 2-16]. 2019 * (b) save of CSA, performed by PPE [steps 17-42] 2020 * (c) save of LSCSA, mostly performed by SPU [steps 43-52]. 2021 * 2022 * Returns 0 on success. 2023 * 2,6 if failed to quiece SPU 2024 * 53 if SPU-side of save failed. 2025 */ 2026 2027 rc = quiece_spu(prev, spu); /* Steps 2-16. */ 2028 switch (rc) { 2029 default: 2030 case 2: 2031 case 6: 2032 harvest(prev, spu); 2033 return rc; 2034 break; 2035 case 0: 2036 break; 2037 } 2038 save_csa(prev, spu); /* Steps 17-43. */ 2039 save_lscsa(prev, spu); /* Steps 44-53. */ 2040 return check_save_status(prev, spu); /* Step 54. */ 2041 } 2042 2043 static int __do_spu_restore(struct spu_state *next, struct spu *spu) 2044 { 2045 int rc; 2046 2047 /* 2048 * SPU context restore can be broken into three phases: 2049 * 2050 * (a) harvest (or reset) SPU [steps 2-24]. 2051 * (b) restore LSCSA [steps 25-40], mostly performed by SPU. 2052 * (c) restore CSA [steps 41-76], performed by PPE. 2053 * 2054 * The 'harvest' step is not performed here, but rather 2055 * as needed below. 2056 */ 2057 2058 restore_lscsa(next, spu); /* Steps 24-39. */ 2059 rc = check_restore_status(next, spu); /* Step 40. */ 2060 switch (rc) { 2061 default: 2062 /* Failed. Return now. */ 2063 return rc; 2064 break; 2065 case 0: 2066 /* Fall through to next step. */ 2067 break; 2068 } 2069 restore_csa(next, spu); 2070 2071 return 0; 2072 } 2073 2074 /** 2075 * spu_save - SPU context save, with locking. 2076 * @prev: pointer to SPU context save area, to be saved. 2077 * @spu: pointer to SPU iomem structure. 2078 * 2079 * Acquire locks, perform the save operation then return. 2080 */ 2081 int spu_save(struct spu_state *prev, struct spu *spu) 2082 { 2083 int rc; 2084 2085 acquire_spu_lock(spu); /* Step 1. */ 2086 rc = __do_spu_save(prev, spu); /* Steps 2-53. */ 2087 release_spu_lock(spu); 2088 if (rc != 0 && rc != 2 && rc != 6) { 2089 panic("%s failed on SPU[%d], rc=%d.\n", 2090 __func__, spu->number, rc); 2091 } 2092 return 0; 2093 } 2094 EXPORT_SYMBOL_GPL(spu_save); 2095 2096 /** 2097 * spu_restore - SPU context restore, with harvest and locking. 2098 * @new: pointer to SPU context save area, to be restored. 2099 * @spu: pointer to SPU iomem structure. 2100 * 2101 * Perform harvest + restore, as we may not be coming 2102 * from a previous successful save operation, and the 2103 * hardware state is unknown. 2104 */ 2105 int spu_restore(struct spu_state *new, struct spu *spu) 2106 { 2107 int rc; 2108 2109 acquire_spu_lock(spu); 2110 harvest(NULL, spu); 2111 spu->slb_replace = 0; 2112 rc = __do_spu_restore(new, spu); 2113 release_spu_lock(spu); 2114 if (rc) { 2115 panic("%s failed on SPU[%d] rc=%d.\n", 2116 __func__, spu->number, rc); 2117 } 2118 return rc; 2119 } 2120 EXPORT_SYMBOL_GPL(spu_restore); 2121 2122 static void init_prob(struct spu_state *csa) 2123 { 2124 csa->spu_chnlcnt_RW[9] = 1; 2125 csa->spu_chnlcnt_RW[21] = 16; 2126 csa->spu_chnlcnt_RW[23] = 1; 2127 csa->spu_chnlcnt_RW[28] = 1; 2128 csa->spu_chnlcnt_RW[30] = 1; 2129 csa->prob.spu_runcntl_RW = SPU_RUNCNTL_STOP; 2130 csa->prob.mb_stat_R = 0x000400; 2131 } 2132 2133 static void init_priv1(struct spu_state *csa) 2134 { 2135 /* Enable decode, relocate, tlbie response, master runcntl. */ 2136 csa->priv1.mfc_sr1_RW = MFC_STATE1_LOCAL_STORAGE_DECODE_MASK | 2137 MFC_STATE1_MASTER_RUN_CONTROL_MASK | 2138 MFC_STATE1_PROBLEM_STATE_MASK | 2139 MFC_STATE1_RELOCATE_MASK | MFC_STATE1_BUS_TLBIE_MASK; 2140 2141 /* Enable OS-specific set of interrupts. */ 2142 csa->priv1.int_mask_class0_RW = CLASS0_ENABLE_DMA_ALIGNMENT_INTR | 2143 CLASS0_ENABLE_INVALID_DMA_COMMAND_INTR | 2144 CLASS0_ENABLE_SPU_ERROR_INTR; 2145 csa->priv1.int_mask_class1_RW = CLASS1_ENABLE_SEGMENT_FAULT_INTR | 2146 CLASS1_ENABLE_STORAGE_FAULT_INTR; 2147 csa->priv1.int_mask_class2_RW = CLASS2_ENABLE_SPU_STOP_INTR | 2148 CLASS2_ENABLE_SPU_HALT_INTR | 2149 CLASS2_ENABLE_SPU_DMA_TAG_GROUP_COMPLETE_INTR; 2150 } 2151 2152 static void init_priv2(struct spu_state *csa) 2153 { 2154 csa->priv2.spu_lslr_RW = LS_ADDR_MASK; 2155 csa->priv2.mfc_control_RW = MFC_CNTL_RESUME_DMA_QUEUE | 2156 MFC_CNTL_NORMAL_DMA_QUEUE_OPERATION | 2157 MFC_CNTL_DMA_QUEUES_EMPTY_MASK; 2158 } 2159 2160 /** 2161 * spu_alloc_csa - allocate and initialize an SPU context save area. 2162 * 2163 * Allocate and initialize the contents of an SPU context save area. 2164 * This includes enabling address translation, interrupt masks, etc., 2165 * as appropriate for the given OS environment. 2166 * 2167 * Note that storage for the 'lscsa' is allocated separately, 2168 * as it is by far the largest of the context save regions, 2169 * and may need to be pinned or otherwise specially aligned. 2170 */ 2171 int spu_init_csa(struct spu_state *csa) 2172 { 2173 int rc; 2174 2175 if (!csa) 2176 return -EINVAL; 2177 memset(csa, 0, sizeof(struct spu_state)); 2178 2179 rc = spu_alloc_lscsa(csa); 2180 if (rc) 2181 return rc; 2182 2183 spin_lock_init(&csa->register_lock); 2184 2185 init_prob(csa); 2186 init_priv1(csa); 2187 init_priv2(csa); 2188 2189 return 0; 2190 } 2191 2192 void spu_fini_csa(struct spu_state *csa) 2193 { 2194 spu_free_lscsa(csa); 2195 } 2196