1 /* 2 * spu_switch.c 3 * 4 * (C) Copyright IBM Corp. 2005 5 * 6 * Author: Mark Nutter <mnutter@us.ibm.com> 7 * 8 * Host-side part of SPU context switch sequence outlined in 9 * Synergistic Processor Element, Book IV. 10 * 11 * A fully premptive switch of an SPE is very expensive in terms 12 * of time and system resources. SPE Book IV indicates that SPE 13 * allocation should follow a "serially reusable device" model, 14 * in which the SPE is assigned a task until it completes. When 15 * this is not possible, this sequence may be used to premptively 16 * save, and then later (optionally) restore the context of a 17 * program executing on an SPE. 18 * 19 * 20 * This program is free software; you can redistribute it and/or modify 21 * it under the terms of the GNU General Public License as published by 22 * the Free Software Foundation; either version 2, or (at your option) 23 * any later version. 24 * 25 * This program is distributed in the hope that it will be useful, 26 * but WITHOUT ANY WARRANTY; without even the implied warranty of 27 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 28 * GNU General Public License for more details. 29 * 30 * You should have received a copy of the GNU General Public License 31 * along with this program; if not, write to the Free Software 32 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 33 */ 34 35 #include <linux/config.h> 36 #include <linux/module.h> 37 #include <linux/errno.h> 38 #include <linux/sched.h> 39 #include <linux/kernel.h> 40 #include <linux/mm.h> 41 #include <linux/vmalloc.h> 42 #include <linux/smp.h> 43 #include <linux/smp_lock.h> 44 #include <linux/stddef.h> 45 #include <linux/unistd.h> 46 47 #include <asm/io.h> 48 #include <asm/spu.h> 49 #include <asm/spu_csa.h> 50 #include <asm/mmu_context.h> 51 52 #include "spu_save_dump.h" 53 #include "spu_restore_dump.h" 54 55 #if 0 56 #define POLL_WHILE_TRUE(_c) { \ 57 do { \ 58 } while (_c); \ 59 } 60 #else 61 #define RELAX_SPIN_COUNT 1000 62 #define POLL_WHILE_TRUE(_c) { \ 63 do { \ 64 int _i; \ 65 for (_i=0; _i<RELAX_SPIN_COUNT && (_c); _i++) { \ 66 cpu_relax(); \ 67 } \ 68 if (unlikely(_c)) yield(); \ 69 else break; \ 70 } while (_c); \ 71 } 72 #endif /* debug */ 73 74 #define POLL_WHILE_FALSE(_c) POLL_WHILE_TRUE(!(_c)) 75 76 static inline void acquire_spu_lock(struct spu *spu) 77 { 78 /* Save, Step 1: 79 * Restore, Step 1: 80 * Acquire SPU-specific mutual exclusion lock. 81 * TBD. 82 */ 83 } 84 85 static inline void release_spu_lock(struct spu *spu) 86 { 87 /* Restore, Step 76: 88 * Release SPU-specific mutual exclusion lock. 89 * TBD. 90 */ 91 } 92 93 static inline int check_spu_isolate(struct spu_state *csa, struct spu *spu) 94 { 95 struct spu_problem __iomem *prob = spu->problem; 96 u32 isolate_state; 97 98 /* Save, Step 2: 99 * Save, Step 6: 100 * If SPU_Status[E,L,IS] any field is '1', this 101 * SPU is in isolate state and cannot be context 102 * saved at this time. 103 */ 104 isolate_state = SPU_STATUS_ISOLATED_STATE | 105 SPU_STATUS_ISOLATED_LOAD_STAUTUS | SPU_STATUS_ISOLATED_EXIT_STAUTUS; 106 return (in_be32(&prob->spu_status_R) & isolate_state) ? 1 : 0; 107 } 108 109 static inline void disable_interrupts(struct spu_state *csa, struct spu *spu) 110 { 111 /* Save, Step 3: 112 * Restore, Step 2: 113 * Save INT_Mask_class0 in CSA. 114 * Write INT_MASK_class0 with value of 0. 115 * Save INT_Mask_class1 in CSA. 116 * Write INT_MASK_class1 with value of 0. 117 * Save INT_Mask_class2 in CSA. 118 * Write INT_MASK_class2 with value of 0. 119 */ 120 spin_lock_irq(&spu->register_lock); 121 if (csa) { 122 csa->priv1.int_mask_class0_RW = spu_int_mask_get(spu, 0); 123 csa->priv1.int_mask_class1_RW = spu_int_mask_get(spu, 1); 124 csa->priv1.int_mask_class2_RW = spu_int_mask_get(spu, 2); 125 } 126 spu_int_mask_set(spu, 0, 0ul); 127 spu_int_mask_set(spu, 1, 0ul); 128 spu_int_mask_set(spu, 2, 0ul); 129 eieio(); 130 spin_unlock_irq(&spu->register_lock); 131 } 132 133 static inline void set_watchdog_timer(struct spu_state *csa, struct spu *spu) 134 { 135 /* Save, Step 4: 136 * Restore, Step 25. 137 * Set a software watchdog timer, which specifies the 138 * maximum allowable time for a context save sequence. 139 * 140 * For present, this implementation will not set a global 141 * watchdog timer, as virtualization & variable system load 142 * may cause unpredictable execution times. 143 */ 144 } 145 146 static inline void inhibit_user_access(struct spu_state *csa, struct spu *spu) 147 { 148 /* Save, Step 5: 149 * Restore, Step 3: 150 * Inhibit user-space access (if provided) to this 151 * SPU by unmapping the virtual pages assigned to 152 * the SPU memory-mapped I/O (MMIO) for problem 153 * state. TBD. 154 */ 155 } 156 157 static inline void set_switch_pending(struct spu_state *csa, struct spu *spu) 158 { 159 /* Save, Step 7: 160 * Restore, Step 5: 161 * Set a software context switch pending flag. 162 */ 163 set_bit(SPU_CONTEXT_SWITCH_PENDING, &spu->flags); 164 mb(); 165 } 166 167 static inline void save_mfc_cntl(struct spu_state *csa, struct spu *spu) 168 { 169 struct spu_priv2 __iomem *priv2 = spu->priv2; 170 171 /* Save, Step 8: 172 * Read and save MFC_CNTL[Ss]. 173 */ 174 if (csa) { 175 csa->priv2.mfc_control_RW = in_be64(&priv2->mfc_control_RW) & 176 MFC_CNTL_SUSPEND_DMA_STATUS_MASK; 177 } 178 } 179 180 static inline void save_spu_runcntl(struct spu_state *csa, struct spu *spu) 181 { 182 struct spu_problem __iomem *prob = spu->problem; 183 184 /* Save, Step 9: 185 * Save SPU_Runcntl in the CSA. This value contains 186 * the "Application Desired State". 187 */ 188 csa->prob.spu_runcntl_RW = in_be32(&prob->spu_runcntl_RW); 189 } 190 191 static inline void save_mfc_sr1(struct spu_state *csa, struct spu *spu) 192 { 193 /* Save, Step 10: 194 * Save MFC_SR1 in the CSA. 195 */ 196 csa->priv1.mfc_sr1_RW = spu_mfc_sr1_get(spu); 197 } 198 199 static inline void save_spu_status(struct spu_state *csa, struct spu *spu) 200 { 201 struct spu_problem __iomem *prob = spu->problem; 202 203 /* Save, Step 11: 204 * Read SPU_Status[R], and save to CSA. 205 */ 206 if ((in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) == 0) { 207 csa->prob.spu_status_R = in_be32(&prob->spu_status_R); 208 } else { 209 u32 stopped; 210 211 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 212 eieio(); 213 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 214 SPU_STATUS_RUNNING); 215 stopped = 216 SPU_STATUS_INVALID_INSTR | SPU_STATUS_SINGLE_STEP | 217 SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP; 218 if ((in_be32(&prob->spu_status_R) & stopped) == 0) 219 csa->prob.spu_status_R = SPU_STATUS_RUNNING; 220 else 221 csa->prob.spu_status_R = in_be32(&prob->spu_status_R); 222 } 223 } 224 225 static inline void save_mfc_decr(struct spu_state *csa, struct spu *spu) 226 { 227 struct spu_priv2 __iomem *priv2 = spu->priv2; 228 229 /* Save, Step 12: 230 * Read MFC_CNTL[Ds]. Update saved copy of 231 * CSA.MFC_CNTL[Ds]. 232 */ 233 if (in_be64(&priv2->mfc_control_RW) & MFC_CNTL_DECREMENTER_RUNNING) { 234 csa->priv2.mfc_control_RW |= MFC_CNTL_DECREMENTER_RUNNING; 235 csa->suspend_time = get_cycles(); 236 out_be64(&priv2->spu_chnlcntptr_RW, 7ULL); 237 eieio(); 238 csa->spu_chnldata_RW[7] = in_be64(&priv2->spu_chnldata_RW); 239 eieio(); 240 } 241 } 242 243 static inline void halt_mfc_decr(struct spu_state *csa, struct spu *spu) 244 { 245 struct spu_priv2 __iomem *priv2 = spu->priv2; 246 247 /* Save, Step 13: 248 * Write MFC_CNTL[Dh] set to a '1' to halt 249 * the decrementer. 250 */ 251 out_be64(&priv2->mfc_control_RW, MFC_CNTL_DECREMENTER_HALTED); 252 eieio(); 253 } 254 255 static inline void save_timebase(struct spu_state *csa, struct spu *spu) 256 { 257 /* Save, Step 14: 258 * Read PPE Timebase High and Timebase low registers 259 * and save in CSA. TBD. 260 */ 261 csa->suspend_time = get_cycles(); 262 } 263 264 static inline void remove_other_spu_access(struct spu_state *csa, 265 struct spu *spu) 266 { 267 /* Save, Step 15: 268 * Remove other SPU access to this SPU by unmapping 269 * this SPU's pages from their address space. TBD. 270 */ 271 } 272 273 static inline void do_mfc_mssync(struct spu_state *csa, struct spu *spu) 274 { 275 struct spu_problem __iomem *prob = spu->problem; 276 277 /* Save, Step 16: 278 * Restore, Step 11. 279 * Write SPU_MSSync register. Poll SPU_MSSync[P] 280 * for a value of 0. 281 */ 282 out_be64(&prob->spc_mssync_RW, 1UL); 283 POLL_WHILE_TRUE(in_be64(&prob->spc_mssync_RW) & MS_SYNC_PENDING); 284 } 285 286 static inline void issue_mfc_tlbie(struct spu_state *csa, struct spu *spu) 287 { 288 /* Save, Step 17: 289 * Restore, Step 12. 290 * Restore, Step 48. 291 * Write TLB_Invalidate_Entry[IS,VPN,L,Lp]=0 register. 292 * Then issue a PPE sync instruction. 293 */ 294 spu_tlb_invalidate(spu); 295 mb(); 296 } 297 298 static inline void handle_pending_interrupts(struct spu_state *csa, 299 struct spu *spu) 300 { 301 /* Save, Step 18: 302 * Handle any pending interrupts from this SPU 303 * here. This is OS or hypervisor specific. One 304 * option is to re-enable interrupts to handle any 305 * pending interrupts, with the interrupt handlers 306 * recognizing the software Context Switch Pending 307 * flag, to ensure the SPU execution or MFC command 308 * queue is not restarted. TBD. 309 */ 310 } 311 312 static inline void save_mfc_queues(struct spu_state *csa, struct spu *spu) 313 { 314 struct spu_priv2 __iomem *priv2 = spu->priv2; 315 int i; 316 317 /* Save, Step 19: 318 * If MFC_Cntl[Se]=0 then save 319 * MFC command queues. 320 */ 321 if ((in_be64(&priv2->mfc_control_RW) & MFC_CNTL_DMA_QUEUES_EMPTY) == 0) { 322 for (i = 0; i < 8; i++) { 323 csa->priv2.puq[i].mfc_cq_data0_RW = 324 in_be64(&priv2->puq[i].mfc_cq_data0_RW); 325 csa->priv2.puq[i].mfc_cq_data1_RW = 326 in_be64(&priv2->puq[i].mfc_cq_data1_RW); 327 csa->priv2.puq[i].mfc_cq_data2_RW = 328 in_be64(&priv2->puq[i].mfc_cq_data2_RW); 329 csa->priv2.puq[i].mfc_cq_data3_RW = 330 in_be64(&priv2->puq[i].mfc_cq_data3_RW); 331 } 332 for (i = 0; i < 16; i++) { 333 csa->priv2.spuq[i].mfc_cq_data0_RW = 334 in_be64(&priv2->spuq[i].mfc_cq_data0_RW); 335 csa->priv2.spuq[i].mfc_cq_data1_RW = 336 in_be64(&priv2->spuq[i].mfc_cq_data1_RW); 337 csa->priv2.spuq[i].mfc_cq_data2_RW = 338 in_be64(&priv2->spuq[i].mfc_cq_data2_RW); 339 csa->priv2.spuq[i].mfc_cq_data3_RW = 340 in_be64(&priv2->spuq[i].mfc_cq_data3_RW); 341 } 342 } 343 } 344 345 static inline void save_ppu_querymask(struct spu_state *csa, struct spu *spu) 346 { 347 struct spu_problem __iomem *prob = spu->problem; 348 349 /* Save, Step 20: 350 * Save the PPU_QueryMask register 351 * in the CSA. 352 */ 353 csa->prob.dma_querymask_RW = in_be32(&prob->dma_querymask_RW); 354 } 355 356 static inline void save_ppu_querytype(struct spu_state *csa, struct spu *spu) 357 { 358 struct spu_problem __iomem *prob = spu->problem; 359 360 /* Save, Step 21: 361 * Save the PPU_QueryType register 362 * in the CSA. 363 */ 364 csa->prob.dma_querytype_RW = in_be32(&prob->dma_querytype_RW); 365 } 366 367 static inline void save_mfc_csr_tsq(struct spu_state *csa, struct spu *spu) 368 { 369 struct spu_priv2 __iomem *priv2 = spu->priv2; 370 371 /* Save, Step 22: 372 * Save the MFC_CSR_TSQ register 373 * in the LSCSA. 374 */ 375 csa->priv2.spu_tag_status_query_RW = 376 in_be64(&priv2->spu_tag_status_query_RW); 377 } 378 379 static inline void save_mfc_csr_cmd(struct spu_state *csa, struct spu *spu) 380 { 381 struct spu_priv2 __iomem *priv2 = spu->priv2; 382 383 /* Save, Step 23: 384 * Save the MFC_CSR_CMD1 and MFC_CSR_CMD2 385 * registers in the CSA. 386 */ 387 csa->priv2.spu_cmd_buf1_RW = in_be64(&priv2->spu_cmd_buf1_RW); 388 csa->priv2.spu_cmd_buf2_RW = in_be64(&priv2->spu_cmd_buf2_RW); 389 } 390 391 static inline void save_mfc_csr_ato(struct spu_state *csa, struct spu *spu) 392 { 393 struct spu_priv2 __iomem *priv2 = spu->priv2; 394 395 /* Save, Step 24: 396 * Save the MFC_CSR_ATO register in 397 * the CSA. 398 */ 399 csa->priv2.spu_atomic_status_RW = in_be64(&priv2->spu_atomic_status_RW); 400 } 401 402 static inline void save_mfc_tclass_id(struct spu_state *csa, struct spu *spu) 403 { 404 /* Save, Step 25: 405 * Save the MFC_TCLASS_ID register in 406 * the CSA. 407 */ 408 csa->priv1.mfc_tclass_id_RW = spu_mfc_tclass_id_get(spu); 409 } 410 411 static inline void set_mfc_tclass_id(struct spu_state *csa, struct spu *spu) 412 { 413 /* Save, Step 26: 414 * Restore, Step 23. 415 * Write the MFC_TCLASS_ID register with 416 * the value 0x10000000. 417 */ 418 spu_mfc_tclass_id_set(spu, 0x10000000); 419 eieio(); 420 } 421 422 static inline void purge_mfc_queue(struct spu_state *csa, struct spu *spu) 423 { 424 struct spu_priv2 __iomem *priv2 = spu->priv2; 425 426 /* Save, Step 27: 427 * Restore, Step 14. 428 * Write MFC_CNTL[Pc]=1 (purge queue). 429 */ 430 out_be64(&priv2->mfc_control_RW, MFC_CNTL_PURGE_DMA_REQUEST); 431 eieio(); 432 } 433 434 static inline void wait_purge_complete(struct spu_state *csa, struct spu *spu) 435 { 436 struct spu_priv2 __iomem *priv2 = spu->priv2; 437 438 /* Save, Step 28: 439 * Poll MFC_CNTL[Ps] until value '11' is read 440 * (purge complete). 441 */ 442 POLL_WHILE_FALSE(in_be64(&priv2->mfc_control_RW) & 443 MFC_CNTL_PURGE_DMA_COMPLETE); 444 } 445 446 static inline void save_mfc_slbs(struct spu_state *csa, struct spu *spu) 447 { 448 struct spu_priv2 __iomem *priv2 = spu->priv2; 449 int i; 450 451 /* Save, Step 29: 452 * If MFC_SR1[R]='1', save SLBs in CSA. 453 */ 454 if (spu_mfc_sr1_get(spu) & MFC_STATE1_RELOCATE_MASK) { 455 csa->priv2.slb_index_W = in_be64(&priv2->slb_index_W); 456 for (i = 0; i < 8; i++) { 457 out_be64(&priv2->slb_index_W, i); 458 eieio(); 459 csa->slb_esid_RW[i] = in_be64(&priv2->slb_esid_RW); 460 csa->slb_vsid_RW[i] = in_be64(&priv2->slb_vsid_RW); 461 eieio(); 462 } 463 } 464 } 465 466 static inline void setup_mfc_sr1(struct spu_state *csa, struct spu *spu) 467 { 468 /* Save, Step 30: 469 * Restore, Step 18: 470 * Write MFC_SR1 with MFC_SR1[D=0,S=1] and 471 * MFC_SR1[TL,R,Pr,T] set correctly for the 472 * OS specific environment. 473 * 474 * Implementation note: The SPU-side code 475 * for save/restore is privileged, so the 476 * MFC_SR1[Pr] bit is not set. 477 * 478 */ 479 spu_mfc_sr1_set(spu, (MFC_STATE1_MASTER_RUN_CONTROL_MASK | 480 MFC_STATE1_RELOCATE_MASK | 481 MFC_STATE1_BUS_TLBIE_MASK)); 482 } 483 484 static inline void save_spu_npc(struct spu_state *csa, struct spu *spu) 485 { 486 struct spu_problem __iomem *prob = spu->problem; 487 488 /* Save, Step 31: 489 * Save SPU_NPC in the CSA. 490 */ 491 csa->prob.spu_npc_RW = in_be32(&prob->spu_npc_RW); 492 } 493 494 static inline void save_spu_privcntl(struct spu_state *csa, struct spu *spu) 495 { 496 struct spu_priv2 __iomem *priv2 = spu->priv2; 497 498 /* Save, Step 32: 499 * Save SPU_PrivCntl in the CSA. 500 */ 501 csa->priv2.spu_privcntl_RW = in_be64(&priv2->spu_privcntl_RW); 502 } 503 504 static inline void reset_spu_privcntl(struct spu_state *csa, struct spu *spu) 505 { 506 struct spu_priv2 __iomem *priv2 = spu->priv2; 507 508 /* Save, Step 33: 509 * Restore, Step 16: 510 * Write SPU_PrivCntl[S,Le,A] fields reset to 0. 511 */ 512 out_be64(&priv2->spu_privcntl_RW, 0UL); 513 eieio(); 514 } 515 516 static inline void save_spu_lslr(struct spu_state *csa, struct spu *spu) 517 { 518 struct spu_priv2 __iomem *priv2 = spu->priv2; 519 520 /* Save, Step 34: 521 * Save SPU_LSLR in the CSA. 522 */ 523 csa->priv2.spu_lslr_RW = in_be64(&priv2->spu_lslr_RW); 524 } 525 526 static inline void reset_spu_lslr(struct spu_state *csa, struct spu *spu) 527 { 528 struct spu_priv2 __iomem *priv2 = spu->priv2; 529 530 /* Save, Step 35: 531 * Restore, Step 17. 532 * Reset SPU_LSLR. 533 */ 534 out_be64(&priv2->spu_lslr_RW, LS_ADDR_MASK); 535 eieio(); 536 } 537 538 static inline void save_spu_cfg(struct spu_state *csa, struct spu *spu) 539 { 540 struct spu_priv2 __iomem *priv2 = spu->priv2; 541 542 /* Save, Step 36: 543 * Save SPU_Cfg in the CSA. 544 */ 545 csa->priv2.spu_cfg_RW = in_be64(&priv2->spu_cfg_RW); 546 } 547 548 static inline void save_pm_trace(struct spu_state *csa, struct spu *spu) 549 { 550 /* Save, Step 37: 551 * Save PM_Trace_Tag_Wait_Mask in the CSA. 552 * Not performed by this implementation. 553 */ 554 } 555 556 static inline void save_mfc_rag(struct spu_state *csa, struct spu *spu) 557 { 558 /* Save, Step 38: 559 * Save RA_GROUP_ID register and the 560 * RA_ENABLE reigster in the CSA. 561 */ 562 csa->priv1.resource_allocation_groupID_RW = 563 spu_resource_allocation_groupID_get(spu); 564 csa->priv1.resource_allocation_enable_RW = 565 spu_resource_allocation_enable_get(spu); 566 } 567 568 static inline void save_ppu_mb_stat(struct spu_state *csa, struct spu *spu) 569 { 570 struct spu_problem __iomem *prob = spu->problem; 571 572 /* Save, Step 39: 573 * Save MB_Stat register in the CSA. 574 */ 575 csa->prob.mb_stat_R = in_be32(&prob->mb_stat_R); 576 } 577 578 static inline void save_ppu_mb(struct spu_state *csa, struct spu *spu) 579 { 580 struct spu_problem __iomem *prob = spu->problem; 581 582 /* Save, Step 40: 583 * Save the PPU_MB register in the CSA. 584 */ 585 csa->prob.pu_mb_R = in_be32(&prob->pu_mb_R); 586 } 587 588 static inline void save_ppuint_mb(struct spu_state *csa, struct spu *spu) 589 { 590 struct spu_priv2 __iomem *priv2 = spu->priv2; 591 592 /* Save, Step 41: 593 * Save the PPUINT_MB register in the CSA. 594 */ 595 csa->priv2.puint_mb_R = in_be64(&priv2->puint_mb_R); 596 } 597 598 static inline void save_ch_part1(struct spu_state *csa, struct spu *spu) 599 { 600 struct spu_priv2 __iomem *priv2 = spu->priv2; 601 u64 idx, ch_indices[7] = { 0UL, 1UL, 3UL, 4UL, 24UL, 25UL, 27UL }; 602 int i; 603 604 /* Save, Step 42: 605 * Save the following CH: [0,1,3,4,24,25,27] 606 */ 607 for (i = 0; i < 7; i++) { 608 idx = ch_indices[i]; 609 out_be64(&priv2->spu_chnlcntptr_RW, idx); 610 eieio(); 611 csa->spu_chnldata_RW[idx] = in_be64(&priv2->spu_chnldata_RW); 612 csa->spu_chnlcnt_RW[idx] = in_be64(&priv2->spu_chnlcnt_RW); 613 out_be64(&priv2->spu_chnldata_RW, 0UL); 614 out_be64(&priv2->spu_chnlcnt_RW, 0UL); 615 eieio(); 616 } 617 } 618 619 static inline void save_spu_mb(struct spu_state *csa, struct spu *spu) 620 { 621 struct spu_priv2 __iomem *priv2 = spu->priv2; 622 int i; 623 624 /* Save, Step 43: 625 * Save SPU Read Mailbox Channel. 626 */ 627 out_be64(&priv2->spu_chnlcntptr_RW, 29UL); 628 eieio(); 629 csa->spu_chnlcnt_RW[29] = in_be64(&priv2->spu_chnlcnt_RW); 630 for (i = 0; i < 4; i++) { 631 csa->spu_mailbox_data[i] = in_be64(&priv2->spu_chnldata_RW); 632 } 633 out_be64(&priv2->spu_chnlcnt_RW, 0UL); 634 eieio(); 635 } 636 637 static inline void save_mfc_cmd(struct spu_state *csa, struct spu *spu) 638 { 639 struct spu_priv2 __iomem *priv2 = spu->priv2; 640 641 /* Save, Step 44: 642 * Save MFC_CMD Channel. 643 */ 644 out_be64(&priv2->spu_chnlcntptr_RW, 21UL); 645 eieio(); 646 csa->spu_chnlcnt_RW[21] = in_be64(&priv2->spu_chnlcnt_RW); 647 eieio(); 648 } 649 650 static inline void reset_ch(struct spu_state *csa, struct spu *spu) 651 { 652 struct spu_priv2 __iomem *priv2 = spu->priv2; 653 u64 ch_indices[4] = { 21UL, 23UL, 28UL, 30UL }; 654 u64 ch_counts[4] = { 16UL, 1UL, 1UL, 1UL }; 655 u64 idx; 656 int i; 657 658 /* Save, Step 45: 659 * Reset the following CH: [21, 23, 28, 30] 660 */ 661 for (i = 0; i < 4; i++) { 662 idx = ch_indices[i]; 663 out_be64(&priv2->spu_chnlcntptr_RW, idx); 664 eieio(); 665 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]); 666 eieio(); 667 } 668 } 669 670 static inline void resume_mfc_queue(struct spu_state *csa, struct spu *spu) 671 { 672 struct spu_priv2 __iomem *priv2 = spu->priv2; 673 674 /* Save, Step 46: 675 * Restore, Step 25. 676 * Write MFC_CNTL[Sc]=0 (resume queue processing). 677 */ 678 out_be64(&priv2->mfc_control_RW, MFC_CNTL_RESUME_DMA_QUEUE); 679 } 680 681 static inline void invalidate_slbs(struct spu_state *csa, struct spu *spu) 682 { 683 struct spu_priv2 __iomem *priv2 = spu->priv2; 684 685 /* Save, Step 45: 686 * Restore, Step 19: 687 * If MFC_SR1[R]=1, write 0 to SLB_Invalidate_All. 688 */ 689 if (spu_mfc_sr1_get(spu) & MFC_STATE1_RELOCATE_MASK) { 690 out_be64(&priv2->slb_invalidate_all_W, 0UL); 691 eieio(); 692 } 693 } 694 695 static inline void get_kernel_slb(u64 ea, u64 slb[2]) 696 { 697 slb[0] = (get_kernel_vsid(ea) << SLB_VSID_SHIFT) | SLB_VSID_KERNEL; 698 slb[1] = (ea & ESID_MASK) | SLB_ESID_V; 699 700 /* Large pages are used for kernel text/data, but not vmalloc. */ 701 if (cpu_has_feature(CPU_FTR_16M_PAGE) 702 && REGION_ID(ea) == KERNEL_REGION_ID) 703 slb[0] |= SLB_VSID_L; 704 } 705 706 static inline void load_mfc_slb(struct spu *spu, u64 slb[2], int slbe) 707 { 708 struct spu_priv2 __iomem *priv2 = spu->priv2; 709 710 out_be64(&priv2->slb_index_W, slbe); 711 eieio(); 712 out_be64(&priv2->slb_vsid_RW, slb[0]); 713 out_be64(&priv2->slb_esid_RW, slb[1]); 714 eieio(); 715 } 716 717 static inline void setup_mfc_slbs(struct spu_state *csa, struct spu *spu) 718 { 719 u64 code_slb[2]; 720 u64 lscsa_slb[2]; 721 722 /* Save, Step 47: 723 * Restore, Step 30. 724 * If MFC_SR1[R]=1, write 0 to SLB_Invalidate_All 725 * register, then initialize SLB_VSID and SLB_ESID 726 * to provide access to SPU context save code and 727 * LSCSA. 728 * 729 * This implementation places both the context 730 * switch code and LSCSA in kernel address space. 731 * 732 * Further this implementation assumes that the 733 * MFC_SR1[R]=1 (in other words, assume that 734 * translation is desired by OS environment). 735 */ 736 invalidate_slbs(csa, spu); 737 get_kernel_slb((unsigned long)&spu_save_code[0], code_slb); 738 get_kernel_slb((unsigned long)csa->lscsa, lscsa_slb); 739 load_mfc_slb(spu, code_slb, 0); 740 if ((lscsa_slb[0] != code_slb[0]) || (lscsa_slb[1] != code_slb[1])) 741 load_mfc_slb(spu, lscsa_slb, 1); 742 } 743 744 static inline void set_switch_active(struct spu_state *csa, struct spu *spu) 745 { 746 /* Save, Step 48: 747 * Restore, Step 23. 748 * Change the software context switch pending flag 749 * to context switch active. 750 */ 751 set_bit(SPU_CONTEXT_SWITCH_ACTIVE, &spu->flags); 752 clear_bit(SPU_CONTEXT_SWITCH_PENDING, &spu->flags); 753 mb(); 754 } 755 756 static inline void enable_interrupts(struct spu_state *csa, struct spu *spu) 757 { 758 unsigned long class1_mask = CLASS1_ENABLE_SEGMENT_FAULT_INTR | 759 CLASS1_ENABLE_STORAGE_FAULT_INTR; 760 761 /* Save, Step 49: 762 * Restore, Step 22: 763 * Reset and then enable interrupts, as 764 * needed by OS. 765 * 766 * This implementation enables only class1 767 * (translation) interrupts. 768 */ 769 spin_lock_irq(&spu->register_lock); 770 spu_int_stat_clear(spu, 0, ~0ul); 771 spu_int_stat_clear(spu, 1, ~0ul); 772 spu_int_stat_clear(spu, 2, ~0ul); 773 spu_int_mask_set(spu, 0, 0ul); 774 spu_int_mask_set(spu, 1, class1_mask); 775 spu_int_mask_set(spu, 2, 0ul); 776 spin_unlock_irq(&spu->register_lock); 777 } 778 779 static inline int send_mfc_dma(struct spu *spu, unsigned long ea, 780 unsigned int ls_offset, unsigned int size, 781 unsigned int tag, unsigned int rclass, 782 unsigned int cmd) 783 { 784 struct spu_problem __iomem *prob = spu->problem; 785 union mfc_tag_size_class_cmd command; 786 unsigned int transfer_size; 787 volatile unsigned int status = 0x0; 788 789 while (size > 0) { 790 transfer_size = 791 (size > MFC_MAX_DMA_SIZE) ? MFC_MAX_DMA_SIZE : size; 792 command.u.mfc_size = transfer_size; 793 command.u.mfc_tag = tag; 794 command.u.mfc_rclassid = rclass; 795 command.u.mfc_cmd = cmd; 796 do { 797 out_be32(&prob->mfc_lsa_W, ls_offset); 798 out_be64(&prob->mfc_ea_W, ea); 799 out_be64(&prob->mfc_union_W.all64, command.all64); 800 status = 801 in_be32(&prob->mfc_union_W.by32.mfc_class_cmd32); 802 if (unlikely(status & 0x2)) { 803 cpu_relax(); 804 } 805 } while (status & 0x3); 806 size -= transfer_size; 807 ea += transfer_size; 808 ls_offset += transfer_size; 809 } 810 return 0; 811 } 812 813 static inline void save_ls_16kb(struct spu_state *csa, struct spu *spu) 814 { 815 unsigned long addr = (unsigned long)&csa->lscsa->ls[0]; 816 unsigned int ls_offset = 0x0; 817 unsigned int size = 16384; 818 unsigned int tag = 0; 819 unsigned int rclass = 0; 820 unsigned int cmd = MFC_PUT_CMD; 821 822 /* Save, Step 50: 823 * Issue a DMA command to copy the first 16K bytes 824 * of local storage to the CSA. 825 */ 826 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 827 } 828 829 static inline void set_spu_npc(struct spu_state *csa, struct spu *spu) 830 { 831 struct spu_problem __iomem *prob = spu->problem; 832 833 /* Save, Step 51: 834 * Restore, Step 31. 835 * Write SPU_NPC[IE]=0 and SPU_NPC[LSA] to entry 836 * point address of context save code in local 837 * storage. 838 * 839 * This implementation uses SPU-side save/restore 840 * programs with entry points at LSA of 0. 841 */ 842 out_be32(&prob->spu_npc_RW, 0); 843 eieio(); 844 } 845 846 static inline void set_signot1(struct spu_state *csa, struct spu *spu) 847 { 848 struct spu_problem __iomem *prob = spu->problem; 849 union { 850 u64 ull; 851 u32 ui[2]; 852 } addr64; 853 854 /* Save, Step 52: 855 * Restore, Step 32: 856 * Write SPU_Sig_Notify_1 register with upper 32-bits 857 * of the CSA.LSCSA effective address. 858 */ 859 addr64.ull = (u64) csa->lscsa; 860 out_be32(&prob->signal_notify1, addr64.ui[0]); 861 eieio(); 862 } 863 864 static inline void set_signot2(struct spu_state *csa, struct spu *spu) 865 { 866 struct spu_problem __iomem *prob = spu->problem; 867 union { 868 u64 ull; 869 u32 ui[2]; 870 } addr64; 871 872 /* Save, Step 53: 873 * Restore, Step 33: 874 * Write SPU_Sig_Notify_2 register with lower 32-bits 875 * of the CSA.LSCSA effective address. 876 */ 877 addr64.ull = (u64) csa->lscsa; 878 out_be32(&prob->signal_notify2, addr64.ui[1]); 879 eieio(); 880 } 881 882 static inline void send_save_code(struct spu_state *csa, struct spu *spu) 883 { 884 unsigned long addr = (unsigned long)&spu_save_code[0]; 885 unsigned int ls_offset = 0x0; 886 unsigned int size = sizeof(spu_save_code); 887 unsigned int tag = 0; 888 unsigned int rclass = 0; 889 unsigned int cmd = MFC_GETFS_CMD; 890 891 /* Save, Step 54: 892 * Issue a DMA command to copy context save code 893 * to local storage and start SPU. 894 */ 895 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 896 } 897 898 static inline void set_ppu_querymask(struct spu_state *csa, struct spu *spu) 899 { 900 struct spu_problem __iomem *prob = spu->problem; 901 902 /* Save, Step 55: 903 * Restore, Step 38. 904 * Write PPU_QueryMask=1 (enable Tag Group 0) 905 * and issue eieio instruction. 906 */ 907 out_be32(&prob->dma_querymask_RW, MFC_TAGID_TO_TAGMASK(0)); 908 eieio(); 909 } 910 911 static inline void wait_tag_complete(struct spu_state *csa, struct spu *spu) 912 { 913 struct spu_problem __iomem *prob = spu->problem; 914 u32 mask = MFC_TAGID_TO_TAGMASK(0); 915 unsigned long flags; 916 917 /* Save, Step 56: 918 * Restore, Step 39. 919 * Restore, Step 39. 920 * Restore, Step 46. 921 * Poll PPU_TagStatus[gn] until 01 (Tag group 0 complete) 922 * or write PPU_QueryType[TS]=01 and wait for Tag Group 923 * Complete Interrupt. Write INT_Stat_Class0 or 924 * INT_Stat_Class2 with value of 'handled'. 925 */ 926 POLL_WHILE_FALSE(in_be32(&prob->dma_tagstatus_R) & mask); 927 928 local_irq_save(flags); 929 spu_int_stat_clear(spu, 0, ~(0ul)); 930 spu_int_stat_clear(spu, 2, ~(0ul)); 931 local_irq_restore(flags); 932 } 933 934 static inline void wait_spu_stopped(struct spu_state *csa, struct spu *spu) 935 { 936 struct spu_problem __iomem *prob = spu->problem; 937 unsigned long flags; 938 939 /* Save, Step 57: 940 * Restore, Step 40. 941 * Poll until SPU_Status[R]=0 or wait for SPU Class 0 942 * or SPU Class 2 interrupt. Write INT_Stat_class0 943 * or INT_Stat_class2 with value of handled. 944 */ 945 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING); 946 947 local_irq_save(flags); 948 spu_int_stat_clear(spu, 0, ~(0ul)); 949 spu_int_stat_clear(spu, 2, ~(0ul)); 950 local_irq_restore(flags); 951 } 952 953 static inline int check_save_status(struct spu_state *csa, struct spu *spu) 954 { 955 struct spu_problem __iomem *prob = spu->problem; 956 u32 complete; 957 958 /* Save, Step 54: 959 * If SPU_Status[P]=1 and SPU_Status[SC] = "success", 960 * context save succeeded, otherwise context save 961 * failed. 962 */ 963 complete = ((SPU_SAVE_COMPLETE << SPU_STOP_STATUS_SHIFT) | 964 SPU_STATUS_STOPPED_BY_STOP); 965 return (in_be32(&prob->spu_status_R) != complete) ? 1 : 0; 966 } 967 968 static inline void terminate_spu_app(struct spu_state *csa, struct spu *spu) 969 { 970 /* Restore, Step 4: 971 * If required, notify the "using application" that 972 * the SPU task has been terminated. TBD. 973 */ 974 } 975 976 static inline void suspend_mfc(struct spu_state *csa, struct spu *spu) 977 { 978 struct spu_priv2 __iomem *priv2 = spu->priv2; 979 980 /* Restore, Step 7: 981 * Restore, Step 47. 982 * Write MFC_Cntl[Dh,Sc]='1','1' to suspend 983 * the queue and halt the decrementer. 984 */ 985 out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE | 986 MFC_CNTL_DECREMENTER_HALTED); 987 eieio(); 988 } 989 990 static inline void wait_suspend_mfc_complete(struct spu_state *csa, 991 struct spu *spu) 992 { 993 struct spu_priv2 __iomem *priv2 = spu->priv2; 994 995 /* Restore, Step 8: 996 * Restore, Step 47. 997 * Poll MFC_CNTL[Ss] until 11 is returned. 998 */ 999 POLL_WHILE_FALSE(in_be64(&priv2->mfc_control_RW) & 1000 MFC_CNTL_SUSPEND_COMPLETE); 1001 } 1002 1003 static inline int suspend_spe(struct spu_state *csa, struct spu *spu) 1004 { 1005 struct spu_problem __iomem *prob = spu->problem; 1006 1007 /* Restore, Step 9: 1008 * If SPU_Status[R]=1, stop SPU execution 1009 * and wait for stop to complete. 1010 * 1011 * Returns 1 if SPU_Status[R]=1 on entry. 1012 * 0 otherwise 1013 */ 1014 if (in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) { 1015 if (in_be32(&prob->spu_status_R) & 1016 SPU_STATUS_ISOLATED_EXIT_STAUTUS) { 1017 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1018 SPU_STATUS_RUNNING); 1019 } 1020 if ((in_be32(&prob->spu_status_R) & 1021 SPU_STATUS_ISOLATED_LOAD_STAUTUS) 1022 || (in_be32(&prob->spu_status_R) & 1023 SPU_STATUS_ISOLATED_STATE)) { 1024 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 1025 eieio(); 1026 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1027 SPU_STATUS_RUNNING); 1028 out_be32(&prob->spu_runcntl_RW, 0x2); 1029 eieio(); 1030 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1031 SPU_STATUS_RUNNING); 1032 } 1033 if (in_be32(&prob->spu_status_R) & 1034 SPU_STATUS_WAITING_FOR_CHANNEL) { 1035 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 1036 eieio(); 1037 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1038 SPU_STATUS_RUNNING); 1039 } 1040 return 1; 1041 } 1042 return 0; 1043 } 1044 1045 static inline void clear_spu_status(struct spu_state *csa, struct spu *spu) 1046 { 1047 struct spu_problem __iomem *prob = spu->problem; 1048 1049 /* Restore, Step 10: 1050 * If SPU_Status[R]=0 and SPU_Status[E,L,IS]=1, 1051 * release SPU from isolate state. 1052 */ 1053 if (!(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING)) { 1054 if (in_be32(&prob->spu_status_R) & 1055 SPU_STATUS_ISOLATED_EXIT_STAUTUS) { 1056 spu_mfc_sr1_set(spu, 1057 MFC_STATE1_MASTER_RUN_CONTROL_MASK); 1058 eieio(); 1059 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1060 eieio(); 1061 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1062 SPU_STATUS_RUNNING); 1063 } 1064 if ((in_be32(&prob->spu_status_R) & 1065 SPU_STATUS_ISOLATED_LOAD_STAUTUS) 1066 || (in_be32(&prob->spu_status_R) & 1067 SPU_STATUS_ISOLATED_STATE)) { 1068 spu_mfc_sr1_set(spu, 1069 MFC_STATE1_MASTER_RUN_CONTROL_MASK); 1070 eieio(); 1071 out_be32(&prob->spu_runcntl_RW, 0x2); 1072 eieio(); 1073 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1074 SPU_STATUS_RUNNING); 1075 } 1076 } 1077 } 1078 1079 static inline void reset_ch_part1(struct spu_state *csa, struct spu *spu) 1080 { 1081 struct spu_priv2 __iomem *priv2 = spu->priv2; 1082 u64 ch_indices[7] = { 0UL, 1UL, 3UL, 4UL, 24UL, 25UL, 27UL }; 1083 u64 idx; 1084 int i; 1085 1086 /* Restore, Step 20: 1087 * Reset the following CH: [0,1,3,4,24,25,27] 1088 */ 1089 for (i = 0; i < 7; i++) { 1090 idx = ch_indices[i]; 1091 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1092 eieio(); 1093 out_be64(&priv2->spu_chnldata_RW, 0UL); 1094 out_be64(&priv2->spu_chnlcnt_RW, 0UL); 1095 eieio(); 1096 } 1097 } 1098 1099 static inline void reset_ch_part2(struct spu_state *csa, struct spu *spu) 1100 { 1101 struct spu_priv2 __iomem *priv2 = spu->priv2; 1102 u64 ch_indices[5] = { 21UL, 23UL, 28UL, 29UL, 30UL }; 1103 u64 ch_counts[5] = { 16UL, 1UL, 1UL, 0UL, 1UL }; 1104 u64 idx; 1105 int i; 1106 1107 /* Restore, Step 21: 1108 * Reset the following CH: [21, 23, 28, 29, 30] 1109 */ 1110 for (i = 0; i < 5; i++) { 1111 idx = ch_indices[i]; 1112 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1113 eieio(); 1114 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]); 1115 eieio(); 1116 } 1117 } 1118 1119 static inline void setup_spu_status_part1(struct spu_state *csa, 1120 struct spu *spu) 1121 { 1122 u32 status_P = SPU_STATUS_STOPPED_BY_STOP; 1123 u32 status_I = SPU_STATUS_INVALID_INSTR; 1124 u32 status_H = SPU_STATUS_STOPPED_BY_HALT; 1125 u32 status_S = SPU_STATUS_SINGLE_STEP; 1126 u32 status_S_I = SPU_STATUS_SINGLE_STEP | SPU_STATUS_INVALID_INSTR; 1127 u32 status_S_P = SPU_STATUS_SINGLE_STEP | SPU_STATUS_STOPPED_BY_STOP; 1128 u32 status_P_H = SPU_STATUS_STOPPED_BY_HALT |SPU_STATUS_STOPPED_BY_STOP; 1129 u32 status_P_I = SPU_STATUS_STOPPED_BY_STOP |SPU_STATUS_INVALID_INSTR; 1130 u32 status_code; 1131 1132 /* Restore, Step 27: 1133 * If the CSA.SPU_Status[I,S,H,P]=1 then add the correct 1134 * instruction sequence to the end of the SPU based restore 1135 * code (after the "context restored" stop and signal) to 1136 * restore the correct SPU status. 1137 * 1138 * NOTE: Rather than modifying the SPU executable, we 1139 * instead add a new 'stopped_status' field to the 1140 * LSCSA. The SPU-side restore reads this field and 1141 * takes the appropriate action when exiting. 1142 */ 1143 1144 status_code = 1145 (csa->prob.spu_status_R >> SPU_STOP_STATUS_SHIFT) & 0xFFFF; 1146 if ((csa->prob.spu_status_R & status_P_I) == status_P_I) { 1147 1148 /* SPU_Status[P,I]=1 - Illegal Instruction followed 1149 * by Stop and Signal instruction, followed by 'br -4'. 1150 * 1151 */ 1152 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P_I; 1153 csa->lscsa->stopped_status.slot[1] = status_code; 1154 1155 } else if ((csa->prob.spu_status_R & status_P_H) == status_P_H) { 1156 1157 /* SPU_Status[P,H]=1 - Halt Conditional, followed 1158 * by Stop and Signal instruction, followed by 1159 * 'br -4'. 1160 */ 1161 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P_H; 1162 csa->lscsa->stopped_status.slot[1] = status_code; 1163 1164 } else if ((csa->prob.spu_status_R & status_S_P) == status_S_P) { 1165 1166 /* SPU_Status[S,P]=1 - Stop and Signal instruction 1167 * followed by 'br -4'. 1168 */ 1169 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S_P; 1170 csa->lscsa->stopped_status.slot[1] = status_code; 1171 1172 } else if ((csa->prob.spu_status_R & status_S_I) == status_S_I) { 1173 1174 /* SPU_Status[S,I]=1 - Illegal instruction followed 1175 * by 'br -4'. 1176 */ 1177 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S_I; 1178 csa->lscsa->stopped_status.slot[1] = status_code; 1179 1180 } else if ((csa->prob.spu_status_R & status_P) == status_P) { 1181 1182 /* SPU_Status[P]=1 - Stop and Signal instruction 1183 * followed by 'br -4'. 1184 */ 1185 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P; 1186 csa->lscsa->stopped_status.slot[1] = status_code; 1187 1188 } else if ((csa->prob.spu_status_R & status_H) == status_H) { 1189 1190 /* SPU_Status[H]=1 - Halt Conditional, followed 1191 * by 'br -4'. 1192 */ 1193 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_H; 1194 1195 } else if ((csa->prob.spu_status_R & status_S) == status_S) { 1196 1197 /* SPU_Status[S]=1 - Two nop instructions. 1198 */ 1199 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S; 1200 1201 } else if ((csa->prob.spu_status_R & status_I) == status_I) { 1202 1203 /* SPU_Status[I]=1 - Illegal instruction followed 1204 * by 'br -4'. 1205 */ 1206 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_I; 1207 1208 } 1209 } 1210 1211 static inline void setup_spu_status_part2(struct spu_state *csa, 1212 struct spu *spu) 1213 { 1214 u32 mask; 1215 1216 /* Restore, Step 28: 1217 * If the CSA.SPU_Status[I,S,H,P,R]=0 then 1218 * add a 'br *' instruction to the end of 1219 * the SPU based restore code. 1220 * 1221 * NOTE: Rather than modifying the SPU executable, we 1222 * instead add a new 'stopped_status' field to the 1223 * LSCSA. The SPU-side restore reads this field and 1224 * takes the appropriate action when exiting. 1225 */ 1226 mask = SPU_STATUS_INVALID_INSTR | 1227 SPU_STATUS_SINGLE_STEP | 1228 SPU_STATUS_STOPPED_BY_HALT | 1229 SPU_STATUS_STOPPED_BY_STOP | SPU_STATUS_RUNNING; 1230 if (!(csa->prob.spu_status_R & mask)) { 1231 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_R; 1232 } 1233 } 1234 1235 static inline void restore_mfc_rag(struct spu_state *csa, struct spu *spu) 1236 { 1237 /* Restore, Step 29: 1238 * Restore RA_GROUP_ID register and the 1239 * RA_ENABLE reigster from the CSA. 1240 */ 1241 spu_resource_allocation_groupID_set(spu, 1242 csa->priv1.resource_allocation_groupID_RW); 1243 spu_resource_allocation_enable_set(spu, 1244 csa->priv1.resource_allocation_enable_RW); 1245 } 1246 1247 static inline void send_restore_code(struct spu_state *csa, struct spu *spu) 1248 { 1249 unsigned long addr = (unsigned long)&spu_restore_code[0]; 1250 unsigned int ls_offset = 0x0; 1251 unsigned int size = sizeof(spu_restore_code); 1252 unsigned int tag = 0; 1253 unsigned int rclass = 0; 1254 unsigned int cmd = MFC_GETFS_CMD; 1255 1256 /* Restore, Step 37: 1257 * Issue MFC DMA command to copy context 1258 * restore code to local storage. 1259 */ 1260 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 1261 } 1262 1263 static inline void setup_decr(struct spu_state *csa, struct spu *spu) 1264 { 1265 /* Restore, Step 34: 1266 * If CSA.MFC_CNTL[Ds]=1 (decrementer was 1267 * running) then adjust decrementer, set 1268 * decrementer running status in LSCSA, 1269 * and set decrementer "wrapped" status 1270 * in LSCSA. 1271 */ 1272 if (csa->priv2.mfc_control_RW & MFC_CNTL_DECREMENTER_RUNNING) { 1273 cycles_t resume_time = get_cycles(); 1274 cycles_t delta_time = resume_time - csa->suspend_time; 1275 1276 csa->lscsa->decr.slot[0] = delta_time; 1277 } 1278 } 1279 1280 static inline void setup_ppu_mb(struct spu_state *csa, struct spu *spu) 1281 { 1282 /* Restore, Step 35: 1283 * Copy the CSA.PU_MB data into the LSCSA. 1284 */ 1285 csa->lscsa->ppu_mb.slot[0] = csa->prob.pu_mb_R; 1286 } 1287 1288 static inline void setup_ppuint_mb(struct spu_state *csa, struct spu *spu) 1289 { 1290 /* Restore, Step 36: 1291 * Copy the CSA.PUINT_MB data into the LSCSA. 1292 */ 1293 csa->lscsa->ppuint_mb.slot[0] = csa->priv2.puint_mb_R; 1294 } 1295 1296 static inline int check_restore_status(struct spu_state *csa, struct spu *spu) 1297 { 1298 struct spu_problem __iomem *prob = spu->problem; 1299 u32 complete; 1300 1301 /* Restore, Step 40: 1302 * If SPU_Status[P]=1 and SPU_Status[SC] = "success", 1303 * context restore succeeded, otherwise context restore 1304 * failed. 1305 */ 1306 complete = ((SPU_RESTORE_COMPLETE << SPU_STOP_STATUS_SHIFT) | 1307 SPU_STATUS_STOPPED_BY_STOP); 1308 return (in_be32(&prob->spu_status_R) != complete) ? 1 : 0; 1309 } 1310 1311 static inline void restore_spu_privcntl(struct spu_state *csa, struct spu *spu) 1312 { 1313 struct spu_priv2 __iomem *priv2 = spu->priv2; 1314 1315 /* Restore, Step 41: 1316 * Restore SPU_PrivCntl from the CSA. 1317 */ 1318 out_be64(&priv2->spu_privcntl_RW, csa->priv2.spu_privcntl_RW); 1319 eieio(); 1320 } 1321 1322 static inline void restore_status_part1(struct spu_state *csa, struct spu *spu) 1323 { 1324 struct spu_problem __iomem *prob = spu->problem; 1325 u32 mask; 1326 1327 /* Restore, Step 42: 1328 * If any CSA.SPU_Status[I,S,H,P]=1, then 1329 * restore the error or single step state. 1330 */ 1331 mask = SPU_STATUS_INVALID_INSTR | 1332 SPU_STATUS_SINGLE_STEP | 1333 SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP; 1334 if (csa->prob.spu_status_R & mask) { 1335 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1336 eieio(); 1337 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1338 SPU_STATUS_RUNNING); 1339 } 1340 } 1341 1342 static inline void restore_status_part2(struct spu_state *csa, struct spu *spu) 1343 { 1344 struct spu_problem __iomem *prob = spu->problem; 1345 u32 mask; 1346 1347 /* Restore, Step 43: 1348 * If all CSA.SPU_Status[I,S,H,P,R]=0 then write 1349 * SPU_RunCntl[R0R1]='01', wait for SPU_Status[R]=1, 1350 * then write '00' to SPU_RunCntl[R0R1] and wait 1351 * for SPU_Status[R]=0. 1352 */ 1353 mask = SPU_STATUS_INVALID_INSTR | 1354 SPU_STATUS_SINGLE_STEP | 1355 SPU_STATUS_STOPPED_BY_HALT | 1356 SPU_STATUS_STOPPED_BY_STOP | SPU_STATUS_RUNNING; 1357 if (!(csa->prob.spu_status_R & mask)) { 1358 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1359 eieio(); 1360 POLL_WHILE_FALSE(in_be32(&prob->spu_status_R) & 1361 SPU_STATUS_RUNNING); 1362 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 1363 eieio(); 1364 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1365 SPU_STATUS_RUNNING); 1366 } 1367 } 1368 1369 static inline void restore_ls_16kb(struct spu_state *csa, struct spu *spu) 1370 { 1371 unsigned long addr = (unsigned long)&csa->lscsa->ls[0]; 1372 unsigned int ls_offset = 0x0; 1373 unsigned int size = 16384; 1374 unsigned int tag = 0; 1375 unsigned int rclass = 0; 1376 unsigned int cmd = MFC_GET_CMD; 1377 1378 /* Restore, Step 44: 1379 * Issue a DMA command to restore the first 1380 * 16kb of local storage from CSA. 1381 */ 1382 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 1383 } 1384 1385 static inline void clear_interrupts(struct spu_state *csa, struct spu *spu) 1386 { 1387 /* Restore, Step 49: 1388 * Write INT_MASK_class0 with value of 0. 1389 * Write INT_MASK_class1 with value of 0. 1390 * Write INT_MASK_class2 with value of 0. 1391 * Write INT_STAT_class0 with value of -1. 1392 * Write INT_STAT_class1 with value of -1. 1393 * Write INT_STAT_class2 with value of -1. 1394 */ 1395 spin_lock_irq(&spu->register_lock); 1396 spu_int_mask_set(spu, 0, 0ul); 1397 spu_int_mask_set(spu, 1, 0ul); 1398 spu_int_mask_set(spu, 2, 0ul); 1399 spu_int_stat_clear(spu, 0, ~0ul); 1400 spu_int_stat_clear(spu, 1, ~0ul); 1401 spu_int_stat_clear(spu, 2, ~0ul); 1402 spin_unlock_irq(&spu->register_lock); 1403 } 1404 1405 static inline void restore_mfc_queues(struct spu_state *csa, struct spu *spu) 1406 { 1407 struct spu_priv2 __iomem *priv2 = spu->priv2; 1408 int i; 1409 1410 /* Restore, Step 50: 1411 * If MFC_Cntl[Se]!=0 then restore 1412 * MFC command queues. 1413 */ 1414 if ((csa->priv2.mfc_control_RW & MFC_CNTL_DMA_QUEUES_EMPTY_MASK) == 0) { 1415 for (i = 0; i < 8; i++) { 1416 out_be64(&priv2->puq[i].mfc_cq_data0_RW, 1417 csa->priv2.puq[i].mfc_cq_data0_RW); 1418 out_be64(&priv2->puq[i].mfc_cq_data1_RW, 1419 csa->priv2.puq[i].mfc_cq_data1_RW); 1420 out_be64(&priv2->puq[i].mfc_cq_data2_RW, 1421 csa->priv2.puq[i].mfc_cq_data2_RW); 1422 out_be64(&priv2->puq[i].mfc_cq_data3_RW, 1423 csa->priv2.puq[i].mfc_cq_data3_RW); 1424 } 1425 for (i = 0; i < 16; i++) { 1426 out_be64(&priv2->spuq[i].mfc_cq_data0_RW, 1427 csa->priv2.spuq[i].mfc_cq_data0_RW); 1428 out_be64(&priv2->spuq[i].mfc_cq_data1_RW, 1429 csa->priv2.spuq[i].mfc_cq_data1_RW); 1430 out_be64(&priv2->spuq[i].mfc_cq_data2_RW, 1431 csa->priv2.spuq[i].mfc_cq_data2_RW); 1432 out_be64(&priv2->spuq[i].mfc_cq_data3_RW, 1433 csa->priv2.spuq[i].mfc_cq_data3_RW); 1434 } 1435 } 1436 eieio(); 1437 } 1438 1439 static inline void restore_ppu_querymask(struct spu_state *csa, struct spu *spu) 1440 { 1441 struct spu_problem __iomem *prob = spu->problem; 1442 1443 /* Restore, Step 51: 1444 * Restore the PPU_QueryMask register from CSA. 1445 */ 1446 out_be32(&prob->dma_querymask_RW, csa->prob.dma_querymask_RW); 1447 eieio(); 1448 } 1449 1450 static inline void restore_ppu_querytype(struct spu_state *csa, struct spu *spu) 1451 { 1452 struct spu_problem __iomem *prob = spu->problem; 1453 1454 /* Restore, Step 52: 1455 * Restore the PPU_QueryType register from CSA. 1456 */ 1457 out_be32(&prob->dma_querytype_RW, csa->prob.dma_querytype_RW); 1458 eieio(); 1459 } 1460 1461 static inline void restore_mfc_csr_tsq(struct spu_state *csa, struct spu *spu) 1462 { 1463 struct spu_priv2 __iomem *priv2 = spu->priv2; 1464 1465 /* Restore, Step 53: 1466 * Restore the MFC_CSR_TSQ register from CSA. 1467 */ 1468 out_be64(&priv2->spu_tag_status_query_RW, 1469 csa->priv2.spu_tag_status_query_RW); 1470 eieio(); 1471 } 1472 1473 static inline void restore_mfc_csr_cmd(struct spu_state *csa, struct spu *spu) 1474 { 1475 struct spu_priv2 __iomem *priv2 = spu->priv2; 1476 1477 /* Restore, Step 54: 1478 * Restore the MFC_CSR_CMD1 and MFC_CSR_CMD2 1479 * registers from CSA. 1480 */ 1481 out_be64(&priv2->spu_cmd_buf1_RW, csa->priv2.spu_cmd_buf1_RW); 1482 out_be64(&priv2->spu_cmd_buf2_RW, csa->priv2.spu_cmd_buf2_RW); 1483 eieio(); 1484 } 1485 1486 static inline void restore_mfc_csr_ato(struct spu_state *csa, struct spu *spu) 1487 { 1488 struct spu_priv2 __iomem *priv2 = spu->priv2; 1489 1490 /* Restore, Step 55: 1491 * Restore the MFC_CSR_ATO register from CSA. 1492 */ 1493 out_be64(&priv2->spu_atomic_status_RW, csa->priv2.spu_atomic_status_RW); 1494 } 1495 1496 static inline void restore_mfc_tclass_id(struct spu_state *csa, struct spu *spu) 1497 { 1498 /* Restore, Step 56: 1499 * Restore the MFC_TCLASS_ID register from CSA. 1500 */ 1501 spu_mfc_tclass_id_set(spu, csa->priv1.mfc_tclass_id_RW); 1502 eieio(); 1503 } 1504 1505 static inline void set_llr_event(struct spu_state *csa, struct spu *spu) 1506 { 1507 u64 ch0_cnt, ch0_data; 1508 u64 ch1_data; 1509 1510 /* Restore, Step 57: 1511 * Set the Lock Line Reservation Lost Event by: 1512 * 1. OR CSA.SPU_Event_Status with bit 21 (Lr) set to 1. 1513 * 2. If CSA.SPU_Channel_0_Count=0 and 1514 * CSA.SPU_Wr_Event_Mask[Lr]=1 and 1515 * CSA.SPU_Event_Status[Lr]=0 then set 1516 * CSA.SPU_Event_Status_Count=1. 1517 */ 1518 ch0_cnt = csa->spu_chnlcnt_RW[0]; 1519 ch0_data = csa->spu_chnldata_RW[0]; 1520 ch1_data = csa->spu_chnldata_RW[1]; 1521 csa->spu_chnldata_RW[0] |= MFC_LLR_LOST_EVENT; 1522 if ((ch0_cnt == 0) && !(ch0_data & MFC_LLR_LOST_EVENT) && 1523 (ch1_data & MFC_LLR_LOST_EVENT)) { 1524 csa->spu_chnlcnt_RW[0] = 1; 1525 } 1526 } 1527 1528 static inline void restore_decr_wrapped(struct spu_state *csa, struct spu *spu) 1529 { 1530 /* Restore, Step 58: 1531 * If the status of the CSA software decrementer 1532 * "wrapped" flag is set, OR in a '1' to 1533 * CSA.SPU_Event_Status[Tm]. 1534 */ 1535 if (csa->lscsa->decr_status.slot[0] == 1) { 1536 csa->spu_chnldata_RW[0] |= 0x20; 1537 } 1538 if ((csa->lscsa->decr_status.slot[0] == 1) && 1539 (csa->spu_chnlcnt_RW[0] == 0 && 1540 ((csa->spu_chnldata_RW[2] & 0x20) == 0x0) && 1541 ((csa->spu_chnldata_RW[0] & 0x20) != 0x1))) { 1542 csa->spu_chnlcnt_RW[0] = 1; 1543 } 1544 } 1545 1546 static inline void restore_ch_part1(struct spu_state *csa, struct spu *spu) 1547 { 1548 struct spu_priv2 __iomem *priv2 = spu->priv2; 1549 u64 idx, ch_indices[7] = { 0UL, 1UL, 3UL, 4UL, 24UL, 25UL, 27UL }; 1550 int i; 1551 1552 /* Restore, Step 59: 1553 * Restore the following CH: [0,1,3,4,24,25,27] 1554 */ 1555 for (i = 0; i < 7; i++) { 1556 idx = ch_indices[i]; 1557 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1558 eieio(); 1559 out_be64(&priv2->spu_chnldata_RW, csa->spu_chnldata_RW[idx]); 1560 out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[idx]); 1561 eieio(); 1562 } 1563 } 1564 1565 static inline void restore_ch_part2(struct spu_state *csa, struct spu *spu) 1566 { 1567 struct spu_priv2 __iomem *priv2 = spu->priv2; 1568 u64 ch_indices[3] = { 9UL, 21UL, 23UL }; 1569 u64 ch_counts[3] = { 1UL, 16UL, 1UL }; 1570 u64 idx; 1571 int i; 1572 1573 /* Restore, Step 60: 1574 * Restore the following CH: [9,21,23]. 1575 */ 1576 ch_counts[0] = 1UL; 1577 ch_counts[1] = csa->spu_chnlcnt_RW[21]; 1578 ch_counts[2] = 1UL; 1579 for (i = 0; i < 3; i++) { 1580 idx = ch_indices[i]; 1581 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1582 eieio(); 1583 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]); 1584 eieio(); 1585 } 1586 } 1587 1588 static inline void restore_spu_lslr(struct spu_state *csa, struct spu *spu) 1589 { 1590 struct spu_priv2 __iomem *priv2 = spu->priv2; 1591 1592 /* Restore, Step 61: 1593 * Restore the SPU_LSLR register from CSA. 1594 */ 1595 out_be64(&priv2->spu_lslr_RW, csa->priv2.spu_lslr_RW); 1596 eieio(); 1597 } 1598 1599 static inline void restore_spu_cfg(struct spu_state *csa, struct spu *spu) 1600 { 1601 struct spu_priv2 __iomem *priv2 = spu->priv2; 1602 1603 /* Restore, Step 62: 1604 * Restore the SPU_Cfg register from CSA. 1605 */ 1606 out_be64(&priv2->spu_cfg_RW, csa->priv2.spu_cfg_RW); 1607 eieio(); 1608 } 1609 1610 static inline void restore_pm_trace(struct spu_state *csa, struct spu *spu) 1611 { 1612 /* Restore, Step 63: 1613 * Restore PM_Trace_Tag_Wait_Mask from CSA. 1614 * Not performed by this implementation. 1615 */ 1616 } 1617 1618 static inline void restore_spu_npc(struct spu_state *csa, struct spu *spu) 1619 { 1620 struct spu_problem __iomem *prob = spu->problem; 1621 1622 /* Restore, Step 64: 1623 * Restore SPU_NPC from CSA. 1624 */ 1625 out_be32(&prob->spu_npc_RW, csa->prob.spu_npc_RW); 1626 eieio(); 1627 } 1628 1629 static inline void restore_spu_mb(struct spu_state *csa, struct spu *spu) 1630 { 1631 struct spu_priv2 __iomem *priv2 = spu->priv2; 1632 int i; 1633 1634 /* Restore, Step 65: 1635 * Restore MFC_RdSPU_MB from CSA. 1636 */ 1637 out_be64(&priv2->spu_chnlcntptr_RW, 29UL); 1638 eieio(); 1639 out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[29]); 1640 for (i = 0; i < 4; i++) { 1641 out_be64(&priv2->spu_chnldata_RW, csa->spu_mailbox_data[i]); 1642 } 1643 eieio(); 1644 } 1645 1646 static inline void check_ppu_mb_stat(struct spu_state *csa, struct spu *spu) 1647 { 1648 struct spu_problem __iomem *prob = spu->problem; 1649 u32 dummy = 0; 1650 1651 /* Restore, Step 66: 1652 * If CSA.MB_Stat[P]=0 (mailbox empty) then 1653 * read from the PPU_MB register. 1654 */ 1655 if ((csa->prob.mb_stat_R & 0xFF) == 0) { 1656 dummy = in_be32(&prob->pu_mb_R); 1657 eieio(); 1658 } 1659 } 1660 1661 static inline void check_ppuint_mb_stat(struct spu_state *csa, struct spu *spu) 1662 { 1663 struct spu_priv2 __iomem *priv2 = spu->priv2; 1664 u64 dummy = 0UL; 1665 1666 /* Restore, Step 66: 1667 * If CSA.MB_Stat[I]=0 (mailbox empty) then 1668 * read from the PPUINT_MB register. 1669 */ 1670 if ((csa->prob.mb_stat_R & 0xFF0000) == 0) { 1671 dummy = in_be64(&priv2->puint_mb_R); 1672 eieio(); 1673 spu_int_stat_clear(spu, 2, CLASS2_ENABLE_MAILBOX_INTR); 1674 eieio(); 1675 } 1676 } 1677 1678 static inline void restore_mfc_slbs(struct spu_state *csa, struct spu *spu) 1679 { 1680 struct spu_priv2 __iomem *priv2 = spu->priv2; 1681 int i; 1682 1683 /* Restore, Step 68: 1684 * If MFC_SR1[R]='1', restore SLBs from CSA. 1685 */ 1686 if (csa->priv1.mfc_sr1_RW & MFC_STATE1_RELOCATE_MASK) { 1687 for (i = 0; i < 8; i++) { 1688 out_be64(&priv2->slb_index_W, i); 1689 eieio(); 1690 out_be64(&priv2->slb_esid_RW, csa->slb_esid_RW[i]); 1691 out_be64(&priv2->slb_vsid_RW, csa->slb_vsid_RW[i]); 1692 eieio(); 1693 } 1694 out_be64(&priv2->slb_index_W, csa->priv2.slb_index_W); 1695 eieio(); 1696 } 1697 } 1698 1699 static inline void restore_mfc_sr1(struct spu_state *csa, struct spu *spu) 1700 { 1701 /* Restore, Step 69: 1702 * Restore the MFC_SR1 register from CSA. 1703 */ 1704 spu_mfc_sr1_set(spu, csa->priv1.mfc_sr1_RW); 1705 eieio(); 1706 } 1707 1708 static inline void restore_other_spu_access(struct spu_state *csa, 1709 struct spu *spu) 1710 { 1711 /* Restore, Step 70: 1712 * Restore other SPU mappings to this SPU. TBD. 1713 */ 1714 } 1715 1716 static inline void restore_spu_runcntl(struct spu_state *csa, struct spu *spu) 1717 { 1718 struct spu_problem __iomem *prob = spu->problem; 1719 1720 /* Restore, Step 71: 1721 * If CSA.SPU_Status[R]=1 then write 1722 * SPU_RunCntl[R0R1]='01'. 1723 */ 1724 if (csa->prob.spu_status_R & SPU_STATUS_RUNNING) { 1725 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1726 eieio(); 1727 } 1728 } 1729 1730 static inline void restore_mfc_cntl(struct spu_state *csa, struct spu *spu) 1731 { 1732 struct spu_priv2 __iomem *priv2 = spu->priv2; 1733 1734 /* Restore, Step 72: 1735 * Restore the MFC_CNTL register for the CSA. 1736 */ 1737 out_be64(&priv2->mfc_control_RW, csa->priv2.mfc_control_RW); 1738 eieio(); 1739 } 1740 1741 static inline void enable_user_access(struct spu_state *csa, struct spu *spu) 1742 { 1743 /* Restore, Step 73: 1744 * Enable user-space access (if provided) to this 1745 * SPU by mapping the virtual pages assigned to 1746 * the SPU memory-mapped I/O (MMIO) for problem 1747 * state. TBD. 1748 */ 1749 } 1750 1751 static inline void reset_switch_active(struct spu_state *csa, struct spu *spu) 1752 { 1753 /* Restore, Step 74: 1754 * Reset the "context switch active" flag. 1755 */ 1756 clear_bit(SPU_CONTEXT_SWITCH_ACTIVE, &spu->flags); 1757 mb(); 1758 } 1759 1760 static inline void reenable_interrupts(struct spu_state *csa, struct spu *spu) 1761 { 1762 /* Restore, Step 75: 1763 * Re-enable SPU interrupts. 1764 */ 1765 spin_lock_irq(&spu->register_lock); 1766 spu_int_mask_set(spu, 0, csa->priv1.int_mask_class0_RW); 1767 spu_int_mask_set(spu, 1, csa->priv1.int_mask_class1_RW); 1768 spu_int_mask_set(spu, 2, csa->priv1.int_mask_class2_RW); 1769 spin_unlock_irq(&spu->register_lock); 1770 } 1771 1772 static int quiece_spu(struct spu_state *prev, struct spu *spu) 1773 { 1774 /* 1775 * Combined steps 2-18 of SPU context save sequence, which 1776 * quiesce the SPU state (disable SPU execution, MFC command 1777 * queues, decrementer, SPU interrupts, etc.). 1778 * 1779 * Returns 0 on success. 1780 * 2 if failed step 2. 1781 * 6 if failed step 6. 1782 */ 1783 1784 if (check_spu_isolate(prev, spu)) { /* Step 2. */ 1785 return 2; 1786 } 1787 disable_interrupts(prev, spu); /* Step 3. */ 1788 set_watchdog_timer(prev, spu); /* Step 4. */ 1789 inhibit_user_access(prev, spu); /* Step 5. */ 1790 if (check_spu_isolate(prev, spu)) { /* Step 6. */ 1791 return 6; 1792 } 1793 set_switch_pending(prev, spu); /* Step 7. */ 1794 save_mfc_cntl(prev, spu); /* Step 8. */ 1795 save_spu_runcntl(prev, spu); /* Step 9. */ 1796 save_mfc_sr1(prev, spu); /* Step 10. */ 1797 save_spu_status(prev, spu); /* Step 11. */ 1798 save_mfc_decr(prev, spu); /* Step 12. */ 1799 halt_mfc_decr(prev, spu); /* Step 13. */ 1800 save_timebase(prev, spu); /* Step 14. */ 1801 remove_other_spu_access(prev, spu); /* Step 15. */ 1802 do_mfc_mssync(prev, spu); /* Step 16. */ 1803 issue_mfc_tlbie(prev, spu); /* Step 17. */ 1804 handle_pending_interrupts(prev, spu); /* Step 18. */ 1805 1806 return 0; 1807 } 1808 1809 static void save_csa(struct spu_state *prev, struct spu *spu) 1810 { 1811 /* 1812 * Combine steps 19-44 of SPU context save sequence, which 1813 * save regions of the privileged & problem state areas. 1814 */ 1815 1816 save_mfc_queues(prev, spu); /* Step 19. */ 1817 save_ppu_querymask(prev, spu); /* Step 20. */ 1818 save_ppu_querytype(prev, spu); /* Step 21. */ 1819 save_mfc_csr_tsq(prev, spu); /* Step 22. */ 1820 save_mfc_csr_cmd(prev, spu); /* Step 23. */ 1821 save_mfc_csr_ato(prev, spu); /* Step 24. */ 1822 save_mfc_tclass_id(prev, spu); /* Step 25. */ 1823 set_mfc_tclass_id(prev, spu); /* Step 26. */ 1824 purge_mfc_queue(prev, spu); /* Step 27. */ 1825 wait_purge_complete(prev, spu); /* Step 28. */ 1826 save_mfc_slbs(prev, spu); /* Step 29. */ 1827 setup_mfc_sr1(prev, spu); /* Step 30. */ 1828 save_spu_npc(prev, spu); /* Step 31. */ 1829 save_spu_privcntl(prev, spu); /* Step 32. */ 1830 reset_spu_privcntl(prev, spu); /* Step 33. */ 1831 save_spu_lslr(prev, spu); /* Step 34. */ 1832 reset_spu_lslr(prev, spu); /* Step 35. */ 1833 save_spu_cfg(prev, spu); /* Step 36. */ 1834 save_pm_trace(prev, spu); /* Step 37. */ 1835 save_mfc_rag(prev, spu); /* Step 38. */ 1836 save_ppu_mb_stat(prev, spu); /* Step 39. */ 1837 save_ppu_mb(prev, spu); /* Step 40. */ 1838 save_ppuint_mb(prev, spu); /* Step 41. */ 1839 save_ch_part1(prev, spu); /* Step 42. */ 1840 save_spu_mb(prev, spu); /* Step 43. */ 1841 save_mfc_cmd(prev, spu); /* Step 44. */ 1842 reset_ch(prev, spu); /* Step 45. */ 1843 } 1844 1845 static void save_lscsa(struct spu_state *prev, struct spu *spu) 1846 { 1847 /* 1848 * Perform steps 46-57 of SPU context save sequence, 1849 * which save regions of the local store and register 1850 * file. 1851 */ 1852 1853 resume_mfc_queue(prev, spu); /* Step 46. */ 1854 setup_mfc_slbs(prev, spu); /* Step 47. */ 1855 set_switch_active(prev, spu); /* Step 48. */ 1856 enable_interrupts(prev, spu); /* Step 49. */ 1857 save_ls_16kb(prev, spu); /* Step 50. */ 1858 set_spu_npc(prev, spu); /* Step 51. */ 1859 set_signot1(prev, spu); /* Step 52. */ 1860 set_signot2(prev, spu); /* Step 53. */ 1861 send_save_code(prev, spu); /* Step 54. */ 1862 set_ppu_querymask(prev, spu); /* Step 55. */ 1863 wait_tag_complete(prev, spu); /* Step 56. */ 1864 wait_spu_stopped(prev, spu); /* Step 57. */ 1865 } 1866 1867 static void harvest(struct spu_state *prev, struct spu *spu) 1868 { 1869 /* 1870 * Perform steps 2-25 of SPU context restore sequence, 1871 * which resets an SPU either after a failed save, or 1872 * when using SPU for first time. 1873 */ 1874 1875 disable_interrupts(prev, spu); /* Step 2. */ 1876 inhibit_user_access(prev, spu); /* Step 3. */ 1877 terminate_spu_app(prev, spu); /* Step 4. */ 1878 set_switch_pending(prev, spu); /* Step 5. */ 1879 remove_other_spu_access(prev, spu); /* Step 6. */ 1880 suspend_mfc(prev, spu); /* Step 7. */ 1881 wait_suspend_mfc_complete(prev, spu); /* Step 8. */ 1882 if (!suspend_spe(prev, spu)) /* Step 9. */ 1883 clear_spu_status(prev, spu); /* Step 10. */ 1884 do_mfc_mssync(prev, spu); /* Step 11. */ 1885 issue_mfc_tlbie(prev, spu); /* Step 12. */ 1886 handle_pending_interrupts(prev, spu); /* Step 13. */ 1887 purge_mfc_queue(prev, spu); /* Step 14. */ 1888 wait_purge_complete(prev, spu); /* Step 15. */ 1889 reset_spu_privcntl(prev, spu); /* Step 16. */ 1890 reset_spu_lslr(prev, spu); /* Step 17. */ 1891 setup_mfc_sr1(prev, spu); /* Step 18. */ 1892 invalidate_slbs(prev, spu); /* Step 19. */ 1893 reset_ch_part1(prev, spu); /* Step 20. */ 1894 reset_ch_part2(prev, spu); /* Step 21. */ 1895 enable_interrupts(prev, spu); /* Step 22. */ 1896 set_switch_active(prev, spu); /* Step 23. */ 1897 set_mfc_tclass_id(prev, spu); /* Step 24. */ 1898 resume_mfc_queue(prev, spu); /* Step 25. */ 1899 } 1900 1901 static void restore_lscsa(struct spu_state *next, struct spu *spu) 1902 { 1903 /* 1904 * Perform steps 26-40 of SPU context restore sequence, 1905 * which restores regions of the local store and register 1906 * file. 1907 */ 1908 1909 set_watchdog_timer(next, spu); /* Step 26. */ 1910 setup_spu_status_part1(next, spu); /* Step 27. */ 1911 setup_spu_status_part2(next, spu); /* Step 28. */ 1912 restore_mfc_rag(next, spu); /* Step 29. */ 1913 setup_mfc_slbs(next, spu); /* Step 30. */ 1914 set_spu_npc(next, spu); /* Step 31. */ 1915 set_signot1(next, spu); /* Step 32. */ 1916 set_signot2(next, spu); /* Step 33. */ 1917 setup_decr(next, spu); /* Step 34. */ 1918 setup_ppu_mb(next, spu); /* Step 35. */ 1919 setup_ppuint_mb(next, spu); /* Step 36. */ 1920 send_restore_code(next, spu); /* Step 37. */ 1921 set_ppu_querymask(next, spu); /* Step 38. */ 1922 wait_tag_complete(next, spu); /* Step 39. */ 1923 wait_spu_stopped(next, spu); /* Step 40. */ 1924 } 1925 1926 static void restore_csa(struct spu_state *next, struct spu *spu) 1927 { 1928 /* 1929 * Combine steps 41-76 of SPU context restore sequence, which 1930 * restore regions of the privileged & problem state areas. 1931 */ 1932 1933 restore_spu_privcntl(next, spu); /* Step 41. */ 1934 restore_status_part1(next, spu); /* Step 42. */ 1935 restore_status_part2(next, spu); /* Step 43. */ 1936 restore_ls_16kb(next, spu); /* Step 44. */ 1937 wait_tag_complete(next, spu); /* Step 45. */ 1938 suspend_mfc(next, spu); /* Step 46. */ 1939 wait_suspend_mfc_complete(next, spu); /* Step 47. */ 1940 issue_mfc_tlbie(next, spu); /* Step 48. */ 1941 clear_interrupts(next, spu); /* Step 49. */ 1942 restore_mfc_queues(next, spu); /* Step 50. */ 1943 restore_ppu_querymask(next, spu); /* Step 51. */ 1944 restore_ppu_querytype(next, spu); /* Step 52. */ 1945 restore_mfc_csr_tsq(next, spu); /* Step 53. */ 1946 restore_mfc_csr_cmd(next, spu); /* Step 54. */ 1947 restore_mfc_csr_ato(next, spu); /* Step 55. */ 1948 restore_mfc_tclass_id(next, spu); /* Step 56. */ 1949 set_llr_event(next, spu); /* Step 57. */ 1950 restore_decr_wrapped(next, spu); /* Step 58. */ 1951 restore_ch_part1(next, spu); /* Step 59. */ 1952 restore_ch_part2(next, spu); /* Step 60. */ 1953 restore_spu_lslr(next, spu); /* Step 61. */ 1954 restore_spu_cfg(next, spu); /* Step 62. */ 1955 restore_pm_trace(next, spu); /* Step 63. */ 1956 restore_spu_npc(next, spu); /* Step 64. */ 1957 restore_spu_mb(next, spu); /* Step 65. */ 1958 check_ppu_mb_stat(next, spu); /* Step 66. */ 1959 check_ppuint_mb_stat(next, spu); /* Step 67. */ 1960 restore_mfc_slbs(next, spu); /* Step 68. */ 1961 restore_mfc_sr1(next, spu); /* Step 69. */ 1962 restore_other_spu_access(next, spu); /* Step 70. */ 1963 restore_spu_runcntl(next, spu); /* Step 71. */ 1964 restore_mfc_cntl(next, spu); /* Step 72. */ 1965 enable_user_access(next, spu); /* Step 73. */ 1966 reset_switch_active(next, spu); /* Step 74. */ 1967 reenable_interrupts(next, spu); /* Step 75. */ 1968 } 1969 1970 static int __do_spu_save(struct spu_state *prev, struct spu *spu) 1971 { 1972 int rc; 1973 1974 /* 1975 * SPU context save can be broken into three phases: 1976 * 1977 * (a) quiesce [steps 2-16]. 1978 * (b) save of CSA, performed by PPE [steps 17-42] 1979 * (c) save of LSCSA, mostly performed by SPU [steps 43-52]. 1980 * 1981 * Returns 0 on success. 1982 * 2,6 if failed to quiece SPU 1983 * 53 if SPU-side of save failed. 1984 */ 1985 1986 rc = quiece_spu(prev, spu); /* Steps 2-16. */ 1987 switch (rc) { 1988 default: 1989 case 2: 1990 case 6: 1991 harvest(prev, spu); 1992 return rc; 1993 break; 1994 case 0: 1995 break; 1996 } 1997 save_csa(prev, spu); /* Steps 17-43. */ 1998 save_lscsa(prev, spu); /* Steps 44-53. */ 1999 return check_save_status(prev, spu); /* Step 54. */ 2000 } 2001 2002 static int __do_spu_restore(struct spu_state *next, struct spu *spu) 2003 { 2004 int rc; 2005 2006 /* 2007 * SPU context restore can be broken into three phases: 2008 * 2009 * (a) harvest (or reset) SPU [steps 2-24]. 2010 * (b) restore LSCSA [steps 25-40], mostly performed by SPU. 2011 * (c) restore CSA [steps 41-76], performed by PPE. 2012 * 2013 * The 'harvest' step is not performed here, but rather 2014 * as needed below. 2015 */ 2016 2017 restore_lscsa(next, spu); /* Steps 24-39. */ 2018 rc = check_restore_status(next, spu); /* Step 40. */ 2019 switch (rc) { 2020 default: 2021 /* Failed. Return now. */ 2022 return rc; 2023 break; 2024 case 0: 2025 /* Fall through to next step. */ 2026 break; 2027 } 2028 restore_csa(next, spu); 2029 2030 return 0; 2031 } 2032 2033 /** 2034 * spu_save - SPU context save, with locking. 2035 * @prev: pointer to SPU context save area, to be saved. 2036 * @spu: pointer to SPU iomem structure. 2037 * 2038 * Acquire locks, perform the save operation then return. 2039 */ 2040 int spu_save(struct spu_state *prev, struct spu *spu) 2041 { 2042 int rc; 2043 2044 acquire_spu_lock(spu); /* Step 1. */ 2045 rc = __do_spu_save(prev, spu); /* Steps 2-53. */ 2046 release_spu_lock(spu); 2047 if (rc) { 2048 panic("%s failed on SPU[%d], rc=%d.\n", 2049 __func__, spu->number, rc); 2050 } 2051 return rc; 2052 } 2053 2054 /** 2055 * spu_restore - SPU context restore, with harvest and locking. 2056 * @new: pointer to SPU context save area, to be restored. 2057 * @spu: pointer to SPU iomem structure. 2058 * 2059 * Perform harvest + restore, as we may not be coming 2060 * from a previous succesful save operation, and the 2061 * hardware state is unknown. 2062 */ 2063 int spu_restore(struct spu_state *new, struct spu *spu) 2064 { 2065 int rc; 2066 2067 acquire_spu_lock(spu); 2068 harvest(NULL, spu); 2069 spu->stop_code = 0; 2070 spu->dar = 0; 2071 spu->dsisr = 0; 2072 spu->slb_replace = 0; 2073 spu->class_0_pending = 0; 2074 rc = __do_spu_restore(new, spu); 2075 release_spu_lock(spu); 2076 if (rc) { 2077 panic("%s failed on SPU[%d] rc=%d.\n", 2078 __func__, spu->number, rc); 2079 } 2080 return rc; 2081 } 2082 2083 /** 2084 * spu_harvest - SPU harvest (reset) operation 2085 * @spu: pointer to SPU iomem structure. 2086 * 2087 * Perform SPU harvest (reset) operation. 2088 */ 2089 void spu_harvest(struct spu *spu) 2090 { 2091 acquire_spu_lock(spu); 2092 harvest(NULL, spu); 2093 release_spu_lock(spu); 2094 } 2095 2096 static void init_prob(struct spu_state *csa) 2097 { 2098 csa->spu_chnlcnt_RW[9] = 1; 2099 csa->spu_chnlcnt_RW[21] = 16; 2100 csa->spu_chnlcnt_RW[23] = 1; 2101 csa->spu_chnlcnt_RW[28] = 1; 2102 csa->spu_chnlcnt_RW[30] = 1; 2103 csa->prob.spu_runcntl_RW = SPU_RUNCNTL_STOP; 2104 } 2105 2106 static void init_priv1(struct spu_state *csa) 2107 { 2108 /* Enable decode, relocate, tlbie response, master runcntl. */ 2109 csa->priv1.mfc_sr1_RW = MFC_STATE1_LOCAL_STORAGE_DECODE_MASK | 2110 MFC_STATE1_MASTER_RUN_CONTROL_MASK | 2111 MFC_STATE1_PROBLEM_STATE_MASK | 2112 MFC_STATE1_RELOCATE_MASK | MFC_STATE1_BUS_TLBIE_MASK; 2113 2114 /* Set storage description. */ 2115 csa->priv1.mfc_sdr_RW = mfspr(SPRN_SDR1); 2116 2117 /* Enable OS-specific set of interrupts. */ 2118 csa->priv1.int_mask_class0_RW = CLASS0_ENABLE_DMA_ALIGNMENT_INTR | 2119 CLASS0_ENABLE_INVALID_DMA_COMMAND_INTR | 2120 CLASS0_ENABLE_SPU_ERROR_INTR; 2121 csa->priv1.int_mask_class1_RW = CLASS1_ENABLE_SEGMENT_FAULT_INTR | 2122 CLASS1_ENABLE_STORAGE_FAULT_INTR; 2123 csa->priv1.int_mask_class2_RW = CLASS2_ENABLE_SPU_STOP_INTR | 2124 CLASS2_ENABLE_SPU_HALT_INTR; 2125 } 2126 2127 static void init_priv2(struct spu_state *csa) 2128 { 2129 csa->priv2.spu_lslr_RW = LS_ADDR_MASK; 2130 csa->priv2.mfc_control_RW = MFC_CNTL_RESUME_DMA_QUEUE | 2131 MFC_CNTL_NORMAL_DMA_QUEUE_OPERATION | 2132 MFC_CNTL_DMA_QUEUES_EMPTY_MASK; 2133 } 2134 2135 /** 2136 * spu_alloc_csa - allocate and initialize an SPU context save area. 2137 * 2138 * Allocate and initialize the contents of an SPU context save area. 2139 * This includes enabling address translation, interrupt masks, etc., 2140 * as appropriate for the given OS environment. 2141 * 2142 * Note that storage for the 'lscsa' is allocated separately, 2143 * as it is by far the largest of the context save regions, 2144 * and may need to be pinned or otherwise specially aligned. 2145 */ 2146 void spu_init_csa(struct spu_state *csa) 2147 { 2148 struct spu_lscsa *lscsa; 2149 unsigned char *p; 2150 2151 if (!csa) 2152 return; 2153 memset(csa, 0, sizeof(struct spu_state)); 2154 2155 lscsa = vmalloc(sizeof(struct spu_lscsa)); 2156 if (!lscsa) 2157 return; 2158 2159 memset(lscsa, 0, sizeof(struct spu_lscsa)); 2160 csa->lscsa = lscsa; 2161 csa->register_lock = SPIN_LOCK_UNLOCKED; 2162 2163 /* Set LS pages reserved to allow for user-space mapping. */ 2164 for (p = lscsa->ls; p < lscsa->ls + LS_SIZE; p += PAGE_SIZE) 2165 SetPageReserved(vmalloc_to_page(p)); 2166 2167 init_prob(csa); 2168 init_priv1(csa); 2169 init_priv2(csa); 2170 } 2171 2172 void spu_fini_csa(struct spu_state *csa) 2173 { 2174 /* Clear reserved bit before vfree. */ 2175 unsigned char *p; 2176 for (p = csa->lscsa->ls; p < csa->lscsa->ls + LS_SIZE; p += PAGE_SIZE) 2177 ClearPageReserved(vmalloc_to_page(p)); 2178 2179 vfree(csa->lscsa); 2180 } 2181