1 /*
2  * spu_switch.c
3  *
4  * (C) Copyright IBM Corp. 2005
5  *
6  * Author: Mark Nutter <mnutter@us.ibm.com>
7  *
8  * Host-side part of SPU context switch sequence outlined in
9  * Synergistic Processor Element, Book IV.
10  *
11  * A fully premptive switch of an SPE is very expensive in terms
12  * of time and system resources.  SPE Book IV indicates that SPE
13  * allocation should follow a "serially reusable device" model,
14  * in which the SPE is assigned a task until it completes.  When
15  * this is not possible, this sequence may be used to premptively
16  * save, and then later (optionally) restore the context of a
17  * program executing on an SPE.
18  *
19  *
20  * This program is free software; you can redistribute it and/or modify
21  * it under the terms of the GNU General Public License as published by
22  * the Free Software Foundation; either version 2, or (at your option)
23  * any later version.
24  *
25  * This program is distributed in the hope that it will be useful,
26  * but WITHOUT ANY WARRANTY; without even the implied warranty of
27  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
28  * GNU General Public License for more details.
29  *
30  * You should have received a copy of the GNU General Public License
31  * along with this program; if not, write to the Free Software
32  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
33  */
34 
35 #include <linux/config.h>
36 #include <linux/module.h>
37 #include <linux/errno.h>
38 #include <linux/sched.h>
39 #include <linux/kernel.h>
40 #include <linux/mm.h>
41 #include <linux/vmalloc.h>
42 #include <linux/smp.h>
43 #include <linux/smp_lock.h>
44 #include <linux/stddef.h>
45 #include <linux/unistd.h>
46 
47 #include <asm/io.h>
48 #include <asm/spu.h>
49 #include <asm/spu_csa.h>
50 #include <asm/mmu_context.h>
51 
52 #include "spu_save_dump.h"
53 #include "spu_restore_dump.h"
54 
55 #if 0
56 #define POLL_WHILE_TRUE(_c) {				\
57     do {						\
58     } while (_c);					\
59   }
60 #else
61 #define RELAX_SPIN_COUNT				1000
62 #define POLL_WHILE_TRUE(_c) {				\
63     do {						\
64 	int _i;						\
65 	for (_i=0; _i<RELAX_SPIN_COUNT && (_c); _i++) { \
66 	    cpu_relax();				\
67 	}						\
68 	if (unlikely(_c)) yield();			\
69 	else break;					\
70     } while (_c);					\
71   }
72 #endif				/* debug */
73 
74 #define POLL_WHILE_FALSE(_c) 	POLL_WHILE_TRUE(!(_c))
75 
76 static inline void acquire_spu_lock(struct spu *spu)
77 {
78 	/* Save, Step 1:
79 	 * Restore, Step 1:
80 	 *    Acquire SPU-specific mutual exclusion lock.
81 	 *    TBD.
82 	 */
83 }
84 
85 static inline void release_spu_lock(struct spu *spu)
86 {
87 	/* Restore, Step 76:
88 	 *    Release SPU-specific mutual exclusion lock.
89 	 *    TBD.
90 	 */
91 }
92 
93 static inline int check_spu_isolate(struct spu_state *csa, struct spu *spu)
94 {
95 	struct spu_problem __iomem *prob = spu->problem;
96 	u32 isolate_state;
97 
98 	/* Save, Step 2:
99 	 * Save, Step 6:
100 	 *     If SPU_Status[E,L,IS] any field is '1', this
101 	 *     SPU is in isolate state and cannot be context
102 	 *     saved at this time.
103 	 */
104 	isolate_state = SPU_STATUS_ISOLATED_STATE |
105 	    SPU_STATUS_ISOLATED_LOAD_STAUTUS | SPU_STATUS_ISOLATED_EXIT_STAUTUS;
106 	return (in_be32(&prob->spu_status_R) & isolate_state) ? 1 : 0;
107 }
108 
109 static inline void disable_interrupts(struct spu_state *csa, struct spu *spu)
110 {
111 	/* Save, Step 3:
112 	 * Restore, Step 2:
113 	 *     Save INT_Mask_class0 in CSA.
114 	 *     Write INT_MASK_class0 with value of 0.
115 	 *     Save INT_Mask_class1 in CSA.
116 	 *     Write INT_MASK_class1 with value of 0.
117 	 *     Save INT_Mask_class2 in CSA.
118 	 *     Write INT_MASK_class2 with value of 0.
119 	 */
120 	spin_lock_irq(&spu->register_lock);
121 	if (csa) {
122 		csa->priv1.int_mask_class0_RW = spu_int_mask_get(spu, 0);
123 		csa->priv1.int_mask_class1_RW = spu_int_mask_get(spu, 1);
124 		csa->priv1.int_mask_class2_RW = spu_int_mask_get(spu, 2);
125 	}
126 	spu_int_mask_set(spu, 0, 0ul);
127 	spu_int_mask_set(spu, 1, 0ul);
128 	spu_int_mask_set(spu, 2, 0ul);
129 	eieio();
130 	spin_unlock_irq(&spu->register_lock);
131 }
132 
133 static inline void set_watchdog_timer(struct spu_state *csa, struct spu *spu)
134 {
135 	/* Save, Step 4:
136 	 * Restore, Step 25.
137 	 *    Set a software watchdog timer, which specifies the
138 	 *    maximum allowable time for a context save sequence.
139 	 *
140 	 *    For present, this implementation will not set a global
141 	 *    watchdog timer, as virtualization & variable system load
142 	 *    may cause unpredictable execution times.
143 	 */
144 }
145 
146 static inline void inhibit_user_access(struct spu_state *csa, struct spu *spu)
147 {
148 	/* Save, Step 5:
149 	 * Restore, Step 3:
150 	 *     Inhibit user-space access (if provided) to this
151 	 *     SPU by unmapping the virtual pages assigned to
152 	 *     the SPU memory-mapped I/O (MMIO) for problem
153 	 *     state. TBD.
154 	 */
155 }
156 
157 static inline void set_switch_pending(struct spu_state *csa, struct spu *spu)
158 {
159 	/* Save, Step 7:
160 	 * Restore, Step 5:
161 	 *     Set a software context switch pending flag.
162 	 */
163 	set_bit(SPU_CONTEXT_SWITCH_PENDING, &spu->flags);
164 	mb();
165 }
166 
167 static inline void save_mfc_cntl(struct spu_state *csa, struct spu *spu)
168 {
169 	struct spu_priv2 __iomem *priv2 = spu->priv2;
170 
171 	/* Save, Step 8:
172 	 *     Read and save MFC_CNTL[Ss].
173 	 */
174 	if (csa) {
175 		csa->priv2.mfc_control_RW = in_be64(&priv2->mfc_control_RW) &
176 		    MFC_CNTL_SUSPEND_DMA_STATUS_MASK;
177 	}
178 }
179 
180 static inline void save_spu_runcntl(struct spu_state *csa, struct spu *spu)
181 {
182 	struct spu_problem __iomem *prob = spu->problem;
183 
184 	/* Save, Step 9:
185 	 *     Save SPU_Runcntl in the CSA.  This value contains
186 	 *     the "Application Desired State".
187 	 */
188 	csa->prob.spu_runcntl_RW = in_be32(&prob->spu_runcntl_RW);
189 }
190 
191 static inline void save_mfc_sr1(struct spu_state *csa, struct spu *spu)
192 {
193 	/* Save, Step 10:
194 	 *     Save MFC_SR1 in the CSA.
195 	 */
196 	csa->priv1.mfc_sr1_RW = spu_mfc_sr1_get(spu);
197 }
198 
199 static inline void save_spu_status(struct spu_state *csa, struct spu *spu)
200 {
201 	struct spu_problem __iomem *prob = spu->problem;
202 
203 	/* Save, Step 11:
204 	 *     Read SPU_Status[R], and save to CSA.
205 	 */
206 	if ((in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) == 0) {
207 		csa->prob.spu_status_R = in_be32(&prob->spu_status_R);
208 	} else {
209 		u32 stopped;
210 
211 		out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
212 		eieio();
213 		POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
214 				SPU_STATUS_RUNNING);
215 		stopped =
216 		    SPU_STATUS_INVALID_INSTR | SPU_STATUS_SINGLE_STEP |
217 		    SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP;
218 		if ((in_be32(&prob->spu_status_R) & stopped) == 0)
219 			csa->prob.spu_status_R = SPU_STATUS_RUNNING;
220 		else
221 			csa->prob.spu_status_R = in_be32(&prob->spu_status_R);
222 	}
223 }
224 
225 static inline void save_mfc_decr(struct spu_state *csa, struct spu *spu)
226 {
227 	struct spu_priv2 __iomem *priv2 = spu->priv2;
228 
229 	/* Save, Step 12:
230 	 *     Read MFC_CNTL[Ds].  Update saved copy of
231 	 *     CSA.MFC_CNTL[Ds].
232 	 */
233 	if (in_be64(&priv2->mfc_control_RW) & MFC_CNTL_DECREMENTER_RUNNING) {
234 		csa->priv2.mfc_control_RW |= MFC_CNTL_DECREMENTER_RUNNING;
235 		csa->suspend_time = get_cycles();
236 		out_be64(&priv2->spu_chnlcntptr_RW, 7ULL);
237 		eieio();
238 		csa->spu_chnldata_RW[7] = in_be64(&priv2->spu_chnldata_RW);
239 		eieio();
240 	}
241 }
242 
243 static inline void halt_mfc_decr(struct spu_state *csa, struct spu *spu)
244 {
245 	struct spu_priv2 __iomem *priv2 = spu->priv2;
246 
247 	/* Save, Step 13:
248 	 *     Write MFC_CNTL[Dh] set to a '1' to halt
249 	 *     the decrementer.
250 	 */
251 	out_be64(&priv2->mfc_control_RW, MFC_CNTL_DECREMENTER_HALTED);
252 	eieio();
253 }
254 
255 static inline void save_timebase(struct spu_state *csa, struct spu *spu)
256 {
257 	/* Save, Step 14:
258 	 *    Read PPE Timebase High and Timebase low registers
259 	 *    and save in CSA.  TBD.
260 	 */
261 	csa->suspend_time = get_cycles();
262 }
263 
264 static inline void remove_other_spu_access(struct spu_state *csa,
265 					   struct spu *spu)
266 {
267 	/* Save, Step 15:
268 	 *     Remove other SPU access to this SPU by unmapping
269 	 *     this SPU's pages from their address space.  TBD.
270 	 */
271 }
272 
273 static inline void do_mfc_mssync(struct spu_state *csa, struct spu *spu)
274 {
275 	struct spu_problem __iomem *prob = spu->problem;
276 
277 	/* Save, Step 16:
278 	 * Restore, Step 11.
279 	 *     Write SPU_MSSync register. Poll SPU_MSSync[P]
280 	 *     for a value of 0.
281 	 */
282 	out_be64(&prob->spc_mssync_RW, 1UL);
283 	POLL_WHILE_TRUE(in_be64(&prob->spc_mssync_RW) & MS_SYNC_PENDING);
284 }
285 
286 static inline void issue_mfc_tlbie(struct spu_state *csa, struct spu *spu)
287 {
288 	/* Save, Step 17:
289 	 * Restore, Step 12.
290 	 * Restore, Step 48.
291 	 *     Write TLB_Invalidate_Entry[IS,VPN,L,Lp]=0 register.
292 	 *     Then issue a PPE sync instruction.
293 	 */
294 	spu_tlb_invalidate(spu);
295 	mb();
296 }
297 
298 static inline void handle_pending_interrupts(struct spu_state *csa,
299 					     struct spu *spu)
300 {
301 	/* Save, Step 18:
302 	 *     Handle any pending interrupts from this SPU
303 	 *     here.  This is OS or hypervisor specific.  One
304 	 *     option is to re-enable interrupts to handle any
305 	 *     pending interrupts, with the interrupt handlers
306 	 *     recognizing the software Context Switch Pending
307 	 *     flag, to ensure the SPU execution or MFC command
308 	 *     queue is not restarted.  TBD.
309 	 */
310 }
311 
312 static inline void save_mfc_queues(struct spu_state *csa, struct spu *spu)
313 {
314 	struct spu_priv2 __iomem *priv2 = spu->priv2;
315 	int i;
316 
317 	/* Save, Step 19:
318 	 *     If MFC_Cntl[Se]=0 then save
319 	 *     MFC command queues.
320 	 */
321 	if ((in_be64(&priv2->mfc_control_RW) & MFC_CNTL_DMA_QUEUES_EMPTY) == 0) {
322 		for (i = 0; i < 8; i++) {
323 			csa->priv2.puq[i].mfc_cq_data0_RW =
324 			    in_be64(&priv2->puq[i].mfc_cq_data0_RW);
325 			csa->priv2.puq[i].mfc_cq_data1_RW =
326 			    in_be64(&priv2->puq[i].mfc_cq_data1_RW);
327 			csa->priv2.puq[i].mfc_cq_data2_RW =
328 			    in_be64(&priv2->puq[i].mfc_cq_data2_RW);
329 			csa->priv2.puq[i].mfc_cq_data3_RW =
330 			    in_be64(&priv2->puq[i].mfc_cq_data3_RW);
331 		}
332 		for (i = 0; i < 16; i++) {
333 			csa->priv2.spuq[i].mfc_cq_data0_RW =
334 			    in_be64(&priv2->spuq[i].mfc_cq_data0_RW);
335 			csa->priv2.spuq[i].mfc_cq_data1_RW =
336 			    in_be64(&priv2->spuq[i].mfc_cq_data1_RW);
337 			csa->priv2.spuq[i].mfc_cq_data2_RW =
338 			    in_be64(&priv2->spuq[i].mfc_cq_data2_RW);
339 			csa->priv2.spuq[i].mfc_cq_data3_RW =
340 			    in_be64(&priv2->spuq[i].mfc_cq_data3_RW);
341 		}
342 	}
343 }
344 
345 static inline void save_ppu_querymask(struct spu_state *csa, struct spu *spu)
346 {
347 	struct spu_problem __iomem *prob = spu->problem;
348 
349 	/* Save, Step 20:
350 	 *     Save the PPU_QueryMask register
351 	 *     in the CSA.
352 	 */
353 	csa->prob.dma_querymask_RW = in_be32(&prob->dma_querymask_RW);
354 }
355 
356 static inline void save_ppu_querytype(struct spu_state *csa, struct spu *spu)
357 {
358 	struct spu_problem __iomem *prob = spu->problem;
359 
360 	/* Save, Step 21:
361 	 *     Save the PPU_QueryType register
362 	 *     in the CSA.
363 	 */
364 	csa->prob.dma_querytype_RW = in_be32(&prob->dma_querytype_RW);
365 }
366 
367 static inline void save_mfc_csr_tsq(struct spu_state *csa, struct spu *spu)
368 {
369 	struct spu_priv2 __iomem *priv2 = spu->priv2;
370 
371 	/* Save, Step 22:
372 	 *     Save the MFC_CSR_TSQ register
373 	 *     in the LSCSA.
374 	 */
375 	csa->priv2.spu_tag_status_query_RW =
376 	    in_be64(&priv2->spu_tag_status_query_RW);
377 }
378 
379 static inline void save_mfc_csr_cmd(struct spu_state *csa, struct spu *spu)
380 {
381 	struct spu_priv2 __iomem *priv2 = spu->priv2;
382 
383 	/* Save, Step 23:
384 	 *     Save the MFC_CSR_CMD1 and MFC_CSR_CMD2
385 	 *     registers in the CSA.
386 	 */
387 	csa->priv2.spu_cmd_buf1_RW = in_be64(&priv2->spu_cmd_buf1_RW);
388 	csa->priv2.spu_cmd_buf2_RW = in_be64(&priv2->spu_cmd_buf2_RW);
389 }
390 
391 static inline void save_mfc_csr_ato(struct spu_state *csa, struct spu *spu)
392 {
393 	struct spu_priv2 __iomem *priv2 = spu->priv2;
394 
395 	/* Save, Step 24:
396 	 *     Save the MFC_CSR_ATO register in
397 	 *     the CSA.
398 	 */
399 	csa->priv2.spu_atomic_status_RW = in_be64(&priv2->spu_atomic_status_RW);
400 }
401 
402 static inline void save_mfc_tclass_id(struct spu_state *csa, struct spu *spu)
403 {
404 	/* Save, Step 25:
405 	 *     Save the MFC_TCLASS_ID register in
406 	 *     the CSA.
407 	 */
408 	csa->priv1.mfc_tclass_id_RW = spu_mfc_tclass_id_get(spu);
409 }
410 
411 static inline void set_mfc_tclass_id(struct spu_state *csa, struct spu *spu)
412 {
413 	/* Save, Step 26:
414 	 * Restore, Step 23.
415 	 *     Write the MFC_TCLASS_ID register with
416 	 *     the value 0x10000000.
417 	 */
418 	spu_mfc_tclass_id_set(spu, 0x10000000);
419 	eieio();
420 }
421 
422 static inline void purge_mfc_queue(struct spu_state *csa, struct spu *spu)
423 {
424 	struct spu_priv2 __iomem *priv2 = spu->priv2;
425 
426 	/* Save, Step 27:
427 	 * Restore, Step 14.
428 	 *     Write MFC_CNTL[Pc]=1 (purge queue).
429 	 */
430 	out_be64(&priv2->mfc_control_RW, MFC_CNTL_PURGE_DMA_REQUEST);
431 	eieio();
432 }
433 
434 static inline void wait_purge_complete(struct spu_state *csa, struct spu *spu)
435 {
436 	struct spu_priv2 __iomem *priv2 = spu->priv2;
437 
438 	/* Save, Step 28:
439 	 *     Poll MFC_CNTL[Ps] until value '11' is read
440 	 *     (purge complete).
441 	 */
442 	POLL_WHILE_FALSE(in_be64(&priv2->mfc_control_RW) &
443 			 MFC_CNTL_PURGE_DMA_COMPLETE);
444 }
445 
446 static inline void save_mfc_slbs(struct spu_state *csa, struct spu *spu)
447 {
448 	struct spu_priv2 __iomem *priv2 = spu->priv2;
449 	int i;
450 
451 	/* Save, Step 29:
452 	 *     If MFC_SR1[R]='1', save SLBs in CSA.
453 	 */
454 	if (spu_mfc_sr1_get(spu) & MFC_STATE1_RELOCATE_MASK) {
455 		csa->priv2.slb_index_W = in_be64(&priv2->slb_index_W);
456 		for (i = 0; i < 8; i++) {
457 			out_be64(&priv2->slb_index_W, i);
458 			eieio();
459 			csa->slb_esid_RW[i] = in_be64(&priv2->slb_esid_RW);
460 			csa->slb_vsid_RW[i] = in_be64(&priv2->slb_vsid_RW);
461 			eieio();
462 		}
463 	}
464 }
465 
466 static inline void setup_mfc_sr1(struct spu_state *csa, struct spu *spu)
467 {
468 	/* Save, Step 30:
469 	 * Restore, Step 18:
470 	 *     Write MFC_SR1 with MFC_SR1[D=0,S=1] and
471 	 *     MFC_SR1[TL,R,Pr,T] set correctly for the
472 	 *     OS specific environment.
473 	 *
474 	 *     Implementation note: The SPU-side code
475 	 *     for save/restore is privileged, so the
476 	 *     MFC_SR1[Pr] bit is not set.
477 	 *
478 	 */
479 	spu_mfc_sr1_set(spu, (MFC_STATE1_MASTER_RUN_CONTROL_MASK |
480 			      MFC_STATE1_RELOCATE_MASK |
481 			      MFC_STATE1_BUS_TLBIE_MASK));
482 }
483 
484 static inline void save_spu_npc(struct spu_state *csa, struct spu *spu)
485 {
486 	struct spu_problem __iomem *prob = spu->problem;
487 
488 	/* Save, Step 31:
489 	 *     Save SPU_NPC in the CSA.
490 	 */
491 	csa->prob.spu_npc_RW = in_be32(&prob->spu_npc_RW);
492 }
493 
494 static inline void save_spu_privcntl(struct spu_state *csa, struct spu *spu)
495 {
496 	struct spu_priv2 __iomem *priv2 = spu->priv2;
497 
498 	/* Save, Step 32:
499 	 *     Save SPU_PrivCntl in the CSA.
500 	 */
501 	csa->priv2.spu_privcntl_RW = in_be64(&priv2->spu_privcntl_RW);
502 }
503 
504 static inline void reset_spu_privcntl(struct spu_state *csa, struct spu *spu)
505 {
506 	struct spu_priv2 __iomem *priv2 = spu->priv2;
507 
508 	/* Save, Step 33:
509 	 * Restore, Step 16:
510 	 *     Write SPU_PrivCntl[S,Le,A] fields reset to 0.
511 	 */
512 	out_be64(&priv2->spu_privcntl_RW, 0UL);
513 	eieio();
514 }
515 
516 static inline void save_spu_lslr(struct spu_state *csa, struct spu *spu)
517 {
518 	struct spu_priv2 __iomem *priv2 = spu->priv2;
519 
520 	/* Save, Step 34:
521 	 *     Save SPU_LSLR in the CSA.
522 	 */
523 	csa->priv2.spu_lslr_RW = in_be64(&priv2->spu_lslr_RW);
524 }
525 
526 static inline void reset_spu_lslr(struct spu_state *csa, struct spu *spu)
527 {
528 	struct spu_priv2 __iomem *priv2 = spu->priv2;
529 
530 	/* Save, Step 35:
531 	 * Restore, Step 17.
532 	 *     Reset SPU_LSLR.
533 	 */
534 	out_be64(&priv2->spu_lslr_RW, LS_ADDR_MASK);
535 	eieio();
536 }
537 
538 static inline void save_spu_cfg(struct spu_state *csa, struct spu *spu)
539 {
540 	struct spu_priv2 __iomem *priv2 = spu->priv2;
541 
542 	/* Save, Step 36:
543 	 *     Save SPU_Cfg in the CSA.
544 	 */
545 	csa->priv2.spu_cfg_RW = in_be64(&priv2->spu_cfg_RW);
546 }
547 
548 static inline void save_pm_trace(struct spu_state *csa, struct spu *spu)
549 {
550 	/* Save, Step 37:
551 	 *     Save PM_Trace_Tag_Wait_Mask in the CSA.
552 	 *     Not performed by this implementation.
553 	 */
554 }
555 
556 static inline void save_mfc_rag(struct spu_state *csa, struct spu *spu)
557 {
558 	/* Save, Step 38:
559 	 *     Save RA_GROUP_ID register and the
560 	 *     RA_ENABLE reigster in the CSA.
561 	 */
562 	csa->priv1.resource_allocation_groupID_RW =
563 		spu_resource_allocation_groupID_get(spu);
564 	csa->priv1.resource_allocation_enable_RW =
565 		spu_resource_allocation_enable_get(spu);
566 }
567 
568 static inline void save_ppu_mb_stat(struct spu_state *csa, struct spu *spu)
569 {
570 	struct spu_problem __iomem *prob = spu->problem;
571 
572 	/* Save, Step 39:
573 	 *     Save MB_Stat register in the CSA.
574 	 */
575 	csa->prob.mb_stat_R = in_be32(&prob->mb_stat_R);
576 }
577 
578 static inline void save_ppu_mb(struct spu_state *csa, struct spu *spu)
579 {
580 	struct spu_problem __iomem *prob = spu->problem;
581 
582 	/* Save, Step 40:
583 	 *     Save the PPU_MB register in the CSA.
584 	 */
585 	csa->prob.pu_mb_R = in_be32(&prob->pu_mb_R);
586 }
587 
588 static inline void save_ppuint_mb(struct spu_state *csa, struct spu *spu)
589 {
590 	struct spu_priv2 __iomem *priv2 = spu->priv2;
591 
592 	/* Save, Step 41:
593 	 *     Save the PPUINT_MB register in the CSA.
594 	 */
595 	csa->priv2.puint_mb_R = in_be64(&priv2->puint_mb_R);
596 }
597 
598 static inline void save_ch_part1(struct spu_state *csa, struct spu *spu)
599 {
600 	struct spu_priv2 __iomem *priv2 = spu->priv2;
601 	u64 idx, ch_indices[7] = { 0UL, 1UL, 3UL, 4UL, 24UL, 25UL, 27UL };
602 	int i;
603 
604 	/* Save, Step 42:
605 	 *     Save the following CH: [0,1,3,4,24,25,27]
606 	 */
607 	for (i = 0; i < 7; i++) {
608 		idx = ch_indices[i];
609 		out_be64(&priv2->spu_chnlcntptr_RW, idx);
610 		eieio();
611 		csa->spu_chnldata_RW[idx] = in_be64(&priv2->spu_chnldata_RW);
612 		csa->spu_chnlcnt_RW[idx] = in_be64(&priv2->spu_chnlcnt_RW);
613 		out_be64(&priv2->spu_chnldata_RW, 0UL);
614 		out_be64(&priv2->spu_chnlcnt_RW, 0UL);
615 		eieio();
616 	}
617 }
618 
619 static inline void save_spu_mb(struct spu_state *csa, struct spu *spu)
620 {
621 	struct spu_priv2 __iomem *priv2 = spu->priv2;
622 	int i;
623 
624 	/* Save, Step 43:
625 	 *     Save SPU Read Mailbox Channel.
626 	 */
627 	out_be64(&priv2->spu_chnlcntptr_RW, 29UL);
628 	eieio();
629 	csa->spu_chnlcnt_RW[29] = in_be64(&priv2->spu_chnlcnt_RW);
630 	for (i = 0; i < 4; i++) {
631 		csa->spu_mailbox_data[i] = in_be64(&priv2->spu_chnldata_RW);
632 	}
633 	out_be64(&priv2->spu_chnlcnt_RW, 0UL);
634 	eieio();
635 }
636 
637 static inline void save_mfc_cmd(struct spu_state *csa, struct spu *spu)
638 {
639 	struct spu_priv2 __iomem *priv2 = spu->priv2;
640 
641 	/* Save, Step 44:
642 	 *     Save MFC_CMD Channel.
643 	 */
644 	out_be64(&priv2->spu_chnlcntptr_RW, 21UL);
645 	eieio();
646 	csa->spu_chnlcnt_RW[21] = in_be64(&priv2->spu_chnlcnt_RW);
647 	eieio();
648 }
649 
650 static inline void reset_ch(struct spu_state *csa, struct spu *spu)
651 {
652 	struct spu_priv2 __iomem *priv2 = spu->priv2;
653 	u64 ch_indices[4] = { 21UL, 23UL, 28UL, 30UL };
654 	u64 ch_counts[4] = { 16UL, 1UL, 1UL, 1UL };
655 	u64 idx;
656 	int i;
657 
658 	/* Save, Step 45:
659 	 *     Reset the following CH: [21, 23, 28, 30]
660 	 */
661 	for (i = 0; i < 4; i++) {
662 		idx = ch_indices[i];
663 		out_be64(&priv2->spu_chnlcntptr_RW, idx);
664 		eieio();
665 		out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]);
666 		eieio();
667 	}
668 }
669 
670 static inline void resume_mfc_queue(struct spu_state *csa, struct spu *spu)
671 {
672 	struct spu_priv2 __iomem *priv2 = spu->priv2;
673 
674 	/* Save, Step 46:
675 	 * Restore, Step 25.
676 	 *     Write MFC_CNTL[Sc]=0 (resume queue processing).
677 	 */
678 	out_be64(&priv2->mfc_control_RW, MFC_CNTL_RESUME_DMA_QUEUE);
679 }
680 
681 static inline void invalidate_slbs(struct spu_state *csa, struct spu *spu)
682 {
683 	struct spu_priv2 __iomem *priv2 = spu->priv2;
684 
685 	/* Save, Step 45:
686 	 * Restore, Step 19:
687 	 *     If MFC_SR1[R]=1, write 0 to SLB_Invalidate_All.
688 	 */
689 	if (spu_mfc_sr1_get(spu) & MFC_STATE1_RELOCATE_MASK) {
690 		out_be64(&priv2->slb_invalidate_all_W, 0UL);
691 		eieio();
692 	}
693 }
694 
695 static inline void get_kernel_slb(u64 ea, u64 slb[2])
696 {
697 	slb[0] = (get_kernel_vsid(ea) << SLB_VSID_SHIFT) | SLB_VSID_KERNEL;
698 	slb[1] = (ea & ESID_MASK) | SLB_ESID_V;
699 
700 	/* Large pages are used for kernel text/data, but not vmalloc.  */
701 	if (cpu_has_feature(CPU_FTR_16M_PAGE)
702 	    && REGION_ID(ea) == KERNEL_REGION_ID)
703 		slb[0] |= SLB_VSID_L;
704 }
705 
706 static inline void load_mfc_slb(struct spu *spu, u64 slb[2], int slbe)
707 {
708 	struct spu_priv2 __iomem *priv2 = spu->priv2;
709 
710 	out_be64(&priv2->slb_index_W, slbe);
711 	eieio();
712 	out_be64(&priv2->slb_vsid_RW, slb[0]);
713 	out_be64(&priv2->slb_esid_RW, slb[1]);
714 	eieio();
715 }
716 
717 static inline void setup_mfc_slbs(struct spu_state *csa, struct spu *spu)
718 {
719 	u64 code_slb[2];
720 	u64 lscsa_slb[2];
721 
722 	/* Save, Step 47:
723 	 * Restore, Step 30.
724 	 *     If MFC_SR1[R]=1, write 0 to SLB_Invalidate_All
725 	 *     register, then initialize SLB_VSID and SLB_ESID
726 	 *     to provide access to SPU context save code and
727 	 *     LSCSA.
728 	 *
729 	 *     This implementation places both the context
730 	 *     switch code and LSCSA in kernel address space.
731 	 *
732 	 *     Further this implementation assumes that the
733 	 *     MFC_SR1[R]=1 (in other words, assume that
734 	 *     translation is desired by OS environment).
735 	 */
736 	invalidate_slbs(csa, spu);
737 	get_kernel_slb((unsigned long)&spu_save_code[0], code_slb);
738 	get_kernel_slb((unsigned long)csa->lscsa, lscsa_slb);
739 	load_mfc_slb(spu, code_slb, 0);
740 	if ((lscsa_slb[0] != code_slb[0]) || (lscsa_slb[1] != code_slb[1]))
741 		load_mfc_slb(spu, lscsa_slb, 1);
742 }
743 
744 static inline void set_switch_active(struct spu_state *csa, struct spu *spu)
745 {
746 	/* Save, Step 48:
747 	 * Restore, Step 23.
748 	 *     Change the software context switch pending flag
749 	 *     to context switch active.
750 	 */
751 	set_bit(SPU_CONTEXT_SWITCH_ACTIVE, &spu->flags);
752 	clear_bit(SPU_CONTEXT_SWITCH_PENDING, &spu->flags);
753 	mb();
754 }
755 
756 static inline void enable_interrupts(struct spu_state *csa, struct spu *spu)
757 {
758 	unsigned long class1_mask = CLASS1_ENABLE_SEGMENT_FAULT_INTR |
759 	    CLASS1_ENABLE_STORAGE_FAULT_INTR;
760 
761 	/* Save, Step 49:
762 	 * Restore, Step 22:
763 	 *     Reset and then enable interrupts, as
764 	 *     needed by OS.
765 	 *
766 	 *     This implementation enables only class1
767 	 *     (translation) interrupts.
768 	 */
769 	spin_lock_irq(&spu->register_lock);
770 	spu_int_stat_clear(spu, 0, ~0ul);
771 	spu_int_stat_clear(spu, 1, ~0ul);
772 	spu_int_stat_clear(spu, 2, ~0ul);
773 	spu_int_mask_set(spu, 0, 0ul);
774 	spu_int_mask_set(spu, 1, class1_mask);
775 	spu_int_mask_set(spu, 2, 0ul);
776 	spin_unlock_irq(&spu->register_lock);
777 }
778 
779 static inline int send_mfc_dma(struct spu *spu, unsigned long ea,
780 			       unsigned int ls_offset, unsigned int size,
781 			       unsigned int tag, unsigned int rclass,
782 			       unsigned int cmd)
783 {
784 	struct spu_problem __iomem *prob = spu->problem;
785 	union mfc_tag_size_class_cmd command;
786 	unsigned int transfer_size;
787 	volatile unsigned int status = 0x0;
788 
789 	while (size > 0) {
790 		transfer_size =
791 		    (size > MFC_MAX_DMA_SIZE) ? MFC_MAX_DMA_SIZE : size;
792 		command.u.mfc_size = transfer_size;
793 		command.u.mfc_tag = tag;
794 		command.u.mfc_rclassid = rclass;
795 		command.u.mfc_cmd = cmd;
796 		do {
797 			out_be32(&prob->mfc_lsa_W, ls_offset);
798 			out_be64(&prob->mfc_ea_W, ea);
799 			out_be64(&prob->mfc_union_W.all64, command.all64);
800 			status =
801 			    in_be32(&prob->mfc_union_W.by32.mfc_class_cmd32);
802 			if (unlikely(status & 0x2)) {
803 				cpu_relax();
804 			}
805 		} while (status & 0x3);
806 		size -= transfer_size;
807 		ea += transfer_size;
808 		ls_offset += transfer_size;
809 	}
810 	return 0;
811 }
812 
813 static inline void save_ls_16kb(struct spu_state *csa, struct spu *spu)
814 {
815 	unsigned long addr = (unsigned long)&csa->lscsa->ls[0];
816 	unsigned int ls_offset = 0x0;
817 	unsigned int size = 16384;
818 	unsigned int tag = 0;
819 	unsigned int rclass = 0;
820 	unsigned int cmd = MFC_PUT_CMD;
821 
822 	/* Save, Step 50:
823 	 *     Issue a DMA command to copy the first 16K bytes
824 	 *     of local storage to the CSA.
825 	 */
826 	send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd);
827 }
828 
829 static inline void set_spu_npc(struct spu_state *csa, struct spu *spu)
830 {
831 	struct spu_problem __iomem *prob = spu->problem;
832 
833 	/* Save, Step 51:
834 	 * Restore, Step 31.
835 	 *     Write SPU_NPC[IE]=0 and SPU_NPC[LSA] to entry
836 	 *     point address of context save code in local
837 	 *     storage.
838 	 *
839 	 *     This implementation uses SPU-side save/restore
840 	 *     programs with entry points at LSA of 0.
841 	 */
842 	out_be32(&prob->spu_npc_RW, 0);
843 	eieio();
844 }
845 
846 static inline void set_signot1(struct spu_state *csa, struct spu *spu)
847 {
848 	struct spu_problem __iomem *prob = spu->problem;
849 	union {
850 		u64 ull;
851 		u32 ui[2];
852 	} addr64;
853 
854 	/* Save, Step 52:
855 	 * Restore, Step 32:
856 	 *    Write SPU_Sig_Notify_1 register with upper 32-bits
857 	 *    of the CSA.LSCSA effective address.
858 	 */
859 	addr64.ull = (u64) csa->lscsa;
860 	out_be32(&prob->signal_notify1, addr64.ui[0]);
861 	eieio();
862 }
863 
864 static inline void set_signot2(struct spu_state *csa, struct spu *spu)
865 {
866 	struct spu_problem __iomem *prob = spu->problem;
867 	union {
868 		u64 ull;
869 		u32 ui[2];
870 	} addr64;
871 
872 	/* Save, Step 53:
873 	 * Restore, Step 33:
874 	 *    Write SPU_Sig_Notify_2 register with lower 32-bits
875 	 *    of the CSA.LSCSA effective address.
876 	 */
877 	addr64.ull = (u64) csa->lscsa;
878 	out_be32(&prob->signal_notify2, addr64.ui[1]);
879 	eieio();
880 }
881 
882 static inline void send_save_code(struct spu_state *csa, struct spu *spu)
883 {
884 	unsigned long addr = (unsigned long)&spu_save_code[0];
885 	unsigned int ls_offset = 0x0;
886 	unsigned int size = sizeof(spu_save_code);
887 	unsigned int tag = 0;
888 	unsigned int rclass = 0;
889 	unsigned int cmd = MFC_GETFS_CMD;
890 
891 	/* Save, Step 54:
892 	 *     Issue a DMA command to copy context save code
893 	 *     to local storage and start SPU.
894 	 */
895 	send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd);
896 }
897 
898 static inline void set_ppu_querymask(struct spu_state *csa, struct spu *spu)
899 {
900 	struct spu_problem __iomem *prob = spu->problem;
901 
902 	/* Save, Step 55:
903 	 * Restore, Step 38.
904 	 *     Write PPU_QueryMask=1 (enable Tag Group 0)
905 	 *     and issue eieio instruction.
906 	 */
907 	out_be32(&prob->dma_querymask_RW, MFC_TAGID_TO_TAGMASK(0));
908 	eieio();
909 }
910 
911 static inline void wait_tag_complete(struct spu_state *csa, struct spu *spu)
912 {
913 	struct spu_problem __iomem *prob = spu->problem;
914 	u32 mask = MFC_TAGID_TO_TAGMASK(0);
915 	unsigned long flags;
916 
917 	/* Save, Step 56:
918 	 * Restore, Step 39.
919 	 * Restore, Step 39.
920 	 * Restore, Step 46.
921 	 *     Poll PPU_TagStatus[gn] until 01 (Tag group 0 complete)
922 	 *     or write PPU_QueryType[TS]=01 and wait for Tag Group
923 	 *     Complete Interrupt.  Write INT_Stat_Class0 or
924 	 *     INT_Stat_Class2 with value of 'handled'.
925 	 */
926 	POLL_WHILE_FALSE(in_be32(&prob->dma_tagstatus_R) & mask);
927 
928 	local_irq_save(flags);
929 	spu_int_stat_clear(spu, 0, ~(0ul));
930 	spu_int_stat_clear(spu, 2, ~(0ul));
931 	local_irq_restore(flags);
932 }
933 
934 static inline void wait_spu_stopped(struct spu_state *csa, struct spu *spu)
935 {
936 	struct spu_problem __iomem *prob = spu->problem;
937 	unsigned long flags;
938 
939 	/* Save, Step 57:
940 	 * Restore, Step 40.
941 	 *     Poll until SPU_Status[R]=0 or wait for SPU Class 0
942 	 *     or SPU Class 2 interrupt.  Write INT_Stat_class0
943 	 *     or INT_Stat_class2 with value of handled.
944 	 */
945 	POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING);
946 
947 	local_irq_save(flags);
948 	spu_int_stat_clear(spu, 0, ~(0ul));
949 	spu_int_stat_clear(spu, 2, ~(0ul));
950 	local_irq_restore(flags);
951 }
952 
953 static inline int check_save_status(struct spu_state *csa, struct spu *spu)
954 {
955 	struct spu_problem __iomem *prob = spu->problem;
956 	u32 complete;
957 
958 	/* Save, Step 54:
959 	 *     If SPU_Status[P]=1 and SPU_Status[SC] = "success",
960 	 *     context save succeeded, otherwise context save
961 	 *     failed.
962 	 */
963 	complete = ((SPU_SAVE_COMPLETE << SPU_STOP_STATUS_SHIFT) |
964 		    SPU_STATUS_STOPPED_BY_STOP);
965 	return (in_be32(&prob->spu_status_R) != complete) ? 1 : 0;
966 }
967 
968 static inline void terminate_spu_app(struct spu_state *csa, struct spu *spu)
969 {
970 	/* Restore, Step 4:
971 	 *    If required, notify the "using application" that
972 	 *    the SPU task has been terminated.  TBD.
973 	 */
974 }
975 
976 static inline void suspend_mfc(struct spu_state *csa, struct spu *spu)
977 {
978 	struct spu_priv2 __iomem *priv2 = spu->priv2;
979 
980 	/* Restore, Step 7:
981 	 * Restore, Step 47.
982 	 *     Write MFC_Cntl[Dh,Sc]='1','1' to suspend
983 	 *     the queue and halt the decrementer.
984 	 */
985 	out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE |
986 		 MFC_CNTL_DECREMENTER_HALTED);
987 	eieio();
988 }
989 
990 static inline void wait_suspend_mfc_complete(struct spu_state *csa,
991 					     struct spu *spu)
992 {
993 	struct spu_priv2 __iomem *priv2 = spu->priv2;
994 
995 	/* Restore, Step 8:
996 	 * Restore, Step 47.
997 	 *     Poll MFC_CNTL[Ss] until 11 is returned.
998 	 */
999 	POLL_WHILE_FALSE(in_be64(&priv2->mfc_control_RW) &
1000 			 MFC_CNTL_SUSPEND_COMPLETE);
1001 }
1002 
1003 static inline int suspend_spe(struct spu_state *csa, struct spu *spu)
1004 {
1005 	struct spu_problem __iomem *prob = spu->problem;
1006 
1007 	/* Restore, Step 9:
1008 	 *    If SPU_Status[R]=1, stop SPU execution
1009 	 *    and wait for stop to complete.
1010 	 *
1011 	 *    Returns       1 if SPU_Status[R]=1 on entry.
1012 	 *                  0 otherwise
1013 	 */
1014 	if (in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) {
1015 		if (in_be32(&prob->spu_status_R) &
1016 		    SPU_STATUS_ISOLATED_EXIT_STAUTUS) {
1017 			POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1018 					SPU_STATUS_RUNNING);
1019 		}
1020 		if ((in_be32(&prob->spu_status_R) &
1021 		     SPU_STATUS_ISOLATED_LOAD_STAUTUS)
1022 		    || (in_be32(&prob->spu_status_R) &
1023 			SPU_STATUS_ISOLATED_STATE)) {
1024 			out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
1025 			eieio();
1026 			POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1027 					SPU_STATUS_RUNNING);
1028 			out_be32(&prob->spu_runcntl_RW, 0x2);
1029 			eieio();
1030 			POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1031 					SPU_STATUS_RUNNING);
1032 		}
1033 		if (in_be32(&prob->spu_status_R) &
1034 		    SPU_STATUS_WAITING_FOR_CHANNEL) {
1035 			out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
1036 			eieio();
1037 			POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1038 					SPU_STATUS_RUNNING);
1039 		}
1040 		return 1;
1041 	}
1042 	return 0;
1043 }
1044 
1045 static inline void clear_spu_status(struct spu_state *csa, struct spu *spu)
1046 {
1047 	struct spu_problem __iomem *prob = spu->problem;
1048 
1049 	/* Restore, Step 10:
1050 	 *    If SPU_Status[R]=0 and SPU_Status[E,L,IS]=1,
1051 	 *    release SPU from isolate state.
1052 	 */
1053 	if (!(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING)) {
1054 		if (in_be32(&prob->spu_status_R) &
1055 		    SPU_STATUS_ISOLATED_EXIT_STAUTUS) {
1056 			spu_mfc_sr1_set(spu,
1057 					MFC_STATE1_MASTER_RUN_CONTROL_MASK);
1058 			eieio();
1059 			out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE);
1060 			eieio();
1061 			POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1062 					SPU_STATUS_RUNNING);
1063 		}
1064 		if ((in_be32(&prob->spu_status_R) &
1065 		     SPU_STATUS_ISOLATED_LOAD_STAUTUS)
1066 		    || (in_be32(&prob->spu_status_R) &
1067 			SPU_STATUS_ISOLATED_STATE)) {
1068 			spu_mfc_sr1_set(spu,
1069 					MFC_STATE1_MASTER_RUN_CONTROL_MASK);
1070 			eieio();
1071 			out_be32(&prob->spu_runcntl_RW, 0x2);
1072 			eieio();
1073 			POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1074 					SPU_STATUS_RUNNING);
1075 		}
1076 	}
1077 }
1078 
1079 static inline void reset_ch_part1(struct spu_state *csa, struct spu *spu)
1080 {
1081 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1082 	u64 ch_indices[7] = { 0UL, 1UL, 3UL, 4UL, 24UL, 25UL, 27UL };
1083 	u64 idx;
1084 	int i;
1085 
1086 	/* Restore, Step 20:
1087 	 *     Reset the following CH: [0,1,3,4,24,25,27]
1088 	 */
1089 	for (i = 0; i < 7; i++) {
1090 		idx = ch_indices[i];
1091 		out_be64(&priv2->spu_chnlcntptr_RW, idx);
1092 		eieio();
1093 		out_be64(&priv2->spu_chnldata_RW, 0UL);
1094 		out_be64(&priv2->spu_chnlcnt_RW, 0UL);
1095 		eieio();
1096 	}
1097 }
1098 
1099 static inline void reset_ch_part2(struct spu_state *csa, struct spu *spu)
1100 {
1101 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1102 	u64 ch_indices[5] = { 21UL, 23UL, 28UL, 29UL, 30UL };
1103 	u64 ch_counts[5] = { 16UL, 1UL, 1UL, 0UL, 1UL };
1104 	u64 idx;
1105 	int i;
1106 
1107 	/* Restore, Step 21:
1108 	 *     Reset the following CH: [21, 23, 28, 29, 30]
1109 	 */
1110 	for (i = 0; i < 5; i++) {
1111 		idx = ch_indices[i];
1112 		out_be64(&priv2->spu_chnlcntptr_RW, idx);
1113 		eieio();
1114 		out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]);
1115 		eieio();
1116 	}
1117 }
1118 
1119 static inline void setup_spu_status_part1(struct spu_state *csa,
1120 					  struct spu *spu)
1121 {
1122 	u32 status_P = SPU_STATUS_STOPPED_BY_STOP;
1123 	u32 status_I = SPU_STATUS_INVALID_INSTR;
1124 	u32 status_H = SPU_STATUS_STOPPED_BY_HALT;
1125 	u32 status_S = SPU_STATUS_SINGLE_STEP;
1126 	u32 status_S_I = SPU_STATUS_SINGLE_STEP | SPU_STATUS_INVALID_INSTR;
1127 	u32 status_S_P = SPU_STATUS_SINGLE_STEP | SPU_STATUS_STOPPED_BY_STOP;
1128 	u32 status_P_H = SPU_STATUS_STOPPED_BY_HALT |SPU_STATUS_STOPPED_BY_STOP;
1129 	u32 status_P_I = SPU_STATUS_STOPPED_BY_STOP |SPU_STATUS_INVALID_INSTR;
1130 	u32 status_code;
1131 
1132 	/* Restore, Step 27:
1133 	 *     If the CSA.SPU_Status[I,S,H,P]=1 then add the correct
1134 	 *     instruction sequence to the end of the SPU based restore
1135 	 *     code (after the "context restored" stop and signal) to
1136 	 *     restore the correct SPU status.
1137 	 *
1138 	 *     NOTE: Rather than modifying the SPU executable, we
1139 	 *     instead add a new 'stopped_status' field to the
1140 	 *     LSCSA.  The SPU-side restore reads this field and
1141 	 *     takes the appropriate action when exiting.
1142 	 */
1143 
1144 	status_code =
1145 	    (csa->prob.spu_status_R >> SPU_STOP_STATUS_SHIFT) & 0xFFFF;
1146 	if ((csa->prob.spu_status_R & status_P_I) == status_P_I) {
1147 
1148 		/* SPU_Status[P,I]=1 - Illegal Instruction followed
1149 		 * by Stop and Signal instruction, followed by 'br -4'.
1150 		 *
1151 		 */
1152 		csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P_I;
1153 		csa->lscsa->stopped_status.slot[1] = status_code;
1154 
1155 	} else if ((csa->prob.spu_status_R & status_P_H) == status_P_H) {
1156 
1157 		/* SPU_Status[P,H]=1 - Halt Conditional, followed
1158 		 * by Stop and Signal instruction, followed by
1159 		 * 'br -4'.
1160 		 */
1161 		csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P_H;
1162 		csa->lscsa->stopped_status.slot[1] = status_code;
1163 
1164 	} else if ((csa->prob.spu_status_R & status_S_P) == status_S_P) {
1165 
1166 		/* SPU_Status[S,P]=1 - Stop and Signal instruction
1167 		 * followed by 'br -4'.
1168 		 */
1169 		csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S_P;
1170 		csa->lscsa->stopped_status.slot[1] = status_code;
1171 
1172 	} else if ((csa->prob.spu_status_R & status_S_I) == status_S_I) {
1173 
1174 		/* SPU_Status[S,I]=1 - Illegal instruction followed
1175 		 * by 'br -4'.
1176 		 */
1177 		csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S_I;
1178 		csa->lscsa->stopped_status.slot[1] = status_code;
1179 
1180 	} else if ((csa->prob.spu_status_R & status_P) == status_P) {
1181 
1182 		/* SPU_Status[P]=1 - Stop and Signal instruction
1183 		 * followed by 'br -4'.
1184 		 */
1185 		csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P;
1186 		csa->lscsa->stopped_status.slot[1] = status_code;
1187 
1188 	} else if ((csa->prob.spu_status_R & status_H) == status_H) {
1189 
1190 		/* SPU_Status[H]=1 - Halt Conditional, followed
1191 		 * by 'br -4'.
1192 		 */
1193 		csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_H;
1194 
1195 	} else if ((csa->prob.spu_status_R & status_S) == status_S) {
1196 
1197 		/* SPU_Status[S]=1 - Two nop instructions.
1198 		 */
1199 		csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S;
1200 
1201 	} else if ((csa->prob.spu_status_R & status_I) == status_I) {
1202 
1203 		/* SPU_Status[I]=1 - Illegal instruction followed
1204 		 * by 'br -4'.
1205 		 */
1206 		csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_I;
1207 
1208 	}
1209 }
1210 
1211 static inline void setup_spu_status_part2(struct spu_state *csa,
1212 					  struct spu *spu)
1213 {
1214 	u32 mask;
1215 
1216 	/* Restore, Step 28:
1217 	 *     If the CSA.SPU_Status[I,S,H,P,R]=0 then
1218 	 *     add a 'br *' instruction to the end of
1219 	 *     the SPU based restore code.
1220 	 *
1221 	 *     NOTE: Rather than modifying the SPU executable, we
1222 	 *     instead add a new 'stopped_status' field to the
1223 	 *     LSCSA.  The SPU-side restore reads this field and
1224 	 *     takes the appropriate action when exiting.
1225 	 */
1226 	mask = SPU_STATUS_INVALID_INSTR |
1227 	    SPU_STATUS_SINGLE_STEP |
1228 	    SPU_STATUS_STOPPED_BY_HALT |
1229 	    SPU_STATUS_STOPPED_BY_STOP | SPU_STATUS_RUNNING;
1230 	if (!(csa->prob.spu_status_R & mask)) {
1231 		csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_R;
1232 	}
1233 }
1234 
1235 static inline void restore_mfc_rag(struct spu_state *csa, struct spu *spu)
1236 {
1237 	/* Restore, Step 29:
1238 	 *     Restore RA_GROUP_ID register and the
1239 	 *     RA_ENABLE reigster from the CSA.
1240 	 */
1241 	spu_resource_allocation_groupID_set(spu,
1242 			csa->priv1.resource_allocation_groupID_RW);
1243 	spu_resource_allocation_enable_set(spu,
1244 			csa->priv1.resource_allocation_enable_RW);
1245 }
1246 
1247 static inline void send_restore_code(struct spu_state *csa, struct spu *spu)
1248 {
1249 	unsigned long addr = (unsigned long)&spu_restore_code[0];
1250 	unsigned int ls_offset = 0x0;
1251 	unsigned int size = sizeof(spu_restore_code);
1252 	unsigned int tag = 0;
1253 	unsigned int rclass = 0;
1254 	unsigned int cmd = MFC_GETFS_CMD;
1255 
1256 	/* Restore, Step 37:
1257 	 *     Issue MFC DMA command to copy context
1258 	 *     restore code to local storage.
1259 	 */
1260 	send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd);
1261 }
1262 
1263 static inline void setup_decr(struct spu_state *csa, struct spu *spu)
1264 {
1265 	/* Restore, Step 34:
1266 	 *     If CSA.MFC_CNTL[Ds]=1 (decrementer was
1267 	 *     running) then adjust decrementer, set
1268 	 *     decrementer running status in LSCSA,
1269 	 *     and set decrementer "wrapped" status
1270 	 *     in LSCSA.
1271 	 */
1272 	if (csa->priv2.mfc_control_RW & MFC_CNTL_DECREMENTER_RUNNING) {
1273 		cycles_t resume_time = get_cycles();
1274 		cycles_t delta_time = resume_time - csa->suspend_time;
1275 
1276 		csa->lscsa->decr.slot[0] = delta_time;
1277 	}
1278 }
1279 
1280 static inline void setup_ppu_mb(struct spu_state *csa, struct spu *spu)
1281 {
1282 	/* Restore, Step 35:
1283 	 *     Copy the CSA.PU_MB data into the LSCSA.
1284 	 */
1285 	csa->lscsa->ppu_mb.slot[0] = csa->prob.pu_mb_R;
1286 }
1287 
1288 static inline void setup_ppuint_mb(struct spu_state *csa, struct spu *spu)
1289 {
1290 	/* Restore, Step 36:
1291 	 *     Copy the CSA.PUINT_MB data into the LSCSA.
1292 	 */
1293 	csa->lscsa->ppuint_mb.slot[0] = csa->priv2.puint_mb_R;
1294 }
1295 
1296 static inline int check_restore_status(struct spu_state *csa, struct spu *spu)
1297 {
1298 	struct spu_problem __iomem *prob = spu->problem;
1299 	u32 complete;
1300 
1301 	/* Restore, Step 40:
1302 	 *     If SPU_Status[P]=1 and SPU_Status[SC] = "success",
1303 	 *     context restore succeeded, otherwise context restore
1304 	 *     failed.
1305 	 */
1306 	complete = ((SPU_RESTORE_COMPLETE << SPU_STOP_STATUS_SHIFT) |
1307 		    SPU_STATUS_STOPPED_BY_STOP);
1308 	return (in_be32(&prob->spu_status_R) != complete) ? 1 : 0;
1309 }
1310 
1311 static inline void restore_spu_privcntl(struct spu_state *csa, struct spu *spu)
1312 {
1313 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1314 
1315 	/* Restore, Step 41:
1316 	 *     Restore SPU_PrivCntl from the CSA.
1317 	 */
1318 	out_be64(&priv2->spu_privcntl_RW, csa->priv2.spu_privcntl_RW);
1319 	eieio();
1320 }
1321 
1322 static inline void restore_status_part1(struct spu_state *csa, struct spu *spu)
1323 {
1324 	struct spu_problem __iomem *prob = spu->problem;
1325 	u32 mask;
1326 
1327 	/* Restore, Step 42:
1328 	 *     If any CSA.SPU_Status[I,S,H,P]=1, then
1329 	 *     restore the error or single step state.
1330 	 */
1331 	mask = SPU_STATUS_INVALID_INSTR |
1332 	    SPU_STATUS_SINGLE_STEP |
1333 	    SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP;
1334 	if (csa->prob.spu_status_R & mask) {
1335 		out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE);
1336 		eieio();
1337 		POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1338 				SPU_STATUS_RUNNING);
1339 	}
1340 }
1341 
1342 static inline void restore_status_part2(struct spu_state *csa, struct spu *spu)
1343 {
1344 	struct spu_problem __iomem *prob = spu->problem;
1345 	u32 mask;
1346 
1347 	/* Restore, Step 43:
1348 	 *     If all CSA.SPU_Status[I,S,H,P,R]=0 then write
1349 	 *     SPU_RunCntl[R0R1]='01', wait for SPU_Status[R]=1,
1350 	 *     then write '00' to SPU_RunCntl[R0R1] and wait
1351 	 *     for SPU_Status[R]=0.
1352 	 */
1353 	mask = SPU_STATUS_INVALID_INSTR |
1354 	    SPU_STATUS_SINGLE_STEP |
1355 	    SPU_STATUS_STOPPED_BY_HALT |
1356 	    SPU_STATUS_STOPPED_BY_STOP | SPU_STATUS_RUNNING;
1357 	if (!(csa->prob.spu_status_R & mask)) {
1358 		out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE);
1359 		eieio();
1360 		POLL_WHILE_FALSE(in_be32(&prob->spu_status_R) &
1361 				 SPU_STATUS_RUNNING);
1362 		out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
1363 		eieio();
1364 		POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1365 				SPU_STATUS_RUNNING);
1366 	}
1367 }
1368 
1369 static inline void restore_ls_16kb(struct spu_state *csa, struct spu *spu)
1370 {
1371 	unsigned long addr = (unsigned long)&csa->lscsa->ls[0];
1372 	unsigned int ls_offset = 0x0;
1373 	unsigned int size = 16384;
1374 	unsigned int tag = 0;
1375 	unsigned int rclass = 0;
1376 	unsigned int cmd = MFC_GET_CMD;
1377 
1378 	/* Restore, Step 44:
1379 	 *     Issue a DMA command to restore the first
1380 	 *     16kb of local storage from CSA.
1381 	 */
1382 	send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd);
1383 }
1384 
1385 static inline void clear_interrupts(struct spu_state *csa, struct spu *spu)
1386 {
1387 	/* Restore, Step 49:
1388 	 *     Write INT_MASK_class0 with value of 0.
1389 	 *     Write INT_MASK_class1 with value of 0.
1390 	 *     Write INT_MASK_class2 with value of 0.
1391 	 *     Write INT_STAT_class0 with value of -1.
1392 	 *     Write INT_STAT_class1 with value of -1.
1393 	 *     Write INT_STAT_class2 with value of -1.
1394 	 */
1395 	spin_lock_irq(&spu->register_lock);
1396 	spu_int_mask_set(spu, 0, 0ul);
1397 	spu_int_mask_set(spu, 1, 0ul);
1398 	spu_int_mask_set(spu, 2, 0ul);
1399 	spu_int_stat_clear(spu, 0, ~0ul);
1400 	spu_int_stat_clear(spu, 1, ~0ul);
1401 	spu_int_stat_clear(spu, 2, ~0ul);
1402 	spin_unlock_irq(&spu->register_lock);
1403 }
1404 
1405 static inline void restore_mfc_queues(struct spu_state *csa, struct spu *spu)
1406 {
1407 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1408 	int i;
1409 
1410 	/* Restore, Step 50:
1411 	 *     If MFC_Cntl[Se]!=0 then restore
1412 	 *     MFC command queues.
1413 	 */
1414 	if ((csa->priv2.mfc_control_RW & MFC_CNTL_DMA_QUEUES_EMPTY_MASK) == 0) {
1415 		for (i = 0; i < 8; i++) {
1416 			out_be64(&priv2->puq[i].mfc_cq_data0_RW,
1417 				 csa->priv2.puq[i].mfc_cq_data0_RW);
1418 			out_be64(&priv2->puq[i].mfc_cq_data1_RW,
1419 				 csa->priv2.puq[i].mfc_cq_data1_RW);
1420 			out_be64(&priv2->puq[i].mfc_cq_data2_RW,
1421 				 csa->priv2.puq[i].mfc_cq_data2_RW);
1422 			out_be64(&priv2->puq[i].mfc_cq_data3_RW,
1423 				 csa->priv2.puq[i].mfc_cq_data3_RW);
1424 		}
1425 		for (i = 0; i < 16; i++) {
1426 			out_be64(&priv2->spuq[i].mfc_cq_data0_RW,
1427 				 csa->priv2.spuq[i].mfc_cq_data0_RW);
1428 			out_be64(&priv2->spuq[i].mfc_cq_data1_RW,
1429 				 csa->priv2.spuq[i].mfc_cq_data1_RW);
1430 			out_be64(&priv2->spuq[i].mfc_cq_data2_RW,
1431 				 csa->priv2.spuq[i].mfc_cq_data2_RW);
1432 			out_be64(&priv2->spuq[i].mfc_cq_data3_RW,
1433 				 csa->priv2.spuq[i].mfc_cq_data3_RW);
1434 		}
1435 	}
1436 	eieio();
1437 }
1438 
1439 static inline void restore_ppu_querymask(struct spu_state *csa, struct spu *spu)
1440 {
1441 	struct spu_problem __iomem *prob = spu->problem;
1442 
1443 	/* Restore, Step 51:
1444 	 *     Restore the PPU_QueryMask register from CSA.
1445 	 */
1446 	out_be32(&prob->dma_querymask_RW, csa->prob.dma_querymask_RW);
1447 	eieio();
1448 }
1449 
1450 static inline void restore_ppu_querytype(struct spu_state *csa, struct spu *spu)
1451 {
1452 	struct spu_problem __iomem *prob = spu->problem;
1453 
1454 	/* Restore, Step 52:
1455 	 *     Restore the PPU_QueryType register from CSA.
1456 	 */
1457 	out_be32(&prob->dma_querytype_RW, csa->prob.dma_querytype_RW);
1458 	eieio();
1459 }
1460 
1461 static inline void restore_mfc_csr_tsq(struct spu_state *csa, struct spu *spu)
1462 {
1463 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1464 
1465 	/* Restore, Step 53:
1466 	 *     Restore the MFC_CSR_TSQ register from CSA.
1467 	 */
1468 	out_be64(&priv2->spu_tag_status_query_RW,
1469 		 csa->priv2.spu_tag_status_query_RW);
1470 	eieio();
1471 }
1472 
1473 static inline void restore_mfc_csr_cmd(struct spu_state *csa, struct spu *spu)
1474 {
1475 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1476 
1477 	/* Restore, Step 54:
1478 	 *     Restore the MFC_CSR_CMD1 and MFC_CSR_CMD2
1479 	 *     registers from CSA.
1480 	 */
1481 	out_be64(&priv2->spu_cmd_buf1_RW, csa->priv2.spu_cmd_buf1_RW);
1482 	out_be64(&priv2->spu_cmd_buf2_RW, csa->priv2.spu_cmd_buf2_RW);
1483 	eieio();
1484 }
1485 
1486 static inline void restore_mfc_csr_ato(struct spu_state *csa, struct spu *spu)
1487 {
1488 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1489 
1490 	/* Restore, Step 55:
1491 	 *     Restore the MFC_CSR_ATO register from CSA.
1492 	 */
1493 	out_be64(&priv2->spu_atomic_status_RW, csa->priv2.spu_atomic_status_RW);
1494 }
1495 
1496 static inline void restore_mfc_tclass_id(struct spu_state *csa, struct spu *spu)
1497 {
1498 	/* Restore, Step 56:
1499 	 *     Restore the MFC_TCLASS_ID register from CSA.
1500 	 */
1501 	spu_mfc_tclass_id_set(spu, csa->priv1.mfc_tclass_id_RW);
1502 	eieio();
1503 }
1504 
1505 static inline void set_llr_event(struct spu_state *csa, struct spu *spu)
1506 {
1507 	u64 ch0_cnt, ch0_data;
1508 	u64 ch1_data;
1509 
1510 	/* Restore, Step 57:
1511 	 *    Set the Lock Line Reservation Lost Event by:
1512 	 *      1. OR CSA.SPU_Event_Status with bit 21 (Lr) set to 1.
1513 	 *      2. If CSA.SPU_Channel_0_Count=0 and
1514 	 *         CSA.SPU_Wr_Event_Mask[Lr]=1 and
1515 	 *         CSA.SPU_Event_Status[Lr]=0 then set
1516 	 *         CSA.SPU_Event_Status_Count=1.
1517 	 */
1518 	ch0_cnt = csa->spu_chnlcnt_RW[0];
1519 	ch0_data = csa->spu_chnldata_RW[0];
1520 	ch1_data = csa->spu_chnldata_RW[1];
1521 	csa->spu_chnldata_RW[0] |= MFC_LLR_LOST_EVENT;
1522 	if ((ch0_cnt == 0) && !(ch0_data & MFC_LLR_LOST_EVENT) &&
1523 	    (ch1_data & MFC_LLR_LOST_EVENT)) {
1524 		csa->spu_chnlcnt_RW[0] = 1;
1525 	}
1526 }
1527 
1528 static inline void restore_decr_wrapped(struct spu_state *csa, struct spu *spu)
1529 {
1530 	/* Restore, Step 58:
1531 	 *     If the status of the CSA software decrementer
1532 	 *     "wrapped" flag is set, OR in a '1' to
1533 	 *     CSA.SPU_Event_Status[Tm].
1534 	 */
1535 	if (csa->lscsa->decr_status.slot[0] == 1) {
1536 		csa->spu_chnldata_RW[0] |= 0x20;
1537 	}
1538 	if ((csa->lscsa->decr_status.slot[0] == 1) &&
1539 	    (csa->spu_chnlcnt_RW[0] == 0 &&
1540 	     ((csa->spu_chnldata_RW[2] & 0x20) == 0x0) &&
1541 	     ((csa->spu_chnldata_RW[0] & 0x20) != 0x1))) {
1542 		csa->spu_chnlcnt_RW[0] = 1;
1543 	}
1544 }
1545 
1546 static inline void restore_ch_part1(struct spu_state *csa, struct spu *spu)
1547 {
1548 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1549 	u64 idx, ch_indices[7] = { 0UL, 1UL, 3UL, 4UL, 24UL, 25UL, 27UL };
1550 	int i;
1551 
1552 	/* Restore, Step 59:
1553 	 *     Restore the following CH: [0,1,3,4,24,25,27]
1554 	 */
1555 	for (i = 0; i < 7; i++) {
1556 		idx = ch_indices[i];
1557 		out_be64(&priv2->spu_chnlcntptr_RW, idx);
1558 		eieio();
1559 		out_be64(&priv2->spu_chnldata_RW, csa->spu_chnldata_RW[idx]);
1560 		out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[idx]);
1561 		eieio();
1562 	}
1563 }
1564 
1565 static inline void restore_ch_part2(struct spu_state *csa, struct spu *spu)
1566 {
1567 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1568 	u64 ch_indices[3] = { 9UL, 21UL, 23UL };
1569 	u64 ch_counts[3] = { 1UL, 16UL, 1UL };
1570 	u64 idx;
1571 	int i;
1572 
1573 	/* Restore, Step 60:
1574 	 *     Restore the following CH: [9,21,23].
1575 	 */
1576 	ch_counts[0] = 1UL;
1577 	ch_counts[1] = csa->spu_chnlcnt_RW[21];
1578 	ch_counts[2] = 1UL;
1579 	for (i = 0; i < 3; i++) {
1580 		idx = ch_indices[i];
1581 		out_be64(&priv2->spu_chnlcntptr_RW, idx);
1582 		eieio();
1583 		out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]);
1584 		eieio();
1585 	}
1586 }
1587 
1588 static inline void restore_spu_lslr(struct spu_state *csa, struct spu *spu)
1589 {
1590 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1591 
1592 	/* Restore, Step 61:
1593 	 *     Restore the SPU_LSLR register from CSA.
1594 	 */
1595 	out_be64(&priv2->spu_lslr_RW, csa->priv2.spu_lslr_RW);
1596 	eieio();
1597 }
1598 
1599 static inline void restore_spu_cfg(struct spu_state *csa, struct spu *spu)
1600 {
1601 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1602 
1603 	/* Restore, Step 62:
1604 	 *     Restore the SPU_Cfg register from CSA.
1605 	 */
1606 	out_be64(&priv2->spu_cfg_RW, csa->priv2.spu_cfg_RW);
1607 	eieio();
1608 }
1609 
1610 static inline void restore_pm_trace(struct spu_state *csa, struct spu *spu)
1611 {
1612 	/* Restore, Step 63:
1613 	 *     Restore PM_Trace_Tag_Wait_Mask from CSA.
1614 	 *     Not performed by this implementation.
1615 	 */
1616 }
1617 
1618 static inline void restore_spu_npc(struct spu_state *csa, struct spu *spu)
1619 {
1620 	struct spu_problem __iomem *prob = spu->problem;
1621 
1622 	/* Restore, Step 64:
1623 	 *     Restore SPU_NPC from CSA.
1624 	 */
1625 	out_be32(&prob->spu_npc_RW, csa->prob.spu_npc_RW);
1626 	eieio();
1627 }
1628 
1629 static inline void restore_spu_mb(struct spu_state *csa, struct spu *spu)
1630 {
1631 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1632 	int i;
1633 
1634 	/* Restore, Step 65:
1635 	 *     Restore MFC_RdSPU_MB from CSA.
1636 	 */
1637 	out_be64(&priv2->spu_chnlcntptr_RW, 29UL);
1638 	eieio();
1639 	out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[29]);
1640 	for (i = 0; i < 4; i++) {
1641 		out_be64(&priv2->spu_chnldata_RW, csa->spu_mailbox_data[i]);
1642 	}
1643 	eieio();
1644 }
1645 
1646 static inline void check_ppu_mb_stat(struct spu_state *csa, struct spu *spu)
1647 {
1648 	struct spu_problem __iomem *prob = spu->problem;
1649 	u32 dummy = 0;
1650 
1651 	/* Restore, Step 66:
1652 	 *     If CSA.MB_Stat[P]=0 (mailbox empty) then
1653 	 *     read from the PPU_MB register.
1654 	 */
1655 	if ((csa->prob.mb_stat_R & 0xFF) == 0) {
1656 		dummy = in_be32(&prob->pu_mb_R);
1657 		eieio();
1658 	}
1659 }
1660 
1661 static inline void check_ppuint_mb_stat(struct spu_state *csa, struct spu *spu)
1662 {
1663 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1664 	u64 dummy = 0UL;
1665 
1666 	/* Restore, Step 66:
1667 	 *     If CSA.MB_Stat[I]=0 (mailbox empty) then
1668 	 *     read from the PPUINT_MB register.
1669 	 */
1670 	if ((csa->prob.mb_stat_R & 0xFF0000) == 0) {
1671 		dummy = in_be64(&priv2->puint_mb_R);
1672 		eieio();
1673 		spu_int_stat_clear(spu, 2, CLASS2_ENABLE_MAILBOX_INTR);
1674 		eieio();
1675 	}
1676 }
1677 
1678 static inline void restore_mfc_slbs(struct spu_state *csa, struct spu *spu)
1679 {
1680 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1681 	int i;
1682 
1683 	/* Restore, Step 68:
1684 	 *     If MFC_SR1[R]='1', restore SLBs from CSA.
1685 	 */
1686 	if (csa->priv1.mfc_sr1_RW & MFC_STATE1_RELOCATE_MASK) {
1687 		for (i = 0; i < 8; i++) {
1688 			out_be64(&priv2->slb_index_W, i);
1689 			eieio();
1690 			out_be64(&priv2->slb_esid_RW, csa->slb_esid_RW[i]);
1691 			out_be64(&priv2->slb_vsid_RW, csa->slb_vsid_RW[i]);
1692 			eieio();
1693 		}
1694 		out_be64(&priv2->slb_index_W, csa->priv2.slb_index_W);
1695 		eieio();
1696 	}
1697 }
1698 
1699 static inline void restore_mfc_sr1(struct spu_state *csa, struct spu *spu)
1700 {
1701 	/* Restore, Step 69:
1702 	 *     Restore the MFC_SR1 register from CSA.
1703 	 */
1704 	spu_mfc_sr1_set(spu, csa->priv1.mfc_sr1_RW);
1705 	eieio();
1706 }
1707 
1708 static inline void restore_other_spu_access(struct spu_state *csa,
1709 					    struct spu *spu)
1710 {
1711 	/* Restore, Step 70:
1712 	 *     Restore other SPU mappings to this SPU. TBD.
1713 	 */
1714 }
1715 
1716 static inline void restore_spu_runcntl(struct spu_state *csa, struct spu *spu)
1717 {
1718 	struct spu_problem __iomem *prob = spu->problem;
1719 
1720 	/* Restore, Step 71:
1721 	 *     If CSA.SPU_Status[R]=1 then write
1722 	 *     SPU_RunCntl[R0R1]='01'.
1723 	 */
1724 	if (csa->prob.spu_status_R & SPU_STATUS_RUNNING) {
1725 		out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE);
1726 		eieio();
1727 	}
1728 }
1729 
1730 static inline void restore_mfc_cntl(struct spu_state *csa, struct spu *spu)
1731 {
1732 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1733 
1734 	/* Restore, Step 72:
1735 	 *    Restore the MFC_CNTL register for the CSA.
1736 	 */
1737 	out_be64(&priv2->mfc_control_RW, csa->priv2.mfc_control_RW);
1738 	eieio();
1739 }
1740 
1741 static inline void enable_user_access(struct spu_state *csa, struct spu *spu)
1742 {
1743 	/* Restore, Step 73:
1744 	 *     Enable user-space access (if provided) to this
1745 	 *     SPU by mapping the virtual pages assigned to
1746 	 *     the SPU memory-mapped I/O (MMIO) for problem
1747 	 *     state. TBD.
1748 	 */
1749 }
1750 
1751 static inline void reset_switch_active(struct spu_state *csa, struct spu *spu)
1752 {
1753 	/* Restore, Step 74:
1754 	 *     Reset the "context switch active" flag.
1755 	 */
1756 	clear_bit(SPU_CONTEXT_SWITCH_ACTIVE, &spu->flags);
1757 	mb();
1758 }
1759 
1760 static inline void reenable_interrupts(struct spu_state *csa, struct spu *spu)
1761 {
1762 	/* Restore, Step 75:
1763 	 *     Re-enable SPU interrupts.
1764 	 */
1765 	spin_lock_irq(&spu->register_lock);
1766 	spu_int_mask_set(spu, 0, csa->priv1.int_mask_class0_RW);
1767 	spu_int_mask_set(spu, 1, csa->priv1.int_mask_class1_RW);
1768 	spu_int_mask_set(spu, 2, csa->priv1.int_mask_class2_RW);
1769 	spin_unlock_irq(&spu->register_lock);
1770 }
1771 
1772 static int quiece_spu(struct spu_state *prev, struct spu *spu)
1773 {
1774 	/*
1775 	 * Combined steps 2-18 of SPU context save sequence, which
1776 	 * quiesce the SPU state (disable SPU execution, MFC command
1777 	 * queues, decrementer, SPU interrupts, etc.).
1778 	 *
1779 	 * Returns      0 on success.
1780 	 *              2 if failed step 2.
1781 	 *              6 if failed step 6.
1782 	 */
1783 
1784 	if (check_spu_isolate(prev, spu)) {	/* Step 2. */
1785 		return 2;
1786 	}
1787 	disable_interrupts(prev, spu);	        /* Step 3. */
1788 	set_watchdog_timer(prev, spu);	        /* Step 4. */
1789 	inhibit_user_access(prev, spu);	        /* Step 5. */
1790 	if (check_spu_isolate(prev, spu)) {	/* Step 6. */
1791 		return 6;
1792 	}
1793 	set_switch_pending(prev, spu);	        /* Step 7. */
1794 	save_mfc_cntl(prev, spu);		/* Step 8. */
1795 	save_spu_runcntl(prev, spu);	        /* Step 9. */
1796 	save_mfc_sr1(prev, spu);	        /* Step 10. */
1797 	save_spu_status(prev, spu);	        /* Step 11. */
1798 	save_mfc_decr(prev, spu);	        /* Step 12. */
1799 	halt_mfc_decr(prev, spu);	        /* Step 13. */
1800 	save_timebase(prev, spu);		/* Step 14. */
1801 	remove_other_spu_access(prev, spu);	/* Step 15. */
1802 	do_mfc_mssync(prev, spu);	        /* Step 16. */
1803 	issue_mfc_tlbie(prev, spu);	        /* Step 17. */
1804 	handle_pending_interrupts(prev, spu);	/* Step 18. */
1805 
1806 	return 0;
1807 }
1808 
1809 static void save_csa(struct spu_state *prev, struct spu *spu)
1810 {
1811 	/*
1812 	 * Combine steps 19-44 of SPU context save sequence, which
1813 	 * save regions of the privileged & problem state areas.
1814 	 */
1815 
1816 	save_mfc_queues(prev, spu);	/* Step 19. */
1817 	save_ppu_querymask(prev, spu);	/* Step 20. */
1818 	save_ppu_querytype(prev, spu);	/* Step 21. */
1819 	save_mfc_csr_tsq(prev, spu);	/* Step 22. */
1820 	save_mfc_csr_cmd(prev, spu);	/* Step 23. */
1821 	save_mfc_csr_ato(prev, spu);	/* Step 24. */
1822 	save_mfc_tclass_id(prev, spu);	/* Step 25. */
1823 	set_mfc_tclass_id(prev, spu);	/* Step 26. */
1824 	purge_mfc_queue(prev, spu);	/* Step 27. */
1825 	wait_purge_complete(prev, spu);	/* Step 28. */
1826 	save_mfc_slbs(prev, spu);	/* Step 29. */
1827 	setup_mfc_sr1(prev, spu);	/* Step 30. */
1828 	save_spu_npc(prev, spu);	/* Step 31. */
1829 	save_spu_privcntl(prev, spu);	/* Step 32. */
1830 	reset_spu_privcntl(prev, spu);	/* Step 33. */
1831 	save_spu_lslr(prev, spu);	/* Step 34. */
1832 	reset_spu_lslr(prev, spu);	/* Step 35. */
1833 	save_spu_cfg(prev, spu);	/* Step 36. */
1834 	save_pm_trace(prev, spu);	/* Step 37. */
1835 	save_mfc_rag(prev, spu);	/* Step 38. */
1836 	save_ppu_mb_stat(prev, spu);	/* Step 39. */
1837 	save_ppu_mb(prev, spu);	        /* Step 40. */
1838 	save_ppuint_mb(prev, spu);	/* Step 41. */
1839 	save_ch_part1(prev, spu);	/* Step 42. */
1840 	save_spu_mb(prev, spu);	        /* Step 43. */
1841 	save_mfc_cmd(prev, spu);	/* Step 44. */
1842 	reset_ch(prev, spu);	        /* Step 45. */
1843 }
1844 
1845 static void save_lscsa(struct spu_state *prev, struct spu *spu)
1846 {
1847 	/*
1848 	 * Perform steps 46-57 of SPU context save sequence,
1849 	 * which save regions of the local store and register
1850 	 * file.
1851 	 */
1852 
1853 	resume_mfc_queue(prev, spu);	/* Step 46. */
1854 	setup_mfc_slbs(prev, spu);	/* Step 47. */
1855 	set_switch_active(prev, spu);	/* Step 48. */
1856 	enable_interrupts(prev, spu);	/* Step 49. */
1857 	save_ls_16kb(prev, spu);	/* Step 50. */
1858 	set_spu_npc(prev, spu);	        /* Step 51. */
1859 	set_signot1(prev, spu);		/* Step 52. */
1860 	set_signot2(prev, spu);		/* Step 53. */
1861 	send_save_code(prev, spu);	/* Step 54. */
1862 	set_ppu_querymask(prev, spu);	/* Step 55. */
1863 	wait_tag_complete(prev, spu);	/* Step 56. */
1864 	wait_spu_stopped(prev, spu);	/* Step 57. */
1865 }
1866 
1867 static void harvest(struct spu_state *prev, struct spu *spu)
1868 {
1869 	/*
1870 	 * Perform steps 2-25 of SPU context restore sequence,
1871 	 * which resets an SPU either after a failed save, or
1872 	 * when using SPU for first time.
1873 	 */
1874 
1875 	disable_interrupts(prev, spu);	        /* Step 2.  */
1876 	inhibit_user_access(prev, spu);	        /* Step 3.  */
1877 	terminate_spu_app(prev, spu);	        /* Step 4.  */
1878 	set_switch_pending(prev, spu);	        /* Step 5.  */
1879 	remove_other_spu_access(prev, spu);	/* Step 6.  */
1880 	suspend_mfc(prev, spu);	                /* Step 7.  */
1881 	wait_suspend_mfc_complete(prev, spu);	/* Step 8.  */
1882 	if (!suspend_spe(prev, spu))	        /* Step 9.  */
1883 		clear_spu_status(prev, spu);	/* Step 10. */
1884 	do_mfc_mssync(prev, spu);	        /* Step 11. */
1885 	issue_mfc_tlbie(prev, spu);	        /* Step 12. */
1886 	handle_pending_interrupts(prev, spu);	/* Step 13. */
1887 	purge_mfc_queue(prev, spu);	        /* Step 14. */
1888 	wait_purge_complete(prev, spu);	        /* Step 15. */
1889 	reset_spu_privcntl(prev, spu);	        /* Step 16. */
1890 	reset_spu_lslr(prev, spu);              /* Step 17. */
1891 	setup_mfc_sr1(prev, spu);	        /* Step 18. */
1892 	invalidate_slbs(prev, spu);	        /* Step 19. */
1893 	reset_ch_part1(prev, spu);	        /* Step 20. */
1894 	reset_ch_part2(prev, spu);	        /* Step 21. */
1895 	enable_interrupts(prev, spu);	        /* Step 22. */
1896 	set_switch_active(prev, spu);	        /* Step 23. */
1897 	set_mfc_tclass_id(prev, spu);	        /* Step 24. */
1898 	resume_mfc_queue(prev, spu);	        /* Step 25. */
1899 }
1900 
1901 static void restore_lscsa(struct spu_state *next, struct spu *spu)
1902 {
1903 	/*
1904 	 * Perform steps 26-40 of SPU context restore sequence,
1905 	 * which restores regions of the local store and register
1906 	 * file.
1907 	 */
1908 
1909 	set_watchdog_timer(next, spu);	        /* Step 26. */
1910 	setup_spu_status_part1(next, spu);	/* Step 27. */
1911 	setup_spu_status_part2(next, spu);	/* Step 28. */
1912 	restore_mfc_rag(next, spu);	        /* Step 29. */
1913 	setup_mfc_slbs(next, spu);	        /* Step 30. */
1914 	set_spu_npc(next, spu);	                /* Step 31. */
1915 	set_signot1(next, spu);	                /* Step 32. */
1916 	set_signot2(next, spu);	                /* Step 33. */
1917 	setup_decr(next, spu);	                /* Step 34. */
1918 	setup_ppu_mb(next, spu);	        /* Step 35. */
1919 	setup_ppuint_mb(next, spu);	        /* Step 36. */
1920 	send_restore_code(next, spu);	        /* Step 37. */
1921 	set_ppu_querymask(next, spu);	        /* Step 38. */
1922 	wait_tag_complete(next, spu);	        /* Step 39. */
1923 	wait_spu_stopped(next, spu);	        /* Step 40. */
1924 }
1925 
1926 static void restore_csa(struct spu_state *next, struct spu *spu)
1927 {
1928 	/*
1929 	 * Combine steps 41-76 of SPU context restore sequence, which
1930 	 * restore regions of the privileged & problem state areas.
1931 	 */
1932 
1933 	restore_spu_privcntl(next, spu);	/* Step 41. */
1934 	restore_status_part1(next, spu);	/* Step 42. */
1935 	restore_status_part2(next, spu);	/* Step 43. */
1936 	restore_ls_16kb(next, spu);	        /* Step 44. */
1937 	wait_tag_complete(next, spu);	        /* Step 45. */
1938 	suspend_mfc(next, spu);	                /* Step 46. */
1939 	wait_suspend_mfc_complete(next, spu);	/* Step 47. */
1940 	issue_mfc_tlbie(next, spu);	        /* Step 48. */
1941 	clear_interrupts(next, spu);	        /* Step 49. */
1942 	restore_mfc_queues(next, spu);	        /* Step 50. */
1943 	restore_ppu_querymask(next, spu);	/* Step 51. */
1944 	restore_ppu_querytype(next, spu);	/* Step 52. */
1945 	restore_mfc_csr_tsq(next, spu);	        /* Step 53. */
1946 	restore_mfc_csr_cmd(next, spu);	        /* Step 54. */
1947 	restore_mfc_csr_ato(next, spu);	        /* Step 55. */
1948 	restore_mfc_tclass_id(next, spu);	/* Step 56. */
1949 	set_llr_event(next, spu);	        /* Step 57. */
1950 	restore_decr_wrapped(next, spu);	/* Step 58. */
1951 	restore_ch_part1(next, spu);	        /* Step 59. */
1952 	restore_ch_part2(next, spu);	        /* Step 60. */
1953 	restore_spu_lslr(next, spu);	        /* Step 61. */
1954 	restore_spu_cfg(next, spu);	        /* Step 62. */
1955 	restore_pm_trace(next, spu);	        /* Step 63. */
1956 	restore_spu_npc(next, spu);	        /* Step 64. */
1957 	restore_spu_mb(next, spu);	        /* Step 65. */
1958 	check_ppu_mb_stat(next, spu);	        /* Step 66. */
1959 	check_ppuint_mb_stat(next, spu);	/* Step 67. */
1960 	restore_mfc_slbs(next, spu);	        /* Step 68. */
1961 	restore_mfc_sr1(next, spu);	        /* Step 69. */
1962 	restore_other_spu_access(next, spu);	/* Step 70. */
1963 	restore_spu_runcntl(next, spu);	        /* Step 71. */
1964 	restore_mfc_cntl(next, spu);	        /* Step 72. */
1965 	enable_user_access(next, spu);	        /* Step 73. */
1966 	reset_switch_active(next, spu);	        /* Step 74. */
1967 	reenable_interrupts(next, spu);	        /* Step 75. */
1968 }
1969 
1970 static int __do_spu_save(struct spu_state *prev, struct spu *spu)
1971 {
1972 	int rc;
1973 
1974 	/*
1975 	 * SPU context save can be broken into three phases:
1976 	 *
1977 	 *     (a) quiesce [steps 2-16].
1978 	 *     (b) save of CSA, performed by PPE [steps 17-42]
1979 	 *     (c) save of LSCSA, mostly performed by SPU [steps 43-52].
1980 	 *
1981 	 * Returns      0 on success.
1982 	 *              2,6 if failed to quiece SPU
1983 	 *              53 if SPU-side of save failed.
1984 	 */
1985 
1986 	rc = quiece_spu(prev, spu);	        /* Steps 2-16. */
1987 	switch (rc) {
1988 	default:
1989 	case 2:
1990 	case 6:
1991 		harvest(prev, spu);
1992 		return rc;
1993 		break;
1994 	case 0:
1995 		break;
1996 	}
1997 	save_csa(prev, spu);	                /* Steps 17-43. */
1998 	save_lscsa(prev, spu);	                /* Steps 44-53. */
1999 	return check_save_status(prev, spu);	/* Step 54.     */
2000 }
2001 
2002 static int __do_spu_restore(struct spu_state *next, struct spu *spu)
2003 {
2004 	int rc;
2005 
2006 	/*
2007 	 * SPU context restore can be broken into three phases:
2008 	 *
2009 	 *    (a) harvest (or reset) SPU [steps 2-24].
2010 	 *    (b) restore LSCSA [steps 25-40], mostly performed by SPU.
2011 	 *    (c) restore CSA [steps 41-76], performed by PPE.
2012 	 *
2013 	 * The 'harvest' step is not performed here, but rather
2014 	 * as needed below.
2015 	 */
2016 
2017 	restore_lscsa(next, spu);	        /* Steps 24-39. */
2018 	rc = check_restore_status(next, spu);	/* Step 40.     */
2019 	switch (rc) {
2020 	default:
2021 		/* Failed. Return now. */
2022 		return rc;
2023 		break;
2024 	case 0:
2025 		/* Fall through to next step. */
2026 		break;
2027 	}
2028 	restore_csa(next, spu);
2029 
2030 	return 0;
2031 }
2032 
2033 /**
2034  * spu_save - SPU context save, with locking.
2035  * @prev: pointer to SPU context save area, to be saved.
2036  * @spu: pointer to SPU iomem structure.
2037  *
2038  * Acquire locks, perform the save operation then return.
2039  */
2040 int spu_save(struct spu_state *prev, struct spu *spu)
2041 {
2042 	int rc;
2043 
2044 	acquire_spu_lock(spu);	        /* Step 1.     */
2045 	rc = __do_spu_save(prev, spu);	/* Steps 2-53. */
2046 	release_spu_lock(spu);
2047 	if (rc) {
2048 		panic("%s failed on SPU[%d], rc=%d.\n",
2049 		      __func__, spu->number, rc);
2050 	}
2051 	return rc;
2052 }
2053 
2054 /**
2055  * spu_restore - SPU context restore, with harvest and locking.
2056  * @new: pointer to SPU context save area, to be restored.
2057  * @spu: pointer to SPU iomem structure.
2058  *
2059  * Perform harvest + restore, as we may not be coming
2060  * from a previous succesful save operation, and the
2061  * hardware state is unknown.
2062  */
2063 int spu_restore(struct spu_state *new, struct spu *spu)
2064 {
2065 	int rc;
2066 
2067 	acquire_spu_lock(spu);
2068 	harvest(NULL, spu);
2069 	spu->stop_code = 0;
2070 	spu->dar = 0;
2071 	spu->dsisr = 0;
2072 	spu->slb_replace = 0;
2073 	spu->class_0_pending = 0;
2074 	rc = __do_spu_restore(new, spu);
2075 	release_spu_lock(spu);
2076 	if (rc) {
2077 		panic("%s failed on SPU[%d] rc=%d.\n",
2078 		       __func__, spu->number, rc);
2079 	}
2080 	return rc;
2081 }
2082 
2083 /**
2084  * spu_harvest - SPU harvest (reset) operation
2085  * @spu: pointer to SPU iomem structure.
2086  *
2087  * Perform SPU harvest (reset) operation.
2088  */
2089 void spu_harvest(struct spu *spu)
2090 {
2091 	acquire_spu_lock(spu);
2092 	harvest(NULL, spu);
2093 	release_spu_lock(spu);
2094 }
2095 
2096 static void init_prob(struct spu_state *csa)
2097 {
2098 	csa->spu_chnlcnt_RW[9] = 1;
2099 	csa->spu_chnlcnt_RW[21] = 16;
2100 	csa->spu_chnlcnt_RW[23] = 1;
2101 	csa->spu_chnlcnt_RW[28] = 1;
2102 	csa->spu_chnlcnt_RW[30] = 1;
2103 	csa->prob.spu_runcntl_RW = SPU_RUNCNTL_STOP;
2104 }
2105 
2106 static void init_priv1(struct spu_state *csa)
2107 {
2108 	/* Enable decode, relocate, tlbie response, master runcntl. */
2109 	csa->priv1.mfc_sr1_RW = MFC_STATE1_LOCAL_STORAGE_DECODE_MASK |
2110 	    MFC_STATE1_MASTER_RUN_CONTROL_MASK |
2111 	    MFC_STATE1_PROBLEM_STATE_MASK |
2112 	    MFC_STATE1_RELOCATE_MASK | MFC_STATE1_BUS_TLBIE_MASK;
2113 
2114 	/* Set storage description.  */
2115 	csa->priv1.mfc_sdr_RW = mfspr(SPRN_SDR1);
2116 
2117 	/* Enable OS-specific set of interrupts. */
2118 	csa->priv1.int_mask_class0_RW = CLASS0_ENABLE_DMA_ALIGNMENT_INTR |
2119 	    CLASS0_ENABLE_INVALID_DMA_COMMAND_INTR |
2120 	    CLASS0_ENABLE_SPU_ERROR_INTR;
2121 	csa->priv1.int_mask_class1_RW = CLASS1_ENABLE_SEGMENT_FAULT_INTR |
2122 	    CLASS1_ENABLE_STORAGE_FAULT_INTR;
2123 	csa->priv1.int_mask_class2_RW = CLASS2_ENABLE_SPU_STOP_INTR |
2124 	    CLASS2_ENABLE_SPU_HALT_INTR;
2125 }
2126 
2127 static void init_priv2(struct spu_state *csa)
2128 {
2129 	csa->priv2.spu_lslr_RW = LS_ADDR_MASK;
2130 	csa->priv2.mfc_control_RW = MFC_CNTL_RESUME_DMA_QUEUE |
2131 	    MFC_CNTL_NORMAL_DMA_QUEUE_OPERATION |
2132 	    MFC_CNTL_DMA_QUEUES_EMPTY_MASK;
2133 }
2134 
2135 /**
2136  * spu_alloc_csa - allocate and initialize an SPU context save area.
2137  *
2138  * Allocate and initialize the contents of an SPU context save area.
2139  * This includes enabling address translation, interrupt masks, etc.,
2140  * as appropriate for the given OS environment.
2141  *
2142  * Note that storage for the 'lscsa' is allocated separately,
2143  * as it is by far the largest of the context save regions,
2144  * and may need to be pinned or otherwise specially aligned.
2145  */
2146 void spu_init_csa(struct spu_state *csa)
2147 {
2148 	struct spu_lscsa *lscsa;
2149 	unsigned char *p;
2150 
2151 	if (!csa)
2152 		return;
2153 	memset(csa, 0, sizeof(struct spu_state));
2154 
2155 	lscsa = vmalloc(sizeof(struct spu_lscsa));
2156 	if (!lscsa)
2157 		return;
2158 
2159 	memset(lscsa, 0, sizeof(struct spu_lscsa));
2160 	csa->lscsa = lscsa;
2161 	csa->register_lock = SPIN_LOCK_UNLOCKED;
2162 
2163 	/* Set LS pages reserved to allow for user-space mapping. */
2164 	for (p = lscsa->ls; p < lscsa->ls + LS_SIZE; p += PAGE_SIZE)
2165 		SetPageReserved(vmalloc_to_page(p));
2166 
2167 	init_prob(csa);
2168 	init_priv1(csa);
2169 	init_priv2(csa);
2170 }
2171 
2172 void spu_fini_csa(struct spu_state *csa)
2173 {
2174 	/* Clear reserved bit before vfree. */
2175 	unsigned char *p;
2176 	for (p = csa->lscsa->ls; p < csa->lscsa->ls + LS_SIZE; p += PAGE_SIZE)
2177 		ClearPageReserved(vmalloc_to_page(p));
2178 
2179 	vfree(csa->lscsa);
2180 }
2181