1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * spu_switch.c
4  *
5  * (C) Copyright IBM Corp. 2005
6  *
7  * Author: Mark Nutter <mnutter@us.ibm.com>
8  *
9  * Host-side part of SPU context switch sequence outlined in
10  * Synergistic Processor Element, Book IV.
11  *
12  * A fully premptive switch of an SPE is very expensive in terms
13  * of time and system resources.  SPE Book IV indicates that SPE
14  * allocation should follow a "serially reusable device" model,
15  * in which the SPE is assigned a task until it completes.  When
16  * this is not possible, this sequence may be used to premptively
17  * save, and then later (optionally) restore the context of a
18  * program executing on an SPE.
19  */
20 
21 #include <linux/export.h>
22 #include <linux/errno.h>
23 #include <linux/hardirq.h>
24 #include <linux/sched.h>
25 #include <linux/kernel.h>
26 #include <linux/mm.h>
27 #include <linux/vmalloc.h>
28 #include <linux/smp.h>
29 #include <linux/stddef.h>
30 #include <linux/unistd.h>
31 
32 #include <asm/io.h>
33 #include <asm/spu.h>
34 #include <asm/spu_priv1.h>
35 #include <asm/spu_csa.h>
36 #include <asm/mmu_context.h>
37 
38 #include "spufs.h"
39 
40 #include "spu_save_dump.h"
41 #include "spu_restore_dump.h"
42 
43 #if 0
44 #define POLL_WHILE_TRUE(_c) {				\
45     do {						\
46     } while (_c);					\
47   }
48 #else
49 #define RELAX_SPIN_COUNT				1000
50 #define POLL_WHILE_TRUE(_c) {				\
51     do {						\
52 	int _i;						\
53 	for (_i=0; _i<RELAX_SPIN_COUNT && (_c); _i++) { \
54 	    cpu_relax();				\
55 	}						\
56 	if (unlikely(_c)) yield();			\
57 	else break;					\
58     } while (_c);					\
59   }
60 #endif				/* debug */
61 
62 #define POLL_WHILE_FALSE(_c)	POLL_WHILE_TRUE(!(_c))
63 
64 static inline void acquire_spu_lock(struct spu *spu)
65 {
66 	/* Save, Step 1:
67 	 * Restore, Step 1:
68 	 *    Acquire SPU-specific mutual exclusion lock.
69 	 *    TBD.
70 	 */
71 }
72 
73 static inline void release_spu_lock(struct spu *spu)
74 {
75 	/* Restore, Step 76:
76 	 *    Release SPU-specific mutual exclusion lock.
77 	 *    TBD.
78 	 */
79 }
80 
81 static inline int check_spu_isolate(struct spu_state *csa, struct spu *spu)
82 {
83 	struct spu_problem __iomem *prob = spu->problem;
84 	u32 isolate_state;
85 
86 	/* Save, Step 2:
87 	 * Save, Step 6:
88 	 *     If SPU_Status[E,L,IS] any field is '1', this
89 	 *     SPU is in isolate state and cannot be context
90 	 *     saved at this time.
91 	 */
92 	isolate_state = SPU_STATUS_ISOLATED_STATE |
93 	    SPU_STATUS_ISOLATED_LOAD_STATUS | SPU_STATUS_ISOLATED_EXIT_STATUS;
94 	return (in_be32(&prob->spu_status_R) & isolate_state) ? 1 : 0;
95 }
96 
97 static inline void disable_interrupts(struct spu_state *csa, struct spu *spu)
98 {
99 	/* Save, Step 3:
100 	 * Restore, Step 2:
101 	 *     Save INT_Mask_class0 in CSA.
102 	 *     Write INT_MASK_class0 with value of 0.
103 	 *     Save INT_Mask_class1 in CSA.
104 	 *     Write INT_MASK_class1 with value of 0.
105 	 *     Save INT_Mask_class2 in CSA.
106 	 *     Write INT_MASK_class2 with value of 0.
107 	 *     Synchronize all three interrupts to be sure
108 	 *     we no longer execute a handler on another CPU.
109 	 */
110 	spin_lock_irq(&spu->register_lock);
111 	if (csa) {
112 		csa->priv1.int_mask_class0_RW = spu_int_mask_get(spu, 0);
113 		csa->priv1.int_mask_class1_RW = spu_int_mask_get(spu, 1);
114 		csa->priv1.int_mask_class2_RW = spu_int_mask_get(spu, 2);
115 	}
116 	spu_int_mask_set(spu, 0, 0ul);
117 	spu_int_mask_set(spu, 1, 0ul);
118 	spu_int_mask_set(spu, 2, 0ul);
119 	eieio();
120 	spin_unlock_irq(&spu->register_lock);
121 
122 	/*
123 	 * This flag needs to be set before calling synchronize_irq so
124 	 * that the update will be visible to the relevant handlers
125 	 * via a simple load.
126 	 */
127 	set_bit(SPU_CONTEXT_SWITCH_PENDING, &spu->flags);
128 	clear_bit(SPU_CONTEXT_FAULT_PENDING, &spu->flags);
129 	synchronize_irq(spu->irqs[0]);
130 	synchronize_irq(spu->irqs[1]);
131 	synchronize_irq(spu->irqs[2]);
132 }
133 
134 static inline void set_watchdog_timer(struct spu_state *csa, struct spu *spu)
135 {
136 	/* Save, Step 4:
137 	 * Restore, Step 25.
138 	 *    Set a software watchdog timer, which specifies the
139 	 *    maximum allowable time for a context save sequence.
140 	 *
141 	 *    For present, this implementation will not set a global
142 	 *    watchdog timer, as virtualization & variable system load
143 	 *    may cause unpredictable execution times.
144 	 */
145 }
146 
147 static inline void inhibit_user_access(struct spu_state *csa, struct spu *spu)
148 {
149 	/* Save, Step 5:
150 	 * Restore, Step 3:
151 	 *     Inhibit user-space access (if provided) to this
152 	 *     SPU by unmapping the virtual pages assigned to
153 	 *     the SPU memory-mapped I/O (MMIO) for problem
154 	 *     state. TBD.
155 	 */
156 }
157 
158 static inline void set_switch_pending(struct spu_state *csa, struct spu *spu)
159 {
160 	/* Save, Step 7:
161 	 * Restore, Step 5:
162 	 *     Set a software context switch pending flag.
163 	 *     Done above in Step 3 - disable_interrupts().
164 	 */
165 }
166 
167 static inline void save_mfc_cntl(struct spu_state *csa, struct spu *spu)
168 {
169 	struct spu_priv2 __iomem *priv2 = spu->priv2;
170 
171 	/* Save, Step 8:
172 	 *     Suspend DMA and save MFC_CNTL.
173 	 */
174 	switch (in_be64(&priv2->mfc_control_RW) &
175 	       MFC_CNTL_SUSPEND_DMA_STATUS_MASK) {
176 	case MFC_CNTL_SUSPEND_IN_PROGRESS:
177 		POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) &
178 				  MFC_CNTL_SUSPEND_DMA_STATUS_MASK) ==
179 				 MFC_CNTL_SUSPEND_COMPLETE);
180 		fallthrough;
181 	case MFC_CNTL_SUSPEND_COMPLETE:
182 		if (csa)
183 			csa->priv2.mfc_control_RW =
184 				in_be64(&priv2->mfc_control_RW) |
185 				MFC_CNTL_SUSPEND_DMA_QUEUE;
186 		break;
187 	case MFC_CNTL_NORMAL_DMA_QUEUE_OPERATION:
188 		out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE);
189 		POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) &
190 				  MFC_CNTL_SUSPEND_DMA_STATUS_MASK) ==
191 				 MFC_CNTL_SUSPEND_COMPLETE);
192 		if (csa)
193 			csa->priv2.mfc_control_RW =
194 				in_be64(&priv2->mfc_control_RW) &
195 				~MFC_CNTL_SUSPEND_DMA_QUEUE &
196 				~MFC_CNTL_SUSPEND_MASK;
197 		break;
198 	}
199 }
200 
201 static inline void save_spu_runcntl(struct spu_state *csa, struct spu *spu)
202 {
203 	struct spu_problem __iomem *prob = spu->problem;
204 
205 	/* Save, Step 9:
206 	 *     Save SPU_Runcntl in the CSA.  This value contains
207 	 *     the "Application Desired State".
208 	 */
209 	csa->prob.spu_runcntl_RW = in_be32(&prob->spu_runcntl_RW);
210 }
211 
212 static inline void save_mfc_sr1(struct spu_state *csa, struct spu *spu)
213 {
214 	/* Save, Step 10:
215 	 *     Save MFC_SR1 in the CSA.
216 	 */
217 	csa->priv1.mfc_sr1_RW = spu_mfc_sr1_get(spu);
218 }
219 
220 static inline void save_spu_status(struct spu_state *csa, struct spu *spu)
221 {
222 	struct spu_problem __iomem *prob = spu->problem;
223 
224 	/* Save, Step 11:
225 	 *     Read SPU_Status[R], and save to CSA.
226 	 */
227 	if ((in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) == 0) {
228 		csa->prob.spu_status_R = in_be32(&prob->spu_status_R);
229 	} else {
230 		u32 stopped;
231 
232 		out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
233 		eieio();
234 		POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
235 				SPU_STATUS_RUNNING);
236 		stopped =
237 		    SPU_STATUS_INVALID_INSTR | SPU_STATUS_SINGLE_STEP |
238 		    SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP;
239 		if ((in_be32(&prob->spu_status_R) & stopped) == 0)
240 			csa->prob.spu_status_R = SPU_STATUS_RUNNING;
241 		else
242 			csa->prob.spu_status_R = in_be32(&prob->spu_status_R);
243 	}
244 }
245 
246 static inline void save_mfc_stopped_status(struct spu_state *csa,
247 		struct spu *spu)
248 {
249 	struct spu_priv2 __iomem *priv2 = spu->priv2;
250 	const u64 mask = MFC_CNTL_DECREMENTER_RUNNING |
251 			MFC_CNTL_DMA_QUEUES_EMPTY;
252 
253 	/* Save, Step 12:
254 	 *     Read MFC_CNTL[Ds].  Update saved copy of
255 	 *     CSA.MFC_CNTL[Ds].
256 	 *
257 	 * update: do the same with MFC_CNTL[Q].
258 	 */
259 	csa->priv2.mfc_control_RW &= ~mask;
260 	csa->priv2.mfc_control_RW |= in_be64(&priv2->mfc_control_RW) & mask;
261 }
262 
263 static inline void halt_mfc_decr(struct spu_state *csa, struct spu *spu)
264 {
265 	struct spu_priv2 __iomem *priv2 = spu->priv2;
266 
267 	/* Save, Step 13:
268 	 *     Write MFC_CNTL[Dh] set to a '1' to halt
269 	 *     the decrementer.
270 	 */
271 	out_be64(&priv2->mfc_control_RW,
272 		 MFC_CNTL_DECREMENTER_HALTED | MFC_CNTL_SUSPEND_MASK);
273 	eieio();
274 }
275 
276 static inline void save_timebase(struct spu_state *csa, struct spu *spu)
277 {
278 	/* Save, Step 14:
279 	 *    Read PPE Timebase High and Timebase low registers
280 	 *    and save in CSA.  TBD.
281 	 */
282 	csa->suspend_time = get_cycles();
283 }
284 
285 static inline void remove_other_spu_access(struct spu_state *csa,
286 					   struct spu *spu)
287 {
288 	/* Save, Step 15:
289 	 *     Remove other SPU access to this SPU by unmapping
290 	 *     this SPU's pages from their address space.  TBD.
291 	 */
292 }
293 
294 static inline void do_mfc_mssync(struct spu_state *csa, struct spu *spu)
295 {
296 	struct spu_problem __iomem *prob = spu->problem;
297 
298 	/* Save, Step 16:
299 	 * Restore, Step 11.
300 	 *     Write SPU_MSSync register. Poll SPU_MSSync[P]
301 	 *     for a value of 0.
302 	 */
303 	out_be64(&prob->spc_mssync_RW, 1UL);
304 	POLL_WHILE_TRUE(in_be64(&prob->spc_mssync_RW) & MS_SYNC_PENDING);
305 }
306 
307 static inline void issue_mfc_tlbie(struct spu_state *csa, struct spu *spu)
308 {
309 	/* Save, Step 17:
310 	 * Restore, Step 12.
311 	 * Restore, Step 48.
312 	 *     Write TLB_Invalidate_Entry[IS,VPN,L,Lp]=0 register.
313 	 *     Then issue a PPE sync instruction.
314 	 */
315 	spu_tlb_invalidate(spu);
316 	mb();
317 }
318 
319 static inline void handle_pending_interrupts(struct spu_state *csa,
320 					     struct spu *spu)
321 {
322 	/* Save, Step 18:
323 	 *     Handle any pending interrupts from this SPU
324 	 *     here.  This is OS or hypervisor specific.  One
325 	 *     option is to re-enable interrupts to handle any
326 	 *     pending interrupts, with the interrupt handlers
327 	 *     recognizing the software Context Switch Pending
328 	 *     flag, to ensure the SPU execution or MFC command
329 	 *     queue is not restarted.  TBD.
330 	 */
331 }
332 
333 static inline void save_mfc_queues(struct spu_state *csa, struct spu *spu)
334 {
335 	struct spu_priv2 __iomem *priv2 = spu->priv2;
336 	int i;
337 
338 	/* Save, Step 19:
339 	 *     If MFC_Cntl[Se]=0 then save
340 	 *     MFC command queues.
341 	 */
342 	if ((in_be64(&priv2->mfc_control_RW) & MFC_CNTL_DMA_QUEUES_EMPTY) == 0) {
343 		for (i = 0; i < 8; i++) {
344 			csa->priv2.puq[i].mfc_cq_data0_RW =
345 			    in_be64(&priv2->puq[i].mfc_cq_data0_RW);
346 			csa->priv2.puq[i].mfc_cq_data1_RW =
347 			    in_be64(&priv2->puq[i].mfc_cq_data1_RW);
348 			csa->priv2.puq[i].mfc_cq_data2_RW =
349 			    in_be64(&priv2->puq[i].mfc_cq_data2_RW);
350 			csa->priv2.puq[i].mfc_cq_data3_RW =
351 			    in_be64(&priv2->puq[i].mfc_cq_data3_RW);
352 		}
353 		for (i = 0; i < 16; i++) {
354 			csa->priv2.spuq[i].mfc_cq_data0_RW =
355 			    in_be64(&priv2->spuq[i].mfc_cq_data0_RW);
356 			csa->priv2.spuq[i].mfc_cq_data1_RW =
357 			    in_be64(&priv2->spuq[i].mfc_cq_data1_RW);
358 			csa->priv2.spuq[i].mfc_cq_data2_RW =
359 			    in_be64(&priv2->spuq[i].mfc_cq_data2_RW);
360 			csa->priv2.spuq[i].mfc_cq_data3_RW =
361 			    in_be64(&priv2->spuq[i].mfc_cq_data3_RW);
362 		}
363 	}
364 }
365 
366 static inline void save_ppu_querymask(struct spu_state *csa, struct spu *spu)
367 {
368 	struct spu_problem __iomem *prob = spu->problem;
369 
370 	/* Save, Step 20:
371 	 *     Save the PPU_QueryMask register
372 	 *     in the CSA.
373 	 */
374 	csa->prob.dma_querymask_RW = in_be32(&prob->dma_querymask_RW);
375 }
376 
377 static inline void save_ppu_querytype(struct spu_state *csa, struct spu *spu)
378 {
379 	struct spu_problem __iomem *prob = spu->problem;
380 
381 	/* Save, Step 21:
382 	 *     Save the PPU_QueryType register
383 	 *     in the CSA.
384 	 */
385 	csa->prob.dma_querytype_RW = in_be32(&prob->dma_querytype_RW);
386 }
387 
388 static inline void save_ppu_tagstatus(struct spu_state *csa, struct spu *spu)
389 {
390 	struct spu_problem __iomem *prob = spu->problem;
391 
392 	/* Save the Prxy_TagStatus register in the CSA.
393 	 *
394 	 * It is unnecessary to restore dma_tagstatus_R, however,
395 	 * dma_tagstatus_R in the CSA is accessed via backing_ops, so
396 	 * we must save it.
397 	 */
398 	csa->prob.dma_tagstatus_R = in_be32(&prob->dma_tagstatus_R);
399 }
400 
401 static inline void save_mfc_csr_tsq(struct spu_state *csa, struct spu *spu)
402 {
403 	struct spu_priv2 __iomem *priv2 = spu->priv2;
404 
405 	/* Save, Step 22:
406 	 *     Save the MFC_CSR_TSQ register
407 	 *     in the LSCSA.
408 	 */
409 	csa->priv2.spu_tag_status_query_RW =
410 	    in_be64(&priv2->spu_tag_status_query_RW);
411 }
412 
413 static inline void save_mfc_csr_cmd(struct spu_state *csa, struct spu *spu)
414 {
415 	struct spu_priv2 __iomem *priv2 = spu->priv2;
416 
417 	/* Save, Step 23:
418 	 *     Save the MFC_CSR_CMD1 and MFC_CSR_CMD2
419 	 *     registers in the CSA.
420 	 */
421 	csa->priv2.spu_cmd_buf1_RW = in_be64(&priv2->spu_cmd_buf1_RW);
422 	csa->priv2.spu_cmd_buf2_RW = in_be64(&priv2->spu_cmd_buf2_RW);
423 }
424 
425 static inline void save_mfc_csr_ato(struct spu_state *csa, struct spu *spu)
426 {
427 	struct spu_priv2 __iomem *priv2 = spu->priv2;
428 
429 	/* Save, Step 24:
430 	 *     Save the MFC_CSR_ATO register in
431 	 *     the CSA.
432 	 */
433 	csa->priv2.spu_atomic_status_RW = in_be64(&priv2->spu_atomic_status_RW);
434 }
435 
436 static inline void save_mfc_tclass_id(struct spu_state *csa, struct spu *spu)
437 {
438 	/* Save, Step 25:
439 	 *     Save the MFC_TCLASS_ID register in
440 	 *     the CSA.
441 	 */
442 	csa->priv1.mfc_tclass_id_RW = spu_mfc_tclass_id_get(spu);
443 }
444 
445 static inline void set_mfc_tclass_id(struct spu_state *csa, struct spu *spu)
446 {
447 	/* Save, Step 26:
448 	 * Restore, Step 23.
449 	 *     Write the MFC_TCLASS_ID register with
450 	 *     the value 0x10000000.
451 	 */
452 	spu_mfc_tclass_id_set(spu, 0x10000000);
453 	eieio();
454 }
455 
456 static inline void purge_mfc_queue(struct spu_state *csa, struct spu *spu)
457 {
458 	struct spu_priv2 __iomem *priv2 = spu->priv2;
459 
460 	/* Save, Step 27:
461 	 * Restore, Step 14.
462 	 *     Write MFC_CNTL[Pc]=1 (purge queue).
463 	 */
464 	out_be64(&priv2->mfc_control_RW,
465 			MFC_CNTL_PURGE_DMA_REQUEST |
466 			MFC_CNTL_SUSPEND_MASK);
467 	eieio();
468 }
469 
470 static inline void wait_purge_complete(struct spu_state *csa, struct spu *spu)
471 {
472 	struct spu_priv2 __iomem *priv2 = spu->priv2;
473 
474 	/* Save, Step 28:
475 	 *     Poll MFC_CNTL[Ps] until value '11' is read
476 	 *     (purge complete).
477 	 */
478 	POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) &
479 			 MFC_CNTL_PURGE_DMA_STATUS_MASK) ==
480 			 MFC_CNTL_PURGE_DMA_COMPLETE);
481 }
482 
483 static inline void setup_mfc_sr1(struct spu_state *csa, struct spu *spu)
484 {
485 	/* Save, Step 30:
486 	 * Restore, Step 18:
487 	 *     Write MFC_SR1 with MFC_SR1[D=0,S=1] and
488 	 *     MFC_SR1[TL,R,Pr,T] set correctly for the
489 	 *     OS specific environment.
490 	 *
491 	 *     Implementation note: The SPU-side code
492 	 *     for save/restore is privileged, so the
493 	 *     MFC_SR1[Pr] bit is not set.
494 	 *
495 	 */
496 	spu_mfc_sr1_set(spu, (MFC_STATE1_MASTER_RUN_CONTROL_MASK |
497 			      MFC_STATE1_RELOCATE_MASK |
498 			      MFC_STATE1_BUS_TLBIE_MASK));
499 }
500 
501 static inline void save_spu_npc(struct spu_state *csa, struct spu *spu)
502 {
503 	struct spu_problem __iomem *prob = spu->problem;
504 
505 	/* Save, Step 31:
506 	 *     Save SPU_NPC in the CSA.
507 	 */
508 	csa->prob.spu_npc_RW = in_be32(&prob->spu_npc_RW);
509 }
510 
511 static inline void save_spu_privcntl(struct spu_state *csa, struct spu *spu)
512 {
513 	struct spu_priv2 __iomem *priv2 = spu->priv2;
514 
515 	/* Save, Step 32:
516 	 *     Save SPU_PrivCntl in the CSA.
517 	 */
518 	csa->priv2.spu_privcntl_RW = in_be64(&priv2->spu_privcntl_RW);
519 }
520 
521 static inline void reset_spu_privcntl(struct spu_state *csa, struct spu *spu)
522 {
523 	struct spu_priv2 __iomem *priv2 = spu->priv2;
524 
525 	/* Save, Step 33:
526 	 * Restore, Step 16:
527 	 *     Write SPU_PrivCntl[S,Le,A] fields reset to 0.
528 	 */
529 	out_be64(&priv2->spu_privcntl_RW, 0UL);
530 	eieio();
531 }
532 
533 static inline void save_spu_lslr(struct spu_state *csa, struct spu *spu)
534 {
535 	struct spu_priv2 __iomem *priv2 = spu->priv2;
536 
537 	/* Save, Step 34:
538 	 *     Save SPU_LSLR in the CSA.
539 	 */
540 	csa->priv2.spu_lslr_RW = in_be64(&priv2->spu_lslr_RW);
541 }
542 
543 static inline void reset_spu_lslr(struct spu_state *csa, struct spu *spu)
544 {
545 	struct spu_priv2 __iomem *priv2 = spu->priv2;
546 
547 	/* Save, Step 35:
548 	 * Restore, Step 17.
549 	 *     Reset SPU_LSLR.
550 	 */
551 	out_be64(&priv2->spu_lslr_RW, LS_ADDR_MASK);
552 	eieio();
553 }
554 
555 static inline void save_spu_cfg(struct spu_state *csa, struct spu *spu)
556 {
557 	struct spu_priv2 __iomem *priv2 = spu->priv2;
558 
559 	/* Save, Step 36:
560 	 *     Save SPU_Cfg in the CSA.
561 	 */
562 	csa->priv2.spu_cfg_RW = in_be64(&priv2->spu_cfg_RW);
563 }
564 
565 static inline void save_pm_trace(struct spu_state *csa, struct spu *spu)
566 {
567 	/* Save, Step 37:
568 	 *     Save PM_Trace_Tag_Wait_Mask in the CSA.
569 	 *     Not performed by this implementation.
570 	 */
571 }
572 
573 static inline void save_mfc_rag(struct spu_state *csa, struct spu *spu)
574 {
575 	/* Save, Step 38:
576 	 *     Save RA_GROUP_ID register and the
577 	 *     RA_ENABLE reigster in the CSA.
578 	 */
579 	csa->priv1.resource_allocation_groupID_RW =
580 		spu_resource_allocation_groupID_get(spu);
581 	csa->priv1.resource_allocation_enable_RW =
582 		spu_resource_allocation_enable_get(spu);
583 }
584 
585 static inline void save_ppu_mb_stat(struct spu_state *csa, struct spu *spu)
586 {
587 	struct spu_problem __iomem *prob = spu->problem;
588 
589 	/* Save, Step 39:
590 	 *     Save MB_Stat register in the CSA.
591 	 */
592 	csa->prob.mb_stat_R = in_be32(&prob->mb_stat_R);
593 }
594 
595 static inline void save_ppu_mb(struct spu_state *csa, struct spu *spu)
596 {
597 	struct spu_problem __iomem *prob = spu->problem;
598 
599 	/* Save, Step 40:
600 	 *     Save the PPU_MB register in the CSA.
601 	 */
602 	csa->prob.pu_mb_R = in_be32(&prob->pu_mb_R);
603 }
604 
605 static inline void save_ppuint_mb(struct spu_state *csa, struct spu *spu)
606 {
607 	struct spu_priv2 __iomem *priv2 = spu->priv2;
608 
609 	/* Save, Step 41:
610 	 *     Save the PPUINT_MB register in the CSA.
611 	 */
612 	csa->priv2.puint_mb_R = in_be64(&priv2->puint_mb_R);
613 }
614 
615 static inline void save_ch_part1(struct spu_state *csa, struct spu *spu)
616 {
617 	struct spu_priv2 __iomem *priv2 = spu->priv2;
618 	u64 idx, ch_indices[] = { 0UL, 3UL, 4UL, 24UL, 25UL, 27UL };
619 	int i;
620 
621 	/* Save, Step 42:
622 	 */
623 
624 	/* Save CH 1, without channel count */
625 	out_be64(&priv2->spu_chnlcntptr_RW, 1);
626 	csa->spu_chnldata_RW[1] = in_be64(&priv2->spu_chnldata_RW);
627 
628 	/* Save the following CH: [0,3,4,24,25,27] */
629 	for (i = 0; i < ARRAY_SIZE(ch_indices); i++) {
630 		idx = ch_indices[i];
631 		out_be64(&priv2->spu_chnlcntptr_RW, idx);
632 		eieio();
633 		csa->spu_chnldata_RW[idx] = in_be64(&priv2->spu_chnldata_RW);
634 		csa->spu_chnlcnt_RW[idx] = in_be64(&priv2->spu_chnlcnt_RW);
635 		out_be64(&priv2->spu_chnldata_RW, 0UL);
636 		out_be64(&priv2->spu_chnlcnt_RW, 0UL);
637 		eieio();
638 	}
639 }
640 
641 static inline void save_spu_mb(struct spu_state *csa, struct spu *spu)
642 {
643 	struct spu_priv2 __iomem *priv2 = spu->priv2;
644 	int i;
645 
646 	/* Save, Step 43:
647 	 *     Save SPU Read Mailbox Channel.
648 	 */
649 	out_be64(&priv2->spu_chnlcntptr_RW, 29UL);
650 	eieio();
651 	csa->spu_chnlcnt_RW[29] = in_be64(&priv2->spu_chnlcnt_RW);
652 	for (i = 0; i < 4; i++) {
653 		csa->spu_mailbox_data[i] = in_be64(&priv2->spu_chnldata_RW);
654 	}
655 	out_be64(&priv2->spu_chnlcnt_RW, 0UL);
656 	eieio();
657 }
658 
659 static inline void save_mfc_cmd(struct spu_state *csa, struct spu *spu)
660 {
661 	struct spu_priv2 __iomem *priv2 = spu->priv2;
662 
663 	/* Save, Step 44:
664 	 *     Save MFC_CMD Channel.
665 	 */
666 	out_be64(&priv2->spu_chnlcntptr_RW, 21UL);
667 	eieio();
668 	csa->spu_chnlcnt_RW[21] = in_be64(&priv2->spu_chnlcnt_RW);
669 	eieio();
670 }
671 
672 static inline void reset_ch(struct spu_state *csa, struct spu *spu)
673 {
674 	struct spu_priv2 __iomem *priv2 = spu->priv2;
675 	u64 ch_indices[4] = { 21UL, 23UL, 28UL, 30UL };
676 	u64 ch_counts[4] = { 16UL, 1UL, 1UL, 1UL };
677 	u64 idx;
678 	int i;
679 
680 	/* Save, Step 45:
681 	 *     Reset the following CH: [21, 23, 28, 30]
682 	 */
683 	for (i = 0; i < 4; i++) {
684 		idx = ch_indices[i];
685 		out_be64(&priv2->spu_chnlcntptr_RW, idx);
686 		eieio();
687 		out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]);
688 		eieio();
689 	}
690 }
691 
692 static inline void resume_mfc_queue(struct spu_state *csa, struct spu *spu)
693 {
694 	struct spu_priv2 __iomem *priv2 = spu->priv2;
695 
696 	/* Save, Step 46:
697 	 * Restore, Step 25.
698 	 *     Write MFC_CNTL[Sc]=0 (resume queue processing).
699 	 */
700 	out_be64(&priv2->mfc_control_RW, MFC_CNTL_RESUME_DMA_QUEUE);
701 }
702 
703 static inline void setup_mfc_slbs(struct spu_state *csa, struct spu *spu,
704 		unsigned int *code, int code_size)
705 {
706 	/* Save, Step 47:
707 	 * Restore, Step 30.
708 	 *     If MFC_SR1[R]=1, write 0 to SLB_Invalidate_All
709 	 *     register, then initialize SLB_VSID and SLB_ESID
710 	 *     to provide access to SPU context save code and
711 	 *     LSCSA.
712 	 *
713 	 *     This implementation places both the context
714 	 *     switch code and LSCSA in kernel address space.
715 	 *
716 	 *     Further this implementation assumes that the
717 	 *     MFC_SR1[R]=1 (in other words, assume that
718 	 *     translation is desired by OS environment).
719 	 */
720 	spu_invalidate_slbs(spu);
721 	spu_setup_kernel_slbs(spu, csa->lscsa, code, code_size);
722 }
723 
724 static inline void set_switch_active(struct spu_state *csa, struct spu *spu)
725 {
726 	/* Save, Step 48:
727 	 * Restore, Step 23.
728 	 *     Change the software context switch pending flag
729 	 *     to context switch active.  This implementation does
730 	 *     not uses a switch active flag.
731 	 *
732 	 * Now that we have saved the mfc in the csa, we can add in the
733 	 * restart command if an exception occurred.
734 	 */
735 	if (test_bit(SPU_CONTEXT_FAULT_PENDING, &spu->flags))
736 		csa->priv2.mfc_control_RW |= MFC_CNTL_RESTART_DMA_COMMAND;
737 	clear_bit(SPU_CONTEXT_SWITCH_PENDING, &spu->flags);
738 	mb();
739 }
740 
741 static inline void enable_interrupts(struct spu_state *csa, struct spu *spu)
742 {
743 	unsigned long class1_mask = CLASS1_ENABLE_SEGMENT_FAULT_INTR |
744 	    CLASS1_ENABLE_STORAGE_FAULT_INTR;
745 
746 	/* Save, Step 49:
747 	 * Restore, Step 22:
748 	 *     Reset and then enable interrupts, as
749 	 *     needed by OS.
750 	 *
751 	 *     This implementation enables only class1
752 	 *     (translation) interrupts.
753 	 */
754 	spin_lock_irq(&spu->register_lock);
755 	spu_int_stat_clear(spu, 0, CLASS0_INTR_MASK);
756 	spu_int_stat_clear(spu, 1, CLASS1_INTR_MASK);
757 	spu_int_stat_clear(spu, 2, CLASS2_INTR_MASK);
758 	spu_int_mask_set(spu, 0, 0ul);
759 	spu_int_mask_set(spu, 1, class1_mask);
760 	spu_int_mask_set(spu, 2, 0ul);
761 	spin_unlock_irq(&spu->register_lock);
762 }
763 
764 static inline int send_mfc_dma(struct spu *spu, unsigned long ea,
765 			       unsigned int ls_offset, unsigned int size,
766 			       unsigned int tag, unsigned int rclass,
767 			       unsigned int cmd)
768 {
769 	struct spu_problem __iomem *prob = spu->problem;
770 	union mfc_tag_size_class_cmd command;
771 	unsigned int transfer_size;
772 	volatile unsigned int status = 0x0;
773 
774 	while (size > 0) {
775 		transfer_size =
776 		    (size > MFC_MAX_DMA_SIZE) ? MFC_MAX_DMA_SIZE : size;
777 		command.u.mfc_size = transfer_size;
778 		command.u.mfc_tag = tag;
779 		command.u.mfc_rclassid = rclass;
780 		command.u.mfc_cmd = cmd;
781 		do {
782 			out_be32(&prob->mfc_lsa_W, ls_offset);
783 			out_be64(&prob->mfc_ea_W, ea);
784 			out_be64(&prob->mfc_union_W.all64, command.all64);
785 			status =
786 			    in_be32(&prob->mfc_union_W.by32.mfc_class_cmd32);
787 			if (unlikely(status & 0x2)) {
788 				cpu_relax();
789 			}
790 		} while (status & 0x3);
791 		size -= transfer_size;
792 		ea += transfer_size;
793 		ls_offset += transfer_size;
794 	}
795 	return 0;
796 }
797 
798 static inline void save_ls_16kb(struct spu_state *csa, struct spu *spu)
799 {
800 	unsigned long addr = (unsigned long)&csa->lscsa->ls[0];
801 	unsigned int ls_offset = 0x0;
802 	unsigned int size = 16384;
803 	unsigned int tag = 0;
804 	unsigned int rclass = 0;
805 	unsigned int cmd = MFC_PUT_CMD;
806 
807 	/* Save, Step 50:
808 	 *     Issue a DMA command to copy the first 16K bytes
809 	 *     of local storage to the CSA.
810 	 */
811 	send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd);
812 }
813 
814 static inline void set_spu_npc(struct spu_state *csa, struct spu *spu)
815 {
816 	struct spu_problem __iomem *prob = spu->problem;
817 
818 	/* Save, Step 51:
819 	 * Restore, Step 31.
820 	 *     Write SPU_NPC[IE]=0 and SPU_NPC[LSA] to entry
821 	 *     point address of context save code in local
822 	 *     storage.
823 	 *
824 	 *     This implementation uses SPU-side save/restore
825 	 *     programs with entry points at LSA of 0.
826 	 */
827 	out_be32(&prob->spu_npc_RW, 0);
828 	eieio();
829 }
830 
831 static inline void set_signot1(struct spu_state *csa, struct spu *spu)
832 {
833 	struct spu_problem __iomem *prob = spu->problem;
834 	union {
835 		u64 ull;
836 		u32 ui[2];
837 	} addr64;
838 
839 	/* Save, Step 52:
840 	 * Restore, Step 32:
841 	 *    Write SPU_Sig_Notify_1 register with upper 32-bits
842 	 *    of the CSA.LSCSA effective address.
843 	 */
844 	addr64.ull = (u64) csa->lscsa;
845 	out_be32(&prob->signal_notify1, addr64.ui[0]);
846 	eieio();
847 }
848 
849 static inline void set_signot2(struct spu_state *csa, struct spu *spu)
850 {
851 	struct spu_problem __iomem *prob = spu->problem;
852 	union {
853 		u64 ull;
854 		u32 ui[2];
855 	} addr64;
856 
857 	/* Save, Step 53:
858 	 * Restore, Step 33:
859 	 *    Write SPU_Sig_Notify_2 register with lower 32-bits
860 	 *    of the CSA.LSCSA effective address.
861 	 */
862 	addr64.ull = (u64) csa->lscsa;
863 	out_be32(&prob->signal_notify2, addr64.ui[1]);
864 	eieio();
865 }
866 
867 static inline void send_save_code(struct spu_state *csa, struct spu *spu)
868 {
869 	unsigned long addr = (unsigned long)&spu_save_code[0];
870 	unsigned int ls_offset = 0x0;
871 	unsigned int size = sizeof(spu_save_code);
872 	unsigned int tag = 0;
873 	unsigned int rclass = 0;
874 	unsigned int cmd = MFC_GETFS_CMD;
875 
876 	/* Save, Step 54:
877 	 *     Issue a DMA command to copy context save code
878 	 *     to local storage and start SPU.
879 	 */
880 	send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd);
881 }
882 
883 static inline void set_ppu_querymask(struct spu_state *csa, struct spu *spu)
884 {
885 	struct spu_problem __iomem *prob = spu->problem;
886 
887 	/* Save, Step 55:
888 	 * Restore, Step 38.
889 	 *     Write PPU_QueryMask=1 (enable Tag Group 0)
890 	 *     and issue eieio instruction.
891 	 */
892 	out_be32(&prob->dma_querymask_RW, MFC_TAGID_TO_TAGMASK(0));
893 	eieio();
894 }
895 
896 static inline void wait_tag_complete(struct spu_state *csa, struct spu *spu)
897 {
898 	struct spu_problem __iomem *prob = spu->problem;
899 	u32 mask = MFC_TAGID_TO_TAGMASK(0);
900 	unsigned long flags;
901 
902 	/* Save, Step 56:
903 	 * Restore, Step 39.
904 	 * Restore, Step 39.
905 	 * Restore, Step 46.
906 	 *     Poll PPU_TagStatus[gn] until 01 (Tag group 0 complete)
907 	 *     or write PPU_QueryType[TS]=01 and wait for Tag Group
908 	 *     Complete Interrupt.  Write INT_Stat_Class0 or
909 	 *     INT_Stat_Class2 with value of 'handled'.
910 	 */
911 	POLL_WHILE_FALSE(in_be32(&prob->dma_tagstatus_R) & mask);
912 
913 	local_irq_save(flags);
914 	spu_int_stat_clear(spu, 0, CLASS0_INTR_MASK);
915 	spu_int_stat_clear(spu, 2, CLASS2_INTR_MASK);
916 	local_irq_restore(flags);
917 }
918 
919 static inline void wait_spu_stopped(struct spu_state *csa, struct spu *spu)
920 {
921 	struct spu_problem __iomem *prob = spu->problem;
922 	unsigned long flags;
923 
924 	/* Save, Step 57:
925 	 * Restore, Step 40.
926 	 *     Poll until SPU_Status[R]=0 or wait for SPU Class 0
927 	 *     or SPU Class 2 interrupt.  Write INT_Stat_class0
928 	 *     or INT_Stat_class2 with value of handled.
929 	 */
930 	POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING);
931 
932 	local_irq_save(flags);
933 	spu_int_stat_clear(spu, 0, CLASS0_INTR_MASK);
934 	spu_int_stat_clear(spu, 2, CLASS2_INTR_MASK);
935 	local_irq_restore(flags);
936 }
937 
938 static inline int check_save_status(struct spu_state *csa, struct spu *spu)
939 {
940 	struct spu_problem __iomem *prob = spu->problem;
941 	u32 complete;
942 
943 	/* Save, Step 54:
944 	 *     If SPU_Status[P]=1 and SPU_Status[SC] = "success",
945 	 *     context save succeeded, otherwise context save
946 	 *     failed.
947 	 */
948 	complete = ((SPU_SAVE_COMPLETE << SPU_STOP_STATUS_SHIFT) |
949 		    SPU_STATUS_STOPPED_BY_STOP);
950 	return (in_be32(&prob->spu_status_R) != complete) ? 1 : 0;
951 }
952 
953 static inline void terminate_spu_app(struct spu_state *csa, struct spu *spu)
954 {
955 	/* Restore, Step 4:
956 	 *    If required, notify the "using application" that
957 	 *    the SPU task has been terminated.  TBD.
958 	 */
959 }
960 
961 static inline void suspend_mfc_and_halt_decr(struct spu_state *csa,
962 		struct spu *spu)
963 {
964 	struct spu_priv2 __iomem *priv2 = spu->priv2;
965 
966 	/* Restore, Step 7:
967 	 *     Write MFC_Cntl[Dh,Sc,Sm]='1','1','0' to suspend
968 	 *     the queue and halt the decrementer.
969 	 */
970 	out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE |
971 		 MFC_CNTL_DECREMENTER_HALTED);
972 	eieio();
973 }
974 
975 static inline void wait_suspend_mfc_complete(struct spu_state *csa,
976 					     struct spu *spu)
977 {
978 	struct spu_priv2 __iomem *priv2 = spu->priv2;
979 
980 	/* Restore, Step 8:
981 	 * Restore, Step 47.
982 	 *     Poll MFC_CNTL[Ss] until 11 is returned.
983 	 */
984 	POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) &
985 			 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) ==
986 			 MFC_CNTL_SUSPEND_COMPLETE);
987 }
988 
989 static inline int suspend_spe(struct spu_state *csa, struct spu *spu)
990 {
991 	struct spu_problem __iomem *prob = spu->problem;
992 
993 	/* Restore, Step 9:
994 	 *    If SPU_Status[R]=1, stop SPU execution
995 	 *    and wait for stop to complete.
996 	 *
997 	 *    Returns       1 if SPU_Status[R]=1 on entry.
998 	 *                  0 otherwise
999 	 */
1000 	if (in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) {
1001 		if (in_be32(&prob->spu_status_R) &
1002 		    SPU_STATUS_ISOLATED_EXIT_STATUS) {
1003 			POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1004 					SPU_STATUS_RUNNING);
1005 		}
1006 		if ((in_be32(&prob->spu_status_R) &
1007 		     SPU_STATUS_ISOLATED_LOAD_STATUS)
1008 		    || (in_be32(&prob->spu_status_R) &
1009 			SPU_STATUS_ISOLATED_STATE)) {
1010 			out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
1011 			eieio();
1012 			POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1013 					SPU_STATUS_RUNNING);
1014 			out_be32(&prob->spu_runcntl_RW, 0x2);
1015 			eieio();
1016 			POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1017 					SPU_STATUS_RUNNING);
1018 		}
1019 		if (in_be32(&prob->spu_status_R) &
1020 		    SPU_STATUS_WAITING_FOR_CHANNEL) {
1021 			out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
1022 			eieio();
1023 			POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1024 					SPU_STATUS_RUNNING);
1025 		}
1026 		return 1;
1027 	}
1028 	return 0;
1029 }
1030 
1031 static inline void clear_spu_status(struct spu_state *csa, struct spu *spu)
1032 {
1033 	struct spu_problem __iomem *prob = spu->problem;
1034 
1035 	/* Restore, Step 10:
1036 	 *    If SPU_Status[R]=0 and SPU_Status[E,L,IS]=1,
1037 	 *    release SPU from isolate state.
1038 	 */
1039 	if (!(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING)) {
1040 		if (in_be32(&prob->spu_status_R) &
1041 		    SPU_STATUS_ISOLATED_EXIT_STATUS) {
1042 			spu_mfc_sr1_set(spu,
1043 					MFC_STATE1_MASTER_RUN_CONTROL_MASK);
1044 			eieio();
1045 			out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE);
1046 			eieio();
1047 			POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1048 					SPU_STATUS_RUNNING);
1049 		}
1050 		if ((in_be32(&prob->spu_status_R) &
1051 		     SPU_STATUS_ISOLATED_LOAD_STATUS)
1052 		    || (in_be32(&prob->spu_status_R) &
1053 			SPU_STATUS_ISOLATED_STATE)) {
1054 			spu_mfc_sr1_set(spu,
1055 					MFC_STATE1_MASTER_RUN_CONTROL_MASK);
1056 			eieio();
1057 			out_be32(&prob->spu_runcntl_RW, 0x2);
1058 			eieio();
1059 			POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1060 					SPU_STATUS_RUNNING);
1061 		}
1062 	}
1063 }
1064 
1065 static inline void reset_ch_part1(struct spu_state *csa, struct spu *spu)
1066 {
1067 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1068 	u64 ch_indices[] = { 0UL, 3UL, 4UL, 24UL, 25UL, 27UL };
1069 	u64 idx;
1070 	int i;
1071 
1072 	/* Restore, Step 20:
1073 	 */
1074 
1075 	/* Reset CH 1 */
1076 	out_be64(&priv2->spu_chnlcntptr_RW, 1);
1077 	out_be64(&priv2->spu_chnldata_RW, 0UL);
1078 
1079 	/* Reset the following CH: [0,3,4,24,25,27] */
1080 	for (i = 0; i < ARRAY_SIZE(ch_indices); i++) {
1081 		idx = ch_indices[i];
1082 		out_be64(&priv2->spu_chnlcntptr_RW, idx);
1083 		eieio();
1084 		out_be64(&priv2->spu_chnldata_RW, 0UL);
1085 		out_be64(&priv2->spu_chnlcnt_RW, 0UL);
1086 		eieio();
1087 	}
1088 }
1089 
1090 static inline void reset_ch_part2(struct spu_state *csa, struct spu *spu)
1091 {
1092 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1093 	u64 ch_indices[5] = { 21UL, 23UL, 28UL, 29UL, 30UL };
1094 	u64 ch_counts[5] = { 16UL, 1UL, 1UL, 0UL, 1UL };
1095 	u64 idx;
1096 	int i;
1097 
1098 	/* Restore, Step 21:
1099 	 *     Reset the following CH: [21, 23, 28, 29, 30]
1100 	 */
1101 	for (i = 0; i < 5; i++) {
1102 		idx = ch_indices[i];
1103 		out_be64(&priv2->spu_chnlcntptr_RW, idx);
1104 		eieio();
1105 		out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]);
1106 		eieio();
1107 	}
1108 }
1109 
1110 static inline void setup_spu_status_part1(struct spu_state *csa,
1111 					  struct spu *spu)
1112 {
1113 	u32 status_P = SPU_STATUS_STOPPED_BY_STOP;
1114 	u32 status_I = SPU_STATUS_INVALID_INSTR;
1115 	u32 status_H = SPU_STATUS_STOPPED_BY_HALT;
1116 	u32 status_S = SPU_STATUS_SINGLE_STEP;
1117 	u32 status_S_I = SPU_STATUS_SINGLE_STEP | SPU_STATUS_INVALID_INSTR;
1118 	u32 status_S_P = SPU_STATUS_SINGLE_STEP | SPU_STATUS_STOPPED_BY_STOP;
1119 	u32 status_P_H = SPU_STATUS_STOPPED_BY_HALT |SPU_STATUS_STOPPED_BY_STOP;
1120 	u32 status_P_I = SPU_STATUS_STOPPED_BY_STOP |SPU_STATUS_INVALID_INSTR;
1121 	u32 status_code;
1122 
1123 	/* Restore, Step 27:
1124 	 *     If the CSA.SPU_Status[I,S,H,P]=1 then add the correct
1125 	 *     instruction sequence to the end of the SPU based restore
1126 	 *     code (after the "context restored" stop and signal) to
1127 	 *     restore the correct SPU status.
1128 	 *
1129 	 *     NOTE: Rather than modifying the SPU executable, we
1130 	 *     instead add a new 'stopped_status' field to the
1131 	 *     LSCSA.  The SPU-side restore reads this field and
1132 	 *     takes the appropriate action when exiting.
1133 	 */
1134 
1135 	status_code =
1136 	    (csa->prob.spu_status_R >> SPU_STOP_STATUS_SHIFT) & 0xFFFF;
1137 	if ((csa->prob.spu_status_R & status_P_I) == status_P_I) {
1138 
1139 		/* SPU_Status[P,I]=1 - Illegal Instruction followed
1140 		 * by Stop and Signal instruction, followed by 'br -4'.
1141 		 *
1142 		 */
1143 		csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P_I;
1144 		csa->lscsa->stopped_status.slot[1] = status_code;
1145 
1146 	} else if ((csa->prob.spu_status_R & status_P_H) == status_P_H) {
1147 
1148 		/* SPU_Status[P,H]=1 - Halt Conditional, followed
1149 		 * by Stop and Signal instruction, followed by
1150 		 * 'br -4'.
1151 		 */
1152 		csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P_H;
1153 		csa->lscsa->stopped_status.slot[1] = status_code;
1154 
1155 	} else if ((csa->prob.spu_status_R & status_S_P) == status_S_P) {
1156 
1157 		/* SPU_Status[S,P]=1 - Stop and Signal instruction
1158 		 * followed by 'br -4'.
1159 		 */
1160 		csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S_P;
1161 		csa->lscsa->stopped_status.slot[1] = status_code;
1162 
1163 	} else if ((csa->prob.spu_status_R & status_S_I) == status_S_I) {
1164 
1165 		/* SPU_Status[S,I]=1 - Illegal instruction followed
1166 		 * by 'br -4'.
1167 		 */
1168 		csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S_I;
1169 		csa->lscsa->stopped_status.slot[1] = status_code;
1170 
1171 	} else if ((csa->prob.spu_status_R & status_P) == status_P) {
1172 
1173 		/* SPU_Status[P]=1 - Stop and Signal instruction
1174 		 * followed by 'br -4'.
1175 		 */
1176 		csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P;
1177 		csa->lscsa->stopped_status.slot[1] = status_code;
1178 
1179 	} else if ((csa->prob.spu_status_R & status_H) == status_H) {
1180 
1181 		/* SPU_Status[H]=1 - Halt Conditional, followed
1182 		 * by 'br -4'.
1183 		 */
1184 		csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_H;
1185 
1186 	} else if ((csa->prob.spu_status_R & status_S) == status_S) {
1187 
1188 		/* SPU_Status[S]=1 - Two nop instructions.
1189 		 */
1190 		csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S;
1191 
1192 	} else if ((csa->prob.spu_status_R & status_I) == status_I) {
1193 
1194 		/* SPU_Status[I]=1 - Illegal instruction followed
1195 		 * by 'br -4'.
1196 		 */
1197 		csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_I;
1198 
1199 	}
1200 }
1201 
1202 static inline void setup_spu_status_part2(struct spu_state *csa,
1203 					  struct spu *spu)
1204 {
1205 	u32 mask;
1206 
1207 	/* Restore, Step 28:
1208 	 *     If the CSA.SPU_Status[I,S,H,P,R]=0 then
1209 	 *     add a 'br *' instruction to the end of
1210 	 *     the SPU based restore code.
1211 	 *
1212 	 *     NOTE: Rather than modifying the SPU executable, we
1213 	 *     instead add a new 'stopped_status' field to the
1214 	 *     LSCSA.  The SPU-side restore reads this field and
1215 	 *     takes the appropriate action when exiting.
1216 	 */
1217 	mask = SPU_STATUS_INVALID_INSTR |
1218 	    SPU_STATUS_SINGLE_STEP |
1219 	    SPU_STATUS_STOPPED_BY_HALT |
1220 	    SPU_STATUS_STOPPED_BY_STOP | SPU_STATUS_RUNNING;
1221 	if (!(csa->prob.spu_status_R & mask)) {
1222 		csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_R;
1223 	}
1224 }
1225 
1226 static inline void restore_mfc_rag(struct spu_state *csa, struct spu *spu)
1227 {
1228 	/* Restore, Step 29:
1229 	 *     Restore RA_GROUP_ID register and the
1230 	 *     RA_ENABLE reigster from the CSA.
1231 	 */
1232 	spu_resource_allocation_groupID_set(spu,
1233 			csa->priv1.resource_allocation_groupID_RW);
1234 	spu_resource_allocation_enable_set(spu,
1235 			csa->priv1.resource_allocation_enable_RW);
1236 }
1237 
1238 static inline void send_restore_code(struct spu_state *csa, struct spu *spu)
1239 {
1240 	unsigned long addr = (unsigned long)&spu_restore_code[0];
1241 	unsigned int ls_offset = 0x0;
1242 	unsigned int size = sizeof(spu_restore_code);
1243 	unsigned int tag = 0;
1244 	unsigned int rclass = 0;
1245 	unsigned int cmd = MFC_GETFS_CMD;
1246 
1247 	/* Restore, Step 37:
1248 	 *     Issue MFC DMA command to copy context
1249 	 *     restore code to local storage.
1250 	 */
1251 	send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd);
1252 }
1253 
1254 static inline void setup_decr(struct spu_state *csa, struct spu *spu)
1255 {
1256 	/* Restore, Step 34:
1257 	 *     If CSA.MFC_CNTL[Ds]=1 (decrementer was
1258 	 *     running) then adjust decrementer, set
1259 	 *     decrementer running status in LSCSA,
1260 	 *     and set decrementer "wrapped" status
1261 	 *     in LSCSA.
1262 	 */
1263 	if (csa->priv2.mfc_control_RW & MFC_CNTL_DECREMENTER_RUNNING) {
1264 		cycles_t resume_time = get_cycles();
1265 		cycles_t delta_time = resume_time - csa->suspend_time;
1266 
1267 		csa->lscsa->decr_status.slot[0] = SPU_DECR_STATUS_RUNNING;
1268 		if (csa->lscsa->decr.slot[0] < delta_time) {
1269 			csa->lscsa->decr_status.slot[0] |=
1270 				 SPU_DECR_STATUS_WRAPPED;
1271 		}
1272 
1273 		csa->lscsa->decr.slot[0] -= delta_time;
1274 	} else {
1275 		csa->lscsa->decr_status.slot[0] = 0;
1276 	}
1277 }
1278 
1279 static inline void setup_ppu_mb(struct spu_state *csa, struct spu *spu)
1280 {
1281 	/* Restore, Step 35:
1282 	 *     Copy the CSA.PU_MB data into the LSCSA.
1283 	 */
1284 	csa->lscsa->ppu_mb.slot[0] = csa->prob.pu_mb_R;
1285 }
1286 
1287 static inline void setup_ppuint_mb(struct spu_state *csa, struct spu *spu)
1288 {
1289 	/* Restore, Step 36:
1290 	 *     Copy the CSA.PUINT_MB data into the LSCSA.
1291 	 */
1292 	csa->lscsa->ppuint_mb.slot[0] = csa->priv2.puint_mb_R;
1293 }
1294 
1295 static inline int check_restore_status(struct spu_state *csa, struct spu *spu)
1296 {
1297 	struct spu_problem __iomem *prob = spu->problem;
1298 	u32 complete;
1299 
1300 	/* Restore, Step 40:
1301 	 *     If SPU_Status[P]=1 and SPU_Status[SC] = "success",
1302 	 *     context restore succeeded, otherwise context restore
1303 	 *     failed.
1304 	 */
1305 	complete = ((SPU_RESTORE_COMPLETE << SPU_STOP_STATUS_SHIFT) |
1306 		    SPU_STATUS_STOPPED_BY_STOP);
1307 	return (in_be32(&prob->spu_status_R) != complete) ? 1 : 0;
1308 }
1309 
1310 static inline void restore_spu_privcntl(struct spu_state *csa, struct spu *spu)
1311 {
1312 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1313 
1314 	/* Restore, Step 41:
1315 	 *     Restore SPU_PrivCntl from the CSA.
1316 	 */
1317 	out_be64(&priv2->spu_privcntl_RW, csa->priv2.spu_privcntl_RW);
1318 	eieio();
1319 }
1320 
1321 static inline void restore_status_part1(struct spu_state *csa, struct spu *spu)
1322 {
1323 	struct spu_problem __iomem *prob = spu->problem;
1324 	u32 mask;
1325 
1326 	/* Restore, Step 42:
1327 	 *     If any CSA.SPU_Status[I,S,H,P]=1, then
1328 	 *     restore the error or single step state.
1329 	 */
1330 	mask = SPU_STATUS_INVALID_INSTR |
1331 	    SPU_STATUS_SINGLE_STEP |
1332 	    SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP;
1333 	if (csa->prob.spu_status_R & mask) {
1334 		out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE);
1335 		eieio();
1336 		POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1337 				SPU_STATUS_RUNNING);
1338 	}
1339 }
1340 
1341 static inline void restore_status_part2(struct spu_state *csa, struct spu *spu)
1342 {
1343 	struct spu_problem __iomem *prob = spu->problem;
1344 	u32 mask;
1345 
1346 	/* Restore, Step 43:
1347 	 *     If all CSA.SPU_Status[I,S,H,P,R]=0 then write
1348 	 *     SPU_RunCntl[R0R1]='01', wait for SPU_Status[R]=1,
1349 	 *     then write '00' to SPU_RunCntl[R0R1] and wait
1350 	 *     for SPU_Status[R]=0.
1351 	 */
1352 	mask = SPU_STATUS_INVALID_INSTR |
1353 	    SPU_STATUS_SINGLE_STEP |
1354 	    SPU_STATUS_STOPPED_BY_HALT |
1355 	    SPU_STATUS_STOPPED_BY_STOP | SPU_STATUS_RUNNING;
1356 	if (!(csa->prob.spu_status_R & mask)) {
1357 		out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE);
1358 		eieio();
1359 		POLL_WHILE_FALSE(in_be32(&prob->spu_status_R) &
1360 				 SPU_STATUS_RUNNING);
1361 		out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
1362 		eieio();
1363 		POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1364 				SPU_STATUS_RUNNING);
1365 	}
1366 }
1367 
1368 static inline void restore_ls_16kb(struct spu_state *csa, struct spu *spu)
1369 {
1370 	unsigned long addr = (unsigned long)&csa->lscsa->ls[0];
1371 	unsigned int ls_offset = 0x0;
1372 	unsigned int size = 16384;
1373 	unsigned int tag = 0;
1374 	unsigned int rclass = 0;
1375 	unsigned int cmd = MFC_GET_CMD;
1376 
1377 	/* Restore, Step 44:
1378 	 *     Issue a DMA command to restore the first
1379 	 *     16kb of local storage from CSA.
1380 	 */
1381 	send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd);
1382 }
1383 
1384 static inline void suspend_mfc(struct spu_state *csa, struct spu *spu)
1385 {
1386 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1387 
1388 	/* Restore, Step 47.
1389 	 *     Write MFC_Cntl[Sc,Sm]='1','0' to suspend
1390 	 *     the queue.
1391 	 */
1392 	out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE);
1393 	eieio();
1394 }
1395 
1396 static inline void clear_interrupts(struct spu_state *csa, struct spu *spu)
1397 {
1398 	/* Restore, Step 49:
1399 	 *     Write INT_MASK_class0 with value of 0.
1400 	 *     Write INT_MASK_class1 with value of 0.
1401 	 *     Write INT_MASK_class2 with value of 0.
1402 	 *     Write INT_STAT_class0 with value of -1.
1403 	 *     Write INT_STAT_class1 with value of -1.
1404 	 *     Write INT_STAT_class2 with value of -1.
1405 	 */
1406 	spin_lock_irq(&spu->register_lock);
1407 	spu_int_mask_set(spu, 0, 0ul);
1408 	spu_int_mask_set(spu, 1, 0ul);
1409 	spu_int_mask_set(spu, 2, 0ul);
1410 	spu_int_stat_clear(spu, 0, CLASS0_INTR_MASK);
1411 	spu_int_stat_clear(spu, 1, CLASS1_INTR_MASK);
1412 	spu_int_stat_clear(spu, 2, CLASS2_INTR_MASK);
1413 	spin_unlock_irq(&spu->register_lock);
1414 }
1415 
1416 static inline void restore_mfc_queues(struct spu_state *csa, struct spu *spu)
1417 {
1418 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1419 	int i;
1420 
1421 	/* Restore, Step 50:
1422 	 *     If MFC_Cntl[Se]!=0 then restore
1423 	 *     MFC command queues.
1424 	 */
1425 	if ((csa->priv2.mfc_control_RW & MFC_CNTL_DMA_QUEUES_EMPTY_MASK) == 0) {
1426 		for (i = 0; i < 8; i++) {
1427 			out_be64(&priv2->puq[i].mfc_cq_data0_RW,
1428 				 csa->priv2.puq[i].mfc_cq_data0_RW);
1429 			out_be64(&priv2->puq[i].mfc_cq_data1_RW,
1430 				 csa->priv2.puq[i].mfc_cq_data1_RW);
1431 			out_be64(&priv2->puq[i].mfc_cq_data2_RW,
1432 				 csa->priv2.puq[i].mfc_cq_data2_RW);
1433 			out_be64(&priv2->puq[i].mfc_cq_data3_RW,
1434 				 csa->priv2.puq[i].mfc_cq_data3_RW);
1435 		}
1436 		for (i = 0; i < 16; i++) {
1437 			out_be64(&priv2->spuq[i].mfc_cq_data0_RW,
1438 				 csa->priv2.spuq[i].mfc_cq_data0_RW);
1439 			out_be64(&priv2->spuq[i].mfc_cq_data1_RW,
1440 				 csa->priv2.spuq[i].mfc_cq_data1_RW);
1441 			out_be64(&priv2->spuq[i].mfc_cq_data2_RW,
1442 				 csa->priv2.spuq[i].mfc_cq_data2_RW);
1443 			out_be64(&priv2->spuq[i].mfc_cq_data3_RW,
1444 				 csa->priv2.spuq[i].mfc_cq_data3_RW);
1445 		}
1446 	}
1447 	eieio();
1448 }
1449 
1450 static inline void restore_ppu_querymask(struct spu_state *csa, struct spu *spu)
1451 {
1452 	struct spu_problem __iomem *prob = spu->problem;
1453 
1454 	/* Restore, Step 51:
1455 	 *     Restore the PPU_QueryMask register from CSA.
1456 	 */
1457 	out_be32(&prob->dma_querymask_RW, csa->prob.dma_querymask_RW);
1458 	eieio();
1459 }
1460 
1461 static inline void restore_ppu_querytype(struct spu_state *csa, struct spu *spu)
1462 {
1463 	struct spu_problem __iomem *prob = spu->problem;
1464 
1465 	/* Restore, Step 52:
1466 	 *     Restore the PPU_QueryType register from CSA.
1467 	 */
1468 	out_be32(&prob->dma_querytype_RW, csa->prob.dma_querytype_RW);
1469 	eieio();
1470 }
1471 
1472 static inline void restore_mfc_csr_tsq(struct spu_state *csa, struct spu *spu)
1473 {
1474 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1475 
1476 	/* Restore, Step 53:
1477 	 *     Restore the MFC_CSR_TSQ register from CSA.
1478 	 */
1479 	out_be64(&priv2->spu_tag_status_query_RW,
1480 		 csa->priv2.spu_tag_status_query_RW);
1481 	eieio();
1482 }
1483 
1484 static inline void restore_mfc_csr_cmd(struct spu_state *csa, struct spu *spu)
1485 {
1486 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1487 
1488 	/* Restore, Step 54:
1489 	 *     Restore the MFC_CSR_CMD1 and MFC_CSR_CMD2
1490 	 *     registers from CSA.
1491 	 */
1492 	out_be64(&priv2->spu_cmd_buf1_RW, csa->priv2.spu_cmd_buf1_RW);
1493 	out_be64(&priv2->spu_cmd_buf2_RW, csa->priv2.spu_cmd_buf2_RW);
1494 	eieio();
1495 }
1496 
1497 static inline void restore_mfc_csr_ato(struct spu_state *csa, struct spu *spu)
1498 {
1499 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1500 
1501 	/* Restore, Step 55:
1502 	 *     Restore the MFC_CSR_ATO register from CSA.
1503 	 */
1504 	out_be64(&priv2->spu_atomic_status_RW, csa->priv2.spu_atomic_status_RW);
1505 }
1506 
1507 static inline void restore_mfc_tclass_id(struct spu_state *csa, struct spu *spu)
1508 {
1509 	/* Restore, Step 56:
1510 	 *     Restore the MFC_TCLASS_ID register from CSA.
1511 	 */
1512 	spu_mfc_tclass_id_set(spu, csa->priv1.mfc_tclass_id_RW);
1513 	eieio();
1514 }
1515 
1516 static inline void set_llr_event(struct spu_state *csa, struct spu *spu)
1517 {
1518 	u64 ch0_cnt, ch0_data;
1519 	u64 ch1_data;
1520 
1521 	/* Restore, Step 57:
1522 	 *    Set the Lock Line Reservation Lost Event by:
1523 	 *      1. OR CSA.SPU_Event_Status with bit 21 (Lr) set to 1.
1524 	 *      2. If CSA.SPU_Channel_0_Count=0 and
1525 	 *         CSA.SPU_Wr_Event_Mask[Lr]=1 and
1526 	 *         CSA.SPU_Event_Status[Lr]=0 then set
1527 	 *         CSA.SPU_Event_Status_Count=1.
1528 	 */
1529 	ch0_cnt = csa->spu_chnlcnt_RW[0];
1530 	ch0_data = csa->spu_chnldata_RW[0];
1531 	ch1_data = csa->spu_chnldata_RW[1];
1532 	csa->spu_chnldata_RW[0] |= MFC_LLR_LOST_EVENT;
1533 	if ((ch0_cnt == 0) && !(ch0_data & MFC_LLR_LOST_EVENT) &&
1534 	    (ch1_data & MFC_LLR_LOST_EVENT)) {
1535 		csa->spu_chnlcnt_RW[0] = 1;
1536 	}
1537 }
1538 
1539 static inline void restore_decr_wrapped(struct spu_state *csa, struct spu *spu)
1540 {
1541 	/* Restore, Step 58:
1542 	 *     If the status of the CSA software decrementer
1543 	 *     "wrapped" flag is set, OR in a '1' to
1544 	 *     CSA.SPU_Event_Status[Tm].
1545 	 */
1546 	if (!(csa->lscsa->decr_status.slot[0] & SPU_DECR_STATUS_WRAPPED))
1547 		return;
1548 
1549 	if ((csa->spu_chnlcnt_RW[0] == 0) &&
1550 	    (csa->spu_chnldata_RW[1] & 0x20) &&
1551 	    !(csa->spu_chnldata_RW[0] & 0x20))
1552 		csa->spu_chnlcnt_RW[0] = 1;
1553 
1554 	csa->spu_chnldata_RW[0] |= 0x20;
1555 }
1556 
1557 static inline void restore_ch_part1(struct spu_state *csa, struct spu *spu)
1558 {
1559 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1560 	u64 idx, ch_indices[] = { 0UL, 3UL, 4UL, 24UL, 25UL, 27UL };
1561 	int i;
1562 
1563 	/* Restore, Step 59:
1564 	 *	Restore the following CH: [0,3,4,24,25,27]
1565 	 */
1566 	for (i = 0; i < ARRAY_SIZE(ch_indices); i++) {
1567 		idx = ch_indices[i];
1568 		out_be64(&priv2->spu_chnlcntptr_RW, idx);
1569 		eieio();
1570 		out_be64(&priv2->spu_chnldata_RW, csa->spu_chnldata_RW[idx]);
1571 		out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[idx]);
1572 		eieio();
1573 	}
1574 }
1575 
1576 static inline void restore_ch_part2(struct spu_state *csa, struct spu *spu)
1577 {
1578 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1579 	u64 ch_indices[3] = { 9UL, 21UL, 23UL };
1580 	u64 ch_counts[3] = { 1UL, 16UL, 1UL };
1581 	u64 idx;
1582 	int i;
1583 
1584 	/* Restore, Step 60:
1585 	 *     Restore the following CH: [9,21,23].
1586 	 */
1587 	ch_counts[0] = 1UL;
1588 	ch_counts[1] = csa->spu_chnlcnt_RW[21];
1589 	ch_counts[2] = 1UL;
1590 	for (i = 0; i < 3; i++) {
1591 		idx = ch_indices[i];
1592 		out_be64(&priv2->spu_chnlcntptr_RW, idx);
1593 		eieio();
1594 		out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]);
1595 		eieio();
1596 	}
1597 }
1598 
1599 static inline void restore_spu_lslr(struct spu_state *csa, struct spu *spu)
1600 {
1601 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1602 
1603 	/* Restore, Step 61:
1604 	 *     Restore the SPU_LSLR register from CSA.
1605 	 */
1606 	out_be64(&priv2->spu_lslr_RW, csa->priv2.spu_lslr_RW);
1607 	eieio();
1608 }
1609 
1610 static inline void restore_spu_cfg(struct spu_state *csa, struct spu *spu)
1611 {
1612 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1613 
1614 	/* Restore, Step 62:
1615 	 *     Restore the SPU_Cfg register from CSA.
1616 	 */
1617 	out_be64(&priv2->spu_cfg_RW, csa->priv2.spu_cfg_RW);
1618 	eieio();
1619 }
1620 
1621 static inline void restore_pm_trace(struct spu_state *csa, struct spu *spu)
1622 {
1623 	/* Restore, Step 63:
1624 	 *     Restore PM_Trace_Tag_Wait_Mask from CSA.
1625 	 *     Not performed by this implementation.
1626 	 */
1627 }
1628 
1629 static inline void restore_spu_npc(struct spu_state *csa, struct spu *spu)
1630 {
1631 	struct spu_problem __iomem *prob = spu->problem;
1632 
1633 	/* Restore, Step 64:
1634 	 *     Restore SPU_NPC from CSA.
1635 	 */
1636 	out_be32(&prob->spu_npc_RW, csa->prob.spu_npc_RW);
1637 	eieio();
1638 }
1639 
1640 static inline void restore_spu_mb(struct spu_state *csa, struct spu *spu)
1641 {
1642 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1643 	int i;
1644 
1645 	/* Restore, Step 65:
1646 	 *     Restore MFC_RdSPU_MB from CSA.
1647 	 */
1648 	out_be64(&priv2->spu_chnlcntptr_RW, 29UL);
1649 	eieio();
1650 	out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[29]);
1651 	for (i = 0; i < 4; i++) {
1652 		out_be64(&priv2->spu_chnldata_RW, csa->spu_mailbox_data[i]);
1653 	}
1654 	eieio();
1655 }
1656 
1657 static inline void check_ppu_mb_stat(struct spu_state *csa, struct spu *spu)
1658 {
1659 	struct spu_problem __iomem *prob = spu->problem;
1660 
1661 	/* Restore, Step 66:
1662 	 *     If CSA.MB_Stat[P]=0 (mailbox empty) then
1663 	 *     read from the PPU_MB register.
1664 	 */
1665 	if ((csa->prob.mb_stat_R & 0xFF) == 0) {
1666 		in_be32(&prob->pu_mb_R);
1667 		eieio();
1668 	}
1669 }
1670 
1671 static inline void check_ppuint_mb_stat(struct spu_state *csa, struct spu *spu)
1672 {
1673 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1674 
1675 	/* Restore, Step 66:
1676 	 *     If CSA.MB_Stat[I]=0 (mailbox empty) then
1677 	 *     read from the PPUINT_MB register.
1678 	 */
1679 	if ((csa->prob.mb_stat_R & 0xFF0000) == 0) {
1680 		in_be64(&priv2->puint_mb_R);
1681 		eieio();
1682 		spu_int_stat_clear(spu, 2, CLASS2_ENABLE_MAILBOX_INTR);
1683 		eieio();
1684 	}
1685 }
1686 
1687 static inline void restore_mfc_sr1(struct spu_state *csa, struct spu *spu)
1688 {
1689 	/* Restore, Step 69:
1690 	 *     Restore the MFC_SR1 register from CSA.
1691 	 */
1692 	spu_mfc_sr1_set(spu, csa->priv1.mfc_sr1_RW);
1693 	eieio();
1694 }
1695 
1696 static inline void set_int_route(struct spu_state *csa, struct spu *spu)
1697 {
1698 	struct spu_context *ctx = spu->ctx;
1699 
1700 	spu_cpu_affinity_set(spu, ctx->last_ran);
1701 }
1702 
1703 static inline void restore_other_spu_access(struct spu_state *csa,
1704 					    struct spu *spu)
1705 {
1706 	/* Restore, Step 70:
1707 	 *     Restore other SPU mappings to this SPU. TBD.
1708 	 */
1709 }
1710 
1711 static inline void restore_spu_runcntl(struct spu_state *csa, struct spu *spu)
1712 {
1713 	struct spu_problem __iomem *prob = spu->problem;
1714 
1715 	/* Restore, Step 71:
1716 	 *     If CSA.SPU_Status[R]=1 then write
1717 	 *     SPU_RunCntl[R0R1]='01'.
1718 	 */
1719 	if (csa->prob.spu_status_R & SPU_STATUS_RUNNING) {
1720 		out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE);
1721 		eieio();
1722 	}
1723 }
1724 
1725 static inline void restore_mfc_cntl(struct spu_state *csa, struct spu *spu)
1726 {
1727 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1728 
1729 	/* Restore, Step 72:
1730 	 *    Restore the MFC_CNTL register for the CSA.
1731 	 */
1732 	out_be64(&priv2->mfc_control_RW, csa->priv2.mfc_control_RW);
1733 	eieio();
1734 
1735 	/*
1736 	 * The queue is put back into the same state that was evident prior to
1737 	 * the context switch. The suspend flag is added to the saved state in
1738 	 * the csa, if the operational state was suspending or suspended. In
1739 	 * this case, the code that suspended the mfc is responsible for
1740 	 * continuing it. Note that SPE faults do not change the operational
1741 	 * state of the spu.
1742 	 */
1743 }
1744 
1745 static inline void enable_user_access(struct spu_state *csa, struct spu *spu)
1746 {
1747 	/* Restore, Step 73:
1748 	 *     Enable user-space access (if provided) to this
1749 	 *     SPU by mapping the virtual pages assigned to
1750 	 *     the SPU memory-mapped I/O (MMIO) for problem
1751 	 *     state. TBD.
1752 	 */
1753 }
1754 
1755 static inline void reset_switch_active(struct spu_state *csa, struct spu *spu)
1756 {
1757 	/* Restore, Step 74:
1758 	 *     Reset the "context switch active" flag.
1759 	 *     Not performed by this implementation.
1760 	 */
1761 }
1762 
1763 static inline void reenable_interrupts(struct spu_state *csa, struct spu *spu)
1764 {
1765 	/* Restore, Step 75:
1766 	 *     Re-enable SPU interrupts.
1767 	 */
1768 	spin_lock_irq(&spu->register_lock);
1769 	spu_int_mask_set(spu, 0, csa->priv1.int_mask_class0_RW);
1770 	spu_int_mask_set(spu, 1, csa->priv1.int_mask_class1_RW);
1771 	spu_int_mask_set(spu, 2, csa->priv1.int_mask_class2_RW);
1772 	spin_unlock_irq(&spu->register_lock);
1773 }
1774 
1775 static int quiece_spu(struct spu_state *prev, struct spu *spu)
1776 {
1777 	/*
1778 	 * Combined steps 2-18 of SPU context save sequence, which
1779 	 * quiesce the SPU state (disable SPU execution, MFC command
1780 	 * queues, decrementer, SPU interrupts, etc.).
1781 	 *
1782 	 * Returns      0 on success.
1783 	 *              2 if failed step 2.
1784 	 *              6 if failed step 6.
1785 	 */
1786 
1787 	if (check_spu_isolate(prev, spu)) {	/* Step 2. */
1788 		return 2;
1789 	}
1790 	disable_interrupts(prev, spu);	        /* Step 3. */
1791 	set_watchdog_timer(prev, spu);	        /* Step 4. */
1792 	inhibit_user_access(prev, spu);	        /* Step 5. */
1793 	if (check_spu_isolate(prev, spu)) {	/* Step 6. */
1794 		return 6;
1795 	}
1796 	set_switch_pending(prev, spu);	        /* Step 7. */
1797 	save_mfc_cntl(prev, spu);		/* Step 8. */
1798 	save_spu_runcntl(prev, spu);	        /* Step 9. */
1799 	save_mfc_sr1(prev, spu);	        /* Step 10. */
1800 	save_spu_status(prev, spu);	        /* Step 11. */
1801 	save_mfc_stopped_status(prev, spu);     /* Step 12. */
1802 	halt_mfc_decr(prev, spu);	        /* Step 13. */
1803 	save_timebase(prev, spu);		/* Step 14. */
1804 	remove_other_spu_access(prev, spu);	/* Step 15. */
1805 	do_mfc_mssync(prev, spu);	        /* Step 16. */
1806 	issue_mfc_tlbie(prev, spu);	        /* Step 17. */
1807 	handle_pending_interrupts(prev, spu);	/* Step 18. */
1808 
1809 	return 0;
1810 }
1811 
1812 static void save_csa(struct spu_state *prev, struct spu *spu)
1813 {
1814 	/*
1815 	 * Combine steps 19-44 of SPU context save sequence, which
1816 	 * save regions of the privileged & problem state areas.
1817 	 */
1818 
1819 	save_mfc_queues(prev, spu);	/* Step 19. */
1820 	save_ppu_querymask(prev, spu);	/* Step 20. */
1821 	save_ppu_querytype(prev, spu);	/* Step 21. */
1822 	save_ppu_tagstatus(prev, spu);  /* NEW.     */
1823 	save_mfc_csr_tsq(prev, spu);	/* Step 22. */
1824 	save_mfc_csr_cmd(prev, spu);	/* Step 23. */
1825 	save_mfc_csr_ato(prev, spu);	/* Step 24. */
1826 	save_mfc_tclass_id(prev, spu);	/* Step 25. */
1827 	set_mfc_tclass_id(prev, spu);	/* Step 26. */
1828 	save_mfc_cmd(prev, spu);	/* Step 26a - moved from 44. */
1829 	purge_mfc_queue(prev, spu);	/* Step 27. */
1830 	wait_purge_complete(prev, spu);	/* Step 28. */
1831 	setup_mfc_sr1(prev, spu);	/* Step 30. */
1832 	save_spu_npc(prev, spu);	/* Step 31. */
1833 	save_spu_privcntl(prev, spu);	/* Step 32. */
1834 	reset_spu_privcntl(prev, spu);	/* Step 33. */
1835 	save_spu_lslr(prev, spu);	/* Step 34. */
1836 	reset_spu_lslr(prev, spu);	/* Step 35. */
1837 	save_spu_cfg(prev, spu);	/* Step 36. */
1838 	save_pm_trace(prev, spu);	/* Step 37. */
1839 	save_mfc_rag(prev, spu);	/* Step 38. */
1840 	save_ppu_mb_stat(prev, spu);	/* Step 39. */
1841 	save_ppu_mb(prev, spu);	        /* Step 40. */
1842 	save_ppuint_mb(prev, spu);	/* Step 41. */
1843 	save_ch_part1(prev, spu);	/* Step 42. */
1844 	save_spu_mb(prev, spu);	        /* Step 43. */
1845 	reset_ch(prev, spu);	        /* Step 45. */
1846 }
1847 
1848 static void save_lscsa(struct spu_state *prev, struct spu *spu)
1849 {
1850 	/*
1851 	 * Perform steps 46-57 of SPU context save sequence,
1852 	 * which save regions of the local store and register
1853 	 * file.
1854 	 */
1855 
1856 	resume_mfc_queue(prev, spu);	/* Step 46. */
1857 	/* Step 47. */
1858 	setup_mfc_slbs(prev, spu, spu_save_code, sizeof(spu_save_code));
1859 	set_switch_active(prev, spu);	/* Step 48. */
1860 	enable_interrupts(prev, spu);	/* Step 49. */
1861 	save_ls_16kb(prev, spu);	/* Step 50. */
1862 	set_spu_npc(prev, spu);	        /* Step 51. */
1863 	set_signot1(prev, spu);		/* Step 52. */
1864 	set_signot2(prev, spu);		/* Step 53. */
1865 	send_save_code(prev, spu);	/* Step 54. */
1866 	set_ppu_querymask(prev, spu);	/* Step 55. */
1867 	wait_tag_complete(prev, spu);	/* Step 56. */
1868 	wait_spu_stopped(prev, spu);	/* Step 57. */
1869 }
1870 
1871 static void force_spu_isolate_exit(struct spu *spu)
1872 {
1873 	struct spu_problem __iomem *prob = spu->problem;
1874 	struct spu_priv2 __iomem *priv2 = spu->priv2;
1875 
1876 	/* Stop SPE execution and wait for completion. */
1877 	out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
1878 	iobarrier_rw();
1879 	POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING);
1880 
1881 	/* Restart SPE master runcntl. */
1882 	spu_mfc_sr1_set(spu, MFC_STATE1_MASTER_RUN_CONTROL_MASK);
1883 	iobarrier_w();
1884 
1885 	/* Initiate isolate exit request and wait for completion. */
1886 	out_be64(&priv2->spu_privcntl_RW, 4LL);
1887 	iobarrier_w();
1888 	out_be32(&prob->spu_runcntl_RW, 2);
1889 	iobarrier_rw();
1890 	POLL_WHILE_FALSE((in_be32(&prob->spu_status_R)
1891 				& SPU_STATUS_STOPPED_BY_STOP));
1892 
1893 	/* Reset load request to normal. */
1894 	out_be64(&priv2->spu_privcntl_RW, SPU_PRIVCNT_LOAD_REQUEST_NORMAL);
1895 	iobarrier_w();
1896 }
1897 
1898 /**
1899  * stop_spu_isolate
1900  *	Check SPU run-control state and force isolated
1901  *	exit function as necessary.
1902  */
1903 static void stop_spu_isolate(struct spu *spu)
1904 {
1905 	struct spu_problem __iomem *prob = spu->problem;
1906 
1907 	if (in_be32(&prob->spu_status_R) & SPU_STATUS_ISOLATED_STATE) {
1908 		/* The SPU is in isolated state; the only way
1909 		 * to get it out is to perform an isolated
1910 		 * exit (clean) operation.
1911 		 */
1912 		force_spu_isolate_exit(spu);
1913 	}
1914 }
1915 
1916 static void harvest(struct spu_state *prev, struct spu *spu)
1917 {
1918 	/*
1919 	 * Perform steps 2-25 of SPU context restore sequence,
1920 	 * which resets an SPU either after a failed save, or
1921 	 * when using SPU for first time.
1922 	 */
1923 
1924 	disable_interrupts(prev, spu);	        /* Step 2.  */
1925 	inhibit_user_access(prev, spu);	        /* Step 3.  */
1926 	terminate_spu_app(prev, spu);	        /* Step 4.  */
1927 	set_switch_pending(prev, spu);	        /* Step 5.  */
1928 	stop_spu_isolate(spu);			/* NEW.     */
1929 	remove_other_spu_access(prev, spu);	/* Step 6.  */
1930 	suspend_mfc_and_halt_decr(prev, spu);	/* Step 7.  */
1931 	wait_suspend_mfc_complete(prev, spu);	/* Step 8.  */
1932 	if (!suspend_spe(prev, spu))	        /* Step 9.  */
1933 		clear_spu_status(prev, spu);	/* Step 10. */
1934 	do_mfc_mssync(prev, spu);	        /* Step 11. */
1935 	issue_mfc_tlbie(prev, spu);	        /* Step 12. */
1936 	handle_pending_interrupts(prev, spu);	/* Step 13. */
1937 	purge_mfc_queue(prev, spu);	        /* Step 14. */
1938 	wait_purge_complete(prev, spu);	        /* Step 15. */
1939 	reset_spu_privcntl(prev, spu);	        /* Step 16. */
1940 	reset_spu_lslr(prev, spu);              /* Step 17. */
1941 	setup_mfc_sr1(prev, spu);	        /* Step 18. */
1942 	spu_invalidate_slbs(spu);		/* Step 19. */
1943 	reset_ch_part1(prev, spu);	        /* Step 20. */
1944 	reset_ch_part2(prev, spu);	        /* Step 21. */
1945 	enable_interrupts(prev, spu);	        /* Step 22. */
1946 	set_switch_active(prev, spu);	        /* Step 23. */
1947 	set_mfc_tclass_id(prev, spu);	        /* Step 24. */
1948 	resume_mfc_queue(prev, spu);	        /* Step 25. */
1949 }
1950 
1951 static void restore_lscsa(struct spu_state *next, struct spu *spu)
1952 {
1953 	/*
1954 	 * Perform steps 26-40 of SPU context restore sequence,
1955 	 * which restores regions of the local store and register
1956 	 * file.
1957 	 */
1958 
1959 	set_watchdog_timer(next, spu);	        /* Step 26. */
1960 	setup_spu_status_part1(next, spu);	/* Step 27. */
1961 	setup_spu_status_part2(next, spu);	/* Step 28. */
1962 	restore_mfc_rag(next, spu);	        /* Step 29. */
1963 	/* Step 30. */
1964 	setup_mfc_slbs(next, spu, spu_restore_code, sizeof(spu_restore_code));
1965 	set_spu_npc(next, spu);	                /* Step 31. */
1966 	set_signot1(next, spu);	                /* Step 32. */
1967 	set_signot2(next, spu);	                /* Step 33. */
1968 	setup_decr(next, spu);	                /* Step 34. */
1969 	setup_ppu_mb(next, spu);	        /* Step 35. */
1970 	setup_ppuint_mb(next, spu);	        /* Step 36. */
1971 	send_restore_code(next, spu);	        /* Step 37. */
1972 	set_ppu_querymask(next, spu);	        /* Step 38. */
1973 	wait_tag_complete(next, spu);	        /* Step 39. */
1974 	wait_spu_stopped(next, spu);	        /* Step 40. */
1975 }
1976 
1977 static void restore_csa(struct spu_state *next, struct spu *spu)
1978 {
1979 	/*
1980 	 * Combine steps 41-76 of SPU context restore sequence, which
1981 	 * restore regions of the privileged & problem state areas.
1982 	 */
1983 
1984 	restore_spu_privcntl(next, spu);	/* Step 41. */
1985 	restore_status_part1(next, spu);	/* Step 42. */
1986 	restore_status_part2(next, spu);	/* Step 43. */
1987 	restore_ls_16kb(next, spu);	        /* Step 44. */
1988 	wait_tag_complete(next, spu);	        /* Step 45. */
1989 	suspend_mfc(next, spu);	                /* Step 46. */
1990 	wait_suspend_mfc_complete(next, spu);	/* Step 47. */
1991 	issue_mfc_tlbie(next, spu);	        /* Step 48. */
1992 	clear_interrupts(next, spu);	        /* Step 49. */
1993 	restore_mfc_queues(next, spu);	        /* Step 50. */
1994 	restore_ppu_querymask(next, spu);	/* Step 51. */
1995 	restore_ppu_querytype(next, spu);	/* Step 52. */
1996 	restore_mfc_csr_tsq(next, spu);	        /* Step 53. */
1997 	restore_mfc_csr_cmd(next, spu);	        /* Step 54. */
1998 	restore_mfc_csr_ato(next, spu);	        /* Step 55. */
1999 	restore_mfc_tclass_id(next, spu);	/* Step 56. */
2000 	set_llr_event(next, spu);	        /* Step 57. */
2001 	restore_decr_wrapped(next, spu);	/* Step 58. */
2002 	restore_ch_part1(next, spu);	        /* Step 59. */
2003 	restore_ch_part2(next, spu);	        /* Step 60. */
2004 	restore_spu_lslr(next, spu);	        /* Step 61. */
2005 	restore_spu_cfg(next, spu);	        /* Step 62. */
2006 	restore_pm_trace(next, spu);	        /* Step 63. */
2007 	restore_spu_npc(next, spu);	        /* Step 64. */
2008 	restore_spu_mb(next, spu);	        /* Step 65. */
2009 	check_ppu_mb_stat(next, spu);	        /* Step 66. */
2010 	check_ppuint_mb_stat(next, spu);	/* Step 67. */
2011 	spu_invalidate_slbs(spu);		/* Modified Step 68. */
2012 	restore_mfc_sr1(next, spu);	        /* Step 69. */
2013 	set_int_route(next, spu);		/* NEW      */
2014 	restore_other_spu_access(next, spu);	/* Step 70. */
2015 	restore_spu_runcntl(next, spu);	        /* Step 71. */
2016 	restore_mfc_cntl(next, spu);	        /* Step 72. */
2017 	enable_user_access(next, spu);	        /* Step 73. */
2018 	reset_switch_active(next, spu);	        /* Step 74. */
2019 	reenable_interrupts(next, spu);	        /* Step 75. */
2020 }
2021 
2022 static int __do_spu_save(struct spu_state *prev, struct spu *spu)
2023 {
2024 	int rc;
2025 
2026 	/*
2027 	 * SPU context save can be broken into three phases:
2028 	 *
2029 	 *     (a) quiesce [steps 2-16].
2030 	 *     (b) save of CSA, performed by PPE [steps 17-42]
2031 	 *     (c) save of LSCSA, mostly performed by SPU [steps 43-52].
2032 	 *
2033 	 * Returns      0 on success.
2034 	 *              2,6 if failed to quiece SPU
2035 	 *              53 if SPU-side of save failed.
2036 	 */
2037 
2038 	rc = quiece_spu(prev, spu);	        /* Steps 2-16. */
2039 	switch (rc) {
2040 	default:
2041 	case 2:
2042 	case 6:
2043 		harvest(prev, spu);
2044 		return rc;
2045 		break;
2046 	case 0:
2047 		break;
2048 	}
2049 	save_csa(prev, spu);	                /* Steps 17-43. */
2050 	save_lscsa(prev, spu);	                /* Steps 44-53. */
2051 	return check_save_status(prev, spu);	/* Step 54.     */
2052 }
2053 
2054 static int __do_spu_restore(struct spu_state *next, struct spu *spu)
2055 {
2056 	int rc;
2057 
2058 	/*
2059 	 * SPU context restore can be broken into three phases:
2060 	 *
2061 	 *    (a) harvest (or reset) SPU [steps 2-24].
2062 	 *    (b) restore LSCSA [steps 25-40], mostly performed by SPU.
2063 	 *    (c) restore CSA [steps 41-76], performed by PPE.
2064 	 *
2065 	 * The 'harvest' step is not performed here, but rather
2066 	 * as needed below.
2067 	 */
2068 
2069 	restore_lscsa(next, spu);	        /* Steps 24-39. */
2070 	rc = check_restore_status(next, spu);	/* Step 40.     */
2071 	switch (rc) {
2072 	default:
2073 		/* Failed. Return now. */
2074 		return rc;
2075 		break;
2076 	case 0:
2077 		/* Fall through to next step. */
2078 		break;
2079 	}
2080 	restore_csa(next, spu);
2081 
2082 	return 0;
2083 }
2084 
2085 /**
2086  * spu_save - SPU context save, with locking.
2087  * @prev: pointer to SPU context save area, to be saved.
2088  * @spu: pointer to SPU iomem structure.
2089  *
2090  * Acquire locks, perform the save operation then return.
2091  */
2092 int spu_save(struct spu_state *prev, struct spu *spu)
2093 {
2094 	int rc;
2095 
2096 	acquire_spu_lock(spu);	        /* Step 1.     */
2097 	rc = __do_spu_save(prev, spu);	/* Steps 2-53. */
2098 	release_spu_lock(spu);
2099 	if (rc != 0 && rc != 2 && rc != 6) {
2100 		panic("%s failed on SPU[%d], rc=%d.\n",
2101 		      __func__, spu->number, rc);
2102 	}
2103 	return 0;
2104 }
2105 EXPORT_SYMBOL_GPL(spu_save);
2106 
2107 /**
2108  * spu_restore - SPU context restore, with harvest and locking.
2109  * @new: pointer to SPU context save area, to be restored.
2110  * @spu: pointer to SPU iomem structure.
2111  *
2112  * Perform harvest + restore, as we may not be coming
2113  * from a previous successful save operation, and the
2114  * hardware state is unknown.
2115  */
2116 int spu_restore(struct spu_state *new, struct spu *spu)
2117 {
2118 	int rc;
2119 
2120 	acquire_spu_lock(spu);
2121 	harvest(NULL, spu);
2122 	spu->slb_replace = 0;
2123 	rc = __do_spu_restore(new, spu);
2124 	release_spu_lock(spu);
2125 	if (rc) {
2126 		panic("%s failed on SPU[%d] rc=%d.\n",
2127 		       __func__, spu->number, rc);
2128 	}
2129 	return rc;
2130 }
2131 EXPORT_SYMBOL_GPL(spu_restore);
2132 
2133 static void init_prob(struct spu_state *csa)
2134 {
2135 	csa->spu_chnlcnt_RW[9] = 1;
2136 	csa->spu_chnlcnt_RW[21] = 16;
2137 	csa->spu_chnlcnt_RW[23] = 1;
2138 	csa->spu_chnlcnt_RW[28] = 1;
2139 	csa->spu_chnlcnt_RW[30] = 1;
2140 	csa->prob.spu_runcntl_RW = SPU_RUNCNTL_STOP;
2141 	csa->prob.mb_stat_R = 0x000400;
2142 }
2143 
2144 static void init_priv1(struct spu_state *csa)
2145 {
2146 	/* Enable decode, relocate, tlbie response, master runcntl. */
2147 	csa->priv1.mfc_sr1_RW = MFC_STATE1_LOCAL_STORAGE_DECODE_MASK |
2148 	    MFC_STATE1_MASTER_RUN_CONTROL_MASK |
2149 	    MFC_STATE1_PROBLEM_STATE_MASK |
2150 	    MFC_STATE1_RELOCATE_MASK | MFC_STATE1_BUS_TLBIE_MASK;
2151 
2152 	/* Enable OS-specific set of interrupts. */
2153 	csa->priv1.int_mask_class0_RW = CLASS0_ENABLE_DMA_ALIGNMENT_INTR |
2154 	    CLASS0_ENABLE_INVALID_DMA_COMMAND_INTR |
2155 	    CLASS0_ENABLE_SPU_ERROR_INTR;
2156 	csa->priv1.int_mask_class1_RW = CLASS1_ENABLE_SEGMENT_FAULT_INTR |
2157 	    CLASS1_ENABLE_STORAGE_FAULT_INTR;
2158 	csa->priv1.int_mask_class2_RW = CLASS2_ENABLE_SPU_STOP_INTR |
2159 	    CLASS2_ENABLE_SPU_HALT_INTR |
2160 	    CLASS2_ENABLE_SPU_DMA_TAG_GROUP_COMPLETE_INTR;
2161 }
2162 
2163 static void init_priv2(struct spu_state *csa)
2164 {
2165 	csa->priv2.spu_lslr_RW = LS_ADDR_MASK;
2166 	csa->priv2.mfc_control_RW = MFC_CNTL_RESUME_DMA_QUEUE |
2167 	    MFC_CNTL_NORMAL_DMA_QUEUE_OPERATION |
2168 	    MFC_CNTL_DMA_QUEUES_EMPTY_MASK;
2169 }
2170 
2171 /**
2172  * spu_alloc_csa - allocate and initialize an SPU context save area.
2173  *
2174  * Allocate and initialize the contents of an SPU context save area.
2175  * This includes enabling address translation, interrupt masks, etc.,
2176  * as appropriate for the given OS environment.
2177  *
2178  * Note that storage for the 'lscsa' is allocated separately,
2179  * as it is by far the largest of the context save regions,
2180  * and may need to be pinned or otherwise specially aligned.
2181  */
2182 int spu_init_csa(struct spu_state *csa)
2183 {
2184 	int rc;
2185 
2186 	if (!csa)
2187 		return -EINVAL;
2188 	memset(csa, 0, sizeof(struct spu_state));
2189 
2190 	rc = spu_alloc_lscsa(csa);
2191 	if (rc)
2192 		return rc;
2193 
2194 	spin_lock_init(&csa->register_lock);
2195 
2196 	init_prob(csa);
2197 	init_priv1(csa);
2198 	init_priv2(csa);
2199 
2200 	return 0;
2201 }
2202 
2203 void spu_fini_csa(struct spu_state *csa)
2204 {
2205 	spu_free_lscsa(csa);
2206 }
2207