1 #define DEBUG 2 3 #include <linux/wait.h> 4 #include <linux/ptrace.h> 5 6 #include <asm/spu.h> 7 #include <asm/spu_priv1.h> 8 #include <asm/io.h> 9 #include <asm/unistd.h> 10 11 #include "spufs.h" 12 13 /* interrupt-level stop callback function. */ 14 void spufs_stop_callback(struct spu *spu, int irq) 15 { 16 struct spu_context *ctx = spu->ctx; 17 18 /* 19 * It should be impossible to preempt a context while an exception 20 * is being processed, since the context switch code is specially 21 * coded to deal with interrupts ... But, just in case, sanity check 22 * the context pointer. It is OK to return doing nothing since 23 * the exception will be regenerated when the context is resumed. 24 */ 25 if (ctx) { 26 /* Copy exception arguments into module specific structure */ 27 switch(irq) { 28 case 0 : 29 ctx->csa.class_0_pending = spu->class_0_pending; 30 ctx->csa.class_0_dar = spu->class_0_dar; 31 break; 32 case 1 : 33 ctx->csa.class_1_dsisr = spu->class_1_dsisr; 34 ctx->csa.class_1_dar = spu->class_1_dar; 35 break; 36 case 2 : 37 break; 38 } 39 40 /* ensure that the exception status has hit memory before a 41 * thread waiting on the context's stop queue is woken */ 42 smp_wmb(); 43 44 wake_up_all(&ctx->stop_wq); 45 } 46 } 47 48 int spu_stopped(struct spu_context *ctx, u32 *stat) 49 { 50 u64 dsisr; 51 u32 stopped; 52 53 stopped = SPU_STATUS_INVALID_INSTR | SPU_STATUS_SINGLE_STEP | 54 SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP; 55 56 top: 57 *stat = ctx->ops->status_read(ctx); 58 if (*stat & stopped) { 59 /* 60 * If the spu hasn't finished stopping, we need to 61 * re-read the register to get the stopped value. 62 */ 63 if (*stat & SPU_STATUS_RUNNING) 64 goto top; 65 return 1; 66 } 67 68 if (test_bit(SPU_SCHED_NOTIFY_ACTIVE, &ctx->sched_flags)) 69 return 1; 70 71 dsisr = ctx->csa.class_1_dsisr; 72 if (dsisr & (MFC_DSISR_PTE_NOT_FOUND | MFC_DSISR_ACCESS_DENIED)) 73 return 1; 74 75 if (ctx->csa.class_0_pending) 76 return 1; 77 78 return 0; 79 } 80 81 static int spu_setup_isolated(struct spu_context *ctx) 82 { 83 int ret; 84 u64 __iomem *mfc_cntl; 85 u64 sr1; 86 u32 status; 87 unsigned long timeout; 88 const u32 status_loading = SPU_STATUS_RUNNING 89 | SPU_STATUS_ISOLATED_STATE | SPU_STATUS_ISOLATED_LOAD_STATUS; 90 91 ret = -ENODEV; 92 if (!isolated_loader) 93 goto out; 94 95 /* 96 * We need to exclude userspace access to the context. 97 * 98 * To protect against memory access we invalidate all ptes 99 * and make sure the pagefault handlers block on the mutex. 100 */ 101 spu_unmap_mappings(ctx); 102 103 mfc_cntl = &ctx->spu->priv2->mfc_control_RW; 104 105 /* purge the MFC DMA queue to ensure no spurious accesses before we 106 * enter kernel mode */ 107 timeout = jiffies + HZ; 108 out_be64(mfc_cntl, MFC_CNTL_PURGE_DMA_REQUEST); 109 while ((in_be64(mfc_cntl) & MFC_CNTL_PURGE_DMA_STATUS_MASK) 110 != MFC_CNTL_PURGE_DMA_COMPLETE) { 111 if (time_after(jiffies, timeout)) { 112 printk(KERN_ERR "%s: timeout flushing MFC DMA queue\n", 113 __func__); 114 ret = -EIO; 115 goto out; 116 } 117 cond_resched(); 118 } 119 120 /* put the SPE in kernel mode to allow access to the loader */ 121 sr1 = spu_mfc_sr1_get(ctx->spu); 122 sr1 &= ~MFC_STATE1_PROBLEM_STATE_MASK; 123 spu_mfc_sr1_set(ctx->spu, sr1); 124 125 /* start the loader */ 126 ctx->ops->signal1_write(ctx, (unsigned long)isolated_loader >> 32); 127 ctx->ops->signal2_write(ctx, 128 (unsigned long)isolated_loader & 0xffffffff); 129 130 ctx->ops->runcntl_write(ctx, 131 SPU_RUNCNTL_RUNNABLE | SPU_RUNCNTL_ISOLATE); 132 133 ret = 0; 134 timeout = jiffies + HZ; 135 while (((status = ctx->ops->status_read(ctx)) & status_loading) == 136 status_loading) { 137 if (time_after(jiffies, timeout)) { 138 printk(KERN_ERR "%s: timeout waiting for loader\n", 139 __func__); 140 ret = -EIO; 141 goto out_drop_priv; 142 } 143 cond_resched(); 144 } 145 146 if (!(status & SPU_STATUS_RUNNING)) { 147 /* If isolated LOAD has failed: run SPU, we will get a stop-and 148 * signal later. */ 149 pr_debug("%s: isolated LOAD failed\n", __func__); 150 ctx->ops->runcntl_write(ctx, SPU_RUNCNTL_RUNNABLE); 151 ret = -EACCES; 152 goto out_drop_priv; 153 } 154 155 if (!(status & SPU_STATUS_ISOLATED_STATE)) { 156 /* This isn't allowed by the CBEA, but check anyway */ 157 pr_debug("%s: SPU fell out of isolated mode?\n", __func__); 158 ctx->ops->runcntl_write(ctx, SPU_RUNCNTL_STOP); 159 ret = -EINVAL; 160 goto out_drop_priv; 161 } 162 163 out_drop_priv: 164 /* Finished accessing the loader. Drop kernel mode */ 165 sr1 |= MFC_STATE1_PROBLEM_STATE_MASK; 166 spu_mfc_sr1_set(ctx->spu, sr1); 167 168 out: 169 return ret; 170 } 171 172 static int spu_run_init(struct spu_context *ctx, u32 *npc) 173 { 174 unsigned long runcntl = SPU_RUNCNTL_RUNNABLE; 175 int ret; 176 177 spuctx_switch_state(ctx, SPU_UTIL_SYSTEM); 178 179 /* 180 * NOSCHED is synchronous scheduling with respect to the caller. 181 * The caller waits for the context to be loaded. 182 */ 183 if (ctx->flags & SPU_CREATE_NOSCHED) { 184 if (ctx->state == SPU_STATE_SAVED) { 185 ret = spu_activate(ctx, 0); 186 if (ret) 187 return ret; 188 } 189 } 190 191 /* 192 * Apply special setup as required. 193 */ 194 if (ctx->flags & SPU_CREATE_ISOLATE) { 195 if (!(ctx->ops->status_read(ctx) & SPU_STATUS_ISOLATED_STATE)) { 196 ret = spu_setup_isolated(ctx); 197 if (ret) 198 return ret; 199 } 200 201 /* 202 * If userspace has set the runcntrl register (eg, to 203 * issue an isolated exit), we need to re-set it here 204 */ 205 runcntl = ctx->ops->runcntl_read(ctx) & 206 (SPU_RUNCNTL_RUNNABLE | SPU_RUNCNTL_ISOLATE); 207 if (runcntl == 0) 208 runcntl = SPU_RUNCNTL_RUNNABLE; 209 } else { 210 unsigned long privcntl; 211 212 if (test_thread_flag(TIF_SINGLESTEP)) 213 privcntl = SPU_PRIVCNTL_MODE_SINGLE_STEP; 214 else 215 privcntl = SPU_PRIVCNTL_MODE_NORMAL; 216 217 ctx->ops->privcntl_write(ctx, privcntl); 218 ctx->ops->npc_write(ctx, *npc); 219 } 220 221 ctx->ops->runcntl_write(ctx, runcntl); 222 223 if (ctx->flags & SPU_CREATE_NOSCHED) { 224 spuctx_switch_state(ctx, SPU_UTIL_USER); 225 } else { 226 227 if (ctx->state == SPU_STATE_SAVED) { 228 ret = spu_activate(ctx, 0); 229 if (ret) 230 return ret; 231 } else { 232 spuctx_switch_state(ctx, SPU_UTIL_USER); 233 } 234 } 235 236 set_bit(SPU_SCHED_SPU_RUN, &ctx->sched_flags); 237 return 0; 238 } 239 240 static int spu_run_fini(struct spu_context *ctx, u32 *npc, 241 u32 *status) 242 { 243 int ret = 0; 244 245 spu_del_from_rq(ctx); 246 247 *status = ctx->ops->status_read(ctx); 248 *npc = ctx->ops->npc_read(ctx); 249 250 spuctx_switch_state(ctx, SPU_UTIL_IDLE_LOADED); 251 clear_bit(SPU_SCHED_SPU_RUN, &ctx->sched_flags); 252 spu_release(ctx); 253 254 if (signal_pending(current)) 255 ret = -ERESTARTSYS; 256 257 return ret; 258 } 259 260 /* 261 * SPU syscall restarting is tricky because we violate the basic 262 * assumption that the signal handler is running on the interrupted 263 * thread. Here instead, the handler runs on PowerPC user space code, 264 * while the syscall was called from the SPU. 265 * This means we can only do a very rough approximation of POSIX 266 * signal semantics. 267 */ 268 static int spu_handle_restartsys(struct spu_context *ctx, long *spu_ret, 269 unsigned int *npc) 270 { 271 int ret; 272 273 switch (*spu_ret) { 274 case -ERESTARTSYS: 275 case -ERESTARTNOINTR: 276 /* 277 * Enter the regular syscall restarting for 278 * sys_spu_run, then restart the SPU syscall 279 * callback. 280 */ 281 *npc -= 8; 282 ret = -ERESTARTSYS; 283 break; 284 case -ERESTARTNOHAND: 285 case -ERESTART_RESTARTBLOCK: 286 /* 287 * Restart block is too hard for now, just return -EINTR 288 * to the SPU. 289 * ERESTARTNOHAND comes from sys_pause, we also return 290 * -EINTR from there. 291 * Assume that we need to be restarted ourselves though. 292 */ 293 *spu_ret = -EINTR; 294 ret = -ERESTARTSYS; 295 break; 296 default: 297 printk(KERN_WARNING "%s: unexpected return code %ld\n", 298 __func__, *spu_ret); 299 ret = 0; 300 } 301 return ret; 302 } 303 304 static int spu_process_callback(struct spu_context *ctx) 305 { 306 struct spu_syscall_block s; 307 u32 ls_pointer, npc; 308 void __iomem *ls; 309 long spu_ret; 310 int ret; 311 312 /* get syscall block from local store */ 313 npc = ctx->ops->npc_read(ctx) & ~3; 314 ls = (void __iomem *)ctx->ops->get_ls(ctx); 315 ls_pointer = in_be32(ls + npc); 316 if (ls_pointer > (LS_SIZE - sizeof(s))) 317 return -EFAULT; 318 memcpy_fromio(&s, ls + ls_pointer, sizeof(s)); 319 320 /* do actual syscall without pinning the spu */ 321 ret = 0; 322 spu_ret = -ENOSYS; 323 npc += 4; 324 325 if (s.nr_ret < __NR_syscalls) { 326 spu_release(ctx); 327 /* do actual system call from here */ 328 spu_ret = spu_sys_callback(&s); 329 if (spu_ret <= -ERESTARTSYS) { 330 ret = spu_handle_restartsys(ctx, &spu_ret, &npc); 331 } 332 mutex_lock(&ctx->state_mutex); 333 if (ret == -ERESTARTSYS) 334 return ret; 335 } 336 337 /* need to re-get the ls, as it may have changed when we released the 338 * spu */ 339 ls = (void __iomem *)ctx->ops->get_ls(ctx); 340 341 /* write result, jump over indirect pointer */ 342 memcpy_toio(ls + ls_pointer, &spu_ret, sizeof(spu_ret)); 343 ctx->ops->npc_write(ctx, npc); 344 ctx->ops->runcntl_write(ctx, SPU_RUNCNTL_RUNNABLE); 345 return ret; 346 } 347 348 long spufs_run_spu(struct spu_context *ctx, u32 *npc, u32 *event) 349 { 350 int ret; 351 struct spu *spu; 352 u32 status; 353 354 if (mutex_lock_interruptible(&ctx->run_mutex)) 355 return -ERESTARTSYS; 356 357 ctx->event_return = 0; 358 359 ret = spu_acquire(ctx); 360 if (ret) 361 goto out_unlock; 362 363 spu_enable_spu(ctx); 364 365 spu_update_sched_info(ctx); 366 367 ret = spu_run_init(ctx, npc); 368 if (ret) { 369 spu_release(ctx); 370 goto out; 371 } 372 373 do { 374 ret = spufs_wait(ctx->stop_wq, spu_stopped(ctx, &status)); 375 if (unlikely(ret)) { 376 /* 377 * This is nasty: we need the state_mutex for all the 378 * bookkeeping even if the syscall was interrupted by 379 * a signal. ewww. 380 */ 381 mutex_lock(&ctx->state_mutex); 382 break; 383 } 384 spu = ctx->spu; 385 if (unlikely(test_and_clear_bit(SPU_SCHED_NOTIFY_ACTIVE, 386 &ctx->sched_flags))) { 387 if (!(status & SPU_STATUS_STOPPED_BY_STOP)) { 388 spu_switch_notify(spu, ctx); 389 continue; 390 } 391 } 392 393 spuctx_switch_state(ctx, SPU_UTIL_SYSTEM); 394 395 if ((status & SPU_STATUS_STOPPED_BY_STOP) && 396 (status >> SPU_STOP_STATUS_SHIFT == 0x2104)) { 397 ret = spu_process_callback(ctx); 398 if (ret) 399 break; 400 status &= ~SPU_STATUS_STOPPED_BY_STOP; 401 } 402 ret = spufs_handle_class1(ctx); 403 if (ret) 404 break; 405 406 ret = spufs_handle_class0(ctx); 407 if (ret) 408 break; 409 410 if (signal_pending(current)) 411 ret = -ERESTARTSYS; 412 } while (!ret && !(status & (SPU_STATUS_STOPPED_BY_STOP | 413 SPU_STATUS_STOPPED_BY_HALT | 414 SPU_STATUS_SINGLE_STEP))); 415 416 spu_disable_spu(ctx); 417 ret = spu_run_fini(ctx, npc, &status); 418 spu_yield(ctx); 419 420 spu_switch_log_notify(NULL, ctx, SWITCH_LOG_EXIT, status); 421 422 if ((status & SPU_STATUS_STOPPED_BY_STOP) && 423 (((status >> SPU_STOP_STATUS_SHIFT) & 0x3f00) == 0x2100)) 424 ctx->stats.libassist++; 425 426 if ((ret == 0) || 427 ((ret == -ERESTARTSYS) && 428 ((status & SPU_STATUS_STOPPED_BY_HALT) || 429 (status & SPU_STATUS_SINGLE_STEP) || 430 ((status & SPU_STATUS_STOPPED_BY_STOP) && 431 (status >> SPU_STOP_STATUS_SHIFT != 0x2104))))) 432 ret = status; 433 434 /* Note: we don't need to force_sig SIGTRAP on single-step 435 * since we have TIF_SINGLESTEP set, thus the kernel will do 436 * it upon return from the syscall anyawy 437 */ 438 if (unlikely(status & SPU_STATUS_SINGLE_STEP)) 439 ret = -ERESTARTSYS; 440 441 else if (unlikely((status & SPU_STATUS_STOPPED_BY_STOP) 442 && (status >> SPU_STOP_STATUS_SHIFT) == 0x3fff)) { 443 force_sig(SIGTRAP, current); 444 ret = -ERESTARTSYS; 445 } 446 447 out: 448 *event = ctx->event_return; 449 out_unlock: 450 mutex_unlock(&ctx->run_mutex); 451 return ret; 452 } 453