1 #define DEBUG 2 3 #include <linux/wait.h> 4 #include <linux/ptrace.h> 5 6 #include <asm/spu.h> 7 #include <asm/spu_priv1.h> 8 #include <asm/io.h> 9 #include <asm/unistd.h> 10 11 #include "spufs.h" 12 13 /* interrupt-level stop callback function. */ 14 void spufs_stop_callback(struct spu *spu, int irq) 15 { 16 struct spu_context *ctx = spu->ctx; 17 18 /* 19 * It should be impossible to preempt a context while an exception 20 * is being processed, since the context switch code is specially 21 * coded to deal with interrupts ... But, just in case, sanity check 22 * the context pointer. It is OK to return doing nothing since 23 * the exception will be regenerated when the context is resumed. 24 */ 25 if (ctx) { 26 /* Copy exception arguments into module specific structure */ 27 switch(irq) { 28 case 0 : 29 ctx->csa.class_0_pending = spu->class_0_pending; 30 ctx->csa.class_0_dar = spu->class_0_dar; 31 break; 32 case 1 : 33 ctx->csa.class_1_dsisr = spu->class_1_dsisr; 34 ctx->csa.class_1_dar = spu->class_1_dar; 35 break; 36 case 2 : 37 break; 38 } 39 40 /* ensure that the exception status has hit memory before a 41 * thread waiting on the context's stop queue is woken */ 42 smp_wmb(); 43 44 wake_up_all(&ctx->stop_wq); 45 } 46 } 47 48 int spu_stopped(struct spu_context *ctx, u32 *stat) 49 { 50 u64 dsisr; 51 u32 stopped; 52 53 stopped = SPU_STATUS_INVALID_INSTR | SPU_STATUS_SINGLE_STEP | 54 SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP; 55 56 top: 57 *stat = ctx->ops->status_read(ctx); 58 if (*stat & stopped) { 59 /* 60 * If the spu hasn't finished stopping, we need to 61 * re-read the register to get the stopped value. 62 */ 63 if (*stat & SPU_STATUS_RUNNING) 64 goto top; 65 return 1; 66 } 67 68 if (test_bit(SPU_SCHED_NOTIFY_ACTIVE, &ctx->sched_flags)) 69 return 1; 70 71 dsisr = ctx->csa.class_1_dsisr; 72 if (dsisr & (MFC_DSISR_PTE_NOT_FOUND | MFC_DSISR_ACCESS_DENIED)) 73 return 1; 74 75 if (ctx->csa.class_0_pending) 76 return 1; 77 78 return 0; 79 } 80 81 static int spu_setup_isolated(struct spu_context *ctx) 82 { 83 int ret; 84 u64 __iomem *mfc_cntl; 85 u64 sr1; 86 u32 status; 87 unsigned long timeout; 88 const u32 status_loading = SPU_STATUS_RUNNING 89 | SPU_STATUS_ISOLATED_STATE | SPU_STATUS_ISOLATED_LOAD_STATUS; 90 91 ret = -ENODEV; 92 if (!isolated_loader) 93 goto out; 94 95 /* 96 * We need to exclude userspace access to the context. 97 * 98 * To protect against memory access we invalidate all ptes 99 * and make sure the pagefault handlers block on the mutex. 100 */ 101 spu_unmap_mappings(ctx); 102 103 mfc_cntl = &ctx->spu->priv2->mfc_control_RW; 104 105 /* purge the MFC DMA queue to ensure no spurious accesses before we 106 * enter kernel mode */ 107 timeout = jiffies + HZ; 108 out_be64(mfc_cntl, MFC_CNTL_PURGE_DMA_REQUEST); 109 while ((in_be64(mfc_cntl) & MFC_CNTL_PURGE_DMA_STATUS_MASK) 110 != MFC_CNTL_PURGE_DMA_COMPLETE) { 111 if (time_after(jiffies, timeout)) { 112 printk(KERN_ERR "%s: timeout flushing MFC DMA queue\n", 113 __func__); 114 ret = -EIO; 115 goto out; 116 } 117 cond_resched(); 118 } 119 120 /* put the SPE in kernel mode to allow access to the loader */ 121 sr1 = spu_mfc_sr1_get(ctx->spu); 122 sr1 &= ~MFC_STATE1_PROBLEM_STATE_MASK; 123 spu_mfc_sr1_set(ctx->spu, sr1); 124 125 /* start the loader */ 126 ctx->ops->signal1_write(ctx, (unsigned long)isolated_loader >> 32); 127 ctx->ops->signal2_write(ctx, 128 (unsigned long)isolated_loader & 0xffffffff); 129 130 ctx->ops->runcntl_write(ctx, 131 SPU_RUNCNTL_RUNNABLE | SPU_RUNCNTL_ISOLATE); 132 133 ret = 0; 134 timeout = jiffies + HZ; 135 while (((status = ctx->ops->status_read(ctx)) & status_loading) == 136 status_loading) { 137 if (time_after(jiffies, timeout)) { 138 printk(KERN_ERR "%s: timeout waiting for loader\n", 139 __func__); 140 ret = -EIO; 141 goto out_drop_priv; 142 } 143 cond_resched(); 144 } 145 146 if (!(status & SPU_STATUS_RUNNING)) { 147 /* If isolated LOAD has failed: run SPU, we will get a stop-and 148 * signal later. */ 149 pr_debug("%s: isolated LOAD failed\n", __func__); 150 ctx->ops->runcntl_write(ctx, SPU_RUNCNTL_RUNNABLE); 151 ret = -EACCES; 152 goto out_drop_priv; 153 } 154 155 if (!(status & SPU_STATUS_ISOLATED_STATE)) { 156 /* This isn't allowed by the CBEA, but check anyway */ 157 pr_debug("%s: SPU fell out of isolated mode?\n", __func__); 158 ctx->ops->runcntl_write(ctx, SPU_RUNCNTL_STOP); 159 ret = -EINVAL; 160 goto out_drop_priv; 161 } 162 163 out_drop_priv: 164 /* Finished accessing the loader. Drop kernel mode */ 165 sr1 |= MFC_STATE1_PROBLEM_STATE_MASK; 166 spu_mfc_sr1_set(ctx->spu, sr1); 167 168 out: 169 return ret; 170 } 171 172 static int spu_run_init(struct spu_context *ctx, u32 *npc) 173 { 174 unsigned long runcntl = SPU_RUNCNTL_RUNNABLE; 175 int ret; 176 177 spuctx_switch_state(ctx, SPU_UTIL_SYSTEM); 178 179 /* 180 * NOSCHED is synchronous scheduling with respect to the caller. 181 * The caller waits for the context to be loaded. 182 */ 183 if (ctx->flags & SPU_CREATE_NOSCHED) { 184 if (ctx->state == SPU_STATE_SAVED) { 185 ret = spu_activate(ctx, 0); 186 if (ret) 187 return ret; 188 } 189 } 190 191 /* 192 * Apply special setup as required. 193 */ 194 if (ctx->flags & SPU_CREATE_ISOLATE) { 195 if (!(ctx->ops->status_read(ctx) & SPU_STATUS_ISOLATED_STATE)) { 196 ret = spu_setup_isolated(ctx); 197 if (ret) 198 return ret; 199 } 200 201 /* 202 * If userspace has set the runcntrl register (eg, to 203 * issue an isolated exit), we need to re-set it here 204 */ 205 runcntl = ctx->ops->runcntl_read(ctx) & 206 (SPU_RUNCNTL_RUNNABLE | SPU_RUNCNTL_ISOLATE); 207 if (runcntl == 0) 208 runcntl = SPU_RUNCNTL_RUNNABLE; 209 } 210 211 if (ctx->flags & SPU_CREATE_NOSCHED) { 212 spuctx_switch_state(ctx, SPU_UTIL_USER); 213 ctx->ops->runcntl_write(ctx, runcntl); 214 } else { 215 unsigned long privcntl; 216 217 if (test_thread_flag(TIF_SINGLESTEP)) 218 privcntl = SPU_PRIVCNTL_MODE_SINGLE_STEP; 219 else 220 privcntl = SPU_PRIVCNTL_MODE_NORMAL; 221 222 ctx->ops->npc_write(ctx, *npc); 223 ctx->ops->privcntl_write(ctx, privcntl); 224 ctx->ops->runcntl_write(ctx, runcntl); 225 226 if (ctx->state == SPU_STATE_SAVED) { 227 ret = spu_activate(ctx, 0); 228 if (ret) 229 return ret; 230 } else { 231 spuctx_switch_state(ctx, SPU_UTIL_USER); 232 } 233 } 234 235 set_bit(SPU_SCHED_SPU_RUN, &ctx->sched_flags); 236 return 0; 237 } 238 239 static int spu_run_fini(struct spu_context *ctx, u32 *npc, 240 u32 *status) 241 { 242 int ret = 0; 243 244 spu_del_from_rq(ctx); 245 246 *status = ctx->ops->status_read(ctx); 247 *npc = ctx->ops->npc_read(ctx); 248 249 spuctx_switch_state(ctx, SPU_UTIL_IDLE_LOADED); 250 clear_bit(SPU_SCHED_SPU_RUN, &ctx->sched_flags); 251 spu_release(ctx); 252 253 if (signal_pending(current)) 254 ret = -ERESTARTSYS; 255 256 return ret; 257 } 258 259 /* 260 * SPU syscall restarting is tricky because we violate the basic 261 * assumption that the signal handler is running on the interrupted 262 * thread. Here instead, the handler runs on PowerPC user space code, 263 * while the syscall was called from the SPU. 264 * This means we can only do a very rough approximation of POSIX 265 * signal semantics. 266 */ 267 static int spu_handle_restartsys(struct spu_context *ctx, long *spu_ret, 268 unsigned int *npc) 269 { 270 int ret; 271 272 switch (*spu_ret) { 273 case -ERESTARTSYS: 274 case -ERESTARTNOINTR: 275 /* 276 * Enter the regular syscall restarting for 277 * sys_spu_run, then restart the SPU syscall 278 * callback. 279 */ 280 *npc -= 8; 281 ret = -ERESTARTSYS; 282 break; 283 case -ERESTARTNOHAND: 284 case -ERESTART_RESTARTBLOCK: 285 /* 286 * Restart block is too hard for now, just return -EINTR 287 * to the SPU. 288 * ERESTARTNOHAND comes from sys_pause, we also return 289 * -EINTR from there. 290 * Assume that we need to be restarted ourselves though. 291 */ 292 *spu_ret = -EINTR; 293 ret = -ERESTARTSYS; 294 break; 295 default: 296 printk(KERN_WARNING "%s: unexpected return code %ld\n", 297 __func__, *spu_ret); 298 ret = 0; 299 } 300 return ret; 301 } 302 303 static int spu_process_callback(struct spu_context *ctx) 304 { 305 struct spu_syscall_block s; 306 u32 ls_pointer, npc; 307 void __iomem *ls; 308 long spu_ret; 309 int ret; 310 311 /* get syscall block from local store */ 312 npc = ctx->ops->npc_read(ctx) & ~3; 313 ls = (void __iomem *)ctx->ops->get_ls(ctx); 314 ls_pointer = in_be32(ls + npc); 315 if (ls_pointer > (LS_SIZE - sizeof(s))) 316 return -EFAULT; 317 memcpy_fromio(&s, ls + ls_pointer, sizeof(s)); 318 319 /* do actual syscall without pinning the spu */ 320 ret = 0; 321 spu_ret = -ENOSYS; 322 npc += 4; 323 324 if (s.nr_ret < __NR_syscalls) { 325 spu_release(ctx); 326 /* do actual system call from here */ 327 spu_ret = spu_sys_callback(&s); 328 if (spu_ret <= -ERESTARTSYS) { 329 ret = spu_handle_restartsys(ctx, &spu_ret, &npc); 330 } 331 mutex_lock(&ctx->state_mutex); 332 if (ret == -ERESTARTSYS) 333 return ret; 334 } 335 336 /* need to re-get the ls, as it may have changed when we released the 337 * spu */ 338 ls = (void __iomem *)ctx->ops->get_ls(ctx); 339 340 /* write result, jump over indirect pointer */ 341 memcpy_toio(ls + ls_pointer, &spu_ret, sizeof(spu_ret)); 342 ctx->ops->npc_write(ctx, npc); 343 ctx->ops->runcntl_write(ctx, SPU_RUNCNTL_RUNNABLE); 344 return ret; 345 } 346 347 long spufs_run_spu(struct spu_context *ctx, u32 *npc, u32 *event) 348 { 349 int ret; 350 struct spu *spu; 351 u32 status; 352 353 if (mutex_lock_interruptible(&ctx->run_mutex)) 354 return -ERESTARTSYS; 355 356 ctx->event_return = 0; 357 358 ret = spu_acquire(ctx); 359 if (ret) 360 goto out_unlock; 361 362 spu_enable_spu(ctx); 363 364 spu_update_sched_info(ctx); 365 366 ret = spu_run_init(ctx, npc); 367 if (ret) { 368 spu_release(ctx); 369 goto out; 370 } 371 372 do { 373 ret = spufs_wait(ctx->stop_wq, spu_stopped(ctx, &status)); 374 if (unlikely(ret)) { 375 /* 376 * This is nasty: we need the state_mutex for all the 377 * bookkeeping even if the syscall was interrupted by 378 * a signal. ewww. 379 */ 380 mutex_lock(&ctx->state_mutex); 381 break; 382 } 383 spu = ctx->spu; 384 if (unlikely(test_and_clear_bit(SPU_SCHED_NOTIFY_ACTIVE, 385 &ctx->sched_flags))) { 386 if (!(status & SPU_STATUS_STOPPED_BY_STOP)) { 387 spu_switch_notify(spu, ctx); 388 continue; 389 } 390 } 391 392 spuctx_switch_state(ctx, SPU_UTIL_SYSTEM); 393 394 if ((status & SPU_STATUS_STOPPED_BY_STOP) && 395 (status >> SPU_STOP_STATUS_SHIFT == 0x2104)) { 396 ret = spu_process_callback(ctx); 397 if (ret) 398 break; 399 status &= ~SPU_STATUS_STOPPED_BY_STOP; 400 } 401 ret = spufs_handle_class1(ctx); 402 if (ret) 403 break; 404 405 ret = spufs_handle_class0(ctx); 406 if (ret) 407 break; 408 409 if (signal_pending(current)) 410 ret = -ERESTARTSYS; 411 } while (!ret && !(status & (SPU_STATUS_STOPPED_BY_STOP | 412 SPU_STATUS_STOPPED_BY_HALT | 413 SPU_STATUS_SINGLE_STEP))); 414 415 spu_disable_spu(ctx); 416 ret = spu_run_fini(ctx, npc, &status); 417 spu_yield(ctx); 418 419 spu_switch_log_notify(NULL, ctx, SWITCH_LOG_EXIT, status); 420 421 if ((status & SPU_STATUS_STOPPED_BY_STOP) && 422 (((status >> SPU_STOP_STATUS_SHIFT) & 0x3f00) == 0x2100)) 423 ctx->stats.libassist++; 424 425 if ((ret == 0) || 426 ((ret == -ERESTARTSYS) && 427 ((status & SPU_STATUS_STOPPED_BY_HALT) || 428 (status & SPU_STATUS_SINGLE_STEP) || 429 ((status & SPU_STATUS_STOPPED_BY_STOP) && 430 (status >> SPU_STOP_STATUS_SHIFT != 0x2104))))) 431 ret = status; 432 433 /* Note: we don't need to force_sig SIGTRAP on single-step 434 * since we have TIF_SINGLESTEP set, thus the kernel will do 435 * it upon return from the syscall anyawy 436 */ 437 if (unlikely(status & SPU_STATUS_SINGLE_STEP)) 438 ret = -ERESTARTSYS; 439 440 else if (unlikely((status & SPU_STATUS_STOPPED_BY_STOP) 441 && (status >> SPU_STOP_STATUS_SHIFT) == 0x3fff)) { 442 force_sig(SIGTRAP, current); 443 ret = -ERESTARTSYS; 444 } 445 446 out: 447 *event = ctx->event_return; 448 out_unlock: 449 mutex_unlock(&ctx->run_mutex); 450 return ret; 451 } 452