1 #define DEBUG 2 3 #include <linux/wait.h> 4 #include <linux/ptrace.h> 5 6 #include <asm/spu.h> 7 #include <asm/spu_priv1.h> 8 #include <asm/io.h> 9 #include <asm/unistd.h> 10 11 #include "spufs.h" 12 13 /* interrupt-level stop callback function. */ 14 void spufs_stop_callback(struct spu *spu) 15 { 16 struct spu_context *ctx = spu->ctx; 17 18 /* 19 * It should be impossible to preempt a context while an exception 20 * is being processed, since the context switch code is specially 21 * coded to deal with interrupts ... But, just in case, sanity check 22 * the context pointer. It is OK to return doing nothing since 23 * the exception will be regenerated when the context is resumed. 24 */ 25 if (ctx) { 26 /* Copy exception arguments into module specific structure */ 27 ctx->csa.class_0_pending = spu->class_0_pending; 28 ctx->csa.dsisr = spu->dsisr; 29 ctx->csa.dar = spu->dar; 30 31 /* ensure that the exception status has hit memory before a 32 * thread waiting on the context's stop queue is woken */ 33 smp_wmb(); 34 35 wake_up_all(&ctx->stop_wq); 36 } 37 38 /* Clear callback arguments from spu structure */ 39 spu->class_0_pending = 0; 40 spu->dsisr = 0; 41 spu->dar = 0; 42 } 43 44 int spu_stopped(struct spu_context *ctx, u32 *stat) 45 { 46 u64 dsisr; 47 u32 stopped; 48 49 *stat = ctx->ops->status_read(ctx); 50 51 if (test_bit(SPU_SCHED_NOTIFY_ACTIVE, &ctx->sched_flags)) 52 return 1; 53 54 stopped = SPU_STATUS_INVALID_INSTR | SPU_STATUS_SINGLE_STEP | 55 SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP; 56 if (!(*stat & SPU_STATUS_RUNNING) && (*stat & stopped)) 57 return 1; 58 59 dsisr = ctx->csa.dsisr; 60 if (dsisr & (MFC_DSISR_PTE_NOT_FOUND | MFC_DSISR_ACCESS_DENIED)) 61 return 1; 62 63 if (ctx->csa.class_0_pending) 64 return 1; 65 66 return 0; 67 } 68 69 static int spu_setup_isolated(struct spu_context *ctx) 70 { 71 int ret; 72 u64 __iomem *mfc_cntl; 73 u64 sr1; 74 u32 status; 75 unsigned long timeout; 76 const u32 status_loading = SPU_STATUS_RUNNING 77 | SPU_STATUS_ISOLATED_STATE | SPU_STATUS_ISOLATED_LOAD_STATUS; 78 79 ret = -ENODEV; 80 if (!isolated_loader) 81 goto out; 82 83 /* 84 * We need to exclude userspace access to the context. 85 * 86 * To protect against memory access we invalidate all ptes 87 * and make sure the pagefault handlers block on the mutex. 88 */ 89 spu_unmap_mappings(ctx); 90 91 mfc_cntl = &ctx->spu->priv2->mfc_control_RW; 92 93 /* purge the MFC DMA queue to ensure no spurious accesses before we 94 * enter kernel mode */ 95 timeout = jiffies + HZ; 96 out_be64(mfc_cntl, MFC_CNTL_PURGE_DMA_REQUEST); 97 while ((in_be64(mfc_cntl) & MFC_CNTL_PURGE_DMA_STATUS_MASK) 98 != MFC_CNTL_PURGE_DMA_COMPLETE) { 99 if (time_after(jiffies, timeout)) { 100 printk(KERN_ERR "%s: timeout flushing MFC DMA queue\n", 101 __FUNCTION__); 102 ret = -EIO; 103 goto out; 104 } 105 cond_resched(); 106 } 107 108 /* put the SPE in kernel mode to allow access to the loader */ 109 sr1 = spu_mfc_sr1_get(ctx->spu); 110 sr1 &= ~MFC_STATE1_PROBLEM_STATE_MASK; 111 spu_mfc_sr1_set(ctx->spu, sr1); 112 113 /* start the loader */ 114 ctx->ops->signal1_write(ctx, (unsigned long)isolated_loader >> 32); 115 ctx->ops->signal2_write(ctx, 116 (unsigned long)isolated_loader & 0xffffffff); 117 118 ctx->ops->runcntl_write(ctx, 119 SPU_RUNCNTL_RUNNABLE | SPU_RUNCNTL_ISOLATE); 120 121 ret = 0; 122 timeout = jiffies + HZ; 123 while (((status = ctx->ops->status_read(ctx)) & status_loading) == 124 status_loading) { 125 if (time_after(jiffies, timeout)) { 126 printk(KERN_ERR "%s: timeout waiting for loader\n", 127 __FUNCTION__); 128 ret = -EIO; 129 goto out_drop_priv; 130 } 131 cond_resched(); 132 } 133 134 if (!(status & SPU_STATUS_RUNNING)) { 135 /* If isolated LOAD has failed: run SPU, we will get a stop-and 136 * signal later. */ 137 pr_debug("%s: isolated LOAD failed\n", __FUNCTION__); 138 ctx->ops->runcntl_write(ctx, SPU_RUNCNTL_RUNNABLE); 139 ret = -EACCES; 140 goto out_drop_priv; 141 } 142 143 if (!(status & SPU_STATUS_ISOLATED_STATE)) { 144 /* This isn't allowed by the CBEA, but check anyway */ 145 pr_debug("%s: SPU fell out of isolated mode?\n", __FUNCTION__); 146 ctx->ops->runcntl_write(ctx, SPU_RUNCNTL_STOP); 147 ret = -EINVAL; 148 goto out_drop_priv; 149 } 150 151 out_drop_priv: 152 /* Finished accessing the loader. Drop kernel mode */ 153 sr1 |= MFC_STATE1_PROBLEM_STATE_MASK; 154 spu_mfc_sr1_set(ctx->spu, sr1); 155 156 out: 157 return ret; 158 } 159 160 static int spu_run_init(struct spu_context *ctx, u32 *npc) 161 { 162 unsigned long runcntl = SPU_RUNCNTL_RUNNABLE; 163 int ret; 164 165 spuctx_switch_state(ctx, SPU_UTIL_SYSTEM); 166 167 /* 168 * NOSCHED is synchronous scheduling with respect to the caller. 169 * The caller waits for the context to be loaded. 170 */ 171 if (ctx->flags & SPU_CREATE_NOSCHED) { 172 if (ctx->state == SPU_STATE_SAVED) { 173 ret = spu_activate(ctx, 0); 174 if (ret) 175 return ret; 176 } 177 } 178 179 /* 180 * Apply special setup as required. 181 */ 182 if (ctx->flags & SPU_CREATE_ISOLATE) { 183 if (!(ctx->ops->status_read(ctx) & SPU_STATUS_ISOLATED_STATE)) { 184 ret = spu_setup_isolated(ctx); 185 if (ret) 186 return ret; 187 } 188 189 /* 190 * If userspace has set the runcntrl register (eg, to 191 * issue an isolated exit), we need to re-set it here 192 */ 193 runcntl = ctx->ops->runcntl_read(ctx) & 194 (SPU_RUNCNTL_RUNNABLE | SPU_RUNCNTL_ISOLATE); 195 if (runcntl == 0) 196 runcntl = SPU_RUNCNTL_RUNNABLE; 197 } 198 199 if (ctx->flags & SPU_CREATE_NOSCHED) { 200 spuctx_switch_state(ctx, SPU_UTIL_USER); 201 ctx->ops->runcntl_write(ctx, runcntl); 202 } else { 203 unsigned long privcntl; 204 205 if (test_thread_flag(TIF_SINGLESTEP)) 206 privcntl = SPU_PRIVCNTL_MODE_SINGLE_STEP; 207 else 208 privcntl = SPU_PRIVCNTL_MODE_NORMAL; 209 210 ctx->ops->npc_write(ctx, *npc); 211 ctx->ops->privcntl_write(ctx, privcntl); 212 ctx->ops->runcntl_write(ctx, runcntl); 213 214 if (ctx->state == SPU_STATE_SAVED) { 215 ret = spu_activate(ctx, 0); 216 if (ret) 217 return ret; 218 } else { 219 spuctx_switch_state(ctx, SPU_UTIL_USER); 220 } 221 } 222 223 set_bit(SPU_SCHED_SPU_RUN, &ctx->sched_flags); 224 return 0; 225 } 226 227 static int spu_run_fini(struct spu_context *ctx, u32 *npc, 228 u32 *status) 229 { 230 int ret = 0; 231 232 spu_del_from_rq(ctx); 233 234 *status = ctx->ops->status_read(ctx); 235 *npc = ctx->ops->npc_read(ctx); 236 237 spuctx_switch_state(ctx, SPU_UTIL_IDLE_LOADED); 238 clear_bit(SPU_SCHED_SPU_RUN, &ctx->sched_flags); 239 spu_release(ctx); 240 241 if (signal_pending(current)) 242 ret = -ERESTARTSYS; 243 244 return ret; 245 } 246 247 /* 248 * SPU syscall restarting is tricky because we violate the basic 249 * assumption that the signal handler is running on the interrupted 250 * thread. Here instead, the handler runs on PowerPC user space code, 251 * while the syscall was called from the SPU. 252 * This means we can only do a very rough approximation of POSIX 253 * signal semantics. 254 */ 255 static int spu_handle_restartsys(struct spu_context *ctx, long *spu_ret, 256 unsigned int *npc) 257 { 258 int ret; 259 260 switch (*spu_ret) { 261 case -ERESTARTSYS: 262 case -ERESTARTNOINTR: 263 /* 264 * Enter the regular syscall restarting for 265 * sys_spu_run, then restart the SPU syscall 266 * callback. 267 */ 268 *npc -= 8; 269 ret = -ERESTARTSYS; 270 break; 271 case -ERESTARTNOHAND: 272 case -ERESTART_RESTARTBLOCK: 273 /* 274 * Restart block is too hard for now, just return -EINTR 275 * to the SPU. 276 * ERESTARTNOHAND comes from sys_pause, we also return 277 * -EINTR from there. 278 * Assume that we need to be restarted ourselves though. 279 */ 280 *spu_ret = -EINTR; 281 ret = -ERESTARTSYS; 282 break; 283 default: 284 printk(KERN_WARNING "%s: unexpected return code %ld\n", 285 __FUNCTION__, *spu_ret); 286 ret = 0; 287 } 288 return ret; 289 } 290 291 static int spu_process_callback(struct spu_context *ctx) 292 { 293 struct spu_syscall_block s; 294 u32 ls_pointer, npc; 295 void __iomem *ls; 296 long spu_ret; 297 int ret, ret2; 298 299 /* get syscall block from local store */ 300 npc = ctx->ops->npc_read(ctx) & ~3; 301 ls = (void __iomem *)ctx->ops->get_ls(ctx); 302 ls_pointer = in_be32(ls + npc); 303 if (ls_pointer > (LS_SIZE - sizeof(s))) 304 return -EFAULT; 305 memcpy_fromio(&s, ls + ls_pointer, sizeof(s)); 306 307 /* do actual syscall without pinning the spu */ 308 ret = 0; 309 spu_ret = -ENOSYS; 310 npc += 4; 311 312 if (s.nr_ret < __NR_syscalls) { 313 spu_release(ctx); 314 /* do actual system call from here */ 315 spu_ret = spu_sys_callback(&s); 316 if (spu_ret <= -ERESTARTSYS) { 317 ret = spu_handle_restartsys(ctx, &spu_ret, &npc); 318 } 319 ret2 = spu_acquire(ctx); 320 if (ret == -ERESTARTSYS) 321 return ret; 322 if (ret2) 323 return -EINTR; 324 } 325 326 /* write result, jump over indirect pointer */ 327 memcpy_toio(ls + ls_pointer, &spu_ret, sizeof(spu_ret)); 328 ctx->ops->npc_write(ctx, npc); 329 ctx->ops->runcntl_write(ctx, SPU_RUNCNTL_RUNNABLE); 330 return ret; 331 } 332 333 long spufs_run_spu(struct spu_context *ctx, u32 *npc, u32 *event) 334 { 335 int ret; 336 struct spu *spu; 337 u32 status; 338 339 if (mutex_lock_interruptible(&ctx->run_mutex)) 340 return -ERESTARTSYS; 341 342 spu_enable_spu(ctx); 343 ctx->event_return = 0; 344 345 ret = spu_acquire(ctx); 346 if (ret) 347 goto out_unlock; 348 349 spu_update_sched_info(ctx); 350 351 ret = spu_run_init(ctx, npc); 352 if (ret) { 353 spu_release(ctx); 354 goto out; 355 } 356 357 do { 358 ret = spufs_wait(ctx->stop_wq, spu_stopped(ctx, &status)); 359 if (unlikely(ret)) { 360 /* 361 * This is nasty: we need the state_mutex for all the 362 * bookkeeping even if the syscall was interrupted by 363 * a signal. ewww. 364 */ 365 mutex_lock(&ctx->state_mutex); 366 break; 367 } 368 spu = ctx->spu; 369 if (unlikely(test_and_clear_bit(SPU_SCHED_NOTIFY_ACTIVE, 370 &ctx->sched_flags))) { 371 if (!(status & SPU_STATUS_STOPPED_BY_STOP)) { 372 spu_switch_notify(spu, ctx); 373 continue; 374 } 375 } 376 377 spuctx_switch_state(ctx, SPU_UTIL_SYSTEM); 378 379 if ((status & SPU_STATUS_STOPPED_BY_STOP) && 380 (status >> SPU_STOP_STATUS_SHIFT == 0x2104)) { 381 ret = spu_process_callback(ctx); 382 if (ret) 383 break; 384 status &= ~SPU_STATUS_STOPPED_BY_STOP; 385 } 386 ret = spufs_handle_class1(ctx); 387 if (ret) 388 break; 389 390 ret = spufs_handle_class0(ctx); 391 if (ret) 392 break; 393 394 if (signal_pending(current)) 395 ret = -ERESTARTSYS; 396 } while (!ret && !(status & (SPU_STATUS_STOPPED_BY_STOP | 397 SPU_STATUS_STOPPED_BY_HALT | 398 SPU_STATUS_SINGLE_STEP))); 399 400 spu_disable_spu(ctx); 401 ret = spu_run_fini(ctx, npc, &status); 402 spu_yield(ctx); 403 404 if ((status & SPU_STATUS_STOPPED_BY_STOP) && 405 (((status >> SPU_STOP_STATUS_SHIFT) & 0x3f00) == 0x2100)) 406 ctx->stats.libassist++; 407 408 if ((ret == 0) || 409 ((ret == -ERESTARTSYS) && 410 ((status & SPU_STATUS_STOPPED_BY_HALT) || 411 (status & SPU_STATUS_SINGLE_STEP) || 412 ((status & SPU_STATUS_STOPPED_BY_STOP) && 413 (status >> SPU_STOP_STATUS_SHIFT != 0x2104))))) 414 ret = status; 415 416 /* Note: we don't need to force_sig SIGTRAP on single-step 417 * since we have TIF_SINGLESTEP set, thus the kernel will do 418 * it upon return from the syscall anyawy 419 */ 420 if (unlikely(status & SPU_STATUS_SINGLE_STEP)) 421 ret = -ERESTARTSYS; 422 423 else if (unlikely((status & SPU_STATUS_STOPPED_BY_STOP) 424 && (status >> SPU_STOP_STATUS_SHIFT) == 0x3fff)) { 425 force_sig(SIGTRAP, current); 426 ret = -ERESTARTSYS; 427 } 428 429 out: 430 *event = ctx->event_return; 431 out_unlock: 432 mutex_unlock(&ctx->run_mutex); 433 return ret; 434 } 435