1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */ 3 4 #include <linux/interrupt.h> 5 #include <linux/io.h> 6 #include <linux/device.h> 7 #include <linux/slab.h> 8 9 #include <drm/lima_drm.h> 10 11 #include "lima_device.h" 12 #include "lima_pp.h" 13 #include "lima_dlbu.h" 14 #include "lima_bcast.h" 15 #include "lima_vm.h" 16 #include "lima_regs.h" 17 18 #define pp_write(reg, data) writel(data, ip->iomem + reg) 19 #define pp_read(reg) readl(ip->iomem + reg) 20 21 static void lima_pp_handle_irq(struct lima_ip *ip, u32 state) 22 { 23 struct lima_device *dev = ip->dev; 24 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp; 25 26 if (state & LIMA_PP_IRQ_MASK_ERROR) { 27 u32 status = pp_read(LIMA_PP_STATUS); 28 29 dev_err(dev->dev, "pp error irq state=%x status=%x\n", 30 state, status); 31 32 pipe->error = true; 33 34 /* mask all interrupts before hard reset */ 35 pp_write(LIMA_PP_INT_MASK, 0); 36 } 37 38 pp_write(LIMA_PP_INT_CLEAR, state); 39 } 40 41 static irqreturn_t lima_pp_irq_handler(int irq, void *data) 42 { 43 struct lima_ip *ip = data; 44 struct lima_device *dev = ip->dev; 45 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp; 46 u32 state = pp_read(LIMA_PP_INT_STATUS); 47 48 /* for shared irq case */ 49 if (!state) 50 return IRQ_NONE; 51 52 lima_pp_handle_irq(ip, state); 53 54 if (atomic_dec_and_test(&pipe->task)) 55 lima_sched_pipe_task_done(pipe); 56 57 return IRQ_HANDLED; 58 } 59 60 static irqreturn_t lima_pp_bcast_irq_handler(int irq, void *data) 61 { 62 int i; 63 irqreturn_t ret = IRQ_NONE; 64 struct lima_ip *pp_bcast = data; 65 struct lima_device *dev = pp_bcast->dev; 66 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp; 67 struct drm_lima_m450_pp_frame *frame; 68 69 /* for shared irq case */ 70 if (!pipe->current_task) 71 return IRQ_NONE; 72 73 frame = pipe->current_task->frame; 74 75 for (i = 0; i < frame->num_pp; i++) { 76 struct lima_ip *ip = pipe->processor[i]; 77 u32 status, state; 78 79 if (pipe->done & (1 << i)) 80 continue; 81 82 /* status read first in case int state change in the middle 83 * which may miss the interrupt handling 84 */ 85 status = pp_read(LIMA_PP_STATUS); 86 state = pp_read(LIMA_PP_INT_STATUS); 87 88 if (state) { 89 lima_pp_handle_irq(ip, state); 90 ret = IRQ_HANDLED; 91 } else { 92 if (status & LIMA_PP_STATUS_RENDERING_ACTIVE) 93 continue; 94 } 95 96 pipe->done |= (1 << i); 97 if (atomic_dec_and_test(&pipe->task)) 98 lima_sched_pipe_task_done(pipe); 99 } 100 101 return ret; 102 } 103 104 static void lima_pp_soft_reset_async(struct lima_ip *ip) 105 { 106 if (ip->data.async_reset) 107 return; 108 109 pp_write(LIMA_PP_INT_MASK, 0); 110 pp_write(LIMA_PP_INT_RAWSTAT, LIMA_PP_IRQ_MASK_ALL); 111 pp_write(LIMA_PP_CTRL, LIMA_PP_CTRL_SOFT_RESET); 112 ip->data.async_reset = true; 113 } 114 115 static int lima_pp_soft_reset_poll(struct lima_ip *ip) 116 { 117 return !(pp_read(LIMA_PP_STATUS) & LIMA_PP_STATUS_RENDERING_ACTIVE) && 118 pp_read(LIMA_PP_INT_RAWSTAT) == LIMA_PP_IRQ_RESET_COMPLETED; 119 } 120 121 static int lima_pp_soft_reset_async_wait_one(struct lima_ip *ip) 122 { 123 struct lima_device *dev = ip->dev; 124 int ret; 125 126 ret = lima_poll_timeout(ip, lima_pp_soft_reset_poll, 0, 100); 127 if (ret) { 128 dev_err(dev->dev, "pp %s reset time out\n", lima_ip_name(ip)); 129 return ret; 130 } 131 132 pp_write(LIMA_PP_INT_CLEAR, LIMA_PP_IRQ_MASK_ALL); 133 pp_write(LIMA_PP_INT_MASK, LIMA_PP_IRQ_MASK_USED); 134 return 0; 135 } 136 137 static int lima_pp_soft_reset_async_wait(struct lima_ip *ip) 138 { 139 int i, err = 0; 140 141 if (!ip->data.async_reset) 142 return 0; 143 144 if (ip->id == lima_ip_pp_bcast) { 145 struct lima_device *dev = ip->dev; 146 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp; 147 struct drm_lima_m450_pp_frame *frame = pipe->current_task->frame; 148 149 for (i = 0; i < frame->num_pp; i++) 150 err |= lima_pp_soft_reset_async_wait_one(pipe->processor[i]); 151 } else 152 err = lima_pp_soft_reset_async_wait_one(ip); 153 154 ip->data.async_reset = false; 155 return err; 156 } 157 158 static void lima_pp_write_frame(struct lima_ip *ip, u32 *frame, u32 *wb) 159 { 160 int i, j, n = 0; 161 162 for (i = 0; i < LIMA_PP_FRAME_REG_NUM; i++) 163 writel(frame[i], ip->iomem + LIMA_PP_FRAME + i * 4); 164 165 for (i = 0; i < 3; i++) { 166 for (j = 0; j < LIMA_PP_WB_REG_NUM; j++) 167 writel(wb[n++], ip->iomem + LIMA_PP_WB(i) + j * 4); 168 } 169 } 170 171 static int lima_pp_hard_reset_poll(struct lima_ip *ip) 172 { 173 pp_write(LIMA_PP_PERF_CNT_0_LIMIT, 0xC01A0000); 174 return pp_read(LIMA_PP_PERF_CNT_0_LIMIT) == 0xC01A0000; 175 } 176 177 static int lima_pp_hard_reset(struct lima_ip *ip) 178 { 179 struct lima_device *dev = ip->dev; 180 int ret; 181 182 pp_write(LIMA_PP_PERF_CNT_0_LIMIT, 0xC0FFE000); 183 pp_write(LIMA_PP_INT_MASK, 0); 184 pp_write(LIMA_PP_CTRL, LIMA_PP_CTRL_FORCE_RESET); 185 ret = lima_poll_timeout(ip, lima_pp_hard_reset_poll, 10, 100); 186 if (ret) { 187 dev_err(dev->dev, "pp hard reset timeout\n"); 188 return ret; 189 } 190 191 pp_write(LIMA_PP_PERF_CNT_0_LIMIT, 0); 192 pp_write(LIMA_PP_INT_CLEAR, LIMA_PP_IRQ_MASK_ALL); 193 pp_write(LIMA_PP_INT_MASK, LIMA_PP_IRQ_MASK_USED); 194 return 0; 195 } 196 197 static void lima_pp_print_version(struct lima_ip *ip) 198 { 199 u32 version, major, minor; 200 char *name; 201 202 version = pp_read(LIMA_PP_VERSION); 203 major = (version >> 8) & 0xFF; 204 minor = version & 0xFF; 205 switch (version >> 16) { 206 case 0xC807: 207 name = "mali200"; 208 break; 209 case 0xCE07: 210 name = "mali300"; 211 break; 212 case 0xCD07: 213 name = "mali400"; 214 break; 215 case 0xCF07: 216 name = "mali450"; 217 break; 218 default: 219 name = "unknown"; 220 break; 221 } 222 dev_info(ip->dev->dev, "%s - %s version major %d minor %d\n", 223 lima_ip_name(ip), name, major, minor); 224 } 225 226 static int lima_pp_hw_init(struct lima_ip *ip) 227 { 228 ip->data.async_reset = false; 229 lima_pp_soft_reset_async(ip); 230 return lima_pp_soft_reset_async_wait(ip); 231 } 232 233 int lima_pp_resume(struct lima_ip *ip) 234 { 235 return lima_pp_hw_init(ip); 236 } 237 238 void lima_pp_suspend(struct lima_ip *ip) 239 { 240 241 } 242 243 int lima_pp_init(struct lima_ip *ip) 244 { 245 struct lima_device *dev = ip->dev; 246 int err; 247 248 lima_pp_print_version(ip); 249 250 err = lima_pp_hw_init(ip); 251 if (err) 252 return err; 253 254 err = devm_request_irq(dev->dev, ip->irq, lima_pp_irq_handler, 255 IRQF_SHARED, lima_ip_name(ip), ip); 256 if (err) { 257 dev_err(dev->dev, "pp %s fail to request irq\n", 258 lima_ip_name(ip)); 259 return err; 260 } 261 262 dev->pp_version = pp_read(LIMA_PP_VERSION); 263 264 return 0; 265 } 266 267 void lima_pp_fini(struct lima_ip *ip) 268 { 269 struct lima_device *dev = ip->dev; 270 271 devm_free_irq(dev->dev, ip->irq, ip); 272 } 273 274 int lima_pp_bcast_resume(struct lima_ip *ip) 275 { 276 /* PP has been reset by individual PP resume */ 277 ip->data.async_reset = false; 278 return 0; 279 } 280 281 void lima_pp_bcast_suspend(struct lima_ip *ip) 282 { 283 284 } 285 286 int lima_pp_bcast_init(struct lima_ip *ip) 287 { 288 struct lima_device *dev = ip->dev; 289 int err; 290 291 err = devm_request_irq(dev->dev, ip->irq, lima_pp_bcast_irq_handler, 292 IRQF_SHARED, lima_ip_name(ip), ip); 293 if (err) { 294 dev_err(dev->dev, "pp %s fail to request irq\n", 295 lima_ip_name(ip)); 296 return err; 297 } 298 299 return 0; 300 } 301 302 void lima_pp_bcast_fini(struct lima_ip *ip) 303 { 304 struct lima_device *dev = ip->dev; 305 306 devm_free_irq(dev->dev, ip->irq, ip); 307 } 308 309 static int lima_pp_task_validate(struct lima_sched_pipe *pipe, 310 struct lima_sched_task *task) 311 { 312 u32 num_pp; 313 314 if (pipe->bcast_processor) { 315 struct drm_lima_m450_pp_frame *f = task->frame; 316 317 num_pp = f->num_pp; 318 319 if (f->_pad) 320 return -EINVAL; 321 } else { 322 struct drm_lima_m400_pp_frame *f = task->frame; 323 324 num_pp = f->num_pp; 325 } 326 327 if (num_pp == 0 || num_pp > pipe->num_processor) 328 return -EINVAL; 329 330 return 0; 331 } 332 333 static void lima_pp_task_run(struct lima_sched_pipe *pipe, 334 struct lima_sched_task *task) 335 { 336 if (pipe->bcast_processor) { 337 struct drm_lima_m450_pp_frame *frame = task->frame; 338 struct lima_device *dev = pipe->bcast_processor->dev; 339 struct lima_ip *ip = pipe->bcast_processor; 340 int i; 341 342 pipe->done = 0; 343 atomic_set(&pipe->task, frame->num_pp); 344 345 if (frame->use_dlbu) { 346 lima_dlbu_enable(dev, frame->num_pp); 347 348 frame->frame[LIMA_PP_FRAME >> 2] = LIMA_VA_RESERVE_DLBU; 349 lima_dlbu_set_reg(dev->ip + lima_ip_dlbu, frame->dlbu_regs); 350 } else 351 lima_dlbu_disable(dev); 352 353 lima_bcast_enable(dev, frame->num_pp); 354 355 lima_pp_soft_reset_async_wait(ip); 356 357 lima_pp_write_frame(ip, frame->frame, frame->wb); 358 359 for (i = 0; i < frame->num_pp; i++) { 360 struct lima_ip *ip = pipe->processor[i]; 361 362 pp_write(LIMA_PP_STACK, frame->fragment_stack_address[i]); 363 if (!frame->use_dlbu) 364 pp_write(LIMA_PP_FRAME, frame->plbu_array_address[i]); 365 } 366 367 pp_write(LIMA_PP_CTRL, LIMA_PP_CTRL_START_RENDERING); 368 } else { 369 struct drm_lima_m400_pp_frame *frame = task->frame; 370 int i; 371 372 atomic_set(&pipe->task, frame->num_pp); 373 374 for (i = 0; i < frame->num_pp; i++) { 375 struct lima_ip *ip = pipe->processor[i]; 376 377 frame->frame[LIMA_PP_FRAME >> 2] = 378 frame->plbu_array_address[i]; 379 frame->frame[LIMA_PP_STACK >> 2] = 380 frame->fragment_stack_address[i]; 381 382 lima_pp_soft_reset_async_wait(ip); 383 384 lima_pp_write_frame(ip, frame->frame, frame->wb); 385 386 pp_write(LIMA_PP_CTRL, LIMA_PP_CTRL_START_RENDERING); 387 } 388 } 389 } 390 391 static void lima_pp_task_fini(struct lima_sched_pipe *pipe) 392 { 393 if (pipe->bcast_processor) 394 lima_pp_soft_reset_async(pipe->bcast_processor); 395 else { 396 int i; 397 398 for (i = 0; i < pipe->num_processor; i++) 399 lima_pp_soft_reset_async(pipe->processor[i]); 400 } 401 } 402 403 static void lima_pp_task_error(struct lima_sched_pipe *pipe) 404 { 405 int i; 406 407 for (i = 0; i < pipe->num_processor; i++) { 408 struct lima_ip *ip = pipe->processor[i]; 409 410 dev_err(ip->dev->dev, "pp task error %d int_state=%x status=%x\n", 411 i, pp_read(LIMA_PP_INT_STATUS), pp_read(LIMA_PP_STATUS)); 412 413 lima_pp_hard_reset(ip); 414 } 415 416 if (pipe->bcast_processor) 417 lima_bcast_reset(pipe->bcast_processor); 418 } 419 420 static void lima_pp_task_mmu_error(struct lima_sched_pipe *pipe) 421 { 422 if (atomic_dec_and_test(&pipe->task)) 423 lima_sched_pipe_task_done(pipe); 424 } 425 426 static void lima_pp_task_mask_irq(struct lima_sched_pipe *pipe) 427 { 428 int i; 429 430 for (i = 0; i < pipe->num_processor; i++) { 431 struct lima_ip *ip = pipe->processor[i]; 432 433 pp_write(LIMA_PP_INT_MASK, 0); 434 } 435 436 if (pipe->bcast_processor) 437 lima_bcast_mask_irq(pipe->bcast_processor); 438 } 439 440 static struct kmem_cache *lima_pp_task_slab; 441 static int lima_pp_task_slab_refcnt; 442 443 int lima_pp_pipe_init(struct lima_device *dev) 444 { 445 int frame_size; 446 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp; 447 448 if (dev->id == lima_gpu_mali400) 449 frame_size = sizeof(struct drm_lima_m400_pp_frame); 450 else 451 frame_size = sizeof(struct drm_lima_m450_pp_frame); 452 453 if (!lima_pp_task_slab) { 454 lima_pp_task_slab = kmem_cache_create_usercopy( 455 "lima_pp_task", sizeof(struct lima_sched_task) + frame_size, 456 0, SLAB_HWCACHE_ALIGN, sizeof(struct lima_sched_task), 457 frame_size, NULL); 458 if (!lima_pp_task_slab) 459 return -ENOMEM; 460 } 461 lima_pp_task_slab_refcnt++; 462 463 pipe->frame_size = frame_size; 464 pipe->task_slab = lima_pp_task_slab; 465 466 pipe->task_validate = lima_pp_task_validate; 467 pipe->task_run = lima_pp_task_run; 468 pipe->task_fini = lima_pp_task_fini; 469 pipe->task_error = lima_pp_task_error; 470 pipe->task_mmu_error = lima_pp_task_mmu_error; 471 pipe->task_mask_irq = lima_pp_task_mask_irq; 472 473 return 0; 474 } 475 476 void lima_pp_pipe_fini(struct lima_device *dev) 477 { 478 if (!--lima_pp_task_slab_refcnt) { 479 kmem_cache_destroy(lima_pp_task_slab); 480 lima_pp_task_slab = NULL; 481 } 482 } 483