1 /*
2  * Copyright 2013 Red Hat Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Ben Skeggs
23  */
24 #include "priv.h"
25 
26 #include <core/client.h>
27 #include <core/option.h>
28 
29 #include <nvif/class.h>
30 #include <nvif/ioctl.h>
31 #include <nvif/unpack.h>
32 
33 static u8
34 nvkm_pm_count_perfdom(struct nvkm_pm *pm)
35 {
36 	struct nvkm_perfdom *dom;
37 	u8 domain_nr = 0;
38 
39 	list_for_each_entry(dom, &pm->domains, head)
40 		domain_nr++;
41 	return domain_nr;
42 }
43 
44 static u16
45 nvkm_perfdom_count_perfsig(struct nvkm_perfdom *dom)
46 {
47 	u16 signal_nr = 0;
48 	int i;
49 
50 	if (dom) {
51 		for (i = 0; i < dom->signal_nr; i++) {
52 			if (dom->signal[i].name)
53 				signal_nr++;
54 		}
55 	}
56 	return signal_nr;
57 }
58 
59 static struct nvkm_perfdom *
60 nvkm_perfdom_find(struct nvkm_pm *pm, int di)
61 {
62 	struct nvkm_perfdom *dom;
63 	int tmp = 0;
64 
65 	list_for_each_entry(dom, &pm->domains, head) {
66 		if (tmp++ == di)
67 			return dom;
68 	}
69 	return NULL;
70 }
71 
72 struct nvkm_perfsig *
73 nvkm_perfsig_find(struct nvkm_pm *pm, u8 di, u8 si, struct nvkm_perfdom **pdom)
74 {
75 	struct nvkm_perfdom *dom = *pdom;
76 
77 	if (dom == NULL) {
78 		dom = nvkm_perfdom_find(pm, di);
79 		if (dom == NULL)
80 			return NULL;
81 		*pdom = dom;
82 	}
83 
84 	if (!dom->signal[si].name)
85 		return NULL;
86 	return &dom->signal[si];
87 }
88 
89 static u8
90 nvkm_perfsig_count_perfsrc(struct nvkm_perfsig *sig)
91 {
92 	u8 source_nr = 0, i;
93 
94 	for (i = 0; i < ARRAY_SIZE(sig->source); i++) {
95 		if (sig->source[i])
96 			source_nr++;
97 	}
98 	return source_nr;
99 }
100 
101 static struct nvkm_perfsrc *
102 nvkm_perfsrc_find(struct nvkm_pm *pm, struct nvkm_perfsig *sig, int si)
103 {
104 	struct nvkm_perfsrc *src;
105 	bool found = false;
106 	int tmp = 1; /* Sources ID start from 1 */
107 	u8 i;
108 
109 	for (i = 0; i < ARRAY_SIZE(sig->source) && sig->source[i]; i++) {
110 		if (sig->source[i] == si) {
111 			found = true;
112 			break;
113 		}
114 	}
115 
116 	if (found) {
117 		list_for_each_entry(src, &pm->sources, head) {
118 			if (tmp++ == si)
119 				return src;
120 		}
121 	}
122 
123 	return NULL;
124 }
125 
126 static int
127 nvkm_perfsrc_enable(struct nvkm_pm *pm, struct nvkm_perfctr *ctr)
128 {
129 	struct nvkm_subdev *subdev = &pm->engine.subdev;
130 	struct nvkm_device *device = subdev->device;
131 	struct nvkm_perfdom *dom = NULL;
132 	struct nvkm_perfsig *sig;
133 	struct nvkm_perfsrc *src;
134 	u32 mask, value;
135 	int i, j;
136 
137 	for (i = 0; i < 4; i++) {
138 		for (j = 0; j < 8 && ctr->source[i][j]; j++) {
139 			sig = nvkm_perfsig_find(pm, ctr->domain,
140 						ctr->signal[i], &dom);
141 			if (!sig)
142 				return -EINVAL;
143 
144 			src = nvkm_perfsrc_find(pm, sig, ctr->source[i][j]);
145 			if (!src)
146 				return -EINVAL;
147 
148 			/* set enable bit if needed */
149 			mask = value = 0x00000000;
150 			if (src->enable)
151 				mask = value = 0x80000000;
152 			mask  |= (src->mask << src->shift);
153 			value |= ((ctr->source[i][j] >> 32) << src->shift);
154 
155 			/* enable the source */
156 			nvkm_mask(device, src->addr, mask, value);
157 			nvkm_debug(subdev,
158 				   "enabled source %08x %08x %08x\n",
159 				   src->addr, mask, value);
160 		}
161 	}
162 	return 0;
163 }
164 
165 static int
166 nvkm_perfsrc_disable(struct nvkm_pm *pm, struct nvkm_perfctr *ctr)
167 {
168 	struct nvkm_subdev *subdev = &pm->engine.subdev;
169 	struct nvkm_device *device = subdev->device;
170 	struct nvkm_perfdom *dom = NULL;
171 	struct nvkm_perfsig *sig;
172 	struct nvkm_perfsrc *src;
173 	u32 mask;
174 	int i, j;
175 
176 	for (i = 0; i < 4; i++) {
177 		for (j = 0; j < 8 && ctr->source[i][j]; j++) {
178 			sig = nvkm_perfsig_find(pm, ctr->domain,
179 						ctr->signal[i], &dom);
180 			if (!sig)
181 				return -EINVAL;
182 
183 			src = nvkm_perfsrc_find(pm, sig, ctr->source[i][j]);
184 			if (!src)
185 				return -EINVAL;
186 
187 			/* unset enable bit if needed */
188 			mask = 0x00000000;
189 			if (src->enable)
190 				mask = 0x80000000;
191 			mask |= (src->mask << src->shift);
192 
193 			/* disable the source */
194 			nvkm_mask(device, src->addr, mask, 0);
195 			nvkm_debug(subdev, "disabled source %08x %08x\n",
196 				   src->addr, mask);
197 		}
198 	}
199 	return 0;
200 }
201 
202 /*******************************************************************************
203  * Perfdom object classes
204  ******************************************************************************/
205 static int
206 nvkm_perfdom_init(struct nvkm_object *object, void *data, u32 size)
207 {
208 	union {
209 		struct nvif_perfdom_init none;
210 	} *args = data;
211 	struct nvkm_pm *pm = (void *)object->engine;
212 	struct nvkm_perfdom *dom = (void *)object;
213 	int ret, i;
214 
215 	nvif_ioctl(object, "perfdom init size %d\n", size);
216 	if (nvif_unvers(args->none)) {
217 		nvif_ioctl(object, "perfdom init\n");
218 	} else
219 		return ret;
220 
221 	for (i = 0; i < 4; i++) {
222 		if (dom->ctr[i]) {
223 			dom->func->init(pm, dom, dom->ctr[i]);
224 
225 			/* enable sources */
226 			nvkm_perfsrc_enable(pm, dom->ctr[i]);
227 		}
228 	}
229 
230 	/* start next batch of counters for sampling */
231 	dom->func->next(pm, dom);
232 	return 0;
233 }
234 
235 static int
236 nvkm_perfdom_sample(struct nvkm_object *object, void *data, u32 size)
237 {
238 	union {
239 		struct nvif_perfdom_sample none;
240 	} *args = data;
241 	struct nvkm_pm *pm = (void *)object->engine;
242 	struct nvkm_perfdom *dom;
243 	int ret;
244 
245 	nvif_ioctl(object, "perfdom sample size %d\n", size);
246 	if (nvif_unvers(args->none)) {
247 		nvif_ioctl(object, "perfdom sample\n");
248 	} else
249 		return ret;
250 	pm->sequence++;
251 
252 	/* sample previous batch of counters */
253 	list_for_each_entry(dom, &pm->domains, head)
254 		dom->func->next(pm, dom);
255 
256 	return 0;
257 }
258 
259 static int
260 nvkm_perfdom_read(struct nvkm_object *object, void *data, u32 size)
261 {
262 	union {
263 		struct nvif_perfdom_read_v0 v0;
264 	} *args = data;
265 	struct nvkm_pm *pm = (void *)object->engine;
266 	struct nvkm_perfdom *dom = (void *)object;
267 	int ret, i;
268 
269 	nvif_ioctl(object, "perfdom read size %d\n", size);
270 	if (nvif_unpack(args->v0, 0, 0, false)) {
271 		nvif_ioctl(object, "perfdom read vers %d\n", args->v0.version);
272 	} else
273 		return ret;
274 
275 	for (i = 0; i < 4; i++) {
276 		if (dom->ctr[i])
277 			dom->func->read(pm, dom, dom->ctr[i]);
278 	}
279 
280 	if (!dom->clk)
281 		return -EAGAIN;
282 
283 	for (i = 0; i < 4; i++)
284 		if (dom->ctr[i])
285 			args->v0.ctr[i] = dom->ctr[i]->ctr;
286 	args->v0.clk = dom->clk;
287 	return 0;
288 }
289 
290 static int
291 nvkm_perfdom_mthd(struct nvkm_object *object, u32 mthd, void *data, u32 size)
292 {
293 	switch (mthd) {
294 	case NVIF_PERFDOM_V0_INIT:
295 		return nvkm_perfdom_init(object, data, size);
296 	case NVIF_PERFDOM_V0_SAMPLE:
297 		return nvkm_perfdom_sample(object, data, size);
298 	case NVIF_PERFDOM_V0_READ:
299 		return nvkm_perfdom_read(object, data, size);
300 	default:
301 		break;
302 	}
303 	return -EINVAL;
304 }
305 
306 static void
307 nvkm_perfdom_dtor(struct nvkm_object *object)
308 {
309 	struct nvkm_pm *pm = (void *)object->engine;
310 	struct nvkm_perfdom *dom = (void *)object;
311 	int i;
312 
313 	for (i = 0; i < 4; i++) {
314 		struct nvkm_perfctr *ctr = dom->ctr[i];
315 		if (ctr) {
316 			nvkm_perfsrc_disable(pm, ctr);
317 			if (ctr->head.next)
318 				list_del(&ctr->head);
319 		}
320 		kfree(ctr);
321 	}
322 	nvkm_object_destroy(&dom->base);
323 }
324 
325 static int
326 nvkm_perfctr_new(struct nvkm_perfdom *dom, int slot, u8 domain,
327 		 struct nvkm_perfsig *signal[4], u64 source[4][8],
328 		 u16 logic_op, struct nvkm_perfctr **pctr)
329 {
330 	struct nvkm_perfctr *ctr;
331 	int i, j;
332 
333 	if (!dom)
334 		return -EINVAL;
335 
336 	ctr = *pctr = kzalloc(sizeof(*ctr), GFP_KERNEL);
337 	if (!ctr)
338 		return -ENOMEM;
339 
340 	ctr->domain   = domain;
341 	ctr->logic_op = logic_op;
342 	ctr->slot     = slot;
343 	for (i = 0; i < 4; i++) {
344 		if (signal[i]) {
345 			ctr->signal[i] = signal[i] - dom->signal;
346 			for (j = 0; j < 8; j++)
347 				ctr->source[i][j] = source[i][j];
348 		}
349 	}
350 	list_add_tail(&ctr->head, &dom->list);
351 
352 	return 0;
353 }
354 
355 static int
356 nvkm_perfdom_ctor(struct nvkm_object *parent, struct nvkm_object *engine,
357 		  struct nvkm_oclass *oclass, void *data, u32 size,
358 		  struct nvkm_object **pobject)
359 {
360 	union {
361 		struct nvif_perfdom_v0 v0;
362 	} *args = data;
363 	struct nvkm_pm *pm = (void *)engine;
364 	struct nvkm_perfdom *sdom = NULL;
365 	struct nvkm_perfctr *ctr[4] = {};
366 	struct nvkm_perfdom *dom;
367 	int c, s, m;
368 	int ret;
369 
370 	nvif_ioctl(parent, "create perfdom size %d\n", size);
371 	if (nvif_unpack(args->v0, 0, 0, false)) {
372 		nvif_ioctl(parent, "create perfdom vers %d dom %d mode %02x\n",
373 			   args->v0.version, args->v0.domain, args->v0.mode);
374 	} else
375 		return ret;
376 
377 	for (c = 0; c < ARRAY_SIZE(args->v0.ctr); c++) {
378 		struct nvkm_perfsig *sig[4] = {};
379 		u64 src[4][8] = {};
380 
381 		for (s = 0; s < ARRAY_SIZE(args->v0.ctr[c].signal); s++) {
382 			sig[s] = nvkm_perfsig_find(pm, args->v0.domain,
383 						   args->v0.ctr[c].signal[s],
384 						   &sdom);
385 			if (args->v0.ctr[c].signal[s] && !sig[s])
386 				return -EINVAL;
387 
388 			for (m = 0; m < 8; m++) {
389 				src[s][m] = args->v0.ctr[c].source[s][m];
390 				if (src[s][m] && !nvkm_perfsrc_find(pm, sig[s],
391 							            src[s][m]))
392 					return -EINVAL;
393 			}
394 		}
395 
396 		ret = nvkm_perfctr_new(sdom, c, args->v0.domain, sig, src,
397 				       args->v0.ctr[c].logic_op, &ctr[c]);
398 		if (ret)
399 			return ret;
400 	}
401 
402 	if (!sdom)
403 		return -EINVAL;
404 
405 	ret = nvkm_object_create(parent, engine, oclass, 0, &dom);
406 	*pobject = nv_object(dom);
407 	if (ret)
408 		return ret;
409 
410 	dom->func = sdom->func;
411 	dom->addr = sdom->addr;
412 	dom->mode = args->v0.mode;
413 	for (c = 0; c < ARRAY_SIZE(ctr); c++)
414 		dom->ctr[c] = ctr[c];
415 	return 0;
416 }
417 
418 static struct nvkm_ofuncs
419 nvkm_perfdom_ofuncs = {
420 	.ctor = nvkm_perfdom_ctor,
421 	.dtor = nvkm_perfdom_dtor,
422 	.init = _nvkm_object_init,
423 	.fini = _nvkm_object_fini,
424 	.mthd = nvkm_perfdom_mthd,
425 };
426 
427 /*******************************************************************************
428  * Perfmon object classes
429  ******************************************************************************/
430 static int
431 nvkm_perfmon_mthd_query_domain(struct nvkm_object *object, void *data, u32 size)
432 {
433 	union {
434 		struct nvif_perfmon_query_domain_v0 v0;
435 	} *args = data;
436 	struct nvkm_pm *pm = (void *)object->engine;
437 	struct nvkm_perfdom *dom;
438 	u8 domain_nr;
439 	int di, ret;
440 
441 	nvif_ioctl(object, "perfmon query domain size %d\n", size);
442 	if (nvif_unpack(args->v0, 0, 0, false)) {
443 		nvif_ioctl(object, "perfmon domain vers %d iter %02x\n",
444 			   args->v0.version, args->v0.iter);
445 		di = (args->v0.iter & 0xff) - 1;
446 	} else
447 		return ret;
448 
449 	domain_nr = nvkm_pm_count_perfdom(pm);
450 	if (di >= (int)domain_nr)
451 		return -EINVAL;
452 
453 	if (di >= 0) {
454 		dom = nvkm_perfdom_find(pm, di);
455 		if (dom == NULL)
456 			return -EINVAL;
457 
458 		args->v0.id         = di;
459 		args->v0.signal_nr  = nvkm_perfdom_count_perfsig(dom);
460 		strncpy(args->v0.name, dom->name, sizeof(args->v0.name));
461 
462 		/* Currently only global counters (PCOUNTER) are implemented
463 		 * but this will be different for local counters (MP). */
464 		args->v0.counter_nr = 4;
465 	}
466 
467 	if (++di < domain_nr) {
468 		args->v0.iter = ++di;
469 		return 0;
470 	}
471 
472 	args->v0.iter = 0xff;
473 	return 0;
474 }
475 
476 static int
477 nvkm_perfmon_mthd_query_signal(struct nvkm_object *object, void *data, u32 size)
478 {
479 	union {
480 		struct nvif_perfmon_query_signal_v0 v0;
481 	} *args = data;
482 	struct nvkm_device *device = nv_device(object);
483 	struct nvkm_pm *pm = (void *)object->engine;
484 	struct nvkm_perfdom *dom;
485 	struct nvkm_perfsig *sig;
486 	const bool all = nvkm_boolopt(device->cfgopt, "NvPmShowAll", false);
487 	const bool raw = nvkm_boolopt(device->cfgopt, "NvPmUnnamed", all);
488 	int ret, si;
489 
490 	nvif_ioctl(object, "perfmon query signal size %d\n", size);
491 	if (nvif_unpack(args->v0, 0, 0, false)) {
492 		nvif_ioctl(object,
493 			   "perfmon query signal vers %d dom %d iter %04x\n",
494 			   args->v0.version, args->v0.domain, args->v0.iter);
495 		si = (args->v0.iter & 0xffff) - 1;
496 	} else
497 		return ret;
498 
499 	dom = nvkm_perfdom_find(pm, args->v0.domain);
500 	if (dom == NULL || si >= (int)dom->signal_nr)
501 		return -EINVAL;
502 
503 	if (si >= 0) {
504 		sig = &dom->signal[si];
505 		if (raw || !sig->name) {
506 			snprintf(args->v0.name, sizeof(args->v0.name),
507 				 "/%s/%02x", dom->name, si);
508 		} else {
509 			strncpy(args->v0.name, sig->name,
510 				sizeof(args->v0.name));
511 		}
512 
513 		args->v0.signal = si;
514 		args->v0.source_nr = nvkm_perfsig_count_perfsrc(sig);
515 	}
516 
517 	while (++si < dom->signal_nr) {
518 		if (all || dom->signal[si].name) {
519 			args->v0.iter = ++si;
520 			return 0;
521 		}
522 	}
523 
524 	args->v0.iter = 0xffff;
525 	return 0;
526 }
527 
528 static int
529 nvkm_perfmon_mthd_query_source(struct nvkm_object *object, void *data, u32 size)
530 {
531 	union {
532 		struct nvif_perfmon_query_source_v0 v0;
533 	} *args = data;
534 	struct nvkm_pm *pm = (void *)object->engine;
535 	struct nvkm_perfdom *dom = NULL;
536 	struct nvkm_perfsig *sig;
537 	struct nvkm_perfsrc *src;
538 	u8 source_nr = 0;
539 	int si, ret;
540 
541 	nvif_ioctl(object, "perfmon query source size %d\n", size);
542 	if (nvif_unpack(args->v0, 0, 0, false)) {
543 		nvif_ioctl(object,
544 			   "perfmon source vers %d dom %d sig %02x iter %02x\n",
545 			   args->v0.version, args->v0.domain, args->v0.signal,
546 			   args->v0.iter);
547 		si = (args->v0.iter & 0xff) - 1;
548 	} else
549 		return ret;
550 
551 	sig = nvkm_perfsig_find(pm, args->v0.domain, args->v0.signal, &dom);
552 	if (!sig)
553 		return -EINVAL;
554 
555 	source_nr = nvkm_perfsig_count_perfsrc(sig);
556 	if (si >= (int)source_nr)
557 		return -EINVAL;
558 
559 	if (si >= 0) {
560 		src = nvkm_perfsrc_find(pm, sig, sig->source[si]);
561 		if (!src)
562 			return -EINVAL;
563 
564 		args->v0.source = sig->source[si];
565 		args->v0.mask   = src->mask;
566 		strncpy(args->v0.name, src->name, sizeof(args->v0.name));
567 	}
568 
569 	if (++si < source_nr) {
570 		args->v0.iter = ++si;
571 		return 0;
572 	}
573 
574 	args->v0.iter = 0xff;
575 	return 0;
576 }
577 
578 static int
579 nvkm_perfmon_mthd(struct nvkm_object *object, u32 mthd, void *data, u32 size)
580 {
581 	switch (mthd) {
582 	case NVIF_PERFMON_V0_QUERY_DOMAIN:
583 		return nvkm_perfmon_mthd_query_domain(object, data, size);
584 	case NVIF_PERFMON_V0_QUERY_SIGNAL:
585 		return nvkm_perfmon_mthd_query_signal(object, data, size);
586 	case NVIF_PERFMON_V0_QUERY_SOURCE:
587 		return nvkm_perfmon_mthd_query_source(object, data, size);
588 	default:
589 		break;
590 	}
591 	return -EINVAL;
592 }
593 
594 static struct nvkm_oclass
595 nvkm_perfmon_sclass[] = {
596 	{ .handle = NVIF_IOCTL_NEW_V0_PERFDOM,
597 	  .ofuncs = &nvkm_perfdom_ofuncs,
598 	},
599 	{}
600 };
601 
602 static int
603 nvkm_perfmon_ctor(struct nvkm_object *parent, struct nvkm_object *engine,
604 		  struct nvkm_oclass *oclass, void *data, u32 size,
605 		  struct nvkm_object **pobject)
606 {
607 	struct nvkm_parent *perfmon;
608 	int ret = nvkm_parent_create(parent, engine, oclass, 0,
609 				     nvkm_perfmon_sclass, 0, &perfmon);
610 	*pobject = perfmon ? &perfmon->object : NULL;
611 	return ret;
612 }
613 
614 static struct nvkm_ofuncs
615 nvkm_perfmon_ofuncs = {
616 	.ctor = nvkm_perfmon_ctor,
617 	.dtor = _nvkm_parent_dtor,
618 	.init = _nvkm_parent_init,
619 	.fini = _nvkm_parent_fini,
620 	.mthd = nvkm_perfmon_mthd,
621 };
622 
623 struct nvkm_oclass
624 nvkm_pm_sclass[] = {
625 	{
626 	  .handle = NVIF_IOCTL_NEW_V0_PERFMON,
627 	  .ofuncs = &nvkm_perfmon_ofuncs,
628 	},
629 	{},
630 };
631 
632 /*******************************************************************************
633  * PPM context
634  ******************************************************************************/
635 static void
636 nvkm_perfctx_dtor(struct nvkm_object *object)
637 {
638 	struct nvkm_pm *pm = (void *)object->engine;
639 	struct nvkm_perfctx *ctx = (void *)object;
640 
641 	nvkm_gpuobj_destroy(&ctx->base);
642 	mutex_lock(&nv_subdev(pm)->mutex);
643 	if (pm->context == ctx)
644 		pm->context = NULL;
645 	mutex_unlock(&nv_subdev(pm)->mutex);
646 }
647 
648 static int
649 nvkm_perfctx_ctor(struct nvkm_object *parent, struct nvkm_object *engine,
650 		  struct nvkm_oclass *oclass, void *data, u32 size,
651 		  struct nvkm_object **pobject)
652 {
653 	struct nvkm_pm *pm = (void *)engine;
654 	struct nvkm_perfctx *ctx;
655 	int ret;
656 
657 	/* no context needed for perfdom objects... */
658 	if (nv_mclass(parent) != NV_DEVICE) {
659 		atomic_inc(&parent->refcount);
660 		*pobject = parent;
661 		return 1;
662 	}
663 
664 	ret = nvkm_gpuobj_create(parent, engine, oclass, 0, NULL, 0, 0, 0, &ctx);
665 	*pobject = nv_object(ctx);
666 	if (ret)
667 		return ret;
668 
669 	mutex_lock(&nv_subdev(pm)->mutex);
670 	if (pm->context == NULL)
671 		pm->context = ctx;
672 	if (ctx != pm->context)
673 		ret = -EBUSY;
674 	mutex_unlock(&nv_subdev(pm)->mutex);
675 
676 	return ret;
677 }
678 
679 struct nvkm_oclass
680 nvkm_pm_cclass = {
681 	.ofuncs = &(struct nvkm_ofuncs) {
682 		.ctor = nvkm_perfctx_ctor,
683 		.dtor = nvkm_perfctx_dtor,
684 		.init = _nvkm_gpuobj_init,
685 		.fini = _nvkm_gpuobj_fini,
686 	},
687 };
688 
689 /*******************************************************************************
690  * PPM engine/subdev functions
691  ******************************************************************************/
692 int
693 nvkm_perfsrc_new(struct nvkm_pm *pm, struct nvkm_perfsig *sig,
694 		 const struct nvkm_specsrc *spec)
695 {
696 	const struct nvkm_specsrc *ssrc;
697 	const struct nvkm_specmux *smux;
698 	struct nvkm_perfsrc *src;
699 	u8 source_nr = 0;
700 
701 	if (!spec) {
702 		/* No sources are defined for this signal. */
703 		return 0;
704 	}
705 
706 	ssrc = spec;
707 	while (ssrc->name) {
708 		smux = ssrc->mux;
709 		while (smux->name) {
710 			bool found = false;
711 			u8 source_id = 0;
712 			u32 len;
713 
714 			list_for_each_entry(src, &pm->sources, head) {
715 				if (src->addr == ssrc->addr &&
716 				    src->shift == smux->shift) {
717 					found = true;
718 					break;
719 				}
720 				source_id++;
721 			}
722 
723 			if (!found) {
724 				src = kzalloc(sizeof(*src), GFP_KERNEL);
725 				if (!src)
726 					return -ENOMEM;
727 
728 				src->addr   = ssrc->addr;
729 				src->mask   = smux->mask;
730 				src->shift  = smux->shift;
731 				src->enable = smux->enable;
732 
733 				len = strlen(ssrc->name) +
734 				      strlen(smux->name) + 2;
735 				src->name = kzalloc(len, GFP_KERNEL);
736 				if (!src->name) {
737 					kfree(src);
738 					return -ENOMEM;
739 				}
740 				snprintf(src->name, len, "%s_%s", ssrc->name,
741 					 smux->name);
742 
743 				list_add_tail(&src->head, &pm->sources);
744 			}
745 
746 			sig->source[source_nr++] = source_id + 1;
747 			smux++;
748 		}
749 		ssrc++;
750 	}
751 
752 	return 0;
753 }
754 
755 int
756 nvkm_perfdom_new(struct nvkm_pm *pm, const char *name, u32 mask,
757 		 u32 base, u32 size_unit, u32 size_domain,
758 		 const struct nvkm_specdom *spec)
759 {
760 	const struct nvkm_specdom *sdom;
761 	const struct nvkm_specsig *ssig;
762 	struct nvkm_perfdom *dom;
763 	int ret, i;
764 
765 	for (i = 0; i == 0 || mask; i++) {
766 		u32 addr = base + (i * size_unit);
767 		if (i && !(mask & (1 << i)))
768 			continue;
769 
770 		sdom = spec;
771 		while (sdom->signal_nr) {
772 			dom = kzalloc(sizeof(*dom) + sdom->signal_nr *
773 				      sizeof(*dom->signal), GFP_KERNEL);
774 			if (!dom)
775 				return -ENOMEM;
776 
777 			if (mask) {
778 				snprintf(dom->name, sizeof(dom->name),
779 					 "%s/%02x/%02x", name, i,
780 					 (int)(sdom - spec));
781 			} else {
782 				snprintf(dom->name, sizeof(dom->name),
783 					 "%s/%02x", name, (int)(sdom - spec));
784 			}
785 
786 			list_add_tail(&dom->head, &pm->domains);
787 			INIT_LIST_HEAD(&dom->list);
788 			dom->func = sdom->func;
789 			dom->addr = addr;
790 			dom->signal_nr = sdom->signal_nr;
791 
792 			ssig = (sdom++)->signal;
793 			while (ssig->name) {
794 				struct nvkm_perfsig *sig =
795 					&dom->signal[ssig->signal];
796 				sig->name = ssig->name;
797 				ret = nvkm_perfsrc_new(pm, sig, ssig->source);
798 				if (ret)
799 					return ret;
800 				ssig++;
801 			}
802 
803 			addr += size_domain;
804 		}
805 
806 		mask &= ~(1 << i);
807 	}
808 
809 	return 0;
810 }
811 
812 int
813 _nvkm_pm_fini(struct nvkm_object *object, bool suspend)
814 {
815 	struct nvkm_pm *pm = (void *)object;
816 	return nvkm_engine_fini_old(&pm->engine, suspend);
817 }
818 
819 int
820 _nvkm_pm_init(struct nvkm_object *object)
821 {
822 	struct nvkm_pm *pm = (void *)object;
823 	return nvkm_engine_init_old(&pm->engine);
824 }
825 
826 void
827 _nvkm_pm_dtor(struct nvkm_object *object)
828 {
829 	struct nvkm_pm *pm = (void *)object;
830 	struct nvkm_perfdom *dom, *next_dom;
831 	struct nvkm_perfsrc *src, *next_src;
832 
833 	list_for_each_entry_safe(dom, next_dom, &pm->domains, head) {
834 		list_del(&dom->head);
835 		kfree(dom);
836 	}
837 
838 	list_for_each_entry_safe(src, next_src, &pm->sources, head) {
839 		list_del(&src->head);
840 		kfree(src->name);
841 		kfree(src);
842 	}
843 
844 	nvkm_engine_destroy(&pm->engine);
845 }
846 
847 int
848 nvkm_pm_create_(struct nvkm_object *parent, struct nvkm_object *engine,
849 		struct nvkm_oclass *oclass, int length, void **pobject)
850 {
851 	struct nvkm_pm *pm;
852 	int ret;
853 
854 	ret = nvkm_engine_create_(parent, engine, oclass, true, "PPM",
855 				  "pm", length, pobject);
856 	pm = *pobject;
857 	if (ret)
858 		return ret;
859 
860 	INIT_LIST_HEAD(&pm->domains);
861 	INIT_LIST_HEAD(&pm->sources);
862 	return 0;
863 }
864