xref: /openbmc/linux/arch/parisc/kernel/perf.c (revision d863066e)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  *  Parisc performance counters
4  *  Copyright (C) 2001 Randolph Chung <tausq@debian.org>
5  *
6  *  This code is derived, with permission, from HP/UX sources.
7  */
8 
9 /*
10  *  Edited comment from original sources:
11  *
12  *  This driver programs the PCX-U/PCX-W performance counters
13  *  on the PA-RISC 2.0 chips.  The driver keeps all images now
14  *  internally to the kernel to hopefully eliminate the possibility
15  *  of a bad image halting the CPU.  Also, there are different
16  *  images for the PCX-W and later chips vs the PCX-U chips.
17  *
18  *  Only 1 process is allowed to access the driver at any time,
19  *  so the only protection that is needed is at open and close.
20  *  A variable "perf_enabled" is used to hold the state of the
21  *  driver.  The spinlock "perf_lock" is used to protect the
22  *  modification of the state during open/close operations so
23  *  multiple processes don't get into the driver simultaneously.
24  *
25  *  This driver accesses the processor directly vs going through
26  *  the PDC INTRIGUE calls.  This is done to eliminate bugs introduced
27  *  in various PDC revisions.  The code is much more maintainable
28  *  and reliable this way vs having to debug on every version of PDC
29  *  on every box.
30  */
31 
32 #include <linux/capability.h>
33 #include <linux/init.h>
34 #include <linux/proc_fs.h>
35 #include <linux/miscdevice.h>
36 #include <linux/spinlock.h>
37 
38 #include <linux/uaccess.h>
39 #include <asm/perf.h>
40 #include <asm/parisc-device.h>
41 #include <asm/processor.h>
42 #include <asm/runway.h>
43 #include <asm/io.h>		/* for __raw_read() */
44 
45 #include "perf_images.h"
46 
47 #define MAX_RDR_WORDS	24
48 #define PERF_VERSION	2	/* derived from hpux's PI v2 interface */
49 
50 /* definition of RDR regs */
51 struct rdr_tbl_ent {
52 	uint16_t	width;
53 	uint8_t		num_words;
54 	uint8_t		write_control;
55 };
56 
57 static int perf_processor_interface __read_mostly = UNKNOWN_INTF;
58 static int perf_enabled __read_mostly;
59 static DEFINE_SPINLOCK(perf_lock);
60 static struct parisc_device *cpu_device __read_mostly;
61 
62 /* RDRs to write for PCX-W */
63 static const int perf_rdrs_W[] =
64 	{ 0, 1, 4, 5, 6, 15, 16, 17, 18, 20, 21, 22, 23, 24, 25, -1 };
65 
66 /* RDRs to write for PCX-U */
67 static const int perf_rdrs_U[] =
68 	{ 0, 1, 4, 5, 6, 7, 16, 17, 18, 20, 21, 22, 23, 24, 25, -1 };
69 
70 /* RDR register descriptions for PCX-W */
71 static const struct rdr_tbl_ent perf_rdr_tbl_W[] = {
72 	{ 19,	1,	8 },   /* RDR 0 */
73 	{ 16,	1,	16 },  /* RDR 1 */
74 	{ 72,	2,	0 },   /* RDR 2 */
75 	{ 81,	2,	0 },   /* RDR 3 */
76 	{ 328,	6,	0 },   /* RDR 4 */
77 	{ 160,	3,	0 },   /* RDR 5 */
78 	{ 336,	6,	0 },   /* RDR 6 */
79 	{ 164,	3,	0 },   /* RDR 7 */
80 	{ 0,	0,	0 },   /* RDR 8 */
81 	{ 35,	1,	0 },   /* RDR 9 */
82 	{ 6,	1,	0 },   /* RDR 10 */
83 	{ 18,	1,	0 },   /* RDR 11 */
84 	{ 13,	1,	0 },   /* RDR 12 */
85 	{ 8,	1,	0 },   /* RDR 13 */
86 	{ 8,	1,	0 },   /* RDR 14 */
87 	{ 8,	1,	0 },   /* RDR 15 */
88 	{ 1530,	24,	0 },   /* RDR 16 */
89 	{ 16,	1,	0 },   /* RDR 17 */
90 	{ 4,	1,	0 },   /* RDR 18 */
91 	{ 0,	0,	0 },   /* RDR 19 */
92 	{ 152,	3,	24 },  /* RDR 20 */
93 	{ 152,	3,	24 },  /* RDR 21 */
94 	{ 233,	4,	48 },  /* RDR 22 */
95 	{ 233,	4,	48 },  /* RDR 23 */
96 	{ 71,	2,	0 },   /* RDR 24 */
97 	{ 71,	2,	0 },   /* RDR 25 */
98 	{ 11,	1,	0 },   /* RDR 26 */
99 	{ 18,	1,	0 },   /* RDR 27 */
100 	{ 128,	2,	0 },   /* RDR 28 */
101 	{ 0,	0,	0 },   /* RDR 29 */
102 	{ 16,	1,	0 },   /* RDR 30 */
103 	{ 16,	1,	0 },   /* RDR 31 */
104 };
105 
106 /* RDR register descriptions for PCX-U */
107 static const struct rdr_tbl_ent perf_rdr_tbl_U[] = {
108 	{ 19,	1,	8 },              /* RDR 0 */
109 	{ 32,	1,	16 },             /* RDR 1 */
110 	{ 20,	1,	0 },              /* RDR 2 */
111 	{ 0,	0,	0 },              /* RDR 3 */
112 	{ 344,	6,	0 },              /* RDR 4 */
113 	{ 176,	3,	0 },              /* RDR 5 */
114 	{ 336,	6,	0 },              /* RDR 6 */
115 	{ 0,	0,	0 },              /* RDR 7 */
116 	{ 0,	0,	0 },              /* RDR 8 */
117 	{ 0,	0,	0 },              /* RDR 9 */
118 	{ 28,	1,	0 },              /* RDR 10 */
119 	{ 33,	1,	0 },              /* RDR 11 */
120 	{ 0,	0,	0 },              /* RDR 12 */
121 	{ 230,	4,	0 },              /* RDR 13 */
122 	{ 32,	1,	0 },              /* RDR 14 */
123 	{ 128,	2,	0 },              /* RDR 15 */
124 	{ 1494,	24,	0 },              /* RDR 16 */
125 	{ 18,	1,	0 },              /* RDR 17 */
126 	{ 4,	1,	0 },              /* RDR 18 */
127 	{ 0,	0,	0 },              /* RDR 19 */
128 	{ 158,	3,	24 },             /* RDR 20 */
129 	{ 158,	3,	24 },             /* RDR 21 */
130 	{ 194,	4,	48 },             /* RDR 22 */
131 	{ 194,	4,	48 },             /* RDR 23 */
132 	{ 71,	2,	0 },              /* RDR 24 */
133 	{ 71,	2,	0 },              /* RDR 25 */
134 	{ 28,	1,	0 },              /* RDR 26 */
135 	{ 33,	1,	0 },              /* RDR 27 */
136 	{ 88,	2,	0 },              /* RDR 28 */
137 	{ 32,	1,	0 },              /* RDR 29 */
138 	{ 24,	1,	0 },              /* RDR 30 */
139 	{ 16,	1,	0 },              /* RDR 31 */
140 };
141 
142 /*
143  * A non-zero write_control in the above tables is a byte offset into
144  * this array.
145  */
146 static const uint64_t perf_bitmasks[] = {
147 	0x0000000000000000ul,     /* first dbl word must be zero */
148 	0xfdffe00000000000ul,     /* RDR0 bitmask */
149 	0x003f000000000000ul,     /* RDR1 bitmask */
150 	0x00fffffffffffffful,     /* RDR20-RDR21 bitmask (152 bits) */
151 	0xfffffffffffffffful,
152 	0xfffffffc00000000ul,
153 	0xfffffffffffffffful,     /* RDR22-RDR23 bitmask (233 bits) */
154 	0xfffffffffffffffful,
155 	0xfffffffffffffffcul,
156 	0xff00000000000000ul
157 };
158 
159 /*
160  * Write control bitmasks for Pa-8700 processor given
161  * some things have changed slightly.
162  */
163 static const uint64_t perf_bitmasks_piranha[] = {
164 	0x0000000000000000ul,     /* first dbl word must be zero */
165 	0xfdffe00000000000ul,     /* RDR0 bitmask */
166 	0x003f000000000000ul,     /* RDR1 bitmask */
167 	0x00fffffffffffffful,     /* RDR20-RDR21 bitmask (158 bits) */
168 	0xfffffffffffffffful,
169 	0xfffffffc00000000ul,
170 	0xfffffffffffffffful,     /* RDR22-RDR23 bitmask (210 bits) */
171 	0xfffffffffffffffful,
172 	0xfffffffffffffffful,
173 	0xfffc000000000000ul
174 };
175 
176 static const uint64_t *bitmask_array;   /* array of bitmasks to use */
177 
178 /******************************************************************************
179  * Function Prototypes
180  *****************************************************************************/
181 static int perf_config(uint32_t *image_ptr);
182 static int perf_release(struct inode *inode, struct file *file);
183 static int perf_open(struct inode *inode, struct file *file);
184 static ssize_t perf_read(struct file *file, char __user *buf, size_t cnt, loff_t *ppos);
185 static ssize_t perf_write(struct file *file, const char __user *buf,
186 	size_t count, loff_t *ppos);
187 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
188 static void perf_start_counters(void);
189 static int perf_stop_counters(uint32_t *raddr);
190 static const struct rdr_tbl_ent * perf_rdr_get_entry(uint32_t rdr_num);
191 static int perf_rdr_read_ubuf(uint32_t	rdr_num, uint64_t *buffer);
192 static int perf_rdr_clear(uint32_t rdr_num);
193 static int perf_write_image(uint64_t *memaddr);
194 static void perf_rdr_write(uint32_t rdr_num, uint64_t *buffer);
195 
196 /* External Assembly Routines */
197 extern uint64_t perf_rdr_shift_in_W (uint32_t rdr_num, uint16_t width);
198 extern uint64_t perf_rdr_shift_in_U (uint32_t rdr_num, uint16_t width);
199 extern void perf_rdr_shift_out_W (uint32_t rdr_num, uint64_t buffer);
200 extern void perf_rdr_shift_out_U (uint32_t rdr_num, uint64_t buffer);
201 extern void perf_intrigue_enable_perf_counters (void);
202 extern void perf_intrigue_disable_perf_counters (void);
203 
204 /******************************************************************************
205  * Function Definitions
206  *****************************************************************************/
207 
208 
209 /*
210  * configure:
211  *
212  * Configure the cpu with a given data image.  First turn off the counters,
213  * then download the image, then turn the counters back on.
214  */
perf_config(uint32_t * image_ptr)215 static int perf_config(uint32_t *image_ptr)
216 {
217 	long error;
218 	uint32_t raddr[4];
219 
220 	/* Stop the counters*/
221 	error = perf_stop_counters(raddr);
222 	if (error != 0) {
223 		printk("perf_config: perf_stop_counters = %ld\n", error);
224 		return -EINVAL;
225 	}
226 
227 printk("Preparing to write image\n");
228 	/* Write the image to the chip */
229 	error = perf_write_image((uint64_t *)image_ptr);
230 	if (error != 0) {
231 		printk("perf_config: DOWNLOAD = %ld\n", error);
232 		return -EINVAL;
233 	}
234 
235 printk("Preparing to start counters\n");
236 
237 	/* Start the counters */
238 	perf_start_counters();
239 
240 	return sizeof(uint32_t);
241 }
242 
243 /*
244  * Open the device and initialize all of its memory.  The device is only
245  * opened once, but can be "queried" by multiple processes that know its
246  * file descriptor.
247  */
perf_open(struct inode * inode,struct file * file)248 static int perf_open(struct inode *inode, struct file *file)
249 {
250 	spin_lock(&perf_lock);
251 	if (perf_enabled) {
252 		spin_unlock(&perf_lock);
253 		return -EBUSY;
254 	}
255 	perf_enabled = 1;
256  	spin_unlock(&perf_lock);
257 
258 	return 0;
259 }
260 
261 /*
262  * Close the device.
263  */
perf_release(struct inode * inode,struct file * file)264 static int perf_release(struct inode *inode, struct file *file)
265 {
266 	spin_lock(&perf_lock);
267 	perf_enabled = 0;
268 	spin_unlock(&perf_lock);
269 
270 	return 0;
271 }
272 
273 /*
274  * Read does nothing for this driver
275  */
perf_read(struct file * file,char __user * buf,size_t cnt,loff_t * ppos)276 static ssize_t perf_read(struct file *file, char __user *buf, size_t cnt, loff_t *ppos)
277 {
278 	return 0;
279 }
280 
281 /*
282  * write:
283  *
284  * This routine downloads the image to the chip.  It must be
285  * called on the processor that the download should happen
286  * on.
287  */
perf_write(struct file * file,const char __user * buf,size_t count,loff_t * ppos)288 static ssize_t perf_write(struct file *file, const char __user *buf,
289 	size_t count, loff_t *ppos)
290 {
291 	size_t image_size __maybe_unused;
292 	uint32_t image_type;
293 	uint32_t interface_type;
294 	uint32_t test;
295 
296 	if (perf_processor_interface == ONYX_INTF)
297 		image_size = PCXU_IMAGE_SIZE;
298 	else if (perf_processor_interface == CUDA_INTF)
299 		image_size = PCXW_IMAGE_SIZE;
300 	else
301 		return -EFAULT;
302 
303 	if (!perfmon_capable())
304 		return -EACCES;
305 
306 	if (count != sizeof(uint32_t))
307 		return -EIO;
308 
309 	if (copy_from_user(&image_type, buf, sizeof(uint32_t)))
310 		return -EFAULT;
311 
312 	/* Get the interface type and test type */
313    	interface_type = (image_type >> 16) & 0xffff;
314 	test           = (image_type & 0xffff);
315 
316 	/* Make sure everything makes sense */
317 
318 	/* First check the machine type is correct for
319 	   the requested image */
320 	if (((perf_processor_interface == CUDA_INTF) &&
321 			(interface_type != CUDA_INTF)) ||
322 		((perf_processor_interface == ONYX_INTF) &&
323 			(interface_type != ONYX_INTF)))
324 		return -EINVAL;
325 
326 	/* Next check to make sure the requested image
327 	   is valid */
328 	if (((interface_type == CUDA_INTF) &&
329 		       (test >= MAX_CUDA_IMAGES)) ||
330 	    ((interface_type == ONYX_INTF) &&
331 		       (test >= MAX_ONYX_IMAGES)))
332 		return -EINVAL;
333 
334 	/* Copy the image into the processor */
335 	if (interface_type == CUDA_INTF)
336 		return perf_config(cuda_images[test]);
337 	else
338 		return perf_config(onyx_images[test]);
339 
340 	return count;
341 }
342 
343 /*
344  * Patch the images that need to know the IVA addresses.
345  */
perf_patch_images(void)346 static void perf_patch_images(void)
347 {
348 #if 0 /* FIXME!! */
349 /*
350  * NOTE:  this routine is VERY specific to the current TLB image.
351  * If the image is changed, this routine might also need to be changed.
352  */
353 	extern void $i_itlb_miss_2_0();
354 	extern void $i_dtlb_miss_2_0();
355 	extern void PA2_0_iva();
356 
357 	/*
358 	 * We can only use the lower 32-bits, the upper 32-bits should be 0
359 	 * anyway given this is in the kernel
360 	 */
361 	uint32_t itlb_addr  = (uint32_t)&($i_itlb_miss_2_0);
362 	uint32_t dtlb_addr  = (uint32_t)&($i_dtlb_miss_2_0);
363 	uint32_t IVAaddress = (uint32_t)&PA2_0_iva;
364 
365 	if (perf_processor_interface == ONYX_INTF) {
366 		/* clear last 2 bytes */
367 		onyx_images[TLBMISS][15] &= 0xffffff00;
368 		/* set 2 bytes */
369 		onyx_images[TLBMISS][15] |= (0x000000ff&((dtlb_addr) >> 24));
370 		onyx_images[TLBMISS][16] = (dtlb_addr << 8)&0xffffff00;
371 		onyx_images[TLBMISS][17] = itlb_addr;
372 
373 		/* clear last 2 bytes */
374 		onyx_images[TLBHANDMISS][15] &= 0xffffff00;
375 		/* set 2 bytes */
376 		onyx_images[TLBHANDMISS][15] |= (0x000000ff&((dtlb_addr) >> 24));
377 		onyx_images[TLBHANDMISS][16] = (dtlb_addr << 8)&0xffffff00;
378 		onyx_images[TLBHANDMISS][17] = itlb_addr;
379 
380 		/* clear last 2 bytes */
381 		onyx_images[BIG_CPI][15] &= 0xffffff00;
382 		/* set 2 bytes */
383 		onyx_images[BIG_CPI][15] |= (0x000000ff&((dtlb_addr) >> 24));
384 		onyx_images[BIG_CPI][16] = (dtlb_addr << 8)&0xffffff00;
385 		onyx_images[BIG_CPI][17] = itlb_addr;
386 
387 	    onyx_images[PANIC][15] &= 0xffffff00;  /* clear last 2 bytes */
388 	 	onyx_images[PANIC][15] |= (0x000000ff&((IVAaddress) >> 24)); /* set 2 bytes */
389 		onyx_images[PANIC][16] = (IVAaddress << 8)&0xffffff00;
390 
391 
392 	} else if (perf_processor_interface == CUDA_INTF) {
393 		/* Cuda interface */
394 		cuda_images[TLBMISS][16] =
395 			(cuda_images[TLBMISS][16]&0xffff0000) |
396 			((dtlb_addr >> 8)&0x0000ffff);
397 		cuda_images[TLBMISS][17] =
398 			((dtlb_addr << 24)&0xff000000) | ((itlb_addr >> 16)&0x000000ff);
399 		cuda_images[TLBMISS][18] = (itlb_addr << 16)&0xffff0000;
400 
401 		cuda_images[TLBHANDMISS][16] =
402 			(cuda_images[TLBHANDMISS][16]&0xffff0000) |
403 			((dtlb_addr >> 8)&0x0000ffff);
404 		cuda_images[TLBHANDMISS][17] =
405 			((dtlb_addr << 24)&0xff000000) | ((itlb_addr >> 16)&0x000000ff);
406 		cuda_images[TLBHANDMISS][18] = (itlb_addr << 16)&0xffff0000;
407 
408 		cuda_images[BIG_CPI][16] =
409 			(cuda_images[BIG_CPI][16]&0xffff0000) |
410 			((dtlb_addr >> 8)&0x0000ffff);
411 		cuda_images[BIG_CPI][17] =
412 			((dtlb_addr << 24)&0xff000000) | ((itlb_addr >> 16)&0x000000ff);
413 		cuda_images[BIG_CPI][18] = (itlb_addr << 16)&0xffff0000;
414 	} else {
415 		/* Unknown type */
416 	}
417 #endif
418 }
419 
420 
421 /*
422  * ioctl routine
423  * All routines effect the processor that they are executed on.  Thus you
424  * must be running on the processor that you wish to change.
425  */
426 
perf_ioctl(struct file * file,unsigned int cmd,unsigned long arg)427 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
428 {
429 	long error_start;
430 	uint32_t raddr[4];
431 	int error = 0;
432 
433 	switch (cmd) {
434 
435 	    case PA_PERF_ON:
436 			/* Start the counters */
437 			perf_start_counters();
438 			break;
439 
440 	    case PA_PERF_OFF:
441 			error_start = perf_stop_counters(raddr);
442 			if (error_start != 0) {
443 				printk(KERN_ERR "perf_off: perf_stop_counters = %ld\n", error_start);
444 				error = -EFAULT;
445 				break;
446 			}
447 
448 			/* copy out the Counters */
449 			if (copy_to_user((void __user *)arg, raddr,
450 					sizeof (raddr)) != 0) {
451 				error =  -EFAULT;
452 				break;
453 			}
454 			break;
455 
456 	    case PA_PERF_VERSION:
457   	  		/* Return the version # */
458 			error = put_user(PERF_VERSION, (int *)arg);
459 			break;
460 
461 	    default:
462   	 		error = -ENOTTY;
463 	}
464 
465 	return error;
466 }
467 
468 static const struct file_operations perf_fops = {
469 	.llseek = no_llseek,
470 	.read = perf_read,
471 	.write = perf_write,
472 	.unlocked_ioctl = perf_ioctl,
473 	.compat_ioctl = perf_ioctl,
474 	.open = perf_open,
475 	.release = perf_release
476 };
477 
478 static struct miscdevice perf_dev = {
479 	MISC_DYNAMIC_MINOR,
480 	PA_PERF_DEV,
481 	&perf_fops
482 };
483 
484 /*
485  * Initialize the module
486  */
perf_init(void)487 static int __init perf_init(void)
488 {
489 	int ret;
490 
491 	/* Determine correct processor interface to use */
492 	bitmask_array = perf_bitmasks;
493 
494 	if (boot_cpu_data.cpu_type == pcxu ||
495 	    boot_cpu_data.cpu_type == pcxu_) {
496 		perf_processor_interface = ONYX_INTF;
497 	} else if (boot_cpu_data.cpu_type == pcxw ||
498 		 boot_cpu_data.cpu_type == pcxw_ ||
499 		 boot_cpu_data.cpu_type == pcxw2 ||
500 		 boot_cpu_data.cpu_type == mako ||
501 		 boot_cpu_data.cpu_type == mako2) {
502 		perf_processor_interface = CUDA_INTF;
503 		if (boot_cpu_data.cpu_type == pcxw2 ||
504 		    boot_cpu_data.cpu_type == mako ||
505 		    boot_cpu_data.cpu_type == mako2)
506 			bitmask_array = perf_bitmasks_piranha;
507 	} else {
508 		perf_processor_interface = UNKNOWN_INTF;
509 		printk("Performance monitoring counters not supported on this processor\n");
510 		return -ENODEV;
511 	}
512 
513 	ret = misc_register(&perf_dev);
514 	if (ret) {
515 		printk(KERN_ERR "Performance monitoring counters: "
516 			"cannot register misc device.\n");
517 		return ret;
518 	}
519 
520 	/* Patch the images to match the system */
521     	perf_patch_images();
522 
523 	/* TODO: this only lets us access the first cpu.. what to do for SMP? */
524 	cpu_device = per_cpu(cpu_data, 0).dev;
525 	printk("Performance monitoring counters enabled for %s\n",
526 		per_cpu(cpu_data, 0).dev->name);
527 
528 	return 0;
529 }
530 device_initcall(perf_init);
531 
532 /*
533  * perf_start_counters(void)
534  *
535  * Start the counters.
536  */
perf_start_counters(void)537 static void perf_start_counters(void)
538 {
539 	/* Enable performance monitor counters */
540 	perf_intrigue_enable_perf_counters();
541 }
542 
543 /*
544  * perf_stop_counters
545  *
546  * Stop the performance counters and save counts
547  * in a per_processor array.
548  */
perf_stop_counters(uint32_t * raddr)549 static int perf_stop_counters(uint32_t *raddr)
550 {
551 	uint64_t userbuf[MAX_RDR_WORDS];
552 
553 	/* Disable performance counters */
554 	perf_intrigue_disable_perf_counters();
555 
556 	if (perf_processor_interface == ONYX_INTF) {
557 		uint64_t tmp64;
558 		/*
559 		 * Read the counters
560 		 */
561 		if (!perf_rdr_read_ubuf(16, userbuf))
562 			return -13;
563 
564 		/* Counter0 is bits 1398 to 1429 */
565 		tmp64 =  (userbuf[21] << 22) & 0x00000000ffc00000;
566 		tmp64 |= (userbuf[22] >> 42) & 0x00000000003fffff;
567 		/* OR sticky0 (bit 1430) to counter0 bit 32 */
568 		tmp64 |= (userbuf[22] >> 10) & 0x0000000080000000;
569 		raddr[0] = (uint32_t)tmp64;
570 
571 		/* Counter1 is bits 1431 to 1462 */
572 		tmp64 =  (userbuf[22] >> 9) & 0x00000000ffffffff;
573 		/* OR sticky1 (bit 1463) to counter1 bit 32 */
574 		tmp64 |= (userbuf[22] << 23) & 0x0000000080000000;
575 		raddr[1] = (uint32_t)tmp64;
576 
577 		/* Counter2 is bits 1464 to 1495 */
578 		tmp64 =  (userbuf[22] << 24) & 0x00000000ff000000;
579 		tmp64 |= (userbuf[23] >> 40) & 0x0000000000ffffff;
580 		/* OR sticky2 (bit 1496) to counter2 bit 32 */
581 		tmp64 |= (userbuf[23] >> 8) & 0x0000000080000000;
582 		raddr[2] = (uint32_t)tmp64;
583 
584 		/* Counter3 is bits 1497 to 1528 */
585 		tmp64 =  (userbuf[23] >> 7) & 0x00000000ffffffff;
586 		/* OR sticky3 (bit 1529) to counter3 bit 32 */
587 		tmp64 |= (userbuf[23] << 25) & 0x0000000080000000;
588 		raddr[3] = (uint32_t)tmp64;
589 
590 		/*
591 		 * Zero out the counters
592 		 */
593 
594 		/*
595 		 * The counters and sticky-bits comprise the last 132 bits
596 		 * (1398 - 1529) of RDR16 on a U chip.  We'll zero these
597 		 * out the easy way: zero out last 10 bits of dword 21,
598 		 * all of dword 22 and 58 bits (plus 6 don't care bits) of
599 		 * dword 23.
600 		 */
601 		userbuf[21] &= 0xfffffffffffffc00ul;	/* 0 to last 10 bits */
602 		userbuf[22] = 0;
603 		userbuf[23] = 0;
604 
605 		/*
606 		 * Write back the zeroed bytes + the image given
607 		 * the read was destructive.
608 		 */
609 		perf_rdr_write(16, userbuf);
610 	} else {
611 
612 		/*
613 		 * Read RDR-15 which contains the counters and sticky bits
614 		 */
615 		if (!perf_rdr_read_ubuf(15, userbuf)) {
616 			return -13;
617 		}
618 
619 		/*
620 		 * Clear out the counters
621 		 */
622 		perf_rdr_clear(15);
623 
624 		/*
625 		 * Copy the counters
626 		 */
627 		raddr[0] = (uint32_t)((userbuf[0] >> 32) & 0x00000000ffffffffUL);
628 		raddr[1] = (uint32_t)(userbuf[0] & 0x00000000ffffffffUL);
629 		raddr[2] = (uint32_t)((userbuf[1] >> 32) & 0x00000000ffffffffUL);
630 		raddr[3] = (uint32_t)(userbuf[1] & 0x00000000ffffffffUL);
631 	}
632 
633 	return 0;
634 }
635 
636 /*
637  * perf_rdr_get_entry
638  *
639  * Retrieve a pointer to the description of what this
640  * RDR contains.
641  */
perf_rdr_get_entry(uint32_t rdr_num)642 static const struct rdr_tbl_ent * perf_rdr_get_entry(uint32_t rdr_num)
643 {
644 	if (perf_processor_interface == ONYX_INTF) {
645 		return &perf_rdr_tbl_U[rdr_num];
646 	} else {
647 		return &perf_rdr_tbl_W[rdr_num];
648 	}
649 }
650 
651 /*
652  * perf_rdr_read_ubuf
653  *
654  * Read the RDR value into the buffer specified.
655  */
perf_rdr_read_ubuf(uint32_t rdr_num,uint64_t * buffer)656 static int perf_rdr_read_ubuf(uint32_t	rdr_num, uint64_t *buffer)
657 {
658 	uint64_t	data, data_mask = 0;
659 	uint32_t	width, xbits, i;
660 	const struct rdr_tbl_ent *tentry;
661 
662 	tentry = perf_rdr_get_entry(rdr_num);
663 	if ((width = tentry->width) == 0)
664 		return 0;
665 
666 	/* Clear out buffer */
667 	i = tentry->num_words;
668 	while (i--) {
669 		buffer[i] = 0;
670 	}
671 
672 	/* Check for bits an even number of 64 */
673 	if ((xbits = width & 0x03f) != 0) {
674 		data_mask = 1;
675 		data_mask <<= (64 - xbits);
676 		data_mask--;
677 	}
678 
679 	/* Grab all of the data */
680 	i = tentry->num_words;
681 	while (i--) {
682 
683 		if (perf_processor_interface == ONYX_INTF) {
684 			data = perf_rdr_shift_in_U(rdr_num, width);
685 		} else {
686 			data = perf_rdr_shift_in_W(rdr_num, width);
687 		}
688 		if (xbits) {
689 			buffer[i] |= (data << (64 - xbits));
690 			if (i) {
691 				buffer[i-1] |= ((data >> xbits) & data_mask);
692 			}
693 		} else {
694 			buffer[i] = data;
695 		}
696 	}
697 
698 	return 1;
699 }
700 
701 /*
702  * perf_rdr_clear
703  *
704  * Zero out the given RDR register
705  */
perf_rdr_clear(uint32_t rdr_num)706 static int perf_rdr_clear(uint32_t	rdr_num)
707 {
708 	const struct rdr_tbl_ent *tentry;
709 	int32_t		i;
710 
711 	tentry = perf_rdr_get_entry(rdr_num);
712 
713 	if (tentry->width == 0) {
714 		return -1;
715 	}
716 
717 	i = tentry->num_words;
718 	while (i--) {
719 		if (perf_processor_interface == ONYX_INTF) {
720 			perf_rdr_shift_out_U(rdr_num, 0UL);
721 		} else {
722 			perf_rdr_shift_out_W(rdr_num, 0UL);
723 		}
724 	}
725 
726 	return 0;
727 }
728 
729 
730 /*
731  * perf_write_image
732  *
733  * Write the given image out to the processor
734  */
perf_write_image(uint64_t * memaddr)735 static int perf_write_image(uint64_t *memaddr)
736 {
737 	uint64_t buffer[MAX_RDR_WORDS];
738 	uint64_t *bptr;
739 	uint32_t dwords;
740 	const uint32_t *intrigue_rdr;
741 	const uint64_t *intrigue_bitmask;
742 	uint64_t tmp64;
743 	void __iomem *runway;
744 	const struct rdr_tbl_ent *tentry;
745 	int i;
746 
747 	/* Clear out counters */
748 	if (perf_processor_interface == ONYX_INTF) {
749 
750 		perf_rdr_clear(16);
751 
752 		/* Toggle performance monitor */
753 		perf_intrigue_enable_perf_counters();
754 		perf_intrigue_disable_perf_counters();
755 
756 		intrigue_rdr = perf_rdrs_U;
757 	} else {
758 		perf_rdr_clear(15);
759 		intrigue_rdr = perf_rdrs_W;
760 	}
761 
762 	/* Write all RDRs */
763 	while (*intrigue_rdr != -1) {
764 		tentry = perf_rdr_get_entry(*intrigue_rdr);
765 		perf_rdr_read_ubuf(*intrigue_rdr, buffer);
766 		bptr   = &buffer[0];
767 		dwords = tentry->num_words;
768 		if (tentry->write_control) {
769 			intrigue_bitmask = &bitmask_array[tentry->write_control >> 3];
770 			while (dwords--) {
771 				tmp64 = *intrigue_bitmask & *memaddr++;
772 				tmp64 |= (~(*intrigue_bitmask++)) & *bptr;
773 				*bptr++ = tmp64;
774 			}
775 		} else {
776 			while (dwords--) {
777 				*bptr++ = *memaddr++;
778 			}
779 		}
780 
781 		perf_rdr_write(*intrigue_rdr, buffer);
782 		intrigue_rdr++;
783 	}
784 
785 	/*
786 	 * Now copy out the Runway stuff which is not in RDRs
787 	 */
788 
789 	if (cpu_device == NULL)
790 	{
791 		printk(KERN_ERR "write_image: cpu_device not yet initialized!\n");
792 		return -1;
793 	}
794 
795 	runway = ioremap(cpu_device->hpa.start, 4096);
796 	if (!runway) {
797 		pr_err("perf_write_image: ioremap failed!\n");
798 		return -ENOMEM;
799 	}
800 
801 	/* Merge intrigue bits into Runway STATUS 0 */
802 	tmp64 = __raw_readq(runway + RUNWAY_STATUS) & 0xffecfffffffffffful;
803 	__raw_writeq(tmp64 | (*memaddr++ & 0x0013000000000000ul),
804 		     runway + RUNWAY_STATUS);
805 
806 	/* Write RUNWAY DEBUG registers */
807 	for (i = 0; i < 8; i++) {
808 		__raw_writeq(*memaddr++, runway + RUNWAY_DEBUG);
809 	}
810 
811 	return 0;
812 }
813 
814 /*
815  * perf_rdr_write
816  *
817  * Write the given RDR register with the contents
818  * of the given buffer.
819  */
perf_rdr_write(uint32_t rdr_num,uint64_t * buffer)820 static void perf_rdr_write(uint32_t rdr_num, uint64_t *buffer)
821 {
822 	const struct rdr_tbl_ent *tentry;
823 	int32_t		i;
824 
825 printk("perf_rdr_write\n");
826 	tentry = perf_rdr_get_entry(rdr_num);
827 	if (tentry->width == 0) { return; }
828 
829 	i = tentry->num_words;
830 	while (i--) {
831 		if (perf_processor_interface == ONYX_INTF) {
832 			perf_rdr_shift_out_U(rdr_num, buffer[i]);
833 		} else {
834 			perf_rdr_shift_out_W(rdr_num, buffer[i]);
835 		}
836 	}
837 printk("perf_rdr_write done\n");
838 }
839