1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Performance event support - Processor Activity Instrumentation Facility
4  *
5  *  Copyright IBM Corp. 2022
6  *  Author(s): Thomas Richter <tmricht@linux.ibm.com>
7  */
8 #define KMSG_COMPONENT	"pai_crypto"
9 #define pr_fmt(fmt)	KMSG_COMPONENT ": " fmt
10 
11 #include <linux/kernel.h>
12 #include <linux/kernel_stat.h>
13 #include <linux/percpu.h>
14 #include <linux/notifier.h>
15 #include <linux/init.h>
16 #include <linux/export.h>
17 #include <linux/io.h>
18 #include <linux/perf_event.h>
19 
20 #include <asm/ctl_reg.h>
21 #include <asm/pai.h>
22 #include <asm/debug.h>
23 
24 static debug_info_t *cfm_dbg;
25 static unsigned int paicrypt_cnt;	/* Size of the mapped counter sets */
26 					/* extracted with QPACI instruction */
27 
28 DEFINE_STATIC_KEY_FALSE(pai_key);
29 
30 struct pai_userdata {
31 	u16 num;
32 	u64 value;
33 } __packed;
34 
35 struct paicrypt_map {
36 	unsigned long *page;		/* Page for CPU to store counters */
37 	struct pai_userdata *save;	/* Page to store no-zero counters */
38 	unsigned int active_events;	/* # of PAI crypto users */
39 	refcount_t refcnt;		/* Reference count mapped buffers */
40 	enum paievt_mode mode;		/* Type of event */
41 	struct perf_event *event;	/* Perf event for sampling */
42 };
43 
44 static DEFINE_PER_CPU(struct paicrypt_map, paicrypt_map);
45 
46 /* Release the PMU if event is the last perf event */
47 static DEFINE_MUTEX(pai_reserve_mutex);
48 
49 /* Adjust usage counters and remove allocated memory when all users are
50  * gone.
51  */
52 static void paicrypt_event_destroy(struct perf_event *event)
53 {
54 	struct paicrypt_map *cpump = per_cpu_ptr(&paicrypt_map, event->cpu);
55 
56 	cpump->event = NULL;
57 	static_branch_dec(&pai_key);
58 	mutex_lock(&pai_reserve_mutex);
59 	debug_sprintf_event(cfm_dbg, 5, "%s event %#llx cpu %d users %d"
60 			    " mode %d refcnt %u\n", __func__,
61 			    event->attr.config, event->cpu,
62 			    cpump->active_events, cpump->mode,
63 			    refcount_read(&cpump->refcnt));
64 	if (refcount_dec_and_test(&cpump->refcnt)) {
65 		debug_sprintf_event(cfm_dbg, 4, "%s page %#lx save %p\n",
66 				    __func__, (unsigned long)cpump->page,
67 				    cpump->save);
68 		free_page((unsigned long)cpump->page);
69 		cpump->page = NULL;
70 		kvfree(cpump->save);
71 		cpump->save = NULL;
72 		cpump->mode = PAI_MODE_NONE;
73 	}
74 	mutex_unlock(&pai_reserve_mutex);
75 }
76 
77 static u64 paicrypt_getctr(struct paicrypt_map *cpump, int nr, bool kernel)
78 {
79 	if (kernel)
80 		nr += PAI_CRYPTO_MAXCTR;
81 	return cpump->page[nr];
82 }
83 
84 /* Read the counter values. Return value from location in CMP. For event
85  * CRYPTO_ALL sum up all events.
86  */
87 static u64 paicrypt_getdata(struct perf_event *event, bool kernel)
88 {
89 	struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map);
90 	u64 sum = 0;
91 	int i;
92 
93 	if (event->attr.config != PAI_CRYPTO_BASE) {
94 		return paicrypt_getctr(cpump,
95 				       event->attr.config - PAI_CRYPTO_BASE,
96 				       kernel);
97 	}
98 
99 	for (i = 1; i <= paicrypt_cnt; i++) {
100 		u64 val = paicrypt_getctr(cpump, i, kernel);
101 
102 		if (!val)
103 			continue;
104 		sum += val;
105 	}
106 	return sum;
107 }
108 
109 static u64 paicrypt_getall(struct perf_event *event)
110 {
111 	u64 sum = 0;
112 
113 	if (!event->attr.exclude_kernel)
114 		sum += paicrypt_getdata(event, true);
115 	if (!event->attr.exclude_user)
116 		sum += paicrypt_getdata(event, false);
117 
118 	return sum;
119 }
120 
121 /* Used to avoid races in checking concurrent access of counting and
122  * sampling for crypto events
123  *
124  * Only one instance of event pai_crypto/CRYPTO_ALL/ for sampling is
125  * allowed and when this event is running, no counting event is allowed.
126  * Several counting events are allowed in parallel, but no sampling event
127  * is allowed while one (or more) counting events are running.
128  *
129  * This function is called in process context and it is save to block.
130  * When the event initialization functions fails, no other call back will
131  * be invoked.
132  *
133  * Allocate the memory for the event.
134  */
135 static int paicrypt_busy(struct perf_event_attr *a, struct paicrypt_map *cpump)
136 {
137 	int rc = 0;
138 
139 	mutex_lock(&pai_reserve_mutex);
140 	if (a->sample_period) {		/* Sampling requested */
141 		if (cpump->mode != PAI_MODE_NONE)
142 			rc = -EBUSY;	/* ... sampling/counting active */
143 	} else {			/* Counting requested */
144 		if (cpump->mode == PAI_MODE_SAMPLING)
145 			rc = -EBUSY;	/* ... and sampling active */
146 	}
147 	if (rc)
148 		goto unlock;
149 
150 	/* Allocate memory for counter page and counter extraction.
151 	 * Only the first counting event has to allocate a page.
152 	 */
153 	if (cpump->page) {
154 		refcount_inc(&cpump->refcnt);
155 		goto unlock;
156 	}
157 
158 	rc = -ENOMEM;
159 	cpump->page = (unsigned long *)get_zeroed_page(GFP_KERNEL);
160 	if (!cpump->page)
161 		goto unlock;
162 	cpump->save = kvmalloc_array(paicrypt_cnt + 1,
163 				     sizeof(struct pai_userdata), GFP_KERNEL);
164 	if (!cpump->save) {
165 		free_page((unsigned long)cpump->page);
166 		cpump->page = NULL;
167 		goto unlock;
168 	}
169 	rc = 0;
170 	refcount_set(&cpump->refcnt, 1);
171 
172 unlock:
173 	/* If rc is non-zero, do not set mode and reference count */
174 	if (!rc) {
175 		cpump->mode = a->sample_period ? PAI_MODE_SAMPLING
176 					       : PAI_MODE_COUNTING;
177 	}
178 	debug_sprintf_event(cfm_dbg, 5, "%s sample_period %#llx users %d"
179 			    " mode %d refcnt %u page %#lx save %p rc %d\n",
180 			    __func__, a->sample_period, cpump->active_events,
181 			    cpump->mode, refcount_read(&cpump->refcnt),
182 			    (unsigned long)cpump->page, cpump->save, rc);
183 	mutex_unlock(&pai_reserve_mutex);
184 	return rc;
185 }
186 
187 /* Might be called on different CPU than the one the event is intended for. */
188 static int paicrypt_event_init(struct perf_event *event)
189 {
190 	struct perf_event_attr *a = &event->attr;
191 	struct paicrypt_map *cpump;
192 	int rc;
193 
194 	/* PAI crypto PMU registered as PERF_TYPE_RAW, check event type */
195 	if (a->type != PERF_TYPE_RAW && event->pmu->type != a->type)
196 		return -ENOENT;
197 	/* PAI crypto event must be in valid range */
198 	if (a->config < PAI_CRYPTO_BASE ||
199 	    a->config > PAI_CRYPTO_BASE + paicrypt_cnt)
200 		return -EINVAL;
201 	/* Allow only CPU wide operation, no process context for now. */
202 	if (event->hw.target || event->cpu == -1)
203 		return -ENOENT;
204 	/* Allow only CRYPTO_ALL for sampling. */
205 	if (a->sample_period && a->config != PAI_CRYPTO_BASE)
206 		return -EINVAL;
207 
208 	cpump = per_cpu_ptr(&paicrypt_map, event->cpu);
209 	rc = paicrypt_busy(a, cpump);
210 	if (rc)
211 		return rc;
212 
213 	/* Event initialization sets last_tag to 0. When later on the events
214 	 * are deleted and re-added, do not reset the event count value to zero.
215 	 * Events are added, deleted and re-added when 2 or more events
216 	 * are active at the same time.
217 	 */
218 	event->hw.last_tag = 0;
219 	cpump->event = event;
220 	event->destroy = paicrypt_event_destroy;
221 
222 	if (a->sample_period) {
223 		a->sample_period = 1;
224 		a->freq = 0;
225 		/* Register for paicrypt_sched_task() to be called */
226 		event->attach_state |= PERF_ATTACH_SCHED_CB;
227 		/* Add raw data which contain the memory mapped counters */
228 		a->sample_type |= PERF_SAMPLE_RAW;
229 		/* Turn off inheritance */
230 		a->inherit = 0;
231 	}
232 
233 	static_branch_inc(&pai_key);
234 	return 0;
235 }
236 
237 static void paicrypt_read(struct perf_event *event)
238 {
239 	u64 prev, new, delta;
240 
241 	prev = local64_read(&event->hw.prev_count);
242 	new = paicrypt_getall(event);
243 	local64_set(&event->hw.prev_count, new);
244 	delta = (prev <= new) ? new - prev
245 			      : (-1ULL - prev) + new + 1;	 /* overflow */
246 	local64_add(delta, &event->count);
247 }
248 
249 static void paicrypt_start(struct perf_event *event, int flags)
250 {
251 	u64 sum;
252 
253 	if (!event->hw.last_tag) {
254 		event->hw.last_tag = 1;
255 		sum = paicrypt_getall(event);		/* Get current value */
256 		local64_set(&event->count, 0);
257 		local64_set(&event->hw.prev_count, sum);
258 	}
259 }
260 
261 static int paicrypt_add(struct perf_event *event, int flags)
262 {
263 	struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map);
264 	unsigned long ccd;
265 
266 	if (++cpump->active_events == 1) {
267 		ccd = virt_to_phys(cpump->page) | PAI_CRYPTO_KERNEL_OFFSET;
268 		WRITE_ONCE(S390_lowcore.ccd, ccd);
269 		__ctl_set_bit(0, 50);
270 	}
271 	cpump->event = event;
272 	if (flags & PERF_EF_START && !event->attr.sample_period) {
273 		/* Only counting needs initial counter value */
274 		paicrypt_start(event, PERF_EF_RELOAD);
275 	}
276 	event->hw.state = 0;
277 	if (event->attr.sample_period)
278 		perf_sched_cb_inc(event->pmu);
279 	return 0;
280 }
281 
282 static void paicrypt_stop(struct perf_event *event, int flags)
283 {
284 	paicrypt_read(event);
285 	event->hw.state = PERF_HES_STOPPED;
286 }
287 
288 static void paicrypt_del(struct perf_event *event, int flags)
289 {
290 	struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map);
291 
292 	if (event->attr.sample_period)
293 		perf_sched_cb_dec(event->pmu);
294 	if (!event->attr.sample_period)
295 		/* Only counting needs to read counter */
296 		paicrypt_stop(event, PERF_EF_UPDATE);
297 	if (--cpump->active_events == 0) {
298 		__ctl_clear_bit(0, 50);
299 		WRITE_ONCE(S390_lowcore.ccd, 0);
300 	}
301 }
302 
303 /* Create raw data and save it in buffer. Returns number of bytes copied.
304  * Saves only positive counter entries of the form
305  * 2 bytes: Number of counter
306  * 8 bytes: Value of counter
307  */
308 static size_t paicrypt_copy(struct pai_userdata *userdata,
309 			    struct paicrypt_map *cpump,
310 			    bool exclude_user, bool exclude_kernel)
311 {
312 	int i, outidx = 0;
313 
314 	for (i = 1; i <= paicrypt_cnt; i++) {
315 		u64 val = 0;
316 
317 		if (!exclude_kernel)
318 			val += paicrypt_getctr(cpump, i, true);
319 		if (!exclude_user)
320 			val += paicrypt_getctr(cpump, i, false);
321 		if (val) {
322 			userdata[outidx].num = i;
323 			userdata[outidx].value = val;
324 			outidx++;
325 		}
326 	}
327 	return outidx * sizeof(struct pai_userdata);
328 }
329 
330 static int paicrypt_push_sample(void)
331 {
332 	struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map);
333 	struct perf_event *event = cpump->event;
334 	struct perf_sample_data data;
335 	struct perf_raw_record raw;
336 	struct pt_regs regs;
337 	size_t rawsize;
338 	int overflow;
339 
340 	if (!cpump->event)		/* No event active */
341 		return 0;
342 	rawsize = paicrypt_copy(cpump->save, cpump,
343 				cpump->event->attr.exclude_user,
344 				cpump->event->attr.exclude_kernel);
345 	if (!rawsize)			/* No incremented counters */
346 		return 0;
347 
348 	/* Setup perf sample */
349 	memset(&regs, 0, sizeof(regs));
350 	memset(&raw, 0, sizeof(raw));
351 	memset(&data, 0, sizeof(data));
352 	perf_sample_data_init(&data, 0, event->hw.last_period);
353 	if (event->attr.sample_type & PERF_SAMPLE_TID) {
354 		data.tid_entry.pid = task_tgid_nr(current);
355 		data.tid_entry.tid = task_pid_nr(current);
356 	}
357 	if (event->attr.sample_type & PERF_SAMPLE_TIME)
358 		data.time = event->clock();
359 	if (event->attr.sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER))
360 		data.id = event->id;
361 	if (event->attr.sample_type & PERF_SAMPLE_CPU) {
362 		data.cpu_entry.cpu = smp_processor_id();
363 		data.cpu_entry.reserved = 0;
364 	}
365 	if (event->attr.sample_type & PERF_SAMPLE_RAW) {
366 		raw.frag.size = rawsize;
367 		raw.frag.data = cpump->save;
368 		perf_sample_save_raw_data(&data, &raw);
369 	}
370 
371 	overflow = perf_event_overflow(event, &data, &regs);
372 	perf_event_update_userpage(event);
373 	/* Clear lowcore page after read */
374 	memset(cpump->page, 0, PAGE_SIZE);
375 	return overflow;
376 }
377 
378 /* Called on schedule-in and schedule-out. No access to event structure,
379  * but for sampling only event CRYPTO_ALL is allowed.
380  */
381 static void paicrypt_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in)
382 {
383 	/* We started with a clean page on event installation. So read out
384 	 * results on schedule_out and if page was dirty, clear values.
385 	 */
386 	if (!sched_in)
387 		paicrypt_push_sample();
388 }
389 
390 /* Attribute definitions for paicrypt interface. As with other CPU
391  * Measurement Facilities, there is one attribute per mapped counter.
392  * The number of mapped counters may vary per machine generation. Use
393  * the QUERY PROCESSOR ACTIVITY COUNTER INFORMATION (QPACI) instruction
394  * to determine the number of mapped counters. The instructions returns
395  * a positive number, which is the highest number of supported counters.
396  * All counters less than this number are also supported, there are no
397  * holes. A returned number of zero means no support for mapped counters.
398  *
399  * The identification of the counter is a unique number. The chosen range
400  * is 0x1000 + offset in mapped kernel page.
401  * All CPU Measurement Facility counters identifiers must be unique and
402  * the numbers from 0 to 496 are already used for the CPU Measurement
403  * Counter facility. Numbers 0xb0000, 0xbc000 and 0xbd000 are already
404  * used for the CPU Measurement Sampling facility.
405  */
406 PMU_FORMAT_ATTR(event, "config:0-63");
407 
408 static struct attribute *paicrypt_format_attr[] = {
409 	&format_attr_event.attr,
410 	NULL,
411 };
412 
413 static struct attribute_group paicrypt_events_group = {
414 	.name = "events",
415 	.attrs = NULL			/* Filled in attr_event_init() */
416 };
417 
418 static struct attribute_group paicrypt_format_group = {
419 	.name = "format",
420 	.attrs = paicrypt_format_attr,
421 };
422 
423 static const struct attribute_group *paicrypt_attr_groups[] = {
424 	&paicrypt_events_group,
425 	&paicrypt_format_group,
426 	NULL,
427 };
428 
429 /* Performance monitoring unit for mapped counters */
430 static struct pmu paicrypt = {
431 	.task_ctx_nr  = perf_invalid_context,
432 	.event_init   = paicrypt_event_init,
433 	.add	      = paicrypt_add,
434 	.del	      = paicrypt_del,
435 	.start	      = paicrypt_start,
436 	.stop	      = paicrypt_stop,
437 	.read	      = paicrypt_read,
438 	.sched_task   = paicrypt_sched_task,
439 	.attr_groups  = paicrypt_attr_groups
440 };
441 
442 /* List of symbolic PAI counter names. */
443 static const char * const paicrypt_ctrnames[] = {
444 	[0] = "CRYPTO_ALL",
445 	[1] = "KM_DEA",
446 	[2] = "KM_TDEA_128",
447 	[3] = "KM_TDEA_192",
448 	[4] = "KM_ENCRYPTED_DEA",
449 	[5] = "KM_ENCRYPTED_TDEA_128",
450 	[6] = "KM_ENCRYPTED_TDEA_192",
451 	[7] = "KM_AES_128",
452 	[8] = "KM_AES_192",
453 	[9] = "KM_AES_256",
454 	[10] = "KM_ENCRYPTED_AES_128",
455 	[11] = "KM_ENCRYPTED_AES_192",
456 	[12] = "KM_ENCRYPTED_AES_256",
457 	[13] = "KM_XTS_AES_128",
458 	[14] = "KM_XTS_AES_256",
459 	[15] = "KM_XTS_ENCRYPTED_AES_128",
460 	[16] = "KM_XTS_ENCRYPTED_AES_256",
461 	[17] = "KMC_DEA",
462 	[18] = "KMC_TDEA_128",
463 	[19] = "KMC_TDEA_192",
464 	[20] = "KMC_ENCRYPTED_DEA",
465 	[21] = "KMC_ENCRYPTED_TDEA_128",
466 	[22] = "KMC_ENCRYPTED_TDEA_192",
467 	[23] = "KMC_AES_128",
468 	[24] = "KMC_AES_192",
469 	[25] = "KMC_AES_256",
470 	[26] = "KMC_ENCRYPTED_AES_128",
471 	[27] = "KMC_ENCRYPTED_AES_192",
472 	[28] = "KMC_ENCRYPTED_AES_256",
473 	[29] = "KMC_PRNG",
474 	[30] = "KMA_GCM_AES_128",
475 	[31] = "KMA_GCM_AES_192",
476 	[32] = "KMA_GCM_AES_256",
477 	[33] = "KMA_GCM_ENCRYPTED_AES_128",
478 	[34] = "KMA_GCM_ENCRYPTED_AES_192",
479 	[35] = "KMA_GCM_ENCRYPTED_AES_256",
480 	[36] = "KMF_DEA",
481 	[37] = "KMF_TDEA_128",
482 	[38] = "KMF_TDEA_192",
483 	[39] = "KMF_ENCRYPTED_DEA",
484 	[40] = "KMF_ENCRYPTED_TDEA_128",
485 	[41] = "KMF_ENCRYPTED_TDEA_192",
486 	[42] = "KMF_AES_128",
487 	[43] = "KMF_AES_192",
488 	[44] = "KMF_AES_256",
489 	[45] = "KMF_ENCRYPTED_AES_128",
490 	[46] = "KMF_ENCRYPTED_AES_192",
491 	[47] = "KMF_ENCRYPTED_AES_256",
492 	[48] = "KMCTR_DEA",
493 	[49] = "KMCTR_TDEA_128",
494 	[50] = "KMCTR_TDEA_192",
495 	[51] = "KMCTR_ENCRYPTED_DEA",
496 	[52] = "KMCTR_ENCRYPTED_TDEA_128",
497 	[53] = "KMCTR_ENCRYPTED_TDEA_192",
498 	[54] = "KMCTR_AES_128",
499 	[55] = "KMCTR_AES_192",
500 	[56] = "KMCTR_AES_256",
501 	[57] = "KMCTR_ENCRYPTED_AES_128",
502 	[58] = "KMCTR_ENCRYPTED_AES_192",
503 	[59] = "KMCTR_ENCRYPTED_AES_256",
504 	[60] = "KMO_DEA",
505 	[61] = "KMO_TDEA_128",
506 	[62] = "KMO_TDEA_192",
507 	[63] = "KMO_ENCRYPTED_DEA",
508 	[64] = "KMO_ENCRYPTED_TDEA_128",
509 	[65] = "KMO_ENCRYPTED_TDEA_192",
510 	[66] = "KMO_AES_128",
511 	[67] = "KMO_AES_192",
512 	[68] = "KMO_AES_256",
513 	[69] = "KMO_ENCRYPTED_AES_128",
514 	[70] = "KMO_ENCRYPTED_AES_192",
515 	[71] = "KMO_ENCRYPTED_AES_256",
516 	[72] = "KIMD_SHA_1",
517 	[73] = "KIMD_SHA_256",
518 	[74] = "KIMD_SHA_512",
519 	[75] = "KIMD_SHA3_224",
520 	[76] = "KIMD_SHA3_256",
521 	[77] = "KIMD_SHA3_384",
522 	[78] = "KIMD_SHA3_512",
523 	[79] = "KIMD_SHAKE_128",
524 	[80] = "KIMD_SHAKE_256",
525 	[81] = "KIMD_GHASH",
526 	[82] = "KLMD_SHA_1",
527 	[83] = "KLMD_SHA_256",
528 	[84] = "KLMD_SHA_512",
529 	[85] = "KLMD_SHA3_224",
530 	[86] = "KLMD_SHA3_256",
531 	[87] = "KLMD_SHA3_384",
532 	[88] = "KLMD_SHA3_512",
533 	[89] = "KLMD_SHAKE_128",
534 	[90] = "KLMD_SHAKE_256",
535 	[91] = "KMAC_DEA",
536 	[92] = "KMAC_TDEA_128",
537 	[93] = "KMAC_TDEA_192",
538 	[94] = "KMAC_ENCRYPTED_DEA",
539 	[95] = "KMAC_ENCRYPTED_TDEA_128",
540 	[96] = "KMAC_ENCRYPTED_TDEA_192",
541 	[97] = "KMAC_AES_128",
542 	[98] = "KMAC_AES_192",
543 	[99] = "KMAC_AES_256",
544 	[100] = "KMAC_ENCRYPTED_AES_128",
545 	[101] = "KMAC_ENCRYPTED_AES_192",
546 	[102] = "KMAC_ENCRYPTED_AES_256",
547 	[103] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_DEA",
548 	[104] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_128",
549 	[105] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_192",
550 	[106] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_DEA",
551 	[107] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_128",
552 	[108] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_192",
553 	[109] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_128",
554 	[110] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_192",
555 	[111] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_256",
556 	[112] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_128",
557 	[113] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_192",
558 	[114] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_256A",
559 	[115] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_128",
560 	[116] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_256",
561 	[117] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_128",
562 	[118] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_256",
563 	[119] = "PCC_SCALAR_MULTIPLY_P256",
564 	[120] = "PCC_SCALAR_MULTIPLY_P384",
565 	[121] = "PCC_SCALAR_MULTIPLY_P521",
566 	[122] = "PCC_SCALAR_MULTIPLY_ED25519",
567 	[123] = "PCC_SCALAR_MULTIPLY_ED448",
568 	[124] = "PCC_SCALAR_MULTIPLY_X25519",
569 	[125] = "PCC_SCALAR_MULTIPLY_X448",
570 	[126] = "PRNO_SHA_512_DRNG",
571 	[127] = "PRNO_TRNG_QUERY_RAW_TO_CONDITIONED_RATIO",
572 	[128] = "PRNO_TRNG",
573 	[129] = "KDSA_ECDSA_VERIFY_P256",
574 	[130] = "KDSA_ECDSA_VERIFY_P384",
575 	[131] = "KDSA_ECDSA_VERIFY_P521",
576 	[132] = "KDSA_ECDSA_SIGN_P256",
577 	[133] = "KDSA_ECDSA_SIGN_P384",
578 	[134] = "KDSA_ECDSA_SIGN_P521",
579 	[135] = "KDSA_ENCRYPTED_ECDSA_SIGN_P256",
580 	[136] = "KDSA_ENCRYPTED_ECDSA_SIGN_P384",
581 	[137] = "KDSA_ENCRYPTED_ECDSA_SIGN_P521",
582 	[138] = "KDSA_EDDSA_VERIFY_ED25519",
583 	[139] = "KDSA_EDDSA_VERIFY_ED448",
584 	[140] = "KDSA_EDDSA_SIGN_ED25519",
585 	[141] = "KDSA_EDDSA_SIGN_ED448",
586 	[142] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED25519",
587 	[143] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED448",
588 	[144] = "PCKMO_ENCRYPT_DEA_KEY",
589 	[145] = "PCKMO_ENCRYPT_TDEA_128_KEY",
590 	[146] = "PCKMO_ENCRYPT_TDEA_192_KEY",
591 	[147] = "PCKMO_ENCRYPT_AES_128_KEY",
592 	[148] = "PCKMO_ENCRYPT_AES_192_KEY",
593 	[149] = "PCKMO_ENCRYPT_AES_256_KEY",
594 	[150] = "PCKMO_ENCRYPT_ECC_P256_KEY",
595 	[151] = "PCKMO_ENCRYPT_ECC_P384_KEY",
596 	[152] = "PCKMO_ENCRYPT_ECC_P521_KEY",
597 	[153] = "PCKMO_ENCRYPT_ECC_ED25519_KEY",
598 	[154] = "PCKMO_ENCRYPT_ECC_ED448_KEY",
599 	[155] = "IBM_RESERVED_155",
600 	[156] = "IBM_RESERVED_156",
601 };
602 
603 static void __init attr_event_free(struct attribute **attrs, int num)
604 {
605 	struct perf_pmu_events_attr *pa;
606 	int i;
607 
608 	for (i = 0; i < num; i++) {
609 		struct device_attribute *dap;
610 
611 		dap = container_of(attrs[i], struct device_attribute, attr);
612 		pa = container_of(dap, struct perf_pmu_events_attr, attr);
613 		kfree(pa);
614 	}
615 	kfree(attrs);
616 }
617 
618 static int __init attr_event_init_one(struct attribute **attrs, int num)
619 {
620 	struct perf_pmu_events_attr *pa;
621 
622 	pa = kzalloc(sizeof(*pa), GFP_KERNEL);
623 	if (!pa)
624 		return -ENOMEM;
625 
626 	sysfs_attr_init(&pa->attr.attr);
627 	pa->id = PAI_CRYPTO_BASE + num;
628 	pa->attr.attr.name = paicrypt_ctrnames[num];
629 	pa->attr.attr.mode = 0444;
630 	pa->attr.show = cpumf_events_sysfs_show;
631 	pa->attr.store = NULL;
632 	attrs[num] = &pa->attr.attr;
633 	return 0;
634 }
635 
636 /* Create PMU sysfs event attributes on the fly. */
637 static int __init attr_event_init(void)
638 {
639 	struct attribute **attrs;
640 	int ret, i;
641 
642 	attrs = kmalloc_array(ARRAY_SIZE(paicrypt_ctrnames) + 1, sizeof(*attrs),
643 			      GFP_KERNEL);
644 	if (!attrs)
645 		return -ENOMEM;
646 	for (i = 0; i < ARRAY_SIZE(paicrypt_ctrnames); i++) {
647 		ret = attr_event_init_one(attrs, i);
648 		if (ret) {
649 			attr_event_free(attrs, i);
650 			return ret;
651 		}
652 	}
653 	attrs[i] = NULL;
654 	paicrypt_events_group.attrs = attrs;
655 	return 0;
656 }
657 
658 static int __init paicrypt_init(void)
659 {
660 	struct qpaci_info_block ib;
661 	int rc;
662 
663 	if (!test_facility(196))
664 		return 0;
665 
666 	qpaci(&ib);
667 	paicrypt_cnt = ib.num_cc;
668 	if (paicrypt_cnt == 0)
669 		return 0;
670 	if (paicrypt_cnt >= PAI_CRYPTO_MAXCTR)
671 		paicrypt_cnt = PAI_CRYPTO_MAXCTR - 1;
672 
673 	rc = attr_event_init();		/* Export known PAI crypto events */
674 	if (rc) {
675 		pr_err("Creation of PMU pai_crypto /sysfs failed\n");
676 		return rc;
677 	}
678 
679 	/* Setup s390dbf facility */
680 	cfm_dbg = debug_register(KMSG_COMPONENT, 2, 256, 128);
681 	if (!cfm_dbg) {
682 		pr_err("Registration of s390dbf pai_crypto failed\n");
683 		return -ENOMEM;
684 	}
685 	debug_register_view(cfm_dbg, &debug_sprintf_view);
686 
687 	rc = perf_pmu_register(&paicrypt, "pai_crypto", -1);
688 	if (rc) {
689 		pr_err("Registering the pai_crypto PMU failed with rc=%i\n",
690 		       rc);
691 		debug_unregister_view(cfm_dbg, &debug_sprintf_view);
692 		debug_unregister(cfm_dbg);
693 		return rc;
694 	}
695 	return 0;
696 }
697 
698 device_initcall(paicrypt_init);
699