1 // SPDX-License-Identifier: GPL-2.0-only OR MIT
2 /*
3 * Apple RTKit IPC library
4 * Copyright (C) The Asahi Linux Contributors
5 */
6
7 #include "rtkit-internal.h"
8
9 enum {
10 APPLE_RTKIT_PWR_STATE_OFF = 0x00, /* power off, cannot be restarted */
11 APPLE_RTKIT_PWR_STATE_SLEEP = 0x01, /* sleeping, can be restarted */
12 APPLE_RTKIT_PWR_STATE_IDLE = 0x201, /* sleeping, retain state */
13 APPLE_RTKIT_PWR_STATE_QUIESCED = 0x10, /* running but no communication */
14 APPLE_RTKIT_PWR_STATE_ON = 0x20, /* normal operating state */
15 };
16
17 enum {
18 APPLE_RTKIT_EP_MGMT = 0,
19 APPLE_RTKIT_EP_CRASHLOG = 1,
20 APPLE_RTKIT_EP_SYSLOG = 2,
21 APPLE_RTKIT_EP_DEBUG = 3,
22 APPLE_RTKIT_EP_IOREPORT = 4,
23 APPLE_RTKIT_EP_OSLOG = 8,
24 };
25
26 #define APPLE_RTKIT_MGMT_TYPE GENMASK_ULL(59, 52)
27
28 enum {
29 APPLE_RTKIT_MGMT_HELLO = 1,
30 APPLE_RTKIT_MGMT_HELLO_REPLY = 2,
31 APPLE_RTKIT_MGMT_STARTEP = 5,
32 APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE = 6,
33 APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE_ACK = 7,
34 APPLE_RTKIT_MGMT_EPMAP = 8,
35 APPLE_RTKIT_MGMT_EPMAP_REPLY = 8,
36 APPLE_RTKIT_MGMT_SET_AP_PWR_STATE = 0xb,
37 APPLE_RTKIT_MGMT_SET_AP_PWR_STATE_ACK = 0xb,
38 };
39
40 #define APPLE_RTKIT_MGMT_HELLO_MINVER GENMASK_ULL(15, 0)
41 #define APPLE_RTKIT_MGMT_HELLO_MAXVER GENMASK_ULL(31, 16)
42
43 #define APPLE_RTKIT_MGMT_EPMAP_LAST BIT_ULL(51)
44 #define APPLE_RTKIT_MGMT_EPMAP_BASE GENMASK_ULL(34, 32)
45 #define APPLE_RTKIT_MGMT_EPMAP_BITMAP GENMASK_ULL(31, 0)
46
47 #define APPLE_RTKIT_MGMT_EPMAP_REPLY_MORE BIT_ULL(0)
48
49 #define APPLE_RTKIT_MGMT_STARTEP_EP GENMASK_ULL(39, 32)
50 #define APPLE_RTKIT_MGMT_STARTEP_FLAG BIT_ULL(1)
51
52 #define APPLE_RTKIT_MGMT_PWR_STATE GENMASK_ULL(15, 0)
53
54 #define APPLE_RTKIT_CRASHLOG_CRASH 1
55
56 #define APPLE_RTKIT_BUFFER_REQUEST 1
57 #define APPLE_RTKIT_BUFFER_REQUEST_SIZE GENMASK_ULL(51, 44)
58 #define APPLE_RTKIT_BUFFER_REQUEST_IOVA GENMASK_ULL(43, 0)
59
60 #define APPLE_RTKIT_SYSLOG_TYPE GENMASK_ULL(59, 52)
61
62 #define APPLE_RTKIT_SYSLOG_LOG 5
63
64 #define APPLE_RTKIT_SYSLOG_INIT 8
65 #define APPLE_RTKIT_SYSLOG_N_ENTRIES GENMASK_ULL(7, 0)
66 #define APPLE_RTKIT_SYSLOG_MSG_SIZE GENMASK_ULL(31, 24)
67
68 #define APPLE_RTKIT_OSLOG_TYPE GENMASK_ULL(63, 56)
69 #define APPLE_RTKIT_OSLOG_BUFFER_REQUEST 1
70 #define APPLE_RTKIT_OSLOG_SIZE GENMASK_ULL(55, 36)
71 #define APPLE_RTKIT_OSLOG_IOVA GENMASK_ULL(35, 0)
72
73 #define APPLE_RTKIT_MIN_SUPPORTED_VERSION 11
74 #define APPLE_RTKIT_MAX_SUPPORTED_VERSION 12
75
76 struct apple_rtkit_msg {
77 struct completion *completion;
78 struct apple_mbox_msg mbox_msg;
79 };
80
81 struct apple_rtkit_rx_work {
82 struct apple_rtkit *rtk;
83 u8 ep;
84 u64 msg;
85 struct work_struct work;
86 };
87
apple_rtkit_is_running(struct apple_rtkit * rtk)88 bool apple_rtkit_is_running(struct apple_rtkit *rtk)
89 {
90 if (rtk->crashed)
91 return false;
92 if ((rtk->iop_power_state & 0xff) != APPLE_RTKIT_PWR_STATE_ON)
93 return false;
94 if ((rtk->ap_power_state & 0xff) != APPLE_RTKIT_PWR_STATE_ON)
95 return false;
96 return true;
97 }
98 EXPORT_SYMBOL_GPL(apple_rtkit_is_running);
99
apple_rtkit_is_crashed(struct apple_rtkit * rtk)100 bool apple_rtkit_is_crashed(struct apple_rtkit *rtk)
101 {
102 return rtk->crashed;
103 }
104 EXPORT_SYMBOL_GPL(apple_rtkit_is_crashed);
105
apple_rtkit_management_send(struct apple_rtkit * rtk,u8 type,u64 msg)106 static void apple_rtkit_management_send(struct apple_rtkit *rtk, u8 type,
107 u64 msg)
108 {
109 msg &= ~APPLE_RTKIT_MGMT_TYPE;
110 msg |= FIELD_PREP(APPLE_RTKIT_MGMT_TYPE, type);
111 apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_MGMT, msg, NULL, false);
112 }
113
apple_rtkit_management_rx_hello(struct apple_rtkit * rtk,u64 msg)114 static void apple_rtkit_management_rx_hello(struct apple_rtkit *rtk, u64 msg)
115 {
116 u64 reply;
117
118 int min_ver = FIELD_GET(APPLE_RTKIT_MGMT_HELLO_MINVER, msg);
119 int max_ver = FIELD_GET(APPLE_RTKIT_MGMT_HELLO_MAXVER, msg);
120 int want_ver = min(APPLE_RTKIT_MAX_SUPPORTED_VERSION, max_ver);
121
122 dev_dbg(rtk->dev, "RTKit: Min ver %d, max ver %d\n", min_ver, max_ver);
123
124 if (min_ver > APPLE_RTKIT_MAX_SUPPORTED_VERSION) {
125 dev_err(rtk->dev, "RTKit: Firmware min version %d is too new\n",
126 min_ver);
127 goto abort_boot;
128 }
129
130 if (max_ver < APPLE_RTKIT_MIN_SUPPORTED_VERSION) {
131 dev_err(rtk->dev, "RTKit: Firmware max version %d is too old\n",
132 max_ver);
133 goto abort_boot;
134 }
135
136 dev_info(rtk->dev, "RTKit: Initializing (protocol version %d)\n",
137 want_ver);
138 rtk->version = want_ver;
139
140 reply = FIELD_PREP(APPLE_RTKIT_MGMT_HELLO_MINVER, want_ver);
141 reply |= FIELD_PREP(APPLE_RTKIT_MGMT_HELLO_MAXVER, want_ver);
142 apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_HELLO_REPLY, reply);
143
144 return;
145
146 abort_boot:
147 rtk->boot_result = -EINVAL;
148 complete_all(&rtk->epmap_completion);
149 }
150
apple_rtkit_management_rx_epmap(struct apple_rtkit * rtk,u64 msg)151 static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg)
152 {
153 int i, ep;
154 u64 reply;
155 unsigned long bitmap = FIELD_GET(APPLE_RTKIT_MGMT_EPMAP_BITMAP, msg);
156 u32 base = FIELD_GET(APPLE_RTKIT_MGMT_EPMAP_BASE, msg);
157
158 dev_dbg(rtk->dev,
159 "RTKit: received endpoint bitmap 0x%lx with base 0x%x\n",
160 bitmap, base);
161
162 for_each_set_bit(i, &bitmap, 32) {
163 ep = 32 * base + i;
164 dev_dbg(rtk->dev, "RTKit: Discovered endpoint 0x%02x\n", ep);
165 set_bit(ep, rtk->endpoints);
166 }
167
168 reply = FIELD_PREP(APPLE_RTKIT_MGMT_EPMAP_BASE, base);
169 if (msg & APPLE_RTKIT_MGMT_EPMAP_LAST)
170 reply |= APPLE_RTKIT_MGMT_EPMAP_LAST;
171 else
172 reply |= APPLE_RTKIT_MGMT_EPMAP_REPLY_MORE;
173
174 apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_EPMAP_REPLY, reply);
175
176 if (!(msg & APPLE_RTKIT_MGMT_EPMAP_LAST))
177 return;
178
179 for_each_set_bit(ep, rtk->endpoints, APPLE_RTKIT_APP_ENDPOINT_START) {
180 switch (ep) {
181 /* the management endpoint is started by default */
182 case APPLE_RTKIT_EP_MGMT:
183 break;
184
185 /* without starting these RTKit refuses to boot */
186 case APPLE_RTKIT_EP_SYSLOG:
187 case APPLE_RTKIT_EP_CRASHLOG:
188 case APPLE_RTKIT_EP_DEBUG:
189 case APPLE_RTKIT_EP_IOREPORT:
190 case APPLE_RTKIT_EP_OSLOG:
191 dev_dbg(rtk->dev,
192 "RTKit: Starting system endpoint 0x%02x\n", ep);
193 apple_rtkit_start_ep(rtk, ep);
194 break;
195
196 default:
197 dev_warn(rtk->dev,
198 "RTKit: Unknown system endpoint: 0x%02x\n",
199 ep);
200 }
201 }
202
203 rtk->boot_result = 0;
204 complete_all(&rtk->epmap_completion);
205 }
206
apple_rtkit_management_rx_iop_pwr_ack(struct apple_rtkit * rtk,u64 msg)207 static void apple_rtkit_management_rx_iop_pwr_ack(struct apple_rtkit *rtk,
208 u64 msg)
209 {
210 unsigned int new_state = FIELD_GET(APPLE_RTKIT_MGMT_PWR_STATE, msg);
211
212 dev_dbg(rtk->dev, "RTKit: IOP power state transition: 0x%x -> 0x%x\n",
213 rtk->iop_power_state, new_state);
214 rtk->iop_power_state = new_state;
215
216 complete_all(&rtk->iop_pwr_ack_completion);
217 }
218
apple_rtkit_management_rx_ap_pwr_ack(struct apple_rtkit * rtk,u64 msg)219 static void apple_rtkit_management_rx_ap_pwr_ack(struct apple_rtkit *rtk,
220 u64 msg)
221 {
222 unsigned int new_state = FIELD_GET(APPLE_RTKIT_MGMT_PWR_STATE, msg);
223
224 dev_dbg(rtk->dev, "RTKit: AP power state transition: 0x%x -> 0x%x\n",
225 rtk->ap_power_state, new_state);
226 rtk->ap_power_state = new_state;
227
228 complete_all(&rtk->ap_pwr_ack_completion);
229 }
230
apple_rtkit_management_rx(struct apple_rtkit * rtk,u64 msg)231 static void apple_rtkit_management_rx(struct apple_rtkit *rtk, u64 msg)
232 {
233 u8 type = FIELD_GET(APPLE_RTKIT_MGMT_TYPE, msg);
234
235 switch (type) {
236 case APPLE_RTKIT_MGMT_HELLO:
237 apple_rtkit_management_rx_hello(rtk, msg);
238 break;
239 case APPLE_RTKIT_MGMT_EPMAP:
240 apple_rtkit_management_rx_epmap(rtk, msg);
241 break;
242 case APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE_ACK:
243 apple_rtkit_management_rx_iop_pwr_ack(rtk, msg);
244 break;
245 case APPLE_RTKIT_MGMT_SET_AP_PWR_STATE_ACK:
246 apple_rtkit_management_rx_ap_pwr_ack(rtk, msg);
247 break;
248 default:
249 dev_warn(
250 rtk->dev,
251 "RTKit: unknown management message: 0x%llx (type: 0x%02x)\n",
252 msg, type);
253 }
254 }
255
apple_rtkit_common_rx_get_buffer(struct apple_rtkit * rtk,struct apple_rtkit_shmem * buffer,u8 ep,u64 msg)256 static int apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk,
257 struct apple_rtkit_shmem *buffer,
258 u8 ep, u64 msg)
259 {
260 u64 reply;
261 int err;
262
263 /* The different size vs. IOVA shifts look odd but are indeed correct this way */
264 if (ep == APPLE_RTKIT_EP_OSLOG) {
265 buffer->size = FIELD_GET(APPLE_RTKIT_OSLOG_SIZE, msg);
266 buffer->iova = FIELD_GET(APPLE_RTKIT_OSLOG_IOVA, msg) << 12;
267 } else {
268 buffer->size = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_SIZE, msg) << 12;
269 buffer->iova = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg);
270 }
271
272 buffer->buffer = NULL;
273 buffer->iomem = NULL;
274 buffer->is_mapped = false;
275
276 dev_dbg(rtk->dev, "RTKit: buffer request for 0x%zx bytes at %pad\n",
277 buffer->size, &buffer->iova);
278
279 if (buffer->iova &&
280 (!rtk->ops->shmem_setup || !rtk->ops->shmem_destroy)) {
281 err = -EINVAL;
282 goto error;
283 }
284
285 if (rtk->ops->shmem_setup) {
286 err = rtk->ops->shmem_setup(rtk->cookie, buffer);
287 if (err)
288 goto error;
289 } else {
290 buffer->buffer = dma_alloc_coherent(rtk->dev, buffer->size,
291 &buffer->iova, GFP_KERNEL);
292 if (!buffer->buffer) {
293 err = -ENOMEM;
294 goto error;
295 }
296 }
297
298 if (!buffer->is_mapped) {
299 /* oslog uses different fields and needs a shifted IOVA instead of size */
300 if (ep == APPLE_RTKIT_EP_OSLOG) {
301 reply = FIELD_PREP(APPLE_RTKIT_OSLOG_TYPE,
302 APPLE_RTKIT_OSLOG_BUFFER_REQUEST);
303 reply |= FIELD_PREP(APPLE_RTKIT_OSLOG_SIZE, buffer->size);
304 reply |= FIELD_PREP(APPLE_RTKIT_OSLOG_IOVA,
305 buffer->iova >> 12);
306 } else {
307 reply = FIELD_PREP(APPLE_RTKIT_SYSLOG_TYPE,
308 APPLE_RTKIT_BUFFER_REQUEST);
309 reply |= FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_SIZE,
310 buffer->size >> 12);
311 reply |= FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_IOVA,
312 buffer->iova);
313 }
314 apple_rtkit_send_message(rtk, ep, reply, NULL, false);
315 }
316
317 return 0;
318
319 error:
320 buffer->buffer = NULL;
321 buffer->iomem = NULL;
322 buffer->iova = 0;
323 buffer->size = 0;
324 buffer->is_mapped = false;
325 return err;
326 }
327
apple_rtkit_free_buffer(struct apple_rtkit * rtk,struct apple_rtkit_shmem * bfr)328 static void apple_rtkit_free_buffer(struct apple_rtkit *rtk,
329 struct apple_rtkit_shmem *bfr)
330 {
331 if (bfr->size == 0)
332 return;
333
334 if (rtk->ops->shmem_destroy)
335 rtk->ops->shmem_destroy(rtk->cookie, bfr);
336 else if (bfr->buffer)
337 dma_free_coherent(rtk->dev, bfr->size, bfr->buffer, bfr->iova);
338
339 bfr->buffer = NULL;
340 bfr->iomem = NULL;
341 bfr->iova = 0;
342 bfr->size = 0;
343 bfr->is_mapped = false;
344 }
345
apple_rtkit_memcpy(struct apple_rtkit * rtk,void * dst,struct apple_rtkit_shmem * bfr,size_t offset,size_t len)346 static void apple_rtkit_memcpy(struct apple_rtkit *rtk, void *dst,
347 struct apple_rtkit_shmem *bfr, size_t offset,
348 size_t len)
349 {
350 if (bfr->iomem)
351 memcpy_fromio(dst, bfr->iomem + offset, len);
352 else
353 memcpy(dst, bfr->buffer + offset, len);
354 }
355
apple_rtkit_crashlog_rx(struct apple_rtkit * rtk,u64 msg)356 static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg)
357 {
358 u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg);
359 u8 *bfr;
360
361 if (type != APPLE_RTKIT_CRASHLOG_CRASH) {
362 dev_warn(rtk->dev, "RTKit: Unknown crashlog message: %llx\n",
363 msg);
364 return;
365 }
366
367 if (!rtk->crashlog_buffer.size) {
368 apple_rtkit_common_rx_get_buffer(rtk, &rtk->crashlog_buffer,
369 APPLE_RTKIT_EP_CRASHLOG, msg);
370 return;
371 }
372
373 dev_err(rtk->dev, "RTKit: co-processor has crashed\n");
374
375 /*
376 * create a shadow copy here to make sure the co-processor isn't able
377 * to change the log while we're dumping it. this also ensures
378 * the buffer is in normal memory and not iomem for e.g. the SMC
379 */
380 bfr = kzalloc(rtk->crashlog_buffer.size, GFP_KERNEL);
381 if (bfr) {
382 apple_rtkit_memcpy(rtk, bfr, &rtk->crashlog_buffer, 0,
383 rtk->crashlog_buffer.size);
384 apple_rtkit_crashlog_dump(rtk, bfr, rtk->crashlog_buffer.size);
385 kfree(bfr);
386 } else {
387 dev_err(rtk->dev,
388 "RTKit: Couldn't allocate crashlog shadow buffer\n");
389 }
390
391 rtk->crashed = true;
392 if (rtk->ops->crashed)
393 rtk->ops->crashed(rtk->cookie);
394 }
395
apple_rtkit_ioreport_rx(struct apple_rtkit * rtk,u64 msg)396 static void apple_rtkit_ioreport_rx(struct apple_rtkit *rtk, u64 msg)
397 {
398 u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg);
399
400 switch (type) {
401 case APPLE_RTKIT_BUFFER_REQUEST:
402 apple_rtkit_common_rx_get_buffer(rtk, &rtk->ioreport_buffer,
403 APPLE_RTKIT_EP_IOREPORT, msg);
404 break;
405 /* unknown, must be ACKed or the co-processor will hang */
406 case 0x8:
407 case 0xc:
408 apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_IOREPORT, msg,
409 NULL, false);
410 break;
411 default:
412 dev_warn(rtk->dev, "RTKit: Unknown ioreport message: %llx\n",
413 msg);
414 }
415 }
416
apple_rtkit_syslog_rx_init(struct apple_rtkit * rtk,u64 msg)417 static void apple_rtkit_syslog_rx_init(struct apple_rtkit *rtk, u64 msg)
418 {
419 rtk->syslog_n_entries = FIELD_GET(APPLE_RTKIT_SYSLOG_N_ENTRIES, msg);
420 rtk->syslog_msg_size = FIELD_GET(APPLE_RTKIT_SYSLOG_MSG_SIZE, msg);
421
422 rtk->syslog_msg_buffer = kzalloc(rtk->syslog_msg_size, GFP_KERNEL);
423
424 dev_dbg(rtk->dev,
425 "RTKit: syslog initialized: entries: %zd, msg_size: %zd\n",
426 rtk->syslog_n_entries, rtk->syslog_msg_size);
427 }
428
should_crop_syslog_char(char c)429 static bool should_crop_syslog_char(char c)
430 {
431 return c == '\n' || c == '\r' || c == ' ' || c == '\0';
432 }
433
apple_rtkit_syslog_rx_log(struct apple_rtkit * rtk,u64 msg)434 static void apple_rtkit_syslog_rx_log(struct apple_rtkit *rtk, u64 msg)
435 {
436 u8 idx = msg & 0xff;
437 char log_context[24];
438 size_t entry_size = 0x20 + rtk->syslog_msg_size;
439 int msglen;
440
441 if (!rtk->syslog_msg_buffer) {
442 dev_warn(
443 rtk->dev,
444 "RTKit: received syslog message but no syslog_msg_buffer\n");
445 goto done;
446 }
447 if (!rtk->syslog_buffer.size) {
448 dev_warn(
449 rtk->dev,
450 "RTKit: received syslog message but syslog_buffer.size is zero\n");
451 goto done;
452 }
453 if (!rtk->syslog_buffer.buffer && !rtk->syslog_buffer.iomem) {
454 dev_warn(
455 rtk->dev,
456 "RTKit: received syslog message but no syslog_buffer.buffer or syslog_buffer.iomem\n");
457 goto done;
458 }
459 if (idx > rtk->syslog_n_entries) {
460 dev_warn(rtk->dev, "RTKit: syslog index %d out of range\n",
461 idx);
462 goto done;
463 }
464
465 apple_rtkit_memcpy(rtk, log_context, &rtk->syslog_buffer,
466 idx * entry_size + 8, sizeof(log_context));
467 apple_rtkit_memcpy(rtk, rtk->syslog_msg_buffer, &rtk->syslog_buffer,
468 idx * entry_size + 8 + sizeof(log_context),
469 rtk->syslog_msg_size);
470
471 log_context[sizeof(log_context) - 1] = 0;
472
473 msglen = rtk->syslog_msg_size - 1;
474 while (msglen > 0 &&
475 should_crop_syslog_char(rtk->syslog_msg_buffer[msglen - 1]))
476 msglen--;
477
478 rtk->syslog_msg_buffer[msglen] = 0;
479 dev_info(rtk->dev, "RTKit: syslog message: %s: %s\n", log_context,
480 rtk->syslog_msg_buffer);
481
482 done:
483 apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_SYSLOG, msg, NULL, false);
484 }
485
apple_rtkit_syslog_rx(struct apple_rtkit * rtk,u64 msg)486 static void apple_rtkit_syslog_rx(struct apple_rtkit *rtk, u64 msg)
487 {
488 u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg);
489
490 switch (type) {
491 case APPLE_RTKIT_BUFFER_REQUEST:
492 apple_rtkit_common_rx_get_buffer(rtk, &rtk->syslog_buffer,
493 APPLE_RTKIT_EP_SYSLOG, msg);
494 break;
495 case APPLE_RTKIT_SYSLOG_INIT:
496 apple_rtkit_syslog_rx_init(rtk, msg);
497 break;
498 case APPLE_RTKIT_SYSLOG_LOG:
499 apple_rtkit_syslog_rx_log(rtk, msg);
500 break;
501 default:
502 dev_warn(rtk->dev, "RTKit: Unknown syslog message: %llx\n",
503 msg);
504 }
505 }
506
apple_rtkit_oslog_rx(struct apple_rtkit * rtk,u64 msg)507 static void apple_rtkit_oslog_rx(struct apple_rtkit *rtk, u64 msg)
508 {
509 u8 type = FIELD_GET(APPLE_RTKIT_OSLOG_TYPE, msg);
510
511 switch (type) {
512 case APPLE_RTKIT_OSLOG_BUFFER_REQUEST:
513 apple_rtkit_common_rx_get_buffer(rtk, &rtk->oslog_buffer,
514 APPLE_RTKIT_EP_OSLOG, msg);
515 break;
516 default:
517 dev_warn(rtk->dev, "RTKit: Unknown oslog message: %llx\n",
518 msg);
519 }
520 }
521
apple_rtkit_rx_work(struct work_struct * work)522 static void apple_rtkit_rx_work(struct work_struct *work)
523 {
524 struct apple_rtkit_rx_work *rtk_work =
525 container_of(work, struct apple_rtkit_rx_work, work);
526 struct apple_rtkit *rtk = rtk_work->rtk;
527
528 switch (rtk_work->ep) {
529 case APPLE_RTKIT_EP_MGMT:
530 apple_rtkit_management_rx(rtk, rtk_work->msg);
531 break;
532 case APPLE_RTKIT_EP_CRASHLOG:
533 apple_rtkit_crashlog_rx(rtk, rtk_work->msg);
534 break;
535 case APPLE_RTKIT_EP_SYSLOG:
536 apple_rtkit_syslog_rx(rtk, rtk_work->msg);
537 break;
538 case APPLE_RTKIT_EP_IOREPORT:
539 apple_rtkit_ioreport_rx(rtk, rtk_work->msg);
540 break;
541 case APPLE_RTKIT_EP_OSLOG:
542 apple_rtkit_oslog_rx(rtk, rtk_work->msg);
543 break;
544 case APPLE_RTKIT_APP_ENDPOINT_START ... 0xff:
545 if (rtk->ops->recv_message)
546 rtk->ops->recv_message(rtk->cookie, rtk_work->ep,
547 rtk_work->msg);
548 else
549 dev_warn(
550 rtk->dev,
551 "Received unexpected message to EP%02d: %llx\n",
552 rtk_work->ep, rtk_work->msg);
553 break;
554 default:
555 dev_warn(rtk->dev,
556 "RTKit: message to unknown endpoint %02x: %llx\n",
557 rtk_work->ep, rtk_work->msg);
558 }
559
560 kfree(rtk_work);
561 }
562
apple_rtkit_rx(struct mbox_client * cl,void * mssg)563 static void apple_rtkit_rx(struct mbox_client *cl, void *mssg)
564 {
565 struct apple_rtkit *rtk = container_of(cl, struct apple_rtkit, mbox_cl);
566 struct apple_mbox_msg *msg = mssg;
567 struct apple_rtkit_rx_work *work;
568 u8 ep = msg->msg1;
569
570 /*
571 * The message was read from a MMIO FIFO and we have to make
572 * sure all reads from buffers sent with that message happen
573 * afterwards.
574 */
575 dma_rmb();
576
577 if (!test_bit(ep, rtk->endpoints))
578 dev_warn(rtk->dev,
579 "RTKit: Message to undiscovered endpoint 0x%02x\n",
580 ep);
581
582 if (ep >= APPLE_RTKIT_APP_ENDPOINT_START &&
583 rtk->ops->recv_message_early &&
584 rtk->ops->recv_message_early(rtk->cookie, ep, msg->msg0))
585 return;
586
587 work = kzalloc(sizeof(*work), GFP_ATOMIC);
588 if (!work)
589 return;
590
591 work->rtk = rtk;
592 work->ep = ep;
593 work->msg = msg->msg0;
594 INIT_WORK(&work->work, apple_rtkit_rx_work);
595 queue_work(rtk->wq, &work->work);
596 }
597
apple_rtkit_tx_done(struct mbox_client * cl,void * mssg,int r)598 static void apple_rtkit_tx_done(struct mbox_client *cl, void *mssg, int r)
599 {
600 struct apple_rtkit_msg *msg =
601 container_of(mssg, struct apple_rtkit_msg, mbox_msg);
602
603 if (r == -ETIME)
604 return;
605
606 if (msg->completion)
607 complete(msg->completion);
608 kfree(msg);
609 }
610
apple_rtkit_send_message(struct apple_rtkit * rtk,u8 ep,u64 message,struct completion * completion,bool atomic)611 int apple_rtkit_send_message(struct apple_rtkit *rtk, u8 ep, u64 message,
612 struct completion *completion, bool atomic)
613 {
614 struct apple_rtkit_msg *msg;
615 int ret;
616 gfp_t flags;
617
618 if (rtk->crashed)
619 return -EINVAL;
620 if (ep >= APPLE_RTKIT_APP_ENDPOINT_START &&
621 !apple_rtkit_is_running(rtk))
622 return -EINVAL;
623
624 if (atomic)
625 flags = GFP_ATOMIC;
626 else
627 flags = GFP_KERNEL;
628
629 msg = kzalloc(sizeof(*msg), flags);
630 if (!msg)
631 return -ENOMEM;
632
633 msg->mbox_msg.msg0 = message;
634 msg->mbox_msg.msg1 = ep;
635 msg->completion = completion;
636
637 /*
638 * The message will be sent with a MMIO write. We need the barrier
639 * here to ensure any previous writes to buffers are visible to the
640 * device before that MMIO write happens.
641 */
642 dma_wmb();
643
644 ret = mbox_send_message(rtk->mbox_chan, &msg->mbox_msg);
645 if (ret < 0) {
646 kfree(msg);
647 return ret;
648 }
649
650 return 0;
651 }
652 EXPORT_SYMBOL_GPL(apple_rtkit_send_message);
653
apple_rtkit_send_message_wait(struct apple_rtkit * rtk,u8 ep,u64 message,unsigned long timeout,bool atomic)654 int apple_rtkit_send_message_wait(struct apple_rtkit *rtk, u8 ep, u64 message,
655 unsigned long timeout, bool atomic)
656 {
657 DECLARE_COMPLETION_ONSTACK(completion);
658 int ret;
659 long t;
660
661 ret = apple_rtkit_send_message(rtk, ep, message, &completion, atomic);
662 if (ret < 0)
663 return ret;
664
665 if (atomic) {
666 ret = mbox_flush(rtk->mbox_chan, timeout);
667 if (ret < 0)
668 return ret;
669
670 if (try_wait_for_completion(&completion))
671 return 0;
672
673 return -ETIME;
674 } else {
675 t = wait_for_completion_interruptible_timeout(
676 &completion, msecs_to_jiffies(timeout));
677 if (t < 0)
678 return t;
679 else if (t == 0)
680 return -ETIME;
681 return 0;
682 }
683 }
684 EXPORT_SYMBOL_GPL(apple_rtkit_send_message_wait);
685
apple_rtkit_poll(struct apple_rtkit * rtk)686 int apple_rtkit_poll(struct apple_rtkit *rtk)
687 {
688 return mbox_client_peek_data(rtk->mbox_chan);
689 }
690 EXPORT_SYMBOL_GPL(apple_rtkit_poll);
691
apple_rtkit_start_ep(struct apple_rtkit * rtk,u8 endpoint)692 int apple_rtkit_start_ep(struct apple_rtkit *rtk, u8 endpoint)
693 {
694 u64 msg;
695
696 if (!test_bit(endpoint, rtk->endpoints))
697 return -EINVAL;
698 if (endpoint >= APPLE_RTKIT_APP_ENDPOINT_START &&
699 !apple_rtkit_is_running(rtk))
700 return -EINVAL;
701
702 msg = FIELD_PREP(APPLE_RTKIT_MGMT_STARTEP_EP, endpoint);
703 msg |= APPLE_RTKIT_MGMT_STARTEP_FLAG;
704 apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_STARTEP, msg);
705
706 return 0;
707 }
708 EXPORT_SYMBOL_GPL(apple_rtkit_start_ep);
709
apple_rtkit_request_mbox_chan(struct apple_rtkit * rtk)710 static int apple_rtkit_request_mbox_chan(struct apple_rtkit *rtk)
711 {
712 if (rtk->mbox_name)
713 rtk->mbox_chan = mbox_request_channel_byname(&rtk->mbox_cl,
714 rtk->mbox_name);
715 else
716 rtk->mbox_chan =
717 mbox_request_channel(&rtk->mbox_cl, rtk->mbox_idx);
718
719 if (IS_ERR(rtk->mbox_chan))
720 return PTR_ERR(rtk->mbox_chan);
721 return 0;
722 }
723
apple_rtkit_init(struct device * dev,void * cookie,const char * mbox_name,int mbox_idx,const struct apple_rtkit_ops * ops)724 struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie,
725 const char *mbox_name, int mbox_idx,
726 const struct apple_rtkit_ops *ops)
727 {
728 struct apple_rtkit *rtk;
729 int ret;
730
731 if (!ops)
732 return ERR_PTR(-EINVAL);
733
734 rtk = kzalloc(sizeof(*rtk), GFP_KERNEL);
735 if (!rtk)
736 return ERR_PTR(-ENOMEM);
737
738 rtk->dev = dev;
739 rtk->cookie = cookie;
740 rtk->ops = ops;
741
742 init_completion(&rtk->epmap_completion);
743 init_completion(&rtk->iop_pwr_ack_completion);
744 init_completion(&rtk->ap_pwr_ack_completion);
745
746 bitmap_zero(rtk->endpoints, APPLE_RTKIT_MAX_ENDPOINTS);
747 set_bit(APPLE_RTKIT_EP_MGMT, rtk->endpoints);
748
749 rtk->mbox_name = mbox_name;
750 rtk->mbox_idx = mbox_idx;
751 rtk->mbox_cl.dev = dev;
752 rtk->mbox_cl.tx_block = false;
753 rtk->mbox_cl.knows_txdone = false;
754 rtk->mbox_cl.rx_callback = &apple_rtkit_rx;
755 rtk->mbox_cl.tx_done = &apple_rtkit_tx_done;
756
757 rtk->wq = alloc_ordered_workqueue("rtkit-%s", WQ_HIGHPRI | WQ_MEM_RECLAIM,
758 dev_name(rtk->dev));
759 if (!rtk->wq) {
760 ret = -ENOMEM;
761 goto free_rtk;
762 }
763
764 ret = apple_rtkit_request_mbox_chan(rtk);
765 if (ret)
766 goto destroy_wq;
767
768 return rtk;
769
770 destroy_wq:
771 destroy_workqueue(rtk->wq);
772 free_rtk:
773 kfree(rtk);
774 return ERR_PTR(ret);
775 }
776 EXPORT_SYMBOL_GPL(apple_rtkit_init);
777
apple_rtkit_wait_for_completion(struct completion * c)778 static int apple_rtkit_wait_for_completion(struct completion *c)
779 {
780 long t;
781
782 t = wait_for_completion_interruptible_timeout(c,
783 msecs_to_jiffies(1000));
784 if (t < 0)
785 return t;
786 else if (t == 0)
787 return -ETIME;
788 else
789 return 0;
790 }
791
apple_rtkit_reinit(struct apple_rtkit * rtk)792 int apple_rtkit_reinit(struct apple_rtkit *rtk)
793 {
794 /* make sure we don't handle any messages while reinitializing */
795 mbox_free_channel(rtk->mbox_chan);
796 flush_workqueue(rtk->wq);
797
798 apple_rtkit_free_buffer(rtk, &rtk->ioreport_buffer);
799 apple_rtkit_free_buffer(rtk, &rtk->crashlog_buffer);
800 apple_rtkit_free_buffer(rtk, &rtk->oslog_buffer);
801 apple_rtkit_free_buffer(rtk, &rtk->syslog_buffer);
802
803 kfree(rtk->syslog_msg_buffer);
804
805 rtk->syslog_msg_buffer = NULL;
806 rtk->syslog_n_entries = 0;
807 rtk->syslog_msg_size = 0;
808
809 bitmap_zero(rtk->endpoints, APPLE_RTKIT_MAX_ENDPOINTS);
810 set_bit(APPLE_RTKIT_EP_MGMT, rtk->endpoints);
811
812 reinit_completion(&rtk->epmap_completion);
813 reinit_completion(&rtk->iop_pwr_ack_completion);
814 reinit_completion(&rtk->ap_pwr_ack_completion);
815
816 rtk->crashed = false;
817 rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_OFF;
818 rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_OFF;
819
820 return apple_rtkit_request_mbox_chan(rtk);
821 }
822 EXPORT_SYMBOL_GPL(apple_rtkit_reinit);
823
apple_rtkit_set_ap_power_state(struct apple_rtkit * rtk,unsigned int state)824 static int apple_rtkit_set_ap_power_state(struct apple_rtkit *rtk,
825 unsigned int state)
826 {
827 u64 msg;
828 int ret;
829
830 reinit_completion(&rtk->ap_pwr_ack_completion);
831
832 msg = FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, state);
833 apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE,
834 msg);
835
836 ret = apple_rtkit_wait_for_completion(&rtk->ap_pwr_ack_completion);
837 if (ret)
838 return ret;
839
840 if (rtk->ap_power_state != state)
841 return -EINVAL;
842 return 0;
843 }
844
apple_rtkit_set_iop_power_state(struct apple_rtkit * rtk,unsigned int state)845 static int apple_rtkit_set_iop_power_state(struct apple_rtkit *rtk,
846 unsigned int state)
847 {
848 u64 msg;
849 int ret;
850
851 reinit_completion(&rtk->iop_pwr_ack_completion);
852
853 msg = FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, state);
854 apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE,
855 msg);
856
857 ret = apple_rtkit_wait_for_completion(&rtk->iop_pwr_ack_completion);
858 if (ret)
859 return ret;
860
861 if (rtk->iop_power_state != state)
862 return -EINVAL;
863 return 0;
864 }
865
apple_rtkit_boot(struct apple_rtkit * rtk)866 int apple_rtkit_boot(struct apple_rtkit *rtk)
867 {
868 int ret;
869
870 if (apple_rtkit_is_running(rtk))
871 return 0;
872 if (rtk->crashed)
873 return -EINVAL;
874
875 dev_dbg(rtk->dev, "RTKit: waiting for boot to finish\n");
876 ret = apple_rtkit_wait_for_completion(&rtk->epmap_completion);
877 if (ret)
878 return ret;
879 if (rtk->boot_result)
880 return rtk->boot_result;
881
882 dev_dbg(rtk->dev, "RTKit: waiting for IOP power state ACK\n");
883 ret = apple_rtkit_wait_for_completion(&rtk->iop_pwr_ack_completion);
884 if (ret)
885 return ret;
886
887 return apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON);
888 }
889 EXPORT_SYMBOL_GPL(apple_rtkit_boot);
890
apple_rtkit_shutdown(struct apple_rtkit * rtk)891 int apple_rtkit_shutdown(struct apple_rtkit *rtk)
892 {
893 int ret;
894
895 /* if OFF is used here the co-processor will not wake up again */
896 ret = apple_rtkit_set_ap_power_state(rtk,
897 APPLE_RTKIT_PWR_STATE_QUIESCED);
898 if (ret)
899 return ret;
900
901 ret = apple_rtkit_set_iop_power_state(rtk, APPLE_RTKIT_PWR_STATE_SLEEP);
902 if (ret)
903 return ret;
904
905 return apple_rtkit_reinit(rtk);
906 }
907 EXPORT_SYMBOL_GPL(apple_rtkit_shutdown);
908
apple_rtkit_idle(struct apple_rtkit * rtk)909 int apple_rtkit_idle(struct apple_rtkit *rtk)
910 {
911 int ret;
912
913 /* if OFF is used here the co-processor will not wake up again */
914 ret = apple_rtkit_set_ap_power_state(rtk,
915 APPLE_RTKIT_PWR_STATE_IDLE);
916 if (ret)
917 return ret;
918
919 ret = apple_rtkit_set_iop_power_state(rtk, APPLE_RTKIT_PWR_STATE_IDLE);
920 if (ret)
921 return ret;
922
923 rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_IDLE;
924 rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_IDLE;
925 return 0;
926 }
927 EXPORT_SYMBOL_GPL(apple_rtkit_idle);
928
apple_rtkit_quiesce(struct apple_rtkit * rtk)929 int apple_rtkit_quiesce(struct apple_rtkit *rtk)
930 {
931 int ret;
932
933 ret = apple_rtkit_set_ap_power_state(rtk,
934 APPLE_RTKIT_PWR_STATE_QUIESCED);
935 if (ret)
936 return ret;
937
938 ret = apple_rtkit_set_iop_power_state(rtk,
939 APPLE_RTKIT_PWR_STATE_QUIESCED);
940 if (ret)
941 return ret;
942
943 ret = apple_rtkit_reinit(rtk);
944 if (ret)
945 return ret;
946
947 rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_QUIESCED;
948 rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_QUIESCED;
949 return 0;
950 }
951 EXPORT_SYMBOL_GPL(apple_rtkit_quiesce);
952
apple_rtkit_wake(struct apple_rtkit * rtk)953 int apple_rtkit_wake(struct apple_rtkit *rtk)
954 {
955 u64 msg;
956
957 if (apple_rtkit_is_running(rtk))
958 return -EINVAL;
959
960 reinit_completion(&rtk->iop_pwr_ack_completion);
961
962 /*
963 * Use open-coded apple_rtkit_set_iop_power_state since apple_rtkit_boot
964 * will wait for the completion anyway.
965 */
966 msg = FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, APPLE_RTKIT_PWR_STATE_ON);
967 apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE,
968 msg);
969
970 return apple_rtkit_boot(rtk);
971 }
972 EXPORT_SYMBOL_GPL(apple_rtkit_wake);
973
apple_rtkit_free(struct apple_rtkit * rtk)974 void apple_rtkit_free(struct apple_rtkit *rtk)
975 {
976 mbox_free_channel(rtk->mbox_chan);
977 destroy_workqueue(rtk->wq);
978
979 apple_rtkit_free_buffer(rtk, &rtk->ioreport_buffer);
980 apple_rtkit_free_buffer(rtk, &rtk->crashlog_buffer);
981 apple_rtkit_free_buffer(rtk, &rtk->oslog_buffer);
982 apple_rtkit_free_buffer(rtk, &rtk->syslog_buffer);
983
984 kfree(rtk->syslog_msg_buffer);
985 kfree(rtk);
986 }
987 EXPORT_SYMBOL_GPL(apple_rtkit_free);
988
apple_rtkit_free_wrapper(void * data)989 static void apple_rtkit_free_wrapper(void *data)
990 {
991 apple_rtkit_free(data);
992 }
993
devm_apple_rtkit_init(struct device * dev,void * cookie,const char * mbox_name,int mbox_idx,const struct apple_rtkit_ops * ops)994 struct apple_rtkit *devm_apple_rtkit_init(struct device *dev, void *cookie,
995 const char *mbox_name, int mbox_idx,
996 const struct apple_rtkit_ops *ops)
997 {
998 struct apple_rtkit *rtk;
999 int ret;
1000
1001 rtk = apple_rtkit_init(dev, cookie, mbox_name, mbox_idx, ops);
1002 if (IS_ERR(rtk))
1003 return rtk;
1004
1005 ret = devm_add_action_or_reset(dev, apple_rtkit_free_wrapper, rtk);
1006 if (ret)
1007 return ERR_PTR(ret);
1008
1009 return rtk;
1010 }
1011 EXPORT_SYMBOL_GPL(devm_apple_rtkit_init);
1012
1013 MODULE_LICENSE("Dual MIT/GPL");
1014 MODULE_AUTHOR("Sven Peter <sven@svenpeter.dev>");
1015 MODULE_DESCRIPTION("Apple RTKit driver");
1016