1 // SPDX-License-Identifier: GPL-2.0-only OR MIT
2 /*
3  * Bluetooth HCI driver for Broadcom 4377/4378/4387 devices attached via PCIe
4  *
5  * Copyright (C) The Asahi Linux Contributors
6  */
7 
8 #include <linux/async.h>
9 #include <linux/bitfield.h>
10 #include <linux/completion.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/dmi.h>
13 #include <linux/firmware.h>
14 #include <linux/module.h>
15 #include <linux/msi.h>
16 #include <linux/of.h>
17 #include <linux/pci.h>
18 #include <linux/printk.h>
19 
20 #include <asm/unaligned.h>
21 
22 #include <net/bluetooth/bluetooth.h>
23 #include <net/bluetooth/hci_core.h>
24 
25 enum bcm4377_chip {
26 	BCM4377 = 0,
27 	BCM4378,
28 	BCM4387,
29 };
30 
31 #define BCM4377_DEVICE_ID 0x5fa0
32 #define BCM4378_DEVICE_ID 0x5f69
33 #define BCM4387_DEVICE_ID 0x5f71
34 
35 #define BCM4377_TIMEOUT 1000
36 
37 /*
38  * These devices only support DMA transactions inside a 32bit window
39  * (possibly to avoid 64 bit arithmetic). The window size cannot exceed
40  * 0xffffffff but is always aligned down to the previous 0x200 byte boundary
41  * which effectively limits the window to [start, start+0xfffffe00].
42  * We just limit the DMA window to [0, 0xfffffe00] to make sure we don't
43  * run into this limitation.
44  */
45 #define BCM4377_DMA_MASK 0xfffffe00
46 
47 #define BCM4377_PCIECFG_BAR0_WINDOW1	   0x80
48 #define BCM4377_PCIECFG_BAR0_WINDOW2	   0x70
49 #define BCM4377_PCIECFG_BAR0_CORE2_WINDOW1 0x74
50 #define BCM4377_PCIECFG_BAR0_CORE2_WINDOW2 0x78
51 #define BCM4377_PCIECFG_BAR2_WINDOW	   0x84
52 
53 #define BCM4377_PCIECFG_BAR0_CORE2_WINDOW1_DEFAULT 0x18011000
54 #define BCM4377_PCIECFG_BAR2_WINDOW_DEFAULT	   0x19000000
55 
56 #define BCM4377_PCIECFG_SUBSYSTEM_CTRL 0x88
57 
58 #define BCM4377_BAR0_FW_DOORBELL 0x140
59 #define BCM4377_BAR0_RTI_CONTROL 0x144
60 
61 #define BCM4377_BAR0_SLEEP_CONTROL	      0x150
62 #define BCM4377_BAR0_SLEEP_CONTROL_UNQUIESCE  0
63 #define BCM4377_BAR0_SLEEP_CONTROL_AWAKE      2
64 #define BCM4377_BAR0_SLEEP_CONTROL_QUIESCE    3
65 
66 #define BCM4377_BAR0_DOORBELL	    0x174
67 #define BCM4377_BAR0_DOORBELL_VALUE GENMASK(31, 16)
68 #define BCM4377_BAR0_DOORBELL_IDX   GENMASK(15, 8)
69 #define BCM4377_BAR0_DOORBELL_RING  BIT(5)
70 
71 #define BCM4377_BAR0_HOST_WINDOW_LO   0x590
72 #define BCM4377_BAR0_HOST_WINDOW_HI   0x594
73 #define BCM4377_BAR0_HOST_WINDOW_SIZE 0x598
74 
75 #define BCM4377_BAR2_BOOTSTAGE 0x200454
76 
77 #define BCM4377_BAR2_FW_LO   0x200478
78 #define BCM4377_BAR2_FW_HI   0x20047c
79 #define BCM4377_BAR2_FW_SIZE 0x200480
80 
81 #define BCM4377_BAR2_CONTEXT_ADDR_LO 0x20048c
82 #define BCM4377_BAR2_CONTEXT_ADDR_HI 0x200450
83 
84 #define BCM4377_BAR2_RTI_STATUS	     0x20045c
85 #define BCM4377_BAR2_RTI_WINDOW_LO   0x200494
86 #define BCM4377_BAR2_RTI_WINDOW_HI   0x200498
87 #define BCM4377_BAR2_RTI_WINDOW_SIZE 0x20049c
88 
89 #define BCM4377_OTP_SIZE	  0xe0
90 #define BCM4377_OTP_SYS_VENDOR	  0x15
91 #define BCM4377_OTP_CIS		  0x80
92 #define BCM4377_OTP_VENDOR_HDR	  0x00000008
93 #define BCM4377_OTP_MAX_PARAM_LEN 16
94 
95 #define BCM4377_N_TRANSFER_RINGS   9
96 #define BCM4377_N_COMPLETION_RINGS 6
97 
98 #define BCM4377_MAX_RING_SIZE 256
99 
100 #define BCM4377_MSGID_GENERATION GENMASK(15, 8)
101 #define BCM4377_MSGID_ID	 GENMASK(7, 0)
102 
103 #define BCM4377_RING_N_ENTRIES 128
104 
105 #define BCM4377_CONTROL_MSG_SIZE		   0x34
106 #define BCM4377_XFER_RING_MAX_INPLACE_PAYLOAD_SIZE (4 * 0xff)
107 
108 #define MAX_ACL_PAYLOAD_SIZE   (HCI_MAX_FRAME_SIZE + HCI_ACL_HDR_SIZE)
109 #define MAX_SCO_PAYLOAD_SIZE   (HCI_MAX_SCO_SIZE + HCI_SCO_HDR_SIZE)
110 #define MAX_EVENT_PAYLOAD_SIZE (HCI_MAX_EVENT_SIZE + HCI_EVENT_HDR_SIZE)
111 
112 enum bcm4377_otp_params_type {
113 	BCM4377_OTP_BOARD_PARAMS,
114 	BCM4377_OTP_CHIP_PARAMS
115 };
116 
117 enum bcm4377_transfer_ring_id {
118 	BCM4377_XFER_RING_CONTROL = 0,
119 	BCM4377_XFER_RING_HCI_H2D = 1,
120 	BCM4377_XFER_RING_HCI_D2H = 2,
121 	BCM4377_XFER_RING_SCO_H2D = 3,
122 	BCM4377_XFER_RING_SCO_D2H = 4,
123 	BCM4377_XFER_RING_ACL_H2D = 5,
124 	BCM4377_XFER_RING_ACL_D2H = 6,
125 };
126 
127 enum bcm4377_completion_ring_id {
128 	BCM4377_ACK_RING_CONTROL = 0,
129 	BCM4377_ACK_RING_HCI_ACL = 1,
130 	BCM4377_EVENT_RING_HCI_ACL = 2,
131 	BCM4377_ACK_RING_SCO = 3,
132 	BCM4377_EVENT_RING_SCO = 4,
133 };
134 
135 enum bcm4377_doorbell {
136 	BCM4377_DOORBELL_CONTROL = 0,
137 	BCM4377_DOORBELL_HCI_H2D = 1,
138 	BCM4377_DOORBELL_HCI_D2H = 2,
139 	BCM4377_DOORBELL_ACL_H2D = 3,
140 	BCM4377_DOORBELL_ACL_D2H = 4,
141 	BCM4377_DOORBELL_SCO = 6,
142 };
143 
144 /*
145  * Transfer ring entry
146  *
147  * flags: Flags to indicate if the payload is appended or mapped
148  * len: Payload length
149  * payload: Optional payload DMA address
150  * id: Message id to recognize the answer in the completion ring entry
151  */
152 struct bcm4377_xfer_ring_entry {
153 #define BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED	 BIT(0)
154 #define BCM4377_XFER_RING_FLAG_PAYLOAD_IN_FOOTER BIT(1)
155 	u8 flags;
156 	__le16 len;
157 	u8 _unk0;
158 	__le64 payload;
159 	__le16 id;
160 	u8 _unk1[2];
161 } __packed;
162 static_assert(sizeof(struct bcm4377_xfer_ring_entry) == 0x10);
163 
164 /*
165  * Completion ring entry
166  *
167  * flags: Flags to indicate if the payload is appended or mapped. If the payload
168  *        is mapped it can be found in the buffer of the corresponding transfer
169  *        ring message.
170  * ring_id: Transfer ring ID which required this message
171  * msg_id: Message ID specified in transfer ring entry
172  * len: Payload length
173  */
174 struct bcm4377_completion_ring_entry {
175 	u8 flags;
176 	u8 _unk0;
177 	__le16 ring_id;
178 	__le16 msg_id;
179 	__le32 len;
180 	u8 _unk1[6];
181 } __packed;
182 static_assert(sizeof(struct bcm4377_completion_ring_entry) == 0x10);
183 
184 enum bcm4377_control_message_type {
185 	BCM4377_CONTROL_MSG_CREATE_XFER_RING = 1,
186 	BCM4377_CONTROL_MSG_CREATE_COMPLETION_RING = 2,
187 	BCM4377_CONTROL_MSG_DESTROY_XFER_RING = 3,
188 	BCM4377_CONTROL_MSG_DESTROY_COMPLETION_RING = 4,
189 };
190 
191 /*
192  * Control message used to create a completion ring
193  *
194  * msg_type: Must be BCM4377_CONTROL_MSG_CREATE_COMPLETION_RING
195  * header_size: Unknown, but probably reserved space in front of the entry
196  * footer_size: Number of 32 bit words reserved for payloads after the entry
197  * id/id_again: Completion ring index
198  * ring_iova: DMA address of the ring buffer
199  * n_elements: Number of elements inside the ring buffer
200  * msi: MSI index, doesn't work for all rings though and should be zero
201  * intmod_delay: Unknown delay
202  * intmod_bytes: Unknown
203  */
204 struct bcm4377_create_completion_ring_msg {
205 	u8 msg_type;
206 	u8 header_size;
207 	u8 footer_size;
208 	u8 _unk0;
209 	__le16 id;
210 	__le16 id_again;
211 	__le64 ring_iova;
212 	__le16 n_elements;
213 	__le32 unk;
214 	u8 _unk1[6];
215 	__le16 msi;
216 	__le16 intmod_delay;
217 	__le32 intmod_bytes;
218 	__le16 _unk2;
219 	__le32 _unk3;
220 	u8 _unk4[10];
221 } __packed;
222 static_assert(sizeof(struct bcm4377_create_completion_ring_msg) ==
223 	      BCM4377_CONTROL_MSG_SIZE);
224 
225 /*
226  * Control ring message used to destroy a completion ring
227  *
228  * msg_type: Must be BCM4377_CONTROL_MSG_DESTROY_COMPLETION_RING
229  * ring_id: Completion ring to be destroyed
230  */
231 struct bcm4377_destroy_completion_ring_msg {
232 	u8 msg_type;
233 	u8 _pad0;
234 	__le16 ring_id;
235 	u8 _pad1[48];
236 } __packed;
237 static_assert(sizeof(struct bcm4377_destroy_completion_ring_msg) ==
238 	      BCM4377_CONTROL_MSG_SIZE);
239 
240 /*
241  * Control message used to create a transfer ring
242  *
243  * msg_type: Must be BCM4377_CONTROL_MSG_CREATE_XFER_RING
244  * header_size: Number of 32 bit words reserved for unknown content before the
245  *              entry
246  * footer_size: Number of 32 bit words reserved for payloads after the entry
247  * ring_id/ring_id_again: Transfer ring index
248  * ring_iova: DMA address of the ring buffer
249  * n_elements: Number of elements inside the ring buffer
250  * completion_ring_id: Completion ring index for acknowledgements and events
251  * doorbell: Doorbell index used to notify device of new entries
252  * flags: Transfer ring flags
253  *          - virtual: set if there is no associated shared memory and only the
254  *                     corresponding completion ring is used
255  *          - sync: only set for the SCO rings
256  */
257 struct bcm4377_create_transfer_ring_msg {
258 	u8 msg_type;
259 	u8 header_size;
260 	u8 footer_size;
261 	u8 _unk0;
262 	__le16 ring_id;
263 	__le16 ring_id_again;
264 	__le64 ring_iova;
265 	u8 _unk1[8];
266 	__le16 n_elements;
267 	__le16 completion_ring_id;
268 	__le16 doorbell;
269 #define BCM4377_XFER_RING_FLAG_VIRTUAL BIT(7)
270 #define BCM4377_XFER_RING_FLAG_SYNC    BIT(8)
271 	__le16 flags;
272 	u8 _unk2[20];
273 } __packed;
274 static_assert(sizeof(struct bcm4377_create_transfer_ring_msg) ==
275 	      BCM4377_CONTROL_MSG_SIZE);
276 
277 /*
278  * Control ring message used to destroy a transfer ring
279  *
280  * msg_type: Must be BCM4377_CONTROL_MSG_DESTROY_XFER_RING
281  * ring_id: Transfer ring to be destroyed
282  */
283 struct bcm4377_destroy_transfer_ring_msg {
284 	u8 msg_type;
285 	u8 _pad0;
286 	__le16 ring_id;
287 	u8 _pad1[48];
288 } __packed;
289 static_assert(sizeof(struct bcm4377_destroy_transfer_ring_msg) ==
290 	      BCM4377_CONTROL_MSG_SIZE);
291 
292 /*
293  * "Converged IPC" context struct used to make the device aware of all other
294  * shared memory structures. A pointer to this structure is configured inside a
295  * MMIO register.
296  *
297  * version: Protocol version, must be 2.
298  * size: Size of this structure, must be 0x68.
299  * enabled_caps: Enabled capabilities. Unknown bitfield but should be 2.
300  * peripheral_info_addr: DMA address for a 0x20 buffer to which the device will
301  *                       write unknown contents
302  * {completion,xfer}_ring_{tails,heads}_addr: DMA pointers to ring heads/tails
303  * n_completion_rings: Number of completion rings, the firmware only works if
304  *                     this is set to BCM4377_N_COMPLETION_RINGS.
305  * n_xfer_rings: Number of transfer rings, the firmware only works if
306  *               this is set to BCM4377_N_TRANSFER_RINGS.
307  * control_completion_ring_addr: Control completion ring buffer DMA address
308  * control_xfer_ring_addr: Control transfer ring buffer DMA address
309  * control_xfer_ring_n_entries: Number of control transfer ring entries
310  * control_completion_ring_n_entries: Number of control completion ring entries
311  * control_xfer_ring_doorbell: Control transfer ring doorbell
312  * control_completion_ring_doorbell: Control completion ring doorbell,
313  *                                   must be set to 0xffff
314  * control_xfer_ring_msi: Control completion ring MSI index, must be 0
315  * control_completion_ring_msi: Control completion ring MSI index, must be 0.
316  * control_xfer_ring_header_size: Number of 32 bit words reserved in front of
317  *                                every control transfer ring entry
318  * control_xfer_ring_footer_size: Number of 32 bit words reserved after every
319  *                                control transfer ring entry
320  * control_completion_ring_header_size: Number of 32 bit words reserved in front
321  *                                      of every control completion ring entry
322  * control_completion_ring_footer_size: Number of 32 bit words reserved after
323  *                                      every control completion ring entry
324  * scratch_pad: Optional scratch pad DMA address
325  * scratch_pad_size: Scratch pad size
326  */
327 struct bcm4377_context {
328 	__le16 version;
329 	__le16 size;
330 	__le32 enabled_caps;
331 
332 	__le64 peripheral_info_addr;
333 
334 	/* ring heads and tails */
335 	__le64 completion_ring_heads_addr;
336 	__le64 xfer_ring_tails_addr;
337 	__le64 completion_ring_tails_addr;
338 	__le64 xfer_ring_heads_addr;
339 	__le16 n_completion_rings;
340 	__le16 n_xfer_rings;
341 
342 	/* control ring configuration */
343 	__le64 control_completion_ring_addr;
344 	__le64 control_xfer_ring_addr;
345 	__le16 control_xfer_ring_n_entries;
346 	__le16 control_completion_ring_n_entries;
347 	__le16 control_xfer_ring_doorbell;
348 	__le16 control_completion_ring_doorbell;
349 	__le16 control_xfer_ring_msi;
350 	__le16 control_completion_ring_msi;
351 	u8 control_xfer_ring_header_size;
352 	u8 control_xfer_ring_footer_size;
353 	u8 control_completion_ring_header_size;
354 	u8 control_completion_ring_footer_size;
355 
356 	__le16 _unk0;
357 	__le16 _unk1;
358 
359 	__le64 scratch_pad;
360 	__le32 scratch_pad_size;
361 
362 	__le32 _unk3;
363 } __packed;
364 static_assert(sizeof(struct bcm4377_context) == 0x68);
365 
366 #define BCM4378_CALIBRATION_CHUNK_SIZE 0xe6
367 struct bcm4378_hci_send_calibration_cmd {
368 	u8 unk;
369 	__le16 blocks_left;
370 	u8 data[BCM4378_CALIBRATION_CHUNK_SIZE];
371 } __packed;
372 
373 #define BCM4378_PTB_CHUNK_SIZE 0xcf
374 struct bcm4378_hci_send_ptb_cmd {
375 	__le16 blocks_left;
376 	u8 data[BCM4378_PTB_CHUNK_SIZE];
377 } __packed;
378 
379 /*
380  * Shared memory structure used to store the ring head and tail pointers.
381  */
382 struct bcm4377_ring_state {
383 	__le16 completion_ring_head[BCM4377_N_COMPLETION_RINGS];
384 	__le16 completion_ring_tail[BCM4377_N_COMPLETION_RINGS];
385 	__le16 xfer_ring_head[BCM4377_N_TRANSFER_RINGS];
386 	__le16 xfer_ring_tail[BCM4377_N_TRANSFER_RINGS];
387 };
388 
389 /*
390  * A transfer ring can be used in two configurations:
391  *  1) Send control or HCI messages to the device which are then acknowledged
392  *     in the corresponding completion ring
393  *  2) Receiving HCI frames from the devices. In this case the transfer ring
394  *     itself contains empty messages that are acknowledged once data is
395  *     available from the device. If the payloads fit inside the footers
396  *     of the completion ring the transfer ring can be configured to be
397  *     virtual such that it has no ring buffer.
398  *
399  * ring_id: ring index hardcoded in the firmware
400  * doorbell: doorbell index to notify device of new entries
401  * payload_size: optional in-place payload size
402  * mapped_payload_size: optional out-of-place payload size
403  * completion_ring: index of corresponding completion ring
404  * n_entries: number of entries inside this ring
405  * generation: ring generation; incremented on hci_open to detect stale messages
406  * sync: set to true for SCO rings
407  * virtual: set to true if this ring has no entries and is just required to
408  *          setup a corresponding completion ring for device->host messages
409  * d2h_buffers_only: set to true if this ring is only used to provide large
410  *                   buffers used by device->host messages in the completion
411  *                   ring
412  * allow_wait: allow to wait for messages to be acknowledged
413  * enabled: true once the ring has been created and can be used
414  * ring: ring buffer for entries (struct bcm4377_xfer_ring_entry)
415  * ring_dma: DMA address for ring entry buffer
416  * payloads: payload buffer for mapped_payload_size payloads
417  * payloads_dma:DMA address for payload buffer
418  * events: pointer to array of completions if waiting is allowed
419  * msgids: bitmap to keep track of used message ids
420  * lock: Spinlock to protect access to ring structurs used in the irq handler
421  */
422 struct bcm4377_transfer_ring {
423 	enum bcm4377_transfer_ring_id ring_id;
424 	enum bcm4377_doorbell doorbell;
425 	size_t payload_size;
426 	size_t mapped_payload_size;
427 	u8 completion_ring;
428 	u16 n_entries;
429 	u8 generation;
430 
431 	bool sync;
432 	bool virtual;
433 	bool d2h_buffers_only;
434 	bool allow_wait;
435 	bool enabled;
436 
437 	void *ring;
438 	dma_addr_t ring_dma;
439 
440 	void *payloads;
441 	dma_addr_t payloads_dma;
442 
443 	struct completion **events;
444 	DECLARE_BITMAP(msgids, BCM4377_MAX_RING_SIZE);
445 	spinlock_t lock;
446 };
447 
448 /*
449  * A completion ring can be either used to either acknowledge messages sent in
450  * the corresponding transfer ring or to receive messages associated with the
451  * transfer ring. When used to receive messages the transfer ring either
452  * has no ring buffer and is only advanced ("virtual transfer ring") or it
453  * only contains empty DMA buffers to be used for the payloads.
454  *
455  * ring_id: completion ring id, hardcoded in firmware
456  * payload_size: optional payload size after each entry
457  * delay: unknown delay
458  * n_entries: number of entries in this ring
459  * enabled: true once the ring has been created and can be used
460  * ring: ring buffer for entries (struct bcm4377_completion_ring_entry)
461  * ring_dma: DMA address of ring buffer
462  * transfer_rings: bitmap of corresponding transfer ring ids
463  */
464 struct bcm4377_completion_ring {
465 	enum bcm4377_completion_ring_id ring_id;
466 	u16 payload_size;
467 	u16 delay;
468 	u16 n_entries;
469 	bool enabled;
470 
471 	void *ring;
472 	dma_addr_t ring_dma;
473 
474 	unsigned long transfer_rings;
475 };
476 
477 struct bcm4377_data;
478 
479 /*
480  * Chip-specific configuration struct
481  *
482  * id: Chip id (e.g. 0x4377 for BCM4377)
483  * otp_offset: Offset to the start of the OTP inside BAR0
484  * bar0_window1: Backplane address mapped to the first window in BAR0
485  * bar0_window2: Backplane address mapped to the second window in BAR0
486  * bar0_core2_window2: Optional backplane address mapped to the second core's
487  *                     second window in BAR0
488  * has_bar0_core2_window2: Set to true if this chip requires the second core's
489  *                         second window to be configured
490  * clear_pciecfg_subsystem_ctrl_bit19: Set to true if bit 19 in the
491  *                                     vendor-specific subsystem control
492  *                                     register has to be cleared
493  * disable_aspm: Set to true if ASPM must be disabled due to hardware errata
494  * broken_ext_scan: Set to true if the chip erroneously claims to support
495  *                  extended scanning
496  * broken_mws_transport_config: Set to true if the chip erroneously claims to
497  *                              support MWS Transport Configuration
498  * send_calibration: Optional callback to send calibration data
499  * send_ptb: Callback to send "PTB" regulatory/calibration data
500  */
501 struct bcm4377_hw {
502 	unsigned int id;
503 
504 	u32 otp_offset;
505 
506 	u32 bar0_window1;
507 	u32 bar0_window2;
508 	u32 bar0_core2_window2;
509 
510 	unsigned long has_bar0_core2_window2 : 1;
511 	unsigned long clear_pciecfg_subsystem_ctrl_bit19 : 1;
512 	unsigned long disable_aspm : 1;
513 	unsigned long broken_ext_scan : 1;
514 	unsigned long broken_mws_transport_config : 1;
515 	unsigned long broken_le_coded : 1;
516 
517 	int (*send_calibration)(struct bcm4377_data *bcm4377);
518 	int (*send_ptb)(struct bcm4377_data *bcm4377,
519 			const struct firmware *fw);
520 };
521 
522 static const struct bcm4377_hw bcm4377_hw_variants[];
523 static const struct dmi_system_id bcm4377_dmi_board_table[];
524 
525 /*
526  * Private struct associated with each device containing global state
527  *
528  * pdev: Pointer to associated struct pci_dev
529  * hdev: Pointer to associated strucy hci_dev
530  * bar0: iomem pointing to BAR0
531  * bar1: iomem pointing to BAR2
532  * bootstage: Current value of the bootstage
533  * rti_status: Current "RTI" status value
534  * hw: Pointer to chip-specific struct bcm4377_hw
535  * taurus_cal_blob: "Taurus" calibration blob used for some chips
536  * taurus_cal_size: "Taurus" calibration blob size
537  * taurus_beamforming_cal_blob: "Taurus" beamforming calibration blob used for
538  *                              some chips
539  * taurus_beamforming_cal_size: "Taurus" beamforming calibration blob size
540  * stepping: Chip stepping read from OTP; used for firmware selection
541  * vendor: Antenna vendor read from OTP; used for firmware selection
542  * board_type: Board type from FDT or DMI match; used for firmware selection
543  * event: Event for changed bootstage or rti_status; used for booting firmware
544  * ctx: "Converged IPC" context
545  * ctx_dma: "Converged IPC" context DMA address
546  * ring_state: Shared memory buffer containing ring head and tail indexes
547  * ring_state_dma: DMA address for ring_state
548  * {control,hci_acl,sco}_ack_ring: Completion rings used to acknowledge messages
549  * {hci_acl,sco}_event_ring: Completion rings used for device->host messages
550  * control_h2d_ring: Transfer ring used for control messages
551  * {hci,sco,acl}_h2d_ring: Transfer ring used to transfer HCI frames
552  * {hci,sco,acl}_d2h_ring: Transfer ring used to receive HCI frames in the
553  *                         corresponding completion ring
554  */
555 struct bcm4377_data {
556 	struct pci_dev *pdev;
557 	struct hci_dev *hdev;
558 
559 	void __iomem *bar0;
560 	void __iomem *bar2;
561 
562 	u32 bootstage;
563 	u32 rti_status;
564 
565 	const struct bcm4377_hw *hw;
566 
567 	const void *taurus_cal_blob;
568 	int taurus_cal_size;
569 	const void *taurus_beamforming_cal_blob;
570 	int taurus_beamforming_cal_size;
571 
572 	char stepping[BCM4377_OTP_MAX_PARAM_LEN];
573 	char vendor[BCM4377_OTP_MAX_PARAM_LEN];
574 	const char *board_type;
575 
576 	struct completion event;
577 
578 	struct bcm4377_context *ctx;
579 	dma_addr_t ctx_dma;
580 
581 	struct bcm4377_ring_state *ring_state;
582 	dma_addr_t ring_state_dma;
583 
584 	/*
585 	 * The HCI and ACL rings have to be merged because this structure is
586 	 * hardcoded in the firmware.
587 	 */
588 	struct bcm4377_completion_ring control_ack_ring;
589 	struct bcm4377_completion_ring hci_acl_ack_ring;
590 	struct bcm4377_completion_ring hci_acl_event_ring;
591 	struct bcm4377_completion_ring sco_ack_ring;
592 	struct bcm4377_completion_ring sco_event_ring;
593 
594 	struct bcm4377_transfer_ring control_h2d_ring;
595 	struct bcm4377_transfer_ring hci_h2d_ring;
596 	struct bcm4377_transfer_ring hci_d2h_ring;
597 	struct bcm4377_transfer_ring sco_h2d_ring;
598 	struct bcm4377_transfer_ring sco_d2h_ring;
599 	struct bcm4377_transfer_ring acl_h2d_ring;
600 	struct bcm4377_transfer_ring acl_d2h_ring;
601 };
602 
603 static void bcm4377_ring_doorbell(struct bcm4377_data *bcm4377, u8 doorbell,
604 				  u16 val)
605 {
606 	u32 db = 0;
607 
608 	db |= FIELD_PREP(BCM4377_BAR0_DOORBELL_VALUE, val);
609 	db |= FIELD_PREP(BCM4377_BAR0_DOORBELL_IDX, doorbell);
610 	db |= BCM4377_BAR0_DOORBELL_RING;
611 
612 	dev_dbg(&bcm4377->pdev->dev, "write %d to doorbell #%d (0x%x)\n", val,
613 		doorbell, db);
614 	iowrite32(db, bcm4377->bar0 + BCM4377_BAR0_DOORBELL);
615 }
616 
617 static int bcm4377_extract_msgid(struct bcm4377_data *bcm4377,
618 				 struct bcm4377_transfer_ring *ring,
619 				 u16 raw_msgid, u8 *msgid)
620 {
621 	u8 generation = FIELD_GET(BCM4377_MSGID_GENERATION, raw_msgid);
622 	*msgid = FIELD_GET(BCM4377_MSGID_ID, raw_msgid);
623 
624 	if (generation != ring->generation) {
625 		dev_warn(
626 			&bcm4377->pdev->dev,
627 			"invalid message generation %d should be %d in entry for ring %d\n",
628 			generation, ring->generation, ring->ring_id);
629 		return -EINVAL;
630 	}
631 
632 	if (*msgid >= ring->n_entries) {
633 		dev_warn(&bcm4377->pdev->dev,
634 			 "invalid message id in entry for ring %d: %d > %d\n",
635 			 ring->ring_id, *msgid, ring->n_entries);
636 		return -EINVAL;
637 	}
638 
639 	return 0;
640 }
641 
642 static void bcm4377_handle_event(struct bcm4377_data *bcm4377,
643 				 struct bcm4377_transfer_ring *ring,
644 				 u16 raw_msgid, u8 entry_flags, u8 type,
645 				 void *payload, size_t len)
646 {
647 	struct sk_buff *skb;
648 	u16 head;
649 	u8 msgid;
650 	unsigned long flags;
651 
652 	spin_lock_irqsave(&ring->lock, flags);
653 	if (!ring->enabled) {
654 		dev_warn(&bcm4377->pdev->dev,
655 			 "event for disabled transfer ring %d\n",
656 			 ring->ring_id);
657 		goto out;
658 	}
659 
660 	if (ring->d2h_buffers_only &&
661 	    entry_flags & BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED) {
662 		if (bcm4377_extract_msgid(bcm4377, ring, raw_msgid, &msgid))
663 			goto out;
664 
665 		if (len > ring->mapped_payload_size) {
666 			dev_warn(
667 				&bcm4377->pdev->dev,
668 				"invalid payload len in event for ring %d: %zu > %zu\n",
669 				ring->ring_id, len, ring->mapped_payload_size);
670 			goto out;
671 		}
672 
673 		payload = ring->payloads + msgid * ring->mapped_payload_size;
674 	}
675 
676 	skb = bt_skb_alloc(len, GFP_ATOMIC);
677 	if (!skb)
678 		goto out;
679 
680 	memcpy(skb_put(skb, len), payload, len);
681 	hci_skb_pkt_type(skb) = type;
682 	hci_recv_frame(bcm4377->hdev, skb);
683 
684 out:
685 	head = le16_to_cpu(bcm4377->ring_state->xfer_ring_head[ring->ring_id]);
686 	head = (head + 1) % ring->n_entries;
687 	bcm4377->ring_state->xfer_ring_head[ring->ring_id] = cpu_to_le16(head);
688 
689 	bcm4377_ring_doorbell(bcm4377, ring->doorbell, head);
690 
691 	spin_unlock_irqrestore(&ring->lock, flags);
692 }
693 
694 static void bcm4377_handle_ack(struct bcm4377_data *bcm4377,
695 			       struct bcm4377_transfer_ring *ring,
696 			       u16 raw_msgid)
697 {
698 	unsigned long flags;
699 	u8 msgid;
700 
701 	spin_lock_irqsave(&ring->lock, flags);
702 
703 	if (bcm4377_extract_msgid(bcm4377, ring, raw_msgid, &msgid))
704 		goto unlock;
705 
706 	if (!test_bit(msgid, ring->msgids)) {
707 		dev_warn(
708 			&bcm4377->pdev->dev,
709 			"invalid message id in ack for ring %d: %d is not used\n",
710 			ring->ring_id, msgid);
711 		goto unlock;
712 	}
713 
714 	if (ring->allow_wait && ring->events[msgid]) {
715 		complete(ring->events[msgid]);
716 		ring->events[msgid] = NULL;
717 	}
718 
719 	bitmap_release_region(ring->msgids, msgid, ring->n_entries);
720 
721 unlock:
722 	spin_unlock_irqrestore(&ring->lock, flags);
723 }
724 
725 static void bcm4377_handle_completion(struct bcm4377_data *bcm4377,
726 				      struct bcm4377_completion_ring *ring,
727 				      u16 pos)
728 {
729 	struct bcm4377_completion_ring_entry *entry;
730 	u16 msg_id, transfer_ring;
731 	size_t entry_size, data_len;
732 	void *data;
733 
734 	if (pos >= ring->n_entries) {
735 		dev_warn(&bcm4377->pdev->dev,
736 			 "invalid offset %d for completion ring %d\n", pos,
737 			 ring->ring_id);
738 		return;
739 	}
740 
741 	entry_size = sizeof(*entry) + ring->payload_size;
742 	entry = ring->ring + pos * entry_size;
743 	data = ring->ring + pos * entry_size + sizeof(*entry);
744 	data_len = le32_to_cpu(entry->len);
745 	msg_id = le16_to_cpu(entry->msg_id);
746 	transfer_ring = le16_to_cpu(entry->ring_id);
747 
748 	if ((ring->transfer_rings & BIT(transfer_ring)) == 0) {
749 		dev_warn(
750 			&bcm4377->pdev->dev,
751 			"invalid entry at offset %d for transfer ring %d in completion ring %d\n",
752 			pos, transfer_ring, ring->ring_id);
753 		return;
754 	}
755 
756 	dev_dbg(&bcm4377->pdev->dev,
757 		"entry in completion ring %d for transfer ring %d with msg_id %d\n",
758 		ring->ring_id, transfer_ring, msg_id);
759 
760 	switch (transfer_ring) {
761 	case BCM4377_XFER_RING_CONTROL:
762 		bcm4377_handle_ack(bcm4377, &bcm4377->control_h2d_ring, msg_id);
763 		break;
764 	case BCM4377_XFER_RING_HCI_H2D:
765 		bcm4377_handle_ack(bcm4377, &bcm4377->hci_h2d_ring, msg_id);
766 		break;
767 	case BCM4377_XFER_RING_SCO_H2D:
768 		bcm4377_handle_ack(bcm4377, &bcm4377->sco_h2d_ring, msg_id);
769 		break;
770 	case BCM4377_XFER_RING_ACL_H2D:
771 		bcm4377_handle_ack(bcm4377, &bcm4377->acl_h2d_ring, msg_id);
772 		break;
773 
774 	case BCM4377_XFER_RING_HCI_D2H:
775 		bcm4377_handle_event(bcm4377, &bcm4377->hci_d2h_ring, msg_id,
776 				     entry->flags, HCI_EVENT_PKT, data,
777 				     data_len);
778 		break;
779 	case BCM4377_XFER_RING_SCO_D2H:
780 		bcm4377_handle_event(bcm4377, &bcm4377->sco_d2h_ring, msg_id,
781 				     entry->flags, HCI_SCODATA_PKT, data,
782 				     data_len);
783 		break;
784 	case BCM4377_XFER_RING_ACL_D2H:
785 		bcm4377_handle_event(bcm4377, &bcm4377->acl_d2h_ring, msg_id,
786 				     entry->flags, HCI_ACLDATA_PKT, data,
787 				     data_len);
788 		break;
789 
790 	default:
791 		dev_warn(
792 			&bcm4377->pdev->dev,
793 			"entry in completion ring %d for unknown transfer ring %d with msg_id %d\n",
794 			ring->ring_id, transfer_ring, msg_id);
795 	}
796 }
797 
798 static void bcm4377_poll_completion_ring(struct bcm4377_data *bcm4377,
799 					 struct bcm4377_completion_ring *ring)
800 {
801 	u16 tail;
802 	__le16 *heads = bcm4377->ring_state->completion_ring_head;
803 	__le16 *tails = bcm4377->ring_state->completion_ring_tail;
804 
805 	if (!ring->enabled)
806 		return;
807 
808 	tail = le16_to_cpu(tails[ring->ring_id]);
809 	dev_dbg(&bcm4377->pdev->dev,
810 		"completion ring #%d: head: %d, tail: %d\n", ring->ring_id,
811 		le16_to_cpu(heads[ring->ring_id]), tail);
812 
813 	while (tail != le16_to_cpu(READ_ONCE(heads[ring->ring_id]))) {
814 		/*
815 		 * ensure the CPU doesn't speculate through the comparison.
816 		 * otherwise it might already read the (empty) queue entry
817 		 * before the updated head has been loaded and checked.
818 		 */
819 		dma_rmb();
820 
821 		bcm4377_handle_completion(bcm4377, ring, tail);
822 
823 		tail = (tail + 1) % ring->n_entries;
824 		tails[ring->ring_id] = cpu_to_le16(tail);
825 	}
826 }
827 
828 static irqreturn_t bcm4377_irq(int irq, void *data)
829 {
830 	struct bcm4377_data *bcm4377 = data;
831 	u32 bootstage, rti_status;
832 
833 	bootstage = ioread32(bcm4377->bar2 + BCM4377_BAR2_BOOTSTAGE);
834 	rti_status = ioread32(bcm4377->bar2 + BCM4377_BAR2_RTI_STATUS);
835 
836 	if (bootstage != bcm4377->bootstage ||
837 	    rti_status != bcm4377->rti_status) {
838 		dev_dbg(&bcm4377->pdev->dev,
839 			"bootstage = %d -> %d, rti state = %d -> %d\n",
840 			bcm4377->bootstage, bootstage, bcm4377->rti_status,
841 			rti_status);
842 		complete(&bcm4377->event);
843 		bcm4377->bootstage = bootstage;
844 		bcm4377->rti_status = rti_status;
845 	}
846 
847 	if (rti_status > 2)
848 		dev_err(&bcm4377->pdev->dev, "RTI status is %d\n", rti_status);
849 
850 	bcm4377_poll_completion_ring(bcm4377, &bcm4377->control_ack_ring);
851 	bcm4377_poll_completion_ring(bcm4377, &bcm4377->hci_acl_event_ring);
852 	bcm4377_poll_completion_ring(bcm4377, &bcm4377->hci_acl_ack_ring);
853 	bcm4377_poll_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
854 	bcm4377_poll_completion_ring(bcm4377, &bcm4377->sco_event_ring);
855 
856 	return IRQ_HANDLED;
857 }
858 
859 static int bcm4377_enqueue(struct bcm4377_data *bcm4377,
860 			   struct bcm4377_transfer_ring *ring, void *data,
861 			   size_t len, bool wait)
862 {
863 	unsigned long flags;
864 	struct bcm4377_xfer_ring_entry *entry;
865 	void *payload;
866 	size_t offset;
867 	u16 head, tail, new_head;
868 	u16 raw_msgid;
869 	int ret, msgid;
870 	DECLARE_COMPLETION_ONSTACK(event);
871 
872 	if (len > ring->payload_size && len > ring->mapped_payload_size) {
873 		dev_warn(
874 			&bcm4377->pdev->dev,
875 			"payload len %zu is too large for ring %d (max is %zu or %zu)\n",
876 			len, ring->ring_id, ring->payload_size,
877 			ring->mapped_payload_size);
878 		return -EINVAL;
879 	}
880 	if (wait && !ring->allow_wait)
881 		return -EINVAL;
882 	if (ring->virtual)
883 		return -EINVAL;
884 
885 	spin_lock_irqsave(&ring->lock, flags);
886 
887 	head = le16_to_cpu(bcm4377->ring_state->xfer_ring_head[ring->ring_id]);
888 	tail = le16_to_cpu(bcm4377->ring_state->xfer_ring_tail[ring->ring_id]);
889 
890 	new_head = (head + 1) % ring->n_entries;
891 
892 	if (new_head == tail) {
893 		dev_warn(&bcm4377->pdev->dev,
894 			 "can't send message because ring %d is full\n",
895 			 ring->ring_id);
896 		ret = -EINVAL;
897 		goto out;
898 	}
899 
900 	msgid = bitmap_find_free_region(ring->msgids, ring->n_entries, 0);
901 	if (msgid < 0) {
902 		dev_warn(&bcm4377->pdev->dev,
903 			 "can't find message id for ring %d\n", ring->ring_id);
904 		ret = -EINVAL;
905 		goto out;
906 	}
907 
908 	raw_msgid = FIELD_PREP(BCM4377_MSGID_GENERATION, ring->generation);
909 	raw_msgid |= FIELD_PREP(BCM4377_MSGID_ID, msgid);
910 
911 	offset = head * (sizeof(*entry) + ring->payload_size);
912 	entry = ring->ring + offset;
913 
914 	memset(entry, 0, sizeof(*entry));
915 	entry->id = cpu_to_le16(raw_msgid);
916 	entry->len = cpu_to_le16(len);
917 
918 	if (len <= ring->payload_size) {
919 		entry->flags = BCM4377_XFER_RING_FLAG_PAYLOAD_IN_FOOTER;
920 		payload = ring->ring + offset + sizeof(*entry);
921 	} else {
922 		entry->flags = BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED;
923 		entry->payload = cpu_to_le64(ring->payloads_dma +
924 					     msgid * ring->mapped_payload_size);
925 		payload = ring->payloads + msgid * ring->mapped_payload_size;
926 	}
927 
928 	memcpy(payload, data, len);
929 
930 	if (wait)
931 		ring->events[msgid] = &event;
932 
933 	/*
934 	 * The 4377 chips stop responding to any commands as soon as they
935 	 * have been idle for a while. Poking the sleep control register here
936 	 * makes them come alive again.
937 	 */
938 	iowrite32(BCM4377_BAR0_SLEEP_CONTROL_AWAKE,
939 		  bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL);
940 
941 	dev_dbg(&bcm4377->pdev->dev,
942 		"updating head for transfer queue #%d to %d\n", ring->ring_id,
943 		new_head);
944 	bcm4377->ring_state->xfer_ring_head[ring->ring_id] =
945 		cpu_to_le16(new_head);
946 
947 	if (!ring->sync)
948 		bcm4377_ring_doorbell(bcm4377, ring->doorbell, new_head);
949 	ret = 0;
950 
951 out:
952 	spin_unlock_irqrestore(&ring->lock, flags);
953 
954 	if (ret == 0 && wait) {
955 		ret = wait_for_completion_interruptible_timeout(
956 			&event, BCM4377_TIMEOUT);
957 		if (ret == 0)
958 			ret = -ETIMEDOUT;
959 		else if (ret > 0)
960 			ret = 0;
961 
962 		spin_lock_irqsave(&ring->lock, flags);
963 		ring->events[msgid] = NULL;
964 		spin_unlock_irqrestore(&ring->lock, flags);
965 	}
966 
967 	return ret;
968 }
969 
970 static int bcm4377_create_completion_ring(struct bcm4377_data *bcm4377,
971 					  struct bcm4377_completion_ring *ring)
972 {
973 	struct bcm4377_create_completion_ring_msg msg;
974 	int ret;
975 
976 	if (ring->enabled) {
977 		dev_warn(&bcm4377->pdev->dev,
978 			 "completion ring %d already enabled\n", ring->ring_id);
979 		return 0;
980 	}
981 
982 	memset(ring->ring, 0,
983 	       ring->n_entries * (sizeof(struct bcm4377_completion_ring_entry) +
984 				  ring->payload_size));
985 	memset(&msg, 0, sizeof(msg));
986 	msg.msg_type = BCM4377_CONTROL_MSG_CREATE_COMPLETION_RING;
987 	msg.id = cpu_to_le16(ring->ring_id);
988 	msg.id_again = cpu_to_le16(ring->ring_id);
989 	msg.ring_iova = cpu_to_le64(ring->ring_dma);
990 	msg.n_elements = cpu_to_le16(ring->n_entries);
991 	msg.intmod_bytes = cpu_to_le32(0xffffffff);
992 	msg.unk = cpu_to_le32(0xffffffff);
993 	msg.intmod_delay = cpu_to_le16(ring->delay);
994 	msg.footer_size = ring->payload_size / 4;
995 
996 	ret = bcm4377_enqueue(bcm4377, &bcm4377->control_h2d_ring, &msg,
997 			      sizeof(msg), true);
998 	if (!ret)
999 		ring->enabled = true;
1000 
1001 	return ret;
1002 }
1003 
1004 static int bcm4377_destroy_completion_ring(struct bcm4377_data *bcm4377,
1005 					   struct bcm4377_completion_ring *ring)
1006 {
1007 	struct bcm4377_destroy_completion_ring_msg msg;
1008 	int ret;
1009 
1010 	memset(&msg, 0, sizeof(msg));
1011 	msg.msg_type = BCM4377_CONTROL_MSG_DESTROY_COMPLETION_RING;
1012 	msg.ring_id = cpu_to_le16(ring->ring_id);
1013 
1014 	ret = bcm4377_enqueue(bcm4377, &bcm4377->control_h2d_ring, &msg,
1015 			      sizeof(msg), true);
1016 	if (ret)
1017 		dev_warn(&bcm4377->pdev->dev,
1018 			 "failed to destroy completion ring %d\n",
1019 			 ring->ring_id);
1020 
1021 	ring->enabled = false;
1022 	return ret;
1023 }
1024 
1025 static int bcm4377_create_transfer_ring(struct bcm4377_data *bcm4377,
1026 					struct bcm4377_transfer_ring *ring)
1027 {
1028 	struct bcm4377_create_transfer_ring_msg msg;
1029 	u16 flags = 0;
1030 	int ret, i;
1031 	unsigned long spinlock_flags;
1032 
1033 	if (ring->virtual)
1034 		flags |= BCM4377_XFER_RING_FLAG_VIRTUAL;
1035 	if (ring->sync)
1036 		flags |= BCM4377_XFER_RING_FLAG_SYNC;
1037 
1038 	spin_lock_irqsave(&ring->lock, spinlock_flags);
1039 	memset(&msg, 0, sizeof(msg));
1040 	msg.msg_type = BCM4377_CONTROL_MSG_CREATE_XFER_RING;
1041 	msg.ring_id = cpu_to_le16(ring->ring_id);
1042 	msg.ring_id_again = cpu_to_le16(ring->ring_id);
1043 	msg.ring_iova = cpu_to_le64(ring->ring_dma);
1044 	msg.n_elements = cpu_to_le16(ring->n_entries);
1045 	msg.completion_ring_id = cpu_to_le16(ring->completion_ring);
1046 	msg.doorbell = cpu_to_le16(ring->doorbell);
1047 	msg.flags = cpu_to_le16(flags);
1048 	msg.footer_size = ring->payload_size / 4;
1049 
1050 	bcm4377->ring_state->xfer_ring_head[ring->ring_id] = 0;
1051 	bcm4377->ring_state->xfer_ring_tail[ring->ring_id] = 0;
1052 	ring->generation++;
1053 	spin_unlock_irqrestore(&ring->lock, spinlock_flags);
1054 
1055 	ret = bcm4377_enqueue(bcm4377, &bcm4377->control_h2d_ring, &msg,
1056 			      sizeof(msg), true);
1057 
1058 	spin_lock_irqsave(&ring->lock, spinlock_flags);
1059 
1060 	if (ring->d2h_buffers_only) {
1061 		for (i = 0; i < ring->n_entries; ++i) {
1062 			struct bcm4377_xfer_ring_entry *entry =
1063 				ring->ring + i * sizeof(*entry);
1064 			u16 raw_msgid = FIELD_PREP(BCM4377_MSGID_GENERATION,
1065 						   ring->generation);
1066 			raw_msgid |= FIELD_PREP(BCM4377_MSGID_ID, i);
1067 
1068 			memset(entry, 0, sizeof(*entry));
1069 			entry->id = cpu_to_le16(raw_msgid);
1070 			entry->len = cpu_to_le16(ring->mapped_payload_size);
1071 			entry->flags = BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED;
1072 			entry->payload =
1073 				cpu_to_le64(ring->payloads_dma +
1074 					    i * ring->mapped_payload_size);
1075 		}
1076 	}
1077 
1078 	/*
1079 	 * send some messages if this is a device->host ring to allow the device
1080 	 * to reply by acknowledging them in the completion ring
1081 	 */
1082 	if (ring->virtual || ring->d2h_buffers_only) {
1083 		bcm4377->ring_state->xfer_ring_head[ring->ring_id] =
1084 			cpu_to_le16(0xf);
1085 		bcm4377_ring_doorbell(bcm4377, ring->doorbell, 0xf);
1086 	}
1087 
1088 	ring->enabled = true;
1089 	spin_unlock_irqrestore(&ring->lock, spinlock_flags);
1090 
1091 	return ret;
1092 }
1093 
1094 static int bcm4377_destroy_transfer_ring(struct bcm4377_data *bcm4377,
1095 					 struct bcm4377_transfer_ring *ring)
1096 {
1097 	struct bcm4377_destroy_transfer_ring_msg msg;
1098 	int ret;
1099 
1100 	memset(&msg, 0, sizeof(msg));
1101 	msg.msg_type = BCM4377_CONTROL_MSG_DESTROY_XFER_RING;
1102 	msg.ring_id = cpu_to_le16(ring->ring_id);
1103 
1104 	ret = bcm4377_enqueue(bcm4377, &bcm4377->control_h2d_ring, &msg,
1105 			      sizeof(msg), true);
1106 	if (ret)
1107 		dev_warn(&bcm4377->pdev->dev,
1108 			 "failed to destroy transfer ring %d\n", ring->ring_id);
1109 
1110 	ring->enabled = false;
1111 	return ret;
1112 }
1113 
1114 static int __bcm4378_send_calibration_chunk(struct bcm4377_data *bcm4377,
1115 					    const void *data, size_t data_len,
1116 					    u16 blocks_left)
1117 {
1118 	struct bcm4378_hci_send_calibration_cmd cmd;
1119 	struct sk_buff *skb;
1120 
1121 	if (data_len > sizeof(cmd.data))
1122 		return -EINVAL;
1123 
1124 	memset(&cmd, 0, sizeof(cmd));
1125 	cmd.unk = 0x03;
1126 	cmd.blocks_left = cpu_to_le16(blocks_left);
1127 	memcpy(cmd.data, data, data_len);
1128 
1129 	skb = __hci_cmd_sync(bcm4377->hdev, 0xfd97, sizeof(cmd), &cmd,
1130 			     HCI_INIT_TIMEOUT);
1131 	if (IS_ERR(skb))
1132 		return PTR_ERR(skb);
1133 
1134 	kfree_skb(skb);
1135 	return 0;
1136 }
1137 
1138 static int __bcm4378_send_calibration(struct bcm4377_data *bcm4377,
1139 				      const void *data, size_t data_size)
1140 {
1141 	int ret;
1142 	size_t i, left, transfer_len;
1143 	size_t blocks =
1144 		DIV_ROUND_UP(data_size, (size_t)BCM4378_CALIBRATION_CHUNK_SIZE);
1145 
1146 	if (!data) {
1147 		dev_err(&bcm4377->pdev->dev,
1148 			"no calibration data available.\n");
1149 		return -ENOENT;
1150 	}
1151 
1152 	for (i = 0, left = data_size; i < blocks; ++i, left -= transfer_len) {
1153 		transfer_len =
1154 			min_t(size_t, left, BCM4378_CALIBRATION_CHUNK_SIZE);
1155 
1156 		ret = __bcm4378_send_calibration_chunk(
1157 			bcm4377, data + i * BCM4378_CALIBRATION_CHUNK_SIZE,
1158 			transfer_len, blocks - i - 1);
1159 		if (ret) {
1160 			dev_err(&bcm4377->pdev->dev,
1161 				"send calibration chunk failed with %d\n", ret);
1162 			return ret;
1163 		}
1164 	}
1165 
1166 	return 0;
1167 }
1168 
1169 static int bcm4378_send_calibration(struct bcm4377_data *bcm4377)
1170 {
1171 	if ((strcmp(bcm4377->stepping, "b1") == 0) ||
1172 	    strcmp(bcm4377->stepping, "b3") == 0)
1173 		return __bcm4378_send_calibration(
1174 			bcm4377, bcm4377->taurus_beamforming_cal_blob,
1175 			bcm4377->taurus_beamforming_cal_size);
1176 	else
1177 		return __bcm4378_send_calibration(bcm4377,
1178 						  bcm4377->taurus_cal_blob,
1179 						  bcm4377->taurus_cal_size);
1180 }
1181 
1182 static int bcm4387_send_calibration(struct bcm4377_data *bcm4377)
1183 {
1184 	if (strcmp(bcm4377->stepping, "c2") == 0)
1185 		return __bcm4378_send_calibration(
1186 			bcm4377, bcm4377->taurus_beamforming_cal_blob,
1187 			bcm4377->taurus_beamforming_cal_size);
1188 	else
1189 		return __bcm4378_send_calibration(bcm4377,
1190 						  bcm4377->taurus_cal_blob,
1191 						  bcm4377->taurus_cal_size);
1192 }
1193 
1194 static const struct firmware *bcm4377_request_blob(struct bcm4377_data *bcm4377,
1195 						   const char *suffix)
1196 {
1197 	const struct firmware *fw;
1198 	char name0[64], name1[64];
1199 	int ret;
1200 
1201 	snprintf(name0, sizeof(name0), "brcm/brcmbt%04x%s-%s-%s.%s",
1202 		 bcm4377->hw->id, bcm4377->stepping, bcm4377->board_type,
1203 		 bcm4377->vendor, suffix);
1204 	snprintf(name1, sizeof(name1), "brcm/brcmbt%04x%s-%s.%s",
1205 		 bcm4377->hw->id, bcm4377->stepping, bcm4377->board_type,
1206 		 suffix);
1207 	dev_dbg(&bcm4377->pdev->dev, "Trying to load firmware: '%s' or '%s'\n",
1208 		name0, name1);
1209 
1210 	ret = firmware_request_nowarn(&fw, name0, &bcm4377->pdev->dev);
1211 	if (!ret)
1212 		return fw;
1213 	ret = firmware_request_nowarn(&fw, name1, &bcm4377->pdev->dev);
1214 	if (!ret)
1215 		return fw;
1216 
1217 	dev_err(&bcm4377->pdev->dev,
1218 		"Unable to load firmware; tried '%s' and '%s'\n", name0, name1);
1219 	return NULL;
1220 }
1221 
1222 static int bcm4377_send_ptb(struct bcm4377_data *bcm4377,
1223 			    const struct firmware *fw)
1224 {
1225 	struct sk_buff *skb;
1226 
1227 	skb = __hci_cmd_sync(bcm4377->hdev, 0xfd98, fw->size, fw->data,
1228 			     HCI_INIT_TIMEOUT);
1229 	/*
1230 	 * This command seems to always fail on more recent firmware versions
1231 	 * (even in traces taken from the macOS driver). It's unclear why this
1232 	 * happens but because the PTB file contains calibration and/or
1233 	 * regulatory data and may be required on older firmware we still try to
1234 	 * send it here just in case and just ignore if it fails.
1235 	 */
1236 	if (!IS_ERR(skb))
1237 		kfree_skb(skb);
1238 	return 0;
1239 }
1240 
1241 static int bcm4378_send_ptb_chunk(struct bcm4377_data *bcm4377,
1242 				  const void *data, size_t data_len,
1243 				  u16 blocks_left)
1244 {
1245 	struct bcm4378_hci_send_ptb_cmd cmd;
1246 	struct sk_buff *skb;
1247 
1248 	if (data_len > BCM4378_PTB_CHUNK_SIZE)
1249 		return -EINVAL;
1250 
1251 	memset(&cmd, 0, sizeof(cmd));
1252 	cmd.blocks_left = cpu_to_le16(blocks_left);
1253 	memcpy(cmd.data, data, data_len);
1254 
1255 	skb = __hci_cmd_sync(bcm4377->hdev, 0xfe0d, sizeof(cmd), &cmd,
1256 			     HCI_INIT_TIMEOUT);
1257 	if (IS_ERR(skb))
1258 		return PTR_ERR(skb);
1259 
1260 	kfree_skb(skb);
1261 	return 0;
1262 }
1263 
1264 static int bcm4378_send_ptb(struct bcm4377_data *bcm4377,
1265 			    const struct firmware *fw)
1266 {
1267 	size_t chunks = DIV_ROUND_UP(fw->size, (size_t)BCM4378_PTB_CHUNK_SIZE);
1268 	size_t i, left, transfer_len;
1269 	int ret;
1270 
1271 	for (i = 0, left = fw->size; i < chunks; ++i, left -= transfer_len) {
1272 		transfer_len = min_t(size_t, left, BCM4378_PTB_CHUNK_SIZE);
1273 
1274 		dev_dbg(&bcm4377->pdev->dev, "sending ptb chunk %zu/%zu\n",
1275 			i + 1, chunks);
1276 		ret = bcm4378_send_ptb_chunk(
1277 			bcm4377, fw->data + i * BCM4378_PTB_CHUNK_SIZE,
1278 			transfer_len, chunks - i - 1);
1279 		if (ret) {
1280 			dev_err(&bcm4377->pdev->dev,
1281 				"sending ptb chunk %zu failed (%d)", i, ret);
1282 			return ret;
1283 		}
1284 	}
1285 
1286 	return 0;
1287 }
1288 
1289 static int bcm4377_hci_open(struct hci_dev *hdev)
1290 {
1291 	struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1292 	int ret;
1293 
1294 	dev_dbg(&bcm4377->pdev->dev, "creating rings\n");
1295 
1296 	ret = bcm4377_create_completion_ring(bcm4377,
1297 					     &bcm4377->hci_acl_ack_ring);
1298 	if (ret)
1299 		return ret;
1300 	ret = bcm4377_create_completion_ring(bcm4377,
1301 					     &bcm4377->hci_acl_event_ring);
1302 	if (ret)
1303 		goto destroy_hci_acl_ack;
1304 	ret = bcm4377_create_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
1305 	if (ret)
1306 		goto destroy_hci_acl_event;
1307 	ret = bcm4377_create_completion_ring(bcm4377, &bcm4377->sco_event_ring);
1308 	if (ret)
1309 		goto destroy_sco_ack;
1310 	dev_dbg(&bcm4377->pdev->dev,
1311 		"all completion rings successfully created!\n");
1312 
1313 	ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring);
1314 	if (ret)
1315 		goto destroy_sco_event;
1316 	ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring);
1317 	if (ret)
1318 		goto destroy_hci_h2d;
1319 	ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring);
1320 	if (ret)
1321 		goto destroy_hci_d2h;
1322 	ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring);
1323 	if (ret)
1324 		goto destroy_sco_h2d;
1325 	ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring);
1326 	if (ret)
1327 		goto destroy_sco_d2h;
1328 	ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->acl_d2h_ring);
1329 	if (ret)
1330 		goto destroy_acl_h2d;
1331 	dev_dbg(&bcm4377->pdev->dev,
1332 		"all transfer rings successfully created!\n");
1333 
1334 	return 0;
1335 
1336 destroy_acl_h2d:
1337 	bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring);
1338 destroy_sco_d2h:
1339 	bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring);
1340 destroy_sco_h2d:
1341 	bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring);
1342 destroy_hci_d2h:
1343 	bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring);
1344 destroy_hci_h2d:
1345 	bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring);
1346 destroy_sco_event:
1347 	bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_event_ring);
1348 destroy_sco_ack:
1349 	bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
1350 destroy_hci_acl_event:
1351 	bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_event_ring);
1352 destroy_hci_acl_ack:
1353 	bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_ack_ring);
1354 
1355 	dev_err(&bcm4377->pdev->dev, "Creating rings failed with %d\n", ret);
1356 	return ret;
1357 }
1358 
1359 static int bcm4377_hci_close(struct hci_dev *hdev)
1360 {
1361 	struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1362 
1363 	dev_dbg(&bcm4377->pdev->dev, "destroying rings in hci_close\n");
1364 
1365 	bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->acl_d2h_ring);
1366 	bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring);
1367 	bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring);
1368 	bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring);
1369 	bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring);
1370 	bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring);
1371 
1372 	bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_event_ring);
1373 	bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
1374 	bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_event_ring);
1375 	bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_ack_ring);
1376 
1377 	return 0;
1378 }
1379 
1380 static bool bcm4377_is_valid_bdaddr(struct bcm4377_data *bcm4377,
1381 				    bdaddr_t *addr)
1382 {
1383 	if (addr->b[0] != 0x93)
1384 		return true;
1385 	if (addr->b[1] != 0x76)
1386 		return true;
1387 	if (addr->b[2] != 0x00)
1388 		return true;
1389 	if (addr->b[4] != (bcm4377->hw->id & 0xff))
1390 		return true;
1391 	if (addr->b[5] != (bcm4377->hw->id >> 8))
1392 		return true;
1393 	return false;
1394 }
1395 
1396 static int bcm4377_check_bdaddr(struct bcm4377_data *bcm4377)
1397 {
1398 	struct hci_rp_read_bd_addr *bda;
1399 	struct sk_buff *skb;
1400 
1401 	skb = __hci_cmd_sync(bcm4377->hdev, HCI_OP_READ_BD_ADDR, 0, NULL,
1402 			     HCI_INIT_TIMEOUT);
1403 	if (IS_ERR(skb)) {
1404 		int err = PTR_ERR(skb);
1405 
1406 		dev_err(&bcm4377->pdev->dev, "HCI_OP_READ_BD_ADDR failed (%d)",
1407 			err);
1408 		return err;
1409 	}
1410 
1411 	if (skb->len != sizeof(*bda)) {
1412 		dev_err(&bcm4377->pdev->dev,
1413 			"HCI_OP_READ_BD_ADDR reply length invalid");
1414 		kfree_skb(skb);
1415 		return -EIO;
1416 	}
1417 
1418 	bda = (struct hci_rp_read_bd_addr *)skb->data;
1419 	if (!bcm4377_is_valid_bdaddr(bcm4377, &bda->bdaddr))
1420 		set_bit(HCI_QUIRK_USE_BDADDR_PROPERTY, &bcm4377->hdev->quirks);
1421 
1422 	kfree_skb(skb);
1423 	return 0;
1424 }
1425 
1426 static int bcm4377_hci_setup(struct hci_dev *hdev)
1427 {
1428 	struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1429 	const struct firmware *fw;
1430 	int ret;
1431 
1432 	if (bcm4377->hw->send_calibration) {
1433 		ret = bcm4377->hw->send_calibration(bcm4377);
1434 		if (ret)
1435 			return ret;
1436 	}
1437 
1438 	fw = bcm4377_request_blob(bcm4377, "ptb");
1439 	if (!fw) {
1440 		dev_err(&bcm4377->pdev->dev, "failed to load PTB data");
1441 		return -ENOENT;
1442 	}
1443 
1444 	ret = bcm4377->hw->send_ptb(bcm4377, fw);
1445 	release_firmware(fw);
1446 	if (ret)
1447 		return ret;
1448 
1449 	return bcm4377_check_bdaddr(bcm4377);
1450 }
1451 
1452 static int bcm4377_hci_send_frame(struct hci_dev *hdev, struct sk_buff *skb)
1453 {
1454 	struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1455 	struct bcm4377_transfer_ring *ring;
1456 	int ret;
1457 
1458 	switch (hci_skb_pkt_type(skb)) {
1459 	case HCI_COMMAND_PKT:
1460 		hdev->stat.cmd_tx++;
1461 		ring = &bcm4377->hci_h2d_ring;
1462 		break;
1463 
1464 	case HCI_ACLDATA_PKT:
1465 		hdev->stat.acl_tx++;
1466 		ring = &bcm4377->acl_h2d_ring;
1467 		break;
1468 
1469 	case HCI_SCODATA_PKT:
1470 		hdev->stat.sco_tx++;
1471 		ring = &bcm4377->sco_h2d_ring;
1472 		break;
1473 
1474 	default:
1475 		return -EILSEQ;
1476 	}
1477 
1478 	ret = bcm4377_enqueue(bcm4377, ring, skb->data, skb->len, false);
1479 	if (ret < 0) {
1480 		hdev->stat.err_tx++;
1481 		return ret;
1482 	}
1483 
1484 	hdev->stat.byte_tx += skb->len;
1485 	kfree_skb(skb);
1486 	return ret;
1487 }
1488 
1489 static int bcm4377_hci_set_bdaddr(struct hci_dev *hdev, const bdaddr_t *bdaddr)
1490 {
1491 	struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1492 	struct sk_buff *skb;
1493 	int err;
1494 
1495 	skb = __hci_cmd_sync(hdev, 0xfc01, 6, bdaddr, HCI_INIT_TIMEOUT);
1496 	if (IS_ERR(skb)) {
1497 		err = PTR_ERR(skb);
1498 		dev_err(&bcm4377->pdev->dev,
1499 			"Change address command failed (%d)", err);
1500 		return err;
1501 	}
1502 	kfree_skb(skb);
1503 
1504 	return 0;
1505 }
1506 
1507 static int bcm4377_alloc_transfer_ring(struct bcm4377_data *bcm4377,
1508 				       struct bcm4377_transfer_ring *ring)
1509 {
1510 	size_t entry_size;
1511 
1512 	spin_lock_init(&ring->lock);
1513 	ring->payload_size = ALIGN(ring->payload_size, 4);
1514 	ring->mapped_payload_size = ALIGN(ring->mapped_payload_size, 4);
1515 
1516 	if (ring->payload_size > BCM4377_XFER_RING_MAX_INPLACE_PAYLOAD_SIZE)
1517 		return -EINVAL;
1518 	if (ring->n_entries > BCM4377_MAX_RING_SIZE)
1519 		return -EINVAL;
1520 	if (ring->virtual && ring->allow_wait)
1521 		return -EINVAL;
1522 
1523 	if (ring->d2h_buffers_only) {
1524 		if (ring->virtual)
1525 			return -EINVAL;
1526 		if (ring->payload_size)
1527 			return -EINVAL;
1528 		if (!ring->mapped_payload_size)
1529 			return -EINVAL;
1530 	}
1531 	if (ring->virtual)
1532 		return 0;
1533 
1534 	entry_size =
1535 		ring->payload_size + sizeof(struct bcm4377_xfer_ring_entry);
1536 	ring->ring = dmam_alloc_coherent(&bcm4377->pdev->dev,
1537 					 ring->n_entries * entry_size,
1538 					 &ring->ring_dma, GFP_KERNEL);
1539 	if (!ring->ring)
1540 		return -ENOMEM;
1541 
1542 	if (ring->allow_wait) {
1543 		ring->events = devm_kcalloc(&bcm4377->pdev->dev,
1544 					    ring->n_entries,
1545 					    sizeof(*ring->events), GFP_KERNEL);
1546 		if (!ring->events)
1547 			return -ENOMEM;
1548 	}
1549 
1550 	if (ring->mapped_payload_size) {
1551 		ring->payloads = dmam_alloc_coherent(
1552 			&bcm4377->pdev->dev,
1553 			ring->n_entries * ring->mapped_payload_size,
1554 			&ring->payloads_dma, GFP_KERNEL);
1555 		if (!ring->payloads)
1556 			return -ENOMEM;
1557 	}
1558 
1559 	return 0;
1560 }
1561 
1562 static int bcm4377_alloc_completion_ring(struct bcm4377_data *bcm4377,
1563 					 struct bcm4377_completion_ring *ring)
1564 {
1565 	size_t entry_size;
1566 
1567 	ring->payload_size = ALIGN(ring->payload_size, 4);
1568 	if (ring->payload_size > BCM4377_XFER_RING_MAX_INPLACE_PAYLOAD_SIZE)
1569 		return -EINVAL;
1570 	if (ring->n_entries > BCM4377_MAX_RING_SIZE)
1571 		return -EINVAL;
1572 
1573 	entry_size = ring->payload_size +
1574 		     sizeof(struct bcm4377_completion_ring_entry);
1575 
1576 	ring->ring = dmam_alloc_coherent(&bcm4377->pdev->dev,
1577 					 ring->n_entries * entry_size,
1578 					 &ring->ring_dma, GFP_KERNEL);
1579 	if (!ring->ring)
1580 		return -ENOMEM;
1581 	return 0;
1582 }
1583 
1584 static int bcm4377_init_context(struct bcm4377_data *bcm4377)
1585 {
1586 	struct device *dev = &bcm4377->pdev->dev;
1587 	dma_addr_t peripheral_info_dma;
1588 
1589 	bcm4377->ctx = dmam_alloc_coherent(dev, sizeof(*bcm4377->ctx),
1590 					   &bcm4377->ctx_dma, GFP_KERNEL);
1591 	if (!bcm4377->ctx)
1592 		return -ENOMEM;
1593 	memset(bcm4377->ctx, 0, sizeof(*bcm4377->ctx));
1594 
1595 	bcm4377->ring_state =
1596 		dmam_alloc_coherent(dev, sizeof(*bcm4377->ring_state),
1597 				    &bcm4377->ring_state_dma, GFP_KERNEL);
1598 	if (!bcm4377->ring_state)
1599 		return -ENOMEM;
1600 	memset(bcm4377->ring_state, 0, sizeof(*bcm4377->ring_state));
1601 
1602 	bcm4377->ctx->version = cpu_to_le16(1);
1603 	bcm4377->ctx->size = cpu_to_le16(sizeof(*bcm4377->ctx));
1604 	bcm4377->ctx->enabled_caps = cpu_to_le32(2);
1605 
1606 	/*
1607 	 * The BT device will write 0x20 bytes of data to this buffer but
1608 	 * the exact contents are unknown. It only needs to exist for BT
1609 	 * to work such that we can just allocate and then ignore it.
1610 	 */
1611 	if (!dmam_alloc_coherent(&bcm4377->pdev->dev, 0x20,
1612 				 &peripheral_info_dma, GFP_KERNEL))
1613 		return -ENOMEM;
1614 	bcm4377->ctx->peripheral_info_addr = cpu_to_le64(peripheral_info_dma);
1615 
1616 	bcm4377->ctx->xfer_ring_heads_addr = cpu_to_le64(
1617 		bcm4377->ring_state_dma +
1618 		offsetof(struct bcm4377_ring_state, xfer_ring_head));
1619 	bcm4377->ctx->xfer_ring_tails_addr = cpu_to_le64(
1620 		bcm4377->ring_state_dma +
1621 		offsetof(struct bcm4377_ring_state, xfer_ring_tail));
1622 	bcm4377->ctx->completion_ring_heads_addr = cpu_to_le64(
1623 		bcm4377->ring_state_dma +
1624 		offsetof(struct bcm4377_ring_state, completion_ring_head));
1625 	bcm4377->ctx->completion_ring_tails_addr = cpu_to_le64(
1626 		bcm4377->ring_state_dma +
1627 		offsetof(struct bcm4377_ring_state, completion_ring_tail));
1628 
1629 	bcm4377->ctx->n_completion_rings =
1630 		cpu_to_le16(BCM4377_N_COMPLETION_RINGS);
1631 	bcm4377->ctx->n_xfer_rings = cpu_to_le16(BCM4377_N_TRANSFER_RINGS);
1632 
1633 	bcm4377->ctx->control_completion_ring_addr =
1634 		cpu_to_le64(bcm4377->control_ack_ring.ring_dma);
1635 	bcm4377->ctx->control_completion_ring_n_entries =
1636 		cpu_to_le16(bcm4377->control_ack_ring.n_entries);
1637 	bcm4377->ctx->control_completion_ring_doorbell = cpu_to_le16(0xffff);
1638 	bcm4377->ctx->control_completion_ring_msi = 0;
1639 	bcm4377->ctx->control_completion_ring_header_size = 0;
1640 	bcm4377->ctx->control_completion_ring_footer_size = 0;
1641 
1642 	bcm4377->ctx->control_xfer_ring_addr =
1643 		cpu_to_le64(bcm4377->control_h2d_ring.ring_dma);
1644 	bcm4377->ctx->control_xfer_ring_n_entries =
1645 		cpu_to_le16(bcm4377->control_h2d_ring.n_entries);
1646 	bcm4377->ctx->control_xfer_ring_doorbell =
1647 		cpu_to_le16(bcm4377->control_h2d_ring.doorbell);
1648 	bcm4377->ctx->control_xfer_ring_msi = 0;
1649 	bcm4377->ctx->control_xfer_ring_header_size = 0;
1650 	bcm4377->ctx->control_xfer_ring_footer_size =
1651 		bcm4377->control_h2d_ring.payload_size / 4;
1652 
1653 	dev_dbg(&bcm4377->pdev->dev, "context initialized at IOVA %pad",
1654 		&bcm4377->ctx_dma);
1655 
1656 	return 0;
1657 }
1658 
1659 static int bcm4377_prepare_rings(struct bcm4377_data *bcm4377)
1660 {
1661 	int ret;
1662 
1663 	/*
1664 	 * Even though many of these settings appear to be configurable
1665 	 * when sending the "create ring" messages most of these are
1666 	 * actually hardcoded in some (and quite possibly all) firmware versions
1667 	 * and changing them on the host has no effect.
1668 	 * Specifically, this applies to at least the doorbells, the transfer
1669 	 * and completion ring ids and their mapping (e.g. both HCI and ACL
1670 	 * entries will always be queued in completion rings 1 and 2 no matter
1671 	 * what we configure here).
1672 	 */
1673 	bcm4377->control_ack_ring.ring_id = BCM4377_ACK_RING_CONTROL;
1674 	bcm4377->control_ack_ring.n_entries = 32;
1675 	bcm4377->control_ack_ring.transfer_rings =
1676 		BIT(BCM4377_XFER_RING_CONTROL);
1677 
1678 	bcm4377->hci_acl_ack_ring.ring_id = BCM4377_ACK_RING_HCI_ACL;
1679 	bcm4377->hci_acl_ack_ring.n_entries = 2 * BCM4377_RING_N_ENTRIES;
1680 	bcm4377->hci_acl_ack_ring.transfer_rings =
1681 		BIT(BCM4377_XFER_RING_HCI_H2D) | BIT(BCM4377_XFER_RING_ACL_H2D);
1682 	bcm4377->hci_acl_ack_ring.delay = 1000;
1683 
1684 	/*
1685 	 * A payload size of MAX_EVENT_PAYLOAD_SIZE is enough here since large
1686 	 * ACL packets will be transmitted inside buffers mapped via
1687 	 * acl_d2h_ring anyway.
1688 	 */
1689 	bcm4377->hci_acl_event_ring.ring_id = BCM4377_EVENT_RING_HCI_ACL;
1690 	bcm4377->hci_acl_event_ring.payload_size = MAX_EVENT_PAYLOAD_SIZE;
1691 	bcm4377->hci_acl_event_ring.n_entries = 2 * BCM4377_RING_N_ENTRIES;
1692 	bcm4377->hci_acl_event_ring.transfer_rings =
1693 		BIT(BCM4377_XFER_RING_HCI_D2H) | BIT(BCM4377_XFER_RING_ACL_D2H);
1694 	bcm4377->hci_acl_event_ring.delay = 1000;
1695 
1696 	bcm4377->sco_ack_ring.ring_id = BCM4377_ACK_RING_SCO;
1697 	bcm4377->sco_ack_ring.n_entries = BCM4377_RING_N_ENTRIES;
1698 	bcm4377->sco_ack_ring.transfer_rings = BIT(BCM4377_XFER_RING_SCO_H2D);
1699 
1700 	bcm4377->sco_event_ring.ring_id = BCM4377_EVENT_RING_SCO;
1701 	bcm4377->sco_event_ring.payload_size = MAX_SCO_PAYLOAD_SIZE;
1702 	bcm4377->sco_event_ring.n_entries = BCM4377_RING_N_ENTRIES;
1703 	bcm4377->sco_event_ring.transfer_rings = BIT(BCM4377_XFER_RING_SCO_D2H);
1704 
1705 	bcm4377->control_h2d_ring.ring_id = BCM4377_XFER_RING_CONTROL;
1706 	bcm4377->control_h2d_ring.doorbell = BCM4377_DOORBELL_CONTROL;
1707 	bcm4377->control_h2d_ring.payload_size = BCM4377_CONTROL_MSG_SIZE;
1708 	bcm4377->control_h2d_ring.completion_ring = BCM4377_ACK_RING_CONTROL;
1709 	bcm4377->control_h2d_ring.allow_wait = true;
1710 	bcm4377->control_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1711 
1712 	bcm4377->hci_h2d_ring.ring_id = BCM4377_XFER_RING_HCI_H2D;
1713 	bcm4377->hci_h2d_ring.doorbell = BCM4377_DOORBELL_HCI_H2D;
1714 	bcm4377->hci_h2d_ring.payload_size = MAX_EVENT_PAYLOAD_SIZE;
1715 	bcm4377->hci_h2d_ring.completion_ring = BCM4377_ACK_RING_HCI_ACL;
1716 	bcm4377->hci_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1717 
1718 	bcm4377->hci_d2h_ring.ring_id = BCM4377_XFER_RING_HCI_D2H;
1719 	bcm4377->hci_d2h_ring.doorbell = BCM4377_DOORBELL_HCI_D2H;
1720 	bcm4377->hci_d2h_ring.completion_ring = BCM4377_EVENT_RING_HCI_ACL;
1721 	bcm4377->hci_d2h_ring.virtual = true;
1722 	bcm4377->hci_d2h_ring.n_entries = BCM4377_RING_N_ENTRIES;
1723 
1724 	bcm4377->sco_h2d_ring.ring_id = BCM4377_XFER_RING_SCO_H2D;
1725 	bcm4377->sco_h2d_ring.doorbell = BCM4377_DOORBELL_SCO;
1726 	bcm4377->sco_h2d_ring.payload_size = MAX_SCO_PAYLOAD_SIZE;
1727 	bcm4377->sco_h2d_ring.completion_ring = BCM4377_ACK_RING_SCO;
1728 	bcm4377->sco_h2d_ring.sync = true;
1729 	bcm4377->sco_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1730 
1731 	bcm4377->sco_d2h_ring.ring_id = BCM4377_XFER_RING_SCO_D2H;
1732 	bcm4377->sco_d2h_ring.doorbell = BCM4377_DOORBELL_SCO;
1733 	bcm4377->sco_d2h_ring.completion_ring = BCM4377_EVENT_RING_SCO;
1734 	bcm4377->sco_d2h_ring.virtual = true;
1735 	bcm4377->sco_d2h_ring.sync = true;
1736 	bcm4377->sco_d2h_ring.n_entries = BCM4377_RING_N_ENTRIES;
1737 
1738 	/*
1739 	 * This ring has to use mapped_payload_size because the largest ACL
1740 	 * packet doesn't fit inside the largest possible footer
1741 	 */
1742 	bcm4377->acl_h2d_ring.ring_id = BCM4377_XFER_RING_ACL_H2D;
1743 	bcm4377->acl_h2d_ring.doorbell = BCM4377_DOORBELL_ACL_H2D;
1744 	bcm4377->acl_h2d_ring.mapped_payload_size = MAX_ACL_PAYLOAD_SIZE;
1745 	bcm4377->acl_h2d_ring.completion_ring = BCM4377_ACK_RING_HCI_ACL;
1746 	bcm4377->acl_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1747 
1748 	/*
1749 	 * This ring only contains empty buffers to be used by incoming
1750 	 * ACL packets that do not fit inside the footer of hci_acl_event_ring
1751 	 */
1752 	bcm4377->acl_d2h_ring.ring_id = BCM4377_XFER_RING_ACL_D2H;
1753 	bcm4377->acl_d2h_ring.doorbell = BCM4377_DOORBELL_ACL_D2H;
1754 	bcm4377->acl_d2h_ring.completion_ring = BCM4377_EVENT_RING_HCI_ACL;
1755 	bcm4377->acl_d2h_ring.d2h_buffers_only = true;
1756 	bcm4377->acl_d2h_ring.mapped_payload_size = MAX_ACL_PAYLOAD_SIZE;
1757 	bcm4377->acl_d2h_ring.n_entries = BCM4377_RING_N_ENTRIES;
1758 
1759 	/*
1760 	 * no need for any cleanup since this is only called from _probe
1761 	 * and only devres-managed allocations are used
1762 	 */
1763 	ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->control_h2d_ring);
1764 	if (ret)
1765 		return ret;
1766 	ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring);
1767 	if (ret)
1768 		return ret;
1769 	ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring);
1770 	if (ret)
1771 		return ret;
1772 	ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring);
1773 	if (ret)
1774 		return ret;
1775 	ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring);
1776 	if (ret)
1777 		return ret;
1778 	ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring);
1779 	if (ret)
1780 		return ret;
1781 	ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->acl_d2h_ring);
1782 	if (ret)
1783 		return ret;
1784 
1785 	ret = bcm4377_alloc_completion_ring(bcm4377,
1786 					    &bcm4377->control_ack_ring);
1787 	if (ret)
1788 		return ret;
1789 	ret = bcm4377_alloc_completion_ring(bcm4377,
1790 					    &bcm4377->hci_acl_ack_ring);
1791 	if (ret)
1792 		return ret;
1793 	ret = bcm4377_alloc_completion_ring(bcm4377,
1794 					    &bcm4377->hci_acl_event_ring);
1795 	if (ret)
1796 		return ret;
1797 	ret = bcm4377_alloc_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
1798 	if (ret)
1799 		return ret;
1800 	ret = bcm4377_alloc_completion_ring(bcm4377, &bcm4377->sco_event_ring);
1801 	if (ret)
1802 		return ret;
1803 
1804 	dev_dbg(&bcm4377->pdev->dev, "all rings allocated and prepared\n");
1805 
1806 	return 0;
1807 }
1808 
1809 static int bcm4377_boot(struct bcm4377_data *bcm4377)
1810 {
1811 	const struct firmware *fw;
1812 	void *bfr;
1813 	dma_addr_t fw_dma;
1814 	int ret = 0;
1815 	u32 bootstage, rti_status;
1816 
1817 	bootstage = ioread32(bcm4377->bar2 + BCM4377_BAR2_BOOTSTAGE);
1818 	rti_status = ioread32(bcm4377->bar2 + BCM4377_BAR2_RTI_STATUS);
1819 
1820 	if (bootstage != 0) {
1821 		dev_err(&bcm4377->pdev->dev, "bootstage is %d and not 0\n",
1822 			bootstage);
1823 		return -EINVAL;
1824 	}
1825 
1826 	if (rti_status != 0) {
1827 		dev_err(&bcm4377->pdev->dev, "RTI status is %d and not 0\n",
1828 			rti_status);
1829 		return -EINVAL;
1830 	}
1831 
1832 	fw = bcm4377_request_blob(bcm4377, "bin");
1833 	if (!fw) {
1834 		dev_err(&bcm4377->pdev->dev, "Failed to load firmware\n");
1835 		return -ENOENT;
1836 	}
1837 
1838 	bfr = dma_alloc_coherent(&bcm4377->pdev->dev, fw->size, &fw_dma,
1839 				 GFP_KERNEL);
1840 	if (!bfr) {
1841 		ret = -ENOMEM;
1842 		goto out_release_fw;
1843 	}
1844 
1845 	memcpy(bfr, fw->data, fw->size);
1846 
1847 	iowrite32(0, bcm4377->bar0 + BCM4377_BAR0_HOST_WINDOW_LO);
1848 	iowrite32(0, bcm4377->bar0 + BCM4377_BAR0_HOST_WINDOW_HI);
1849 	iowrite32(BCM4377_DMA_MASK,
1850 		  bcm4377->bar0 + BCM4377_BAR0_HOST_WINDOW_SIZE);
1851 
1852 	iowrite32(lower_32_bits(fw_dma), bcm4377->bar2 + BCM4377_BAR2_FW_LO);
1853 	iowrite32(upper_32_bits(fw_dma), bcm4377->bar2 + BCM4377_BAR2_FW_HI);
1854 	iowrite32(fw->size, bcm4377->bar2 + BCM4377_BAR2_FW_SIZE);
1855 	iowrite32(0, bcm4377->bar0 + BCM4377_BAR0_FW_DOORBELL);
1856 
1857 	dev_dbg(&bcm4377->pdev->dev, "waiting for firmware to boot\n");
1858 
1859 	ret = wait_for_completion_interruptible_timeout(&bcm4377->event,
1860 							BCM4377_TIMEOUT);
1861 	if (ret == 0) {
1862 		ret = -ETIMEDOUT;
1863 		goto out_dma_free;
1864 	} else if (ret < 0) {
1865 		goto out_dma_free;
1866 	}
1867 
1868 	if (bcm4377->bootstage != 2) {
1869 		dev_err(&bcm4377->pdev->dev, "boostage %d != 2\n",
1870 			bcm4377->bootstage);
1871 		ret = -ENXIO;
1872 		goto out_dma_free;
1873 	}
1874 
1875 	dev_dbg(&bcm4377->pdev->dev, "firmware has booted (stage = %x)\n",
1876 		bcm4377->bootstage);
1877 	ret = 0;
1878 
1879 out_dma_free:
1880 	dma_free_coherent(&bcm4377->pdev->dev, fw->size, bfr, fw_dma);
1881 out_release_fw:
1882 	release_firmware(fw);
1883 	return ret;
1884 }
1885 
1886 static int bcm4377_setup_rti(struct bcm4377_data *bcm4377)
1887 {
1888 	int ret;
1889 
1890 	dev_dbg(&bcm4377->pdev->dev, "starting RTI\n");
1891 	iowrite32(1, bcm4377->bar0 + BCM4377_BAR0_RTI_CONTROL);
1892 
1893 	ret = wait_for_completion_interruptible_timeout(&bcm4377->event,
1894 							BCM4377_TIMEOUT);
1895 	if (ret == 0) {
1896 		dev_err(&bcm4377->pdev->dev,
1897 			"timed out while waiting for RTI to transition to state 1");
1898 		return -ETIMEDOUT;
1899 	} else if (ret < 0) {
1900 		return ret;
1901 	}
1902 
1903 	if (bcm4377->rti_status != 1) {
1904 		dev_err(&bcm4377->pdev->dev, "RTI did not ack state 1 (%d)\n",
1905 			bcm4377->rti_status);
1906 		return -ENODEV;
1907 	}
1908 	dev_dbg(&bcm4377->pdev->dev, "RTI is in state 1\n");
1909 
1910 	/* allow access to the entire IOVA space again */
1911 	iowrite32(0, bcm4377->bar2 + BCM4377_BAR2_RTI_WINDOW_LO);
1912 	iowrite32(0, bcm4377->bar2 + BCM4377_BAR2_RTI_WINDOW_HI);
1913 	iowrite32(BCM4377_DMA_MASK,
1914 		  bcm4377->bar2 + BCM4377_BAR2_RTI_WINDOW_SIZE);
1915 
1916 	/* setup "Converged IPC" context */
1917 	iowrite32(lower_32_bits(bcm4377->ctx_dma),
1918 		  bcm4377->bar2 + BCM4377_BAR2_CONTEXT_ADDR_LO);
1919 	iowrite32(upper_32_bits(bcm4377->ctx_dma),
1920 		  bcm4377->bar2 + BCM4377_BAR2_CONTEXT_ADDR_HI);
1921 	iowrite32(2, bcm4377->bar0 + BCM4377_BAR0_RTI_CONTROL);
1922 
1923 	ret = wait_for_completion_interruptible_timeout(&bcm4377->event,
1924 							BCM4377_TIMEOUT);
1925 	if (ret == 0) {
1926 		dev_err(&bcm4377->pdev->dev,
1927 			"timed out while waiting for RTI to transition to state 2");
1928 		return -ETIMEDOUT;
1929 	} else if (ret < 0) {
1930 		return ret;
1931 	}
1932 
1933 	if (bcm4377->rti_status != 2) {
1934 		dev_err(&bcm4377->pdev->dev, "RTI did not ack state 2 (%d)\n",
1935 			bcm4377->rti_status);
1936 		return -ENODEV;
1937 	}
1938 
1939 	dev_dbg(&bcm4377->pdev->dev,
1940 		"RTI is in state 2; control ring is ready\n");
1941 	bcm4377->control_ack_ring.enabled = true;
1942 
1943 	return 0;
1944 }
1945 
1946 static int bcm4377_parse_otp_board_params(struct bcm4377_data *bcm4377,
1947 					  char tag, const char *val, size_t len)
1948 {
1949 	if (tag != 'V')
1950 		return 0;
1951 	if (len >= sizeof(bcm4377->vendor))
1952 		return -EINVAL;
1953 
1954 	strscpy(bcm4377->vendor, val, len + 1);
1955 	return 0;
1956 }
1957 
1958 static int bcm4377_parse_otp_chip_params(struct bcm4377_data *bcm4377, char tag,
1959 					 const char *val, size_t len)
1960 {
1961 	size_t idx = 0;
1962 
1963 	if (tag != 's')
1964 		return 0;
1965 	if (len >= sizeof(bcm4377->stepping))
1966 		return -EINVAL;
1967 
1968 	while (len != 0) {
1969 		bcm4377->stepping[idx] = tolower(val[idx]);
1970 		if (val[idx] == '\0')
1971 			return 0;
1972 
1973 		idx++;
1974 		len--;
1975 	}
1976 
1977 	bcm4377->stepping[idx] = '\0';
1978 	return 0;
1979 }
1980 
1981 static int bcm4377_parse_otp_str(struct bcm4377_data *bcm4377, const u8 *str,
1982 				 enum bcm4377_otp_params_type type)
1983 {
1984 	const char *p;
1985 	int ret;
1986 
1987 	p = skip_spaces(str);
1988 	while (*p) {
1989 		char tag = *p++;
1990 		const char *end;
1991 		size_t len;
1992 
1993 		if (*p++ != '=') /* implicit NUL check */
1994 			return -EINVAL;
1995 
1996 		/* *p might be NUL here, if so end == p and len == 0 */
1997 		end = strchrnul(p, ' ');
1998 		len = end - p;
1999 
2000 		/* leave 1 byte for NUL in destination string */
2001 		if (len > (BCM4377_OTP_MAX_PARAM_LEN - 1))
2002 			return -EINVAL;
2003 
2004 		switch (type) {
2005 		case BCM4377_OTP_BOARD_PARAMS:
2006 			ret = bcm4377_parse_otp_board_params(bcm4377, tag, p,
2007 							     len);
2008 			break;
2009 		case BCM4377_OTP_CHIP_PARAMS:
2010 			ret = bcm4377_parse_otp_chip_params(bcm4377, tag, p,
2011 							    len);
2012 			break;
2013 		default:
2014 			ret = -EINVAL;
2015 			break;
2016 		}
2017 
2018 		if (ret)
2019 			return ret;
2020 
2021 		/* Skip to next arg, if any */
2022 		p = skip_spaces(end);
2023 	}
2024 
2025 	return 0;
2026 }
2027 
2028 static int bcm4377_parse_otp_sys_vendor(struct bcm4377_data *bcm4377, u8 *otp,
2029 					size_t size)
2030 {
2031 	int idx = 4;
2032 	const char *chip_params;
2033 	const char *board_params;
2034 	int ret;
2035 
2036 	/* 4-byte header and two empty strings */
2037 	if (size < 6)
2038 		return -EINVAL;
2039 
2040 	if (get_unaligned_le32(otp) != BCM4377_OTP_VENDOR_HDR)
2041 		return -EINVAL;
2042 
2043 	chip_params = &otp[idx];
2044 
2045 	/* Skip first string, including terminator */
2046 	idx += strnlen(chip_params, size - idx) + 1;
2047 	if (idx >= size)
2048 		return -EINVAL;
2049 
2050 	board_params = &otp[idx];
2051 
2052 	/* Skip to terminator of second string */
2053 	idx += strnlen(board_params, size - idx);
2054 	if (idx >= size)
2055 		return -EINVAL;
2056 
2057 	/* At this point both strings are guaranteed NUL-terminated */
2058 	dev_dbg(&bcm4377->pdev->dev,
2059 		"OTP: chip_params='%s' board_params='%s'\n", chip_params,
2060 		board_params);
2061 
2062 	ret = bcm4377_parse_otp_str(bcm4377, chip_params,
2063 				    BCM4377_OTP_CHIP_PARAMS);
2064 	if (ret)
2065 		return ret;
2066 
2067 	ret = bcm4377_parse_otp_str(bcm4377, board_params,
2068 				    BCM4377_OTP_BOARD_PARAMS);
2069 	if (ret)
2070 		return ret;
2071 
2072 	if (!bcm4377->stepping[0] || !bcm4377->vendor[0])
2073 		return -EINVAL;
2074 
2075 	dev_dbg(&bcm4377->pdev->dev, "OTP: stepping=%s, vendor=%s\n",
2076 		bcm4377->stepping, bcm4377->vendor);
2077 	return 0;
2078 }
2079 
2080 static int bcm4377_parse_otp(struct bcm4377_data *bcm4377)
2081 {
2082 	u8 *otp;
2083 	int i;
2084 	int ret = -ENOENT;
2085 
2086 	otp = kzalloc(BCM4377_OTP_SIZE, GFP_KERNEL);
2087 	if (!otp)
2088 		return -ENOMEM;
2089 
2090 	for (i = 0; i < BCM4377_OTP_SIZE; ++i)
2091 		otp[i] = ioread8(bcm4377->bar0 + bcm4377->hw->otp_offset + i);
2092 
2093 	i = 0;
2094 	while (i < (BCM4377_OTP_SIZE - 1)) {
2095 		u8 type = otp[i];
2096 		u8 length = otp[i + 1];
2097 
2098 		if (type == 0)
2099 			break;
2100 
2101 		if ((i + 2 + length) > BCM4377_OTP_SIZE)
2102 			break;
2103 
2104 		switch (type) {
2105 		case BCM4377_OTP_SYS_VENDOR:
2106 			dev_dbg(&bcm4377->pdev->dev,
2107 				"OTP @ 0x%x (%d): SYS_VENDOR", i, length);
2108 			ret = bcm4377_parse_otp_sys_vendor(bcm4377, &otp[i + 2],
2109 							   length);
2110 			break;
2111 		case BCM4377_OTP_CIS:
2112 			dev_dbg(&bcm4377->pdev->dev, "OTP @ 0x%x (%d): CIS", i,
2113 				length);
2114 			break;
2115 		default:
2116 			dev_dbg(&bcm4377->pdev->dev, "OTP @ 0x%x (%d): unknown",
2117 				i, length);
2118 			break;
2119 		}
2120 
2121 		i += 2 + length;
2122 	}
2123 
2124 	kfree(otp);
2125 	return ret;
2126 }
2127 
2128 static int bcm4377_init_cfg(struct bcm4377_data *bcm4377)
2129 {
2130 	int ret;
2131 	u32 ctrl;
2132 
2133 	ret = pci_write_config_dword(bcm4377->pdev,
2134 				     BCM4377_PCIECFG_BAR0_WINDOW1,
2135 				     bcm4377->hw->bar0_window1);
2136 	if (ret)
2137 		return ret;
2138 
2139 	ret = pci_write_config_dword(bcm4377->pdev,
2140 				     BCM4377_PCIECFG_BAR0_WINDOW2,
2141 				     bcm4377->hw->bar0_window2);
2142 	if (ret)
2143 		return ret;
2144 
2145 	ret = pci_write_config_dword(
2146 		bcm4377->pdev, BCM4377_PCIECFG_BAR0_CORE2_WINDOW1,
2147 		BCM4377_PCIECFG_BAR0_CORE2_WINDOW1_DEFAULT);
2148 	if (ret)
2149 		return ret;
2150 
2151 	if (bcm4377->hw->has_bar0_core2_window2) {
2152 		ret = pci_write_config_dword(bcm4377->pdev,
2153 					     BCM4377_PCIECFG_BAR0_CORE2_WINDOW2,
2154 					     bcm4377->hw->bar0_core2_window2);
2155 		if (ret)
2156 			return ret;
2157 	}
2158 
2159 	ret = pci_write_config_dword(bcm4377->pdev, BCM4377_PCIECFG_BAR2_WINDOW,
2160 				     BCM4377_PCIECFG_BAR2_WINDOW_DEFAULT);
2161 	if (ret)
2162 		return ret;
2163 
2164 	ret = pci_read_config_dword(bcm4377->pdev,
2165 				    BCM4377_PCIECFG_SUBSYSTEM_CTRL, &ctrl);
2166 	if (ret)
2167 		return ret;
2168 
2169 	if (bcm4377->hw->clear_pciecfg_subsystem_ctrl_bit19)
2170 		ctrl &= ~BIT(19);
2171 	ctrl |= BIT(16);
2172 
2173 	return pci_write_config_dword(bcm4377->pdev,
2174 				      BCM4377_PCIECFG_SUBSYSTEM_CTRL, ctrl);
2175 }
2176 
2177 static int bcm4377_probe_dmi(struct bcm4377_data *bcm4377)
2178 {
2179 	const struct dmi_system_id *board_type_dmi_id;
2180 
2181 	board_type_dmi_id = dmi_first_match(bcm4377_dmi_board_table);
2182 	if (board_type_dmi_id && board_type_dmi_id->driver_data) {
2183 		bcm4377->board_type = board_type_dmi_id->driver_data;
2184 		dev_dbg(&bcm4377->pdev->dev,
2185 			"found board type via DMI match: %s\n",
2186 			bcm4377->board_type);
2187 	}
2188 
2189 	return 0;
2190 }
2191 
2192 static int bcm4377_probe_of(struct bcm4377_data *bcm4377)
2193 {
2194 	struct device_node *np = bcm4377->pdev->dev.of_node;
2195 	int ret;
2196 
2197 	if (!np)
2198 		return 0;
2199 
2200 	ret = of_property_read_string(np, "brcm,board-type",
2201 				      &bcm4377->board_type);
2202 	if (ret) {
2203 		dev_err(&bcm4377->pdev->dev, "no brcm,board-type property\n");
2204 		return ret;
2205 	}
2206 
2207 	bcm4377->taurus_beamforming_cal_blob =
2208 		of_get_property(np, "brcm,taurus-bf-cal-blob",
2209 				&bcm4377->taurus_beamforming_cal_size);
2210 	if (!bcm4377->taurus_beamforming_cal_blob) {
2211 		dev_err(&bcm4377->pdev->dev,
2212 			"no brcm,taurus-bf-cal-blob property\n");
2213 		return -ENOENT;
2214 	}
2215 	bcm4377->taurus_cal_blob = of_get_property(np, "brcm,taurus-cal-blob",
2216 						   &bcm4377->taurus_cal_size);
2217 	if (!bcm4377->taurus_cal_blob) {
2218 		dev_err(&bcm4377->pdev->dev,
2219 			"no brcm,taurus-cal-blob property\n");
2220 		return -ENOENT;
2221 	}
2222 
2223 	return 0;
2224 }
2225 
2226 static void bcm4377_disable_aspm(struct bcm4377_data *bcm4377)
2227 {
2228 	pci_disable_link_state(bcm4377->pdev,
2229 			       PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1);
2230 
2231 	/*
2232 	 * pci_disable_link_state can fail if either CONFIG_PCIEASPM is disabled
2233 	 * or if the BIOS hasn't handed over control to us. We must *always*
2234 	 * disable ASPM for this device due to hardware errata though.
2235 	 */
2236 	pcie_capability_clear_word(bcm4377->pdev, PCI_EXP_LNKCTL,
2237 				   PCI_EXP_LNKCTL_ASPMC);
2238 }
2239 
2240 static void bcm4377_pci_free_irq_vectors(void *data)
2241 {
2242 	pci_free_irq_vectors(data);
2243 }
2244 
2245 static void bcm4377_hci_free_dev(void *data)
2246 {
2247 	hci_free_dev(data);
2248 }
2249 
2250 static void bcm4377_hci_unregister_dev(void *data)
2251 {
2252 	hci_unregister_dev(data);
2253 }
2254 
2255 static int bcm4377_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2256 {
2257 	struct bcm4377_data *bcm4377;
2258 	struct hci_dev *hdev;
2259 	int ret, irq;
2260 
2261 	ret = dma_set_mask_and_coherent(&pdev->dev, BCM4377_DMA_MASK);
2262 	if (ret)
2263 		return ret;
2264 
2265 	bcm4377 = devm_kzalloc(&pdev->dev, sizeof(*bcm4377), GFP_KERNEL);
2266 	if (!bcm4377)
2267 		return -ENOMEM;
2268 
2269 	bcm4377->pdev = pdev;
2270 	bcm4377->hw = &bcm4377_hw_variants[id->driver_data];
2271 	init_completion(&bcm4377->event);
2272 
2273 	ret = bcm4377_prepare_rings(bcm4377);
2274 	if (ret)
2275 		return ret;
2276 
2277 	ret = bcm4377_init_context(bcm4377);
2278 	if (ret)
2279 		return ret;
2280 
2281 	ret = bcm4377_probe_dmi(bcm4377);
2282 	if (ret)
2283 		return ret;
2284 	ret = bcm4377_probe_of(bcm4377);
2285 	if (ret)
2286 		return ret;
2287 	if (!bcm4377->board_type) {
2288 		dev_err(&pdev->dev, "unable to determine board type\n");
2289 		return -ENODEV;
2290 	}
2291 
2292 	if (bcm4377->hw->disable_aspm)
2293 		bcm4377_disable_aspm(bcm4377);
2294 
2295 	ret = pci_reset_function_locked(pdev);
2296 	if (ret)
2297 		dev_warn(
2298 			&pdev->dev,
2299 			"function level reset failed with %d; trying to continue anyway\n",
2300 			ret);
2301 
2302 	/*
2303 	 * If this number is too low and we try to access any BAR too
2304 	 * early the device will crash. Experiments have shown that
2305 	 * approximately 50 msec is the minimum amount we have to wait.
2306 	 * Let's double that to be safe.
2307 	 */
2308 	msleep(100);
2309 
2310 	ret = pcim_enable_device(pdev);
2311 	if (ret)
2312 		return ret;
2313 	pci_set_master(pdev);
2314 
2315 	ret = bcm4377_init_cfg(bcm4377);
2316 	if (ret)
2317 		return ret;
2318 
2319 	bcm4377->bar0 = pcim_iomap(pdev, 0, 0);
2320 	if (!bcm4377->bar0)
2321 		return -EBUSY;
2322 	bcm4377->bar2 = pcim_iomap(pdev, 2, 0);
2323 	if (!bcm4377->bar2)
2324 		return -EBUSY;
2325 
2326 	ret = bcm4377_parse_otp(bcm4377);
2327 	if (ret) {
2328 		dev_err(&pdev->dev, "Reading OTP failed with %d\n", ret);
2329 		return ret;
2330 	}
2331 
2332 	/*
2333 	 * Legacy interrupts result in an IRQ storm because we don't know where
2334 	 * the interrupt mask and status registers for these chips are.
2335 	 * MSIs are acked automatically instead.
2336 	 */
2337 	ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
2338 	if (ret < 0)
2339 		return -ENODEV;
2340 	ret = devm_add_action_or_reset(&pdev->dev, bcm4377_pci_free_irq_vectors,
2341 				       pdev);
2342 	if (ret)
2343 		return ret;
2344 
2345 	irq = pci_irq_vector(pdev, 0);
2346 	if (irq <= 0)
2347 		return -ENODEV;
2348 
2349 	ret = devm_request_irq(&pdev->dev, irq, bcm4377_irq, 0, "bcm4377",
2350 			       bcm4377);
2351 	if (ret)
2352 		return ret;
2353 
2354 	hdev = hci_alloc_dev();
2355 	if (!hdev)
2356 		return -ENOMEM;
2357 	ret = devm_add_action_or_reset(&pdev->dev, bcm4377_hci_free_dev, hdev);
2358 	if (ret)
2359 		return ret;
2360 
2361 	bcm4377->hdev = hdev;
2362 
2363 	hdev->bus = HCI_PCI;
2364 	hdev->dev_type = HCI_PRIMARY;
2365 	hdev->open = bcm4377_hci_open;
2366 	hdev->close = bcm4377_hci_close;
2367 	hdev->send = bcm4377_hci_send_frame;
2368 	hdev->set_bdaddr = bcm4377_hci_set_bdaddr;
2369 	hdev->setup = bcm4377_hci_setup;
2370 
2371 	if (bcm4377->hw->broken_mws_transport_config)
2372 		set_bit(HCI_QUIRK_BROKEN_MWS_TRANSPORT_CONFIG, &hdev->quirks);
2373 	if (bcm4377->hw->broken_ext_scan)
2374 		set_bit(HCI_QUIRK_BROKEN_EXT_SCAN, &hdev->quirks);
2375 	if (bcm4377->hw->broken_le_coded)
2376 		set_bit(HCI_QUIRK_BROKEN_LE_CODED, &hdev->quirks);
2377 
2378 	pci_set_drvdata(pdev, bcm4377);
2379 	hci_set_drvdata(hdev, bcm4377);
2380 	SET_HCIDEV_DEV(hdev, &pdev->dev);
2381 
2382 	ret = bcm4377_boot(bcm4377);
2383 	if (ret)
2384 		return ret;
2385 
2386 	ret = bcm4377_setup_rti(bcm4377);
2387 	if (ret)
2388 		return ret;
2389 
2390 	ret = hci_register_dev(hdev);
2391 	if (ret)
2392 		return ret;
2393 	return devm_add_action_or_reset(&pdev->dev, bcm4377_hci_unregister_dev,
2394 					hdev);
2395 }
2396 
2397 static int bcm4377_suspend(struct pci_dev *pdev, pm_message_t state)
2398 {
2399 	struct bcm4377_data *bcm4377 = pci_get_drvdata(pdev);
2400 	int ret;
2401 
2402 	ret = hci_suspend_dev(bcm4377->hdev);
2403 	if (ret)
2404 		return ret;
2405 
2406 	iowrite32(BCM4377_BAR0_SLEEP_CONTROL_QUIESCE,
2407 		  bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL);
2408 
2409 	return 0;
2410 }
2411 
2412 static int bcm4377_resume(struct pci_dev *pdev)
2413 {
2414 	struct bcm4377_data *bcm4377 = pci_get_drvdata(pdev);
2415 
2416 	iowrite32(BCM4377_BAR0_SLEEP_CONTROL_UNQUIESCE,
2417 		  bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL);
2418 
2419 	return hci_resume_dev(bcm4377->hdev);
2420 }
2421 
2422 static const struct dmi_system_id bcm4377_dmi_board_table[] = {
2423 	{
2424 		.matches = {
2425 			DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
2426 			DMI_MATCH(DMI_PRODUCT_NAME, "MacBookAir9,1"),
2427 		},
2428 		.driver_data = "apple,formosa",
2429 	},
2430 	{
2431 		.matches = {
2432 			DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
2433 			DMI_MATCH(DMI_PRODUCT_NAME, "MacBookPro15,4"),
2434 		},
2435 		.driver_data = "apple,formosa",
2436 	},
2437 	{
2438 		.matches = {
2439 			DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
2440 			DMI_MATCH(DMI_PRODUCT_NAME, "MacBookPro16,3"),
2441 		},
2442 		.driver_data = "apple,formosa",
2443 	},
2444 	{}
2445 };
2446 
2447 static const struct bcm4377_hw bcm4377_hw_variants[] = {
2448 	[BCM4377] = {
2449 		.id = 0x4377,
2450 		.otp_offset = 0x4120,
2451 		.bar0_window1 = 0x1800b000,
2452 		.bar0_window2 = 0x1810c000,
2453 		.disable_aspm = true,
2454 		.broken_ext_scan = true,
2455 		.send_ptb = bcm4377_send_ptb,
2456 	},
2457 
2458 	[BCM4378] = {
2459 		.id = 0x4378,
2460 		.otp_offset = 0x4120,
2461 		.bar0_window1 = 0x18002000,
2462 		.bar0_window2 = 0x1810a000,
2463 		.bar0_core2_window2 = 0x18107000,
2464 		.has_bar0_core2_window2 = true,
2465 		.broken_mws_transport_config = true,
2466 		.broken_le_coded = true,
2467 		.send_calibration = bcm4378_send_calibration,
2468 		.send_ptb = bcm4378_send_ptb,
2469 	},
2470 
2471 	[BCM4387] = {
2472 		.id = 0x4387,
2473 		.otp_offset = 0x413c,
2474 		.bar0_window1 = 0x18002000,
2475 		.bar0_window2 = 0x18109000,
2476 		.bar0_core2_window2 = 0x18106000,
2477 		.has_bar0_core2_window2 = true,
2478 		.clear_pciecfg_subsystem_ctrl_bit19 = true,
2479 		.broken_mws_transport_config = true,
2480 		.broken_le_coded = true,
2481 		.send_calibration = bcm4387_send_calibration,
2482 		.send_ptb = bcm4378_send_ptb,
2483 	},
2484 };
2485 
2486 #define BCM4377_DEVID_ENTRY(id)                                             \
2487 	{                                                                   \
2488 		PCI_VENDOR_ID_BROADCOM, BCM##id##_DEVICE_ID, PCI_ANY_ID,    \
2489 			PCI_ANY_ID, PCI_CLASS_NETWORK_OTHER << 8, 0xffff00, \
2490 			BCM##id                                             \
2491 	}
2492 
2493 static const struct pci_device_id bcm4377_devid_table[] = {
2494 	BCM4377_DEVID_ENTRY(4377),
2495 	BCM4377_DEVID_ENTRY(4378),
2496 	BCM4377_DEVID_ENTRY(4387),
2497 	{},
2498 };
2499 MODULE_DEVICE_TABLE(pci, bcm4377_devid_table);
2500 
2501 static struct pci_driver bcm4377_pci_driver = {
2502 	.name = "hci_bcm4377",
2503 	.id_table = bcm4377_devid_table,
2504 	.probe = bcm4377_probe,
2505 	.suspend = bcm4377_suspend,
2506 	.resume = bcm4377_resume,
2507 };
2508 module_pci_driver(bcm4377_pci_driver);
2509 
2510 MODULE_AUTHOR("Sven Peter <sven@svenpeter.dev>");
2511 MODULE_DESCRIPTION("Bluetooth support for Broadcom 4377/4378/4387 devices");
2512 MODULE_LICENSE("Dual MIT/GPL");
2513 MODULE_FIRMWARE("brcm/brcmbt4377*.bin");
2514 MODULE_FIRMWARE("brcm/brcmbt4377*.ptb");
2515 MODULE_FIRMWARE("brcm/brcmbt4378*.bin");
2516 MODULE_FIRMWARE("brcm/brcmbt4378*.ptb");
2517 MODULE_FIRMWARE("brcm/brcmbt4387*.bin");
2518 MODULE_FIRMWARE("brcm/brcmbt4387*.ptb");
2519