Loading...
1// SPDX-License-Identifier: GPL-2.0-only OR MIT
2/*
3 * Bluetooth HCI driver for Broadcom 4377/4378/4387 devices attached via PCIe
4 *
5 * Copyright (C) The Asahi Linux Contributors
6 */
7
8#include <linux/async.h>
9#include <linux/bitfield.h>
10#include <linux/completion.h>
11#include <linux/dma-mapping.h>
12#include <linux/dmi.h>
13#include <linux/firmware.h>
14#include <linux/module.h>
15#include <linux/msi.h>
16#include <linux/of.h>
17#include <linux/pci.h>
18#include <linux/printk.h>
19
20#include <asm/unaligned.h>
21
22#include <net/bluetooth/bluetooth.h>
23#include <net/bluetooth/hci_core.h>
24
25enum bcm4377_chip {
26 BCM4377 = 0,
27 BCM4378,
28 BCM4387,
29};
30
31#define BCM4377_DEVICE_ID 0x5fa0
32#define BCM4378_DEVICE_ID 0x5f69
33#define BCM4387_DEVICE_ID 0x5f71
34
35#define BCM4377_TIMEOUT 1000
36
37/*
38 * These devices only support DMA transactions inside a 32bit window
39 * (possibly to avoid 64 bit arithmetic). The window size cannot exceed
40 * 0xffffffff but is always aligned down to the previous 0x200 byte boundary
41 * which effectively limits the window to [start, start+0xfffffe00].
42 * We just limit the DMA window to [0, 0xfffffe00] to make sure we don't
43 * run into this limitation.
44 */
45#define BCM4377_DMA_MASK 0xfffffe00
46
47#define BCM4377_PCIECFG_BAR0_WINDOW1 0x80
48#define BCM4377_PCIECFG_BAR0_WINDOW2 0x70
49#define BCM4377_PCIECFG_BAR0_CORE2_WINDOW1 0x74
50#define BCM4377_PCIECFG_BAR0_CORE2_WINDOW2 0x78
51#define BCM4377_PCIECFG_BAR2_WINDOW 0x84
52
53#define BCM4377_PCIECFG_BAR0_CORE2_WINDOW1_DEFAULT 0x18011000
54#define BCM4377_PCIECFG_BAR2_WINDOW_DEFAULT 0x19000000
55
56#define BCM4377_PCIECFG_SUBSYSTEM_CTRL 0x88
57
58#define BCM4377_BAR0_FW_DOORBELL 0x140
59#define BCM4377_BAR0_RTI_CONTROL 0x144
60
61#define BCM4377_BAR0_SLEEP_CONTROL 0x150
62#define BCM4377_BAR0_SLEEP_CONTROL_UNQUIESCE 0
63#define BCM4377_BAR0_SLEEP_CONTROL_AWAKE 2
64#define BCM4377_BAR0_SLEEP_CONTROL_QUIESCE 3
65
66#define BCM4377_BAR0_DOORBELL 0x174
67#define BCM4377_BAR0_DOORBELL_VALUE GENMASK(31, 16)
68#define BCM4377_BAR0_DOORBELL_IDX GENMASK(15, 8)
69#define BCM4377_BAR0_DOORBELL_RING BIT(5)
70
71#define BCM4377_BAR0_HOST_WINDOW_LO 0x590
72#define BCM4377_BAR0_HOST_WINDOW_HI 0x594
73#define BCM4377_BAR0_HOST_WINDOW_SIZE 0x598
74
75#define BCM4377_BAR2_BOOTSTAGE 0x200454
76
77#define BCM4377_BAR2_FW_LO 0x200478
78#define BCM4377_BAR2_FW_HI 0x20047c
79#define BCM4377_BAR2_FW_SIZE 0x200480
80
81#define BCM4377_BAR2_CONTEXT_ADDR_LO 0x20048c
82#define BCM4377_BAR2_CONTEXT_ADDR_HI 0x200450
83
84#define BCM4377_BAR2_RTI_STATUS 0x20045c
85#define BCM4377_BAR2_RTI_WINDOW_LO 0x200494
86#define BCM4377_BAR2_RTI_WINDOW_HI 0x200498
87#define BCM4377_BAR2_RTI_WINDOW_SIZE 0x20049c
88
89#define BCM4377_OTP_SIZE 0xe0
90#define BCM4377_OTP_SYS_VENDOR 0x15
91#define BCM4377_OTP_CIS 0x80
92#define BCM4377_OTP_VENDOR_HDR 0x00000008
93#define BCM4377_OTP_MAX_PARAM_LEN 16
94
95#define BCM4377_N_TRANSFER_RINGS 9
96#define BCM4377_N_COMPLETION_RINGS 6
97
98#define BCM4377_MAX_RING_SIZE 256
99
100#define BCM4377_MSGID_GENERATION GENMASK(15, 8)
101#define BCM4377_MSGID_ID GENMASK(7, 0)
102
103#define BCM4377_RING_N_ENTRIES 128
104
105#define BCM4377_CONTROL_MSG_SIZE 0x34
106#define BCM4377_XFER_RING_MAX_INPLACE_PAYLOAD_SIZE (4 * 0xff)
107
108#define MAX_ACL_PAYLOAD_SIZE (HCI_MAX_FRAME_SIZE + HCI_ACL_HDR_SIZE)
109#define MAX_SCO_PAYLOAD_SIZE (HCI_MAX_SCO_SIZE + HCI_SCO_HDR_SIZE)
110#define MAX_EVENT_PAYLOAD_SIZE (HCI_MAX_EVENT_SIZE + HCI_EVENT_HDR_SIZE)
111
112enum bcm4377_otp_params_type {
113 BCM4377_OTP_BOARD_PARAMS,
114 BCM4377_OTP_CHIP_PARAMS
115};
116
117enum bcm4377_transfer_ring_id {
118 BCM4377_XFER_RING_CONTROL = 0,
119 BCM4377_XFER_RING_HCI_H2D = 1,
120 BCM4377_XFER_RING_HCI_D2H = 2,
121 BCM4377_XFER_RING_SCO_H2D = 3,
122 BCM4377_XFER_RING_SCO_D2H = 4,
123 BCM4377_XFER_RING_ACL_H2D = 5,
124 BCM4377_XFER_RING_ACL_D2H = 6,
125};
126
127enum bcm4377_completion_ring_id {
128 BCM4377_ACK_RING_CONTROL = 0,
129 BCM4377_ACK_RING_HCI_ACL = 1,
130 BCM4377_EVENT_RING_HCI_ACL = 2,
131 BCM4377_ACK_RING_SCO = 3,
132 BCM4377_EVENT_RING_SCO = 4,
133};
134
135enum bcm4377_doorbell {
136 BCM4377_DOORBELL_CONTROL = 0,
137 BCM4377_DOORBELL_HCI_H2D = 1,
138 BCM4377_DOORBELL_HCI_D2H = 2,
139 BCM4377_DOORBELL_ACL_H2D = 3,
140 BCM4377_DOORBELL_ACL_D2H = 4,
141 BCM4377_DOORBELL_SCO = 6,
142};
143
144/*
145 * Transfer ring entry
146 *
147 * flags: Flags to indicate if the payload is appended or mapped
148 * len: Payload length
149 * payload: Optional payload DMA address
150 * id: Message id to recognize the answer in the completion ring entry
151 */
152struct bcm4377_xfer_ring_entry {
153#define BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED BIT(0)
154#define BCM4377_XFER_RING_FLAG_PAYLOAD_IN_FOOTER BIT(1)
155 u8 flags;
156 __le16 len;
157 u8 _unk0;
158 __le64 payload;
159 __le16 id;
160 u8 _unk1[2];
161} __packed;
162static_assert(sizeof(struct bcm4377_xfer_ring_entry) == 0x10);
163
164/*
165 * Completion ring entry
166 *
167 * flags: Flags to indicate if the payload is appended or mapped. If the payload
168 * is mapped it can be found in the buffer of the corresponding transfer
169 * ring message.
170 * ring_id: Transfer ring ID which required this message
171 * msg_id: Message ID specified in transfer ring entry
172 * len: Payload length
173 */
174struct bcm4377_completion_ring_entry {
175 u8 flags;
176 u8 _unk0;
177 __le16 ring_id;
178 __le16 msg_id;
179 __le32 len;
180 u8 _unk1[6];
181} __packed;
182static_assert(sizeof(struct bcm4377_completion_ring_entry) == 0x10);
183
184enum bcm4377_control_message_type {
185 BCM4377_CONTROL_MSG_CREATE_XFER_RING = 1,
186 BCM4377_CONTROL_MSG_CREATE_COMPLETION_RING = 2,
187 BCM4377_CONTROL_MSG_DESTROY_XFER_RING = 3,
188 BCM4377_CONTROL_MSG_DESTROY_COMPLETION_RING = 4,
189};
190
191/*
192 * Control message used to create a completion ring
193 *
194 * msg_type: Must be BCM4377_CONTROL_MSG_CREATE_COMPLETION_RING
195 * header_size: Unknown, but probably reserved space in front of the entry
196 * footer_size: Number of 32 bit words reserved for payloads after the entry
197 * id/id_again: Completion ring index
198 * ring_iova: DMA address of the ring buffer
199 * n_elements: Number of elements inside the ring buffer
200 * msi: MSI index, doesn't work for all rings though and should be zero
201 * intmod_delay: Unknown delay
202 * intmod_bytes: Unknown
203 */
204struct bcm4377_create_completion_ring_msg {
205 u8 msg_type;
206 u8 header_size;
207 u8 footer_size;
208 u8 _unk0;
209 __le16 id;
210 __le16 id_again;
211 __le64 ring_iova;
212 __le16 n_elements;
213 __le32 unk;
214 u8 _unk1[6];
215 __le16 msi;
216 __le16 intmod_delay;
217 __le32 intmod_bytes;
218 __le16 _unk2;
219 __le32 _unk3;
220 u8 _unk4[10];
221} __packed;
222static_assert(sizeof(struct bcm4377_create_completion_ring_msg) ==
223 BCM4377_CONTROL_MSG_SIZE);
224
225/*
226 * Control ring message used to destroy a completion ring
227 *
228 * msg_type: Must be BCM4377_CONTROL_MSG_DESTROY_COMPLETION_RING
229 * ring_id: Completion ring to be destroyed
230 */
231struct bcm4377_destroy_completion_ring_msg {
232 u8 msg_type;
233 u8 _pad0;
234 __le16 ring_id;
235 u8 _pad1[48];
236} __packed;
237static_assert(sizeof(struct bcm4377_destroy_completion_ring_msg) ==
238 BCM4377_CONTROL_MSG_SIZE);
239
240/*
241 * Control message used to create a transfer ring
242 *
243 * msg_type: Must be BCM4377_CONTROL_MSG_CREATE_XFER_RING
244 * header_size: Number of 32 bit words reserved for unknown content before the
245 * entry
246 * footer_size: Number of 32 bit words reserved for payloads after the entry
247 * ring_id/ring_id_again: Transfer ring index
248 * ring_iova: DMA address of the ring buffer
249 * n_elements: Number of elements inside the ring buffer
250 * completion_ring_id: Completion ring index for acknowledgements and events
251 * doorbell: Doorbell index used to notify device of new entries
252 * flags: Transfer ring flags
253 * - virtual: set if there is no associated shared memory and only the
254 * corresponding completion ring is used
255 * - sync: only set for the SCO rings
256 */
257struct bcm4377_create_transfer_ring_msg {
258 u8 msg_type;
259 u8 header_size;
260 u8 footer_size;
261 u8 _unk0;
262 __le16 ring_id;
263 __le16 ring_id_again;
264 __le64 ring_iova;
265 u8 _unk1[8];
266 __le16 n_elements;
267 __le16 completion_ring_id;
268 __le16 doorbell;
269#define BCM4377_XFER_RING_FLAG_VIRTUAL BIT(7)
270#define BCM4377_XFER_RING_FLAG_SYNC BIT(8)
271 __le16 flags;
272 u8 _unk2[20];
273} __packed;
274static_assert(sizeof(struct bcm4377_create_transfer_ring_msg) ==
275 BCM4377_CONTROL_MSG_SIZE);
276
277/*
278 * Control ring message used to destroy a transfer ring
279 *
280 * msg_type: Must be BCM4377_CONTROL_MSG_DESTROY_XFER_RING
281 * ring_id: Transfer ring to be destroyed
282 */
283struct bcm4377_destroy_transfer_ring_msg {
284 u8 msg_type;
285 u8 _pad0;
286 __le16 ring_id;
287 u8 _pad1[48];
288} __packed;
289static_assert(sizeof(struct bcm4377_destroy_transfer_ring_msg) ==
290 BCM4377_CONTROL_MSG_SIZE);
291
292/*
293 * "Converged IPC" context struct used to make the device aware of all other
294 * shared memory structures. A pointer to this structure is configured inside a
295 * MMIO register.
296 *
297 * version: Protocol version, must be 2.
298 * size: Size of this structure, must be 0x68.
299 * enabled_caps: Enabled capabilities. Unknown bitfield but should be 2.
300 * peripheral_info_addr: DMA address for a 0x20 buffer to which the device will
301 * write unknown contents
302 * {completion,xfer}_ring_{tails,heads}_addr: DMA pointers to ring heads/tails
303 * n_completion_rings: Number of completion rings, the firmware only works if
304 * this is set to BCM4377_N_COMPLETION_RINGS.
305 * n_xfer_rings: Number of transfer rings, the firmware only works if
306 * this is set to BCM4377_N_TRANSFER_RINGS.
307 * control_completion_ring_addr: Control completion ring buffer DMA address
308 * control_xfer_ring_addr: Control transfer ring buffer DMA address
309 * control_xfer_ring_n_entries: Number of control transfer ring entries
310 * control_completion_ring_n_entries: Number of control completion ring entries
311 * control_xfer_ring_doorbell: Control transfer ring doorbell
312 * control_completion_ring_doorbell: Control completion ring doorbell,
313 * must be set to 0xffff
314 * control_xfer_ring_msi: Control completion ring MSI index, must be 0
315 * control_completion_ring_msi: Control completion ring MSI index, must be 0.
316 * control_xfer_ring_header_size: Number of 32 bit words reserved in front of
317 * every control transfer ring entry
318 * control_xfer_ring_footer_size: Number of 32 bit words reserved after every
319 * control transfer ring entry
320 * control_completion_ring_header_size: Number of 32 bit words reserved in front
321 * of every control completion ring entry
322 * control_completion_ring_footer_size: Number of 32 bit words reserved after
323 * every control completion ring entry
324 * scratch_pad: Optional scratch pad DMA address
325 * scratch_pad_size: Scratch pad size
326 */
327struct bcm4377_context {
328 __le16 version;
329 __le16 size;
330 __le32 enabled_caps;
331
332 __le64 peripheral_info_addr;
333
334 /* ring heads and tails */
335 __le64 completion_ring_heads_addr;
336 __le64 xfer_ring_tails_addr;
337 __le64 completion_ring_tails_addr;
338 __le64 xfer_ring_heads_addr;
339 __le16 n_completion_rings;
340 __le16 n_xfer_rings;
341
342 /* control ring configuration */
343 __le64 control_completion_ring_addr;
344 __le64 control_xfer_ring_addr;
345 __le16 control_xfer_ring_n_entries;
346 __le16 control_completion_ring_n_entries;
347 __le16 control_xfer_ring_doorbell;
348 __le16 control_completion_ring_doorbell;
349 __le16 control_xfer_ring_msi;
350 __le16 control_completion_ring_msi;
351 u8 control_xfer_ring_header_size;
352 u8 control_xfer_ring_footer_size;
353 u8 control_completion_ring_header_size;
354 u8 control_completion_ring_footer_size;
355
356 __le16 _unk0;
357 __le16 _unk1;
358
359 __le64 scratch_pad;
360 __le32 scratch_pad_size;
361
362 __le32 _unk3;
363} __packed;
364static_assert(sizeof(struct bcm4377_context) == 0x68);
365
366#define BCM4378_CALIBRATION_CHUNK_SIZE 0xe6
367struct bcm4378_hci_send_calibration_cmd {
368 u8 unk;
369 __le16 blocks_left;
370 u8 data[BCM4378_CALIBRATION_CHUNK_SIZE];
371} __packed;
372
373#define BCM4378_PTB_CHUNK_SIZE 0xcf
374struct bcm4378_hci_send_ptb_cmd {
375 __le16 blocks_left;
376 u8 data[BCM4378_PTB_CHUNK_SIZE];
377} __packed;
378
379/*
380 * Shared memory structure used to store the ring head and tail pointers.
381 */
382struct bcm4377_ring_state {
383 __le16 completion_ring_head[BCM4377_N_COMPLETION_RINGS];
384 __le16 completion_ring_tail[BCM4377_N_COMPLETION_RINGS];
385 __le16 xfer_ring_head[BCM4377_N_TRANSFER_RINGS];
386 __le16 xfer_ring_tail[BCM4377_N_TRANSFER_RINGS];
387};
388
389/*
390 * A transfer ring can be used in two configurations:
391 * 1) Send control or HCI messages to the device which are then acknowledged
392 * in the corresponding completion ring
393 * 2) Receiving HCI frames from the devices. In this case the transfer ring
394 * itself contains empty messages that are acknowledged once data is
395 * available from the device. If the payloads fit inside the footers
396 * of the completion ring the transfer ring can be configured to be
397 * virtual such that it has no ring buffer.
398 *
399 * ring_id: ring index hardcoded in the firmware
400 * doorbell: doorbell index to notify device of new entries
401 * payload_size: optional in-place payload size
402 * mapped_payload_size: optional out-of-place payload size
403 * completion_ring: index of corresponding completion ring
404 * n_entries: number of entries inside this ring
405 * generation: ring generation; incremented on hci_open to detect stale messages
406 * sync: set to true for SCO rings
407 * virtual: set to true if this ring has no entries and is just required to
408 * setup a corresponding completion ring for device->host messages
409 * d2h_buffers_only: set to true if this ring is only used to provide large
410 * buffers used by device->host messages in the completion
411 * ring
412 * allow_wait: allow to wait for messages to be acknowledged
413 * enabled: true once the ring has been created and can be used
414 * ring: ring buffer for entries (struct bcm4377_xfer_ring_entry)
415 * ring_dma: DMA address for ring entry buffer
416 * payloads: payload buffer for mapped_payload_size payloads
417 * payloads_dma:DMA address for payload buffer
418 * events: pointer to array of completions if waiting is allowed
419 * msgids: bitmap to keep track of used message ids
420 * lock: Spinlock to protect access to ring structurs used in the irq handler
421 */
422struct bcm4377_transfer_ring {
423 enum bcm4377_transfer_ring_id ring_id;
424 enum bcm4377_doorbell doorbell;
425 size_t payload_size;
426 size_t mapped_payload_size;
427 u8 completion_ring;
428 u16 n_entries;
429 u8 generation;
430
431 bool sync;
432 bool virtual;
433 bool d2h_buffers_only;
434 bool allow_wait;
435 bool enabled;
436
437 void *ring;
438 dma_addr_t ring_dma;
439
440 void *payloads;
441 dma_addr_t payloads_dma;
442
443 struct completion **events;
444 DECLARE_BITMAP(msgids, BCM4377_MAX_RING_SIZE);
445 spinlock_t lock;
446};
447
448/*
449 * A completion ring can be either used to either acknowledge messages sent in
450 * the corresponding transfer ring or to receive messages associated with the
451 * transfer ring. When used to receive messages the transfer ring either
452 * has no ring buffer and is only advanced ("virtual transfer ring") or it
453 * only contains empty DMA buffers to be used for the payloads.
454 *
455 * ring_id: completion ring id, hardcoded in firmware
456 * payload_size: optional payload size after each entry
457 * delay: unknown delay
458 * n_entries: number of entries in this ring
459 * enabled: true once the ring has been created and can be used
460 * ring: ring buffer for entries (struct bcm4377_completion_ring_entry)
461 * ring_dma: DMA address of ring buffer
462 * transfer_rings: bitmap of corresponding transfer ring ids
463 */
464struct bcm4377_completion_ring {
465 enum bcm4377_completion_ring_id ring_id;
466 u16 payload_size;
467 u16 delay;
468 u16 n_entries;
469 bool enabled;
470
471 void *ring;
472 dma_addr_t ring_dma;
473
474 unsigned long transfer_rings;
475};
476
477struct bcm4377_data;
478
479/*
480 * Chip-specific configuration struct
481 *
482 * id: Chip id (e.g. 0x4377 for BCM4377)
483 * otp_offset: Offset to the start of the OTP inside BAR0
484 * bar0_window1: Backplane address mapped to the first window in BAR0
485 * bar0_window2: Backplane address mapped to the second window in BAR0
486 * bar0_core2_window2: Optional backplane address mapped to the second core's
487 * second window in BAR0
488 * has_bar0_core2_window2: Set to true if this chip requires the second core's
489 * second window to be configured
490 * clear_pciecfg_subsystem_ctrl_bit19: Set to true if bit 19 in the
491 * vendor-specific subsystem control
492 * register has to be cleared
493 * disable_aspm: Set to true if ASPM must be disabled due to hardware errata
494 * broken_ext_scan: Set to true if the chip erroneously claims to support
495 * extended scanning
496 * broken_mws_transport_config: Set to true if the chip erroneously claims to
497 * support MWS Transport Configuration
498 * send_calibration: Optional callback to send calibration data
499 * send_ptb: Callback to send "PTB" regulatory/calibration data
500 */
501struct bcm4377_hw {
502 unsigned int id;
503
504 u32 otp_offset;
505
506 u32 bar0_window1;
507 u32 bar0_window2;
508 u32 bar0_core2_window2;
509
510 unsigned long has_bar0_core2_window2 : 1;
511 unsigned long clear_pciecfg_subsystem_ctrl_bit19 : 1;
512 unsigned long disable_aspm : 1;
513 unsigned long broken_ext_scan : 1;
514 unsigned long broken_mws_transport_config : 1;
515 unsigned long broken_le_coded : 1;
516
517 int (*send_calibration)(struct bcm4377_data *bcm4377);
518 int (*send_ptb)(struct bcm4377_data *bcm4377,
519 const struct firmware *fw);
520};
521
522static const struct bcm4377_hw bcm4377_hw_variants[];
523static const struct dmi_system_id bcm4377_dmi_board_table[];
524
525/*
526 * Private struct associated with each device containing global state
527 *
528 * pdev: Pointer to associated struct pci_dev
529 * hdev: Pointer to associated strucy hci_dev
530 * bar0: iomem pointing to BAR0
531 * bar1: iomem pointing to BAR2
532 * bootstage: Current value of the bootstage
533 * rti_status: Current "RTI" status value
534 * hw: Pointer to chip-specific struct bcm4377_hw
535 * taurus_cal_blob: "Taurus" calibration blob used for some chips
536 * taurus_cal_size: "Taurus" calibration blob size
537 * taurus_beamforming_cal_blob: "Taurus" beamforming calibration blob used for
538 * some chips
539 * taurus_beamforming_cal_size: "Taurus" beamforming calibration blob size
540 * stepping: Chip stepping read from OTP; used for firmware selection
541 * vendor: Antenna vendor read from OTP; used for firmware selection
542 * board_type: Board type from FDT or DMI match; used for firmware selection
543 * event: Event for changed bootstage or rti_status; used for booting firmware
544 * ctx: "Converged IPC" context
545 * ctx_dma: "Converged IPC" context DMA address
546 * ring_state: Shared memory buffer containing ring head and tail indexes
547 * ring_state_dma: DMA address for ring_state
548 * {control,hci_acl,sco}_ack_ring: Completion rings used to acknowledge messages
549 * {hci_acl,sco}_event_ring: Completion rings used for device->host messages
550 * control_h2d_ring: Transfer ring used for control messages
551 * {hci,sco,acl}_h2d_ring: Transfer ring used to transfer HCI frames
552 * {hci,sco,acl}_d2h_ring: Transfer ring used to receive HCI frames in the
553 * corresponding completion ring
554 */
555struct bcm4377_data {
556 struct pci_dev *pdev;
557 struct hci_dev *hdev;
558
559 void __iomem *bar0;
560 void __iomem *bar2;
561
562 u32 bootstage;
563 u32 rti_status;
564
565 const struct bcm4377_hw *hw;
566
567 const void *taurus_cal_blob;
568 int taurus_cal_size;
569 const void *taurus_beamforming_cal_blob;
570 int taurus_beamforming_cal_size;
571
572 char stepping[BCM4377_OTP_MAX_PARAM_LEN];
573 char vendor[BCM4377_OTP_MAX_PARAM_LEN];
574 const char *board_type;
575
576 struct completion event;
577
578 struct bcm4377_context *ctx;
579 dma_addr_t ctx_dma;
580
581 struct bcm4377_ring_state *ring_state;
582 dma_addr_t ring_state_dma;
583
584 /*
585 * The HCI and ACL rings have to be merged because this structure is
586 * hardcoded in the firmware.
587 */
588 struct bcm4377_completion_ring control_ack_ring;
589 struct bcm4377_completion_ring hci_acl_ack_ring;
590 struct bcm4377_completion_ring hci_acl_event_ring;
591 struct bcm4377_completion_ring sco_ack_ring;
592 struct bcm4377_completion_ring sco_event_ring;
593
594 struct bcm4377_transfer_ring control_h2d_ring;
595 struct bcm4377_transfer_ring hci_h2d_ring;
596 struct bcm4377_transfer_ring hci_d2h_ring;
597 struct bcm4377_transfer_ring sco_h2d_ring;
598 struct bcm4377_transfer_ring sco_d2h_ring;
599 struct bcm4377_transfer_ring acl_h2d_ring;
600 struct bcm4377_transfer_ring acl_d2h_ring;
601};
602
603static void bcm4377_ring_doorbell(struct bcm4377_data *bcm4377, u8 doorbell,
604 u16 val)
605{
606 u32 db = 0;
607
608 db |= FIELD_PREP(BCM4377_BAR0_DOORBELL_VALUE, val);
609 db |= FIELD_PREP(BCM4377_BAR0_DOORBELL_IDX, doorbell);
610 db |= BCM4377_BAR0_DOORBELL_RING;
611
612 dev_dbg(&bcm4377->pdev->dev, "write %d to doorbell #%d (0x%x)\n", val,
613 doorbell, db);
614 iowrite32(db, bcm4377->bar0 + BCM4377_BAR0_DOORBELL);
615}
616
617static int bcm4377_extract_msgid(struct bcm4377_data *bcm4377,
618 struct bcm4377_transfer_ring *ring,
619 u16 raw_msgid, u8 *msgid)
620{
621 u8 generation = FIELD_GET(BCM4377_MSGID_GENERATION, raw_msgid);
622 *msgid = FIELD_GET(BCM4377_MSGID_ID, raw_msgid);
623
624 if (generation != ring->generation) {
625 dev_warn(
626 &bcm4377->pdev->dev,
627 "invalid message generation %d should be %d in entry for ring %d\n",
628 generation, ring->generation, ring->ring_id);
629 return -EINVAL;
630 }
631
632 if (*msgid >= ring->n_entries) {
633 dev_warn(&bcm4377->pdev->dev,
634 "invalid message id in entry for ring %d: %d > %d\n",
635 ring->ring_id, *msgid, ring->n_entries);
636 return -EINVAL;
637 }
638
639 return 0;
640}
641
642static void bcm4377_handle_event(struct bcm4377_data *bcm4377,
643 struct bcm4377_transfer_ring *ring,
644 u16 raw_msgid, u8 entry_flags, u8 type,
645 void *payload, size_t len)
646{
647 struct sk_buff *skb;
648 u16 head;
649 u8 msgid;
650 unsigned long flags;
651
652 spin_lock_irqsave(&ring->lock, flags);
653 if (!ring->enabled) {
654 dev_warn(&bcm4377->pdev->dev,
655 "event for disabled transfer ring %d\n",
656 ring->ring_id);
657 goto out;
658 }
659
660 if (ring->d2h_buffers_only &&
661 entry_flags & BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED) {
662 if (bcm4377_extract_msgid(bcm4377, ring, raw_msgid, &msgid))
663 goto out;
664
665 if (len > ring->mapped_payload_size) {
666 dev_warn(
667 &bcm4377->pdev->dev,
668 "invalid payload len in event for ring %d: %zu > %zu\n",
669 ring->ring_id, len, ring->mapped_payload_size);
670 goto out;
671 }
672
673 payload = ring->payloads + msgid * ring->mapped_payload_size;
674 }
675
676 skb = bt_skb_alloc(len, GFP_ATOMIC);
677 if (!skb)
678 goto out;
679
680 memcpy(skb_put(skb, len), payload, len);
681 hci_skb_pkt_type(skb) = type;
682 hci_recv_frame(bcm4377->hdev, skb);
683
684out:
685 head = le16_to_cpu(bcm4377->ring_state->xfer_ring_head[ring->ring_id]);
686 head = (head + 1) % ring->n_entries;
687 bcm4377->ring_state->xfer_ring_head[ring->ring_id] = cpu_to_le16(head);
688
689 bcm4377_ring_doorbell(bcm4377, ring->doorbell, head);
690
691 spin_unlock_irqrestore(&ring->lock, flags);
692}
693
694static void bcm4377_handle_ack(struct bcm4377_data *bcm4377,
695 struct bcm4377_transfer_ring *ring,
696 u16 raw_msgid)
697{
698 unsigned long flags;
699 u8 msgid;
700
701 spin_lock_irqsave(&ring->lock, flags);
702
703 if (bcm4377_extract_msgid(bcm4377, ring, raw_msgid, &msgid))
704 goto unlock;
705
706 if (!test_bit(msgid, ring->msgids)) {
707 dev_warn(
708 &bcm4377->pdev->dev,
709 "invalid message id in ack for ring %d: %d is not used\n",
710 ring->ring_id, msgid);
711 goto unlock;
712 }
713
714 if (ring->allow_wait && ring->events[msgid]) {
715 complete(ring->events[msgid]);
716 ring->events[msgid] = NULL;
717 }
718
719 bitmap_release_region(ring->msgids, msgid, ring->n_entries);
720
721unlock:
722 spin_unlock_irqrestore(&ring->lock, flags);
723}
724
725static void bcm4377_handle_completion(struct bcm4377_data *bcm4377,
726 struct bcm4377_completion_ring *ring,
727 u16 pos)
728{
729 struct bcm4377_completion_ring_entry *entry;
730 u16 msg_id, transfer_ring;
731 size_t entry_size, data_len;
732 void *data;
733
734 if (pos >= ring->n_entries) {
735 dev_warn(&bcm4377->pdev->dev,
736 "invalid offset %d for completion ring %d\n", pos,
737 ring->ring_id);
738 return;
739 }
740
741 entry_size = sizeof(*entry) + ring->payload_size;
742 entry = ring->ring + pos * entry_size;
743 data = ring->ring + pos * entry_size + sizeof(*entry);
744 data_len = le32_to_cpu(entry->len);
745 msg_id = le16_to_cpu(entry->msg_id);
746 transfer_ring = le16_to_cpu(entry->ring_id);
747
748 if ((ring->transfer_rings & BIT(transfer_ring)) == 0) {
749 dev_warn(
750 &bcm4377->pdev->dev,
751 "invalid entry at offset %d for transfer ring %d in completion ring %d\n",
752 pos, transfer_ring, ring->ring_id);
753 return;
754 }
755
756 dev_dbg(&bcm4377->pdev->dev,
757 "entry in completion ring %d for transfer ring %d with msg_id %d\n",
758 ring->ring_id, transfer_ring, msg_id);
759
760 switch (transfer_ring) {
761 case BCM4377_XFER_RING_CONTROL:
762 bcm4377_handle_ack(bcm4377, &bcm4377->control_h2d_ring, msg_id);
763 break;
764 case BCM4377_XFER_RING_HCI_H2D:
765 bcm4377_handle_ack(bcm4377, &bcm4377->hci_h2d_ring, msg_id);
766 break;
767 case BCM4377_XFER_RING_SCO_H2D:
768 bcm4377_handle_ack(bcm4377, &bcm4377->sco_h2d_ring, msg_id);
769 break;
770 case BCM4377_XFER_RING_ACL_H2D:
771 bcm4377_handle_ack(bcm4377, &bcm4377->acl_h2d_ring, msg_id);
772 break;
773
774 case BCM4377_XFER_RING_HCI_D2H:
775 bcm4377_handle_event(bcm4377, &bcm4377->hci_d2h_ring, msg_id,
776 entry->flags, HCI_EVENT_PKT, data,
777 data_len);
778 break;
779 case BCM4377_XFER_RING_SCO_D2H:
780 bcm4377_handle_event(bcm4377, &bcm4377->sco_d2h_ring, msg_id,
781 entry->flags, HCI_SCODATA_PKT, data,
782 data_len);
783 break;
784 case BCM4377_XFER_RING_ACL_D2H:
785 bcm4377_handle_event(bcm4377, &bcm4377->acl_d2h_ring, msg_id,
786 entry->flags, HCI_ACLDATA_PKT, data,
787 data_len);
788 break;
789
790 default:
791 dev_warn(
792 &bcm4377->pdev->dev,
793 "entry in completion ring %d for unknown transfer ring %d with msg_id %d\n",
794 ring->ring_id, transfer_ring, msg_id);
795 }
796}
797
798static void bcm4377_poll_completion_ring(struct bcm4377_data *bcm4377,
799 struct bcm4377_completion_ring *ring)
800{
801 u16 tail;
802 __le16 *heads = bcm4377->ring_state->completion_ring_head;
803 __le16 *tails = bcm4377->ring_state->completion_ring_tail;
804
805 if (!ring->enabled)
806 return;
807
808 tail = le16_to_cpu(tails[ring->ring_id]);
809 dev_dbg(&bcm4377->pdev->dev,
810 "completion ring #%d: head: %d, tail: %d\n", ring->ring_id,
811 le16_to_cpu(heads[ring->ring_id]), tail);
812
813 while (tail != le16_to_cpu(READ_ONCE(heads[ring->ring_id]))) {
814 /*
815 * ensure the CPU doesn't speculate through the comparison.
816 * otherwise it might already read the (empty) queue entry
817 * before the updated head has been loaded and checked.
818 */
819 dma_rmb();
820
821 bcm4377_handle_completion(bcm4377, ring, tail);
822
823 tail = (tail + 1) % ring->n_entries;
824 tails[ring->ring_id] = cpu_to_le16(tail);
825 }
826}
827
828static irqreturn_t bcm4377_irq(int irq, void *data)
829{
830 struct bcm4377_data *bcm4377 = data;
831 u32 bootstage, rti_status;
832
833 bootstage = ioread32(bcm4377->bar2 + BCM4377_BAR2_BOOTSTAGE);
834 rti_status = ioread32(bcm4377->bar2 + BCM4377_BAR2_RTI_STATUS);
835
836 if (bootstage != bcm4377->bootstage ||
837 rti_status != bcm4377->rti_status) {
838 dev_dbg(&bcm4377->pdev->dev,
839 "bootstage = %d -> %d, rti state = %d -> %d\n",
840 bcm4377->bootstage, bootstage, bcm4377->rti_status,
841 rti_status);
842 complete(&bcm4377->event);
843 bcm4377->bootstage = bootstage;
844 bcm4377->rti_status = rti_status;
845 }
846
847 if (rti_status > 2)
848 dev_err(&bcm4377->pdev->dev, "RTI status is %d\n", rti_status);
849
850 bcm4377_poll_completion_ring(bcm4377, &bcm4377->control_ack_ring);
851 bcm4377_poll_completion_ring(bcm4377, &bcm4377->hci_acl_event_ring);
852 bcm4377_poll_completion_ring(bcm4377, &bcm4377->hci_acl_ack_ring);
853 bcm4377_poll_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
854 bcm4377_poll_completion_ring(bcm4377, &bcm4377->sco_event_ring);
855
856 return IRQ_HANDLED;
857}
858
859static int bcm4377_enqueue(struct bcm4377_data *bcm4377,
860 struct bcm4377_transfer_ring *ring, void *data,
861 size_t len, bool wait)
862{
863 unsigned long flags;
864 struct bcm4377_xfer_ring_entry *entry;
865 void *payload;
866 size_t offset;
867 u16 head, tail, new_head;
868 u16 raw_msgid;
869 int ret, msgid;
870 DECLARE_COMPLETION_ONSTACK(event);
871
872 if (len > ring->payload_size && len > ring->mapped_payload_size) {
873 dev_warn(
874 &bcm4377->pdev->dev,
875 "payload len %zu is too large for ring %d (max is %zu or %zu)\n",
876 len, ring->ring_id, ring->payload_size,
877 ring->mapped_payload_size);
878 return -EINVAL;
879 }
880 if (wait && !ring->allow_wait)
881 return -EINVAL;
882 if (ring->virtual)
883 return -EINVAL;
884
885 spin_lock_irqsave(&ring->lock, flags);
886
887 head = le16_to_cpu(bcm4377->ring_state->xfer_ring_head[ring->ring_id]);
888 tail = le16_to_cpu(bcm4377->ring_state->xfer_ring_tail[ring->ring_id]);
889
890 new_head = (head + 1) % ring->n_entries;
891
892 if (new_head == tail) {
893 dev_warn(&bcm4377->pdev->dev,
894 "can't send message because ring %d is full\n",
895 ring->ring_id);
896 ret = -EINVAL;
897 goto out;
898 }
899
900 msgid = bitmap_find_free_region(ring->msgids, ring->n_entries, 0);
901 if (msgid < 0) {
902 dev_warn(&bcm4377->pdev->dev,
903 "can't find message id for ring %d\n", ring->ring_id);
904 ret = -EINVAL;
905 goto out;
906 }
907
908 raw_msgid = FIELD_PREP(BCM4377_MSGID_GENERATION, ring->generation);
909 raw_msgid |= FIELD_PREP(BCM4377_MSGID_ID, msgid);
910
911 offset = head * (sizeof(*entry) + ring->payload_size);
912 entry = ring->ring + offset;
913
914 memset(entry, 0, sizeof(*entry));
915 entry->id = cpu_to_le16(raw_msgid);
916 entry->len = cpu_to_le16(len);
917
918 if (len <= ring->payload_size) {
919 entry->flags = BCM4377_XFER_RING_FLAG_PAYLOAD_IN_FOOTER;
920 payload = ring->ring + offset + sizeof(*entry);
921 } else {
922 entry->flags = BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED;
923 entry->payload = cpu_to_le64(ring->payloads_dma +
924 msgid * ring->mapped_payload_size);
925 payload = ring->payloads + msgid * ring->mapped_payload_size;
926 }
927
928 memcpy(payload, data, len);
929
930 if (wait)
931 ring->events[msgid] = &event;
932
933 /*
934 * The 4377 chips stop responding to any commands as soon as they
935 * have been idle for a while. Poking the sleep control register here
936 * makes them come alive again.
937 */
938 iowrite32(BCM4377_BAR0_SLEEP_CONTROL_AWAKE,
939 bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL);
940
941 dev_dbg(&bcm4377->pdev->dev,
942 "updating head for transfer queue #%d to %d\n", ring->ring_id,
943 new_head);
944 bcm4377->ring_state->xfer_ring_head[ring->ring_id] =
945 cpu_to_le16(new_head);
946
947 if (!ring->sync)
948 bcm4377_ring_doorbell(bcm4377, ring->doorbell, new_head);
949 ret = 0;
950
951out:
952 spin_unlock_irqrestore(&ring->lock, flags);
953
954 if (ret == 0 && wait) {
955 ret = wait_for_completion_interruptible_timeout(
956 &event, BCM4377_TIMEOUT);
957 if (ret == 0)
958 ret = -ETIMEDOUT;
959 else if (ret > 0)
960 ret = 0;
961
962 spin_lock_irqsave(&ring->lock, flags);
963 ring->events[msgid] = NULL;
964 spin_unlock_irqrestore(&ring->lock, flags);
965 }
966
967 return ret;
968}
969
970static int bcm4377_create_completion_ring(struct bcm4377_data *bcm4377,
971 struct bcm4377_completion_ring *ring)
972{
973 struct bcm4377_create_completion_ring_msg msg;
974 int ret;
975
976 if (ring->enabled) {
977 dev_warn(&bcm4377->pdev->dev,
978 "completion ring %d already enabled\n", ring->ring_id);
979 return 0;
980 }
981
982 memset(ring->ring, 0,
983 ring->n_entries * (sizeof(struct bcm4377_completion_ring_entry) +
984 ring->payload_size));
985 memset(&msg, 0, sizeof(msg));
986 msg.msg_type = BCM4377_CONTROL_MSG_CREATE_COMPLETION_RING;
987 msg.id = cpu_to_le16(ring->ring_id);
988 msg.id_again = cpu_to_le16(ring->ring_id);
989 msg.ring_iova = cpu_to_le64(ring->ring_dma);
990 msg.n_elements = cpu_to_le16(ring->n_entries);
991 msg.intmod_bytes = cpu_to_le32(0xffffffff);
992 msg.unk = cpu_to_le32(0xffffffff);
993 msg.intmod_delay = cpu_to_le16(ring->delay);
994 msg.footer_size = ring->payload_size / 4;
995
996 ret = bcm4377_enqueue(bcm4377, &bcm4377->control_h2d_ring, &msg,
997 sizeof(msg), true);
998 if (!ret)
999 ring->enabled = true;
1000
1001 return ret;
1002}
1003
1004static int bcm4377_destroy_completion_ring(struct bcm4377_data *bcm4377,
1005 struct bcm4377_completion_ring *ring)
1006{
1007 struct bcm4377_destroy_completion_ring_msg msg;
1008 int ret;
1009
1010 memset(&msg, 0, sizeof(msg));
1011 msg.msg_type = BCM4377_CONTROL_MSG_DESTROY_COMPLETION_RING;
1012 msg.ring_id = cpu_to_le16(ring->ring_id);
1013
1014 ret = bcm4377_enqueue(bcm4377, &bcm4377->control_h2d_ring, &msg,
1015 sizeof(msg), true);
1016 if (ret)
1017 dev_warn(&bcm4377->pdev->dev,
1018 "failed to destroy completion ring %d\n",
1019 ring->ring_id);
1020
1021 ring->enabled = false;
1022 return ret;
1023}
1024
1025static int bcm4377_create_transfer_ring(struct bcm4377_data *bcm4377,
1026 struct bcm4377_transfer_ring *ring)
1027{
1028 struct bcm4377_create_transfer_ring_msg msg;
1029 u16 flags = 0;
1030 int ret, i;
1031 unsigned long spinlock_flags;
1032
1033 if (ring->virtual)
1034 flags |= BCM4377_XFER_RING_FLAG_VIRTUAL;
1035 if (ring->sync)
1036 flags |= BCM4377_XFER_RING_FLAG_SYNC;
1037
1038 spin_lock_irqsave(&ring->lock, spinlock_flags);
1039 memset(&msg, 0, sizeof(msg));
1040 msg.msg_type = BCM4377_CONTROL_MSG_CREATE_XFER_RING;
1041 msg.ring_id = cpu_to_le16(ring->ring_id);
1042 msg.ring_id_again = cpu_to_le16(ring->ring_id);
1043 msg.ring_iova = cpu_to_le64(ring->ring_dma);
1044 msg.n_elements = cpu_to_le16(ring->n_entries);
1045 msg.completion_ring_id = cpu_to_le16(ring->completion_ring);
1046 msg.doorbell = cpu_to_le16(ring->doorbell);
1047 msg.flags = cpu_to_le16(flags);
1048 msg.footer_size = ring->payload_size / 4;
1049
1050 bcm4377->ring_state->xfer_ring_head[ring->ring_id] = 0;
1051 bcm4377->ring_state->xfer_ring_tail[ring->ring_id] = 0;
1052 ring->generation++;
1053 spin_unlock_irqrestore(&ring->lock, spinlock_flags);
1054
1055 ret = bcm4377_enqueue(bcm4377, &bcm4377->control_h2d_ring, &msg,
1056 sizeof(msg), true);
1057
1058 spin_lock_irqsave(&ring->lock, spinlock_flags);
1059
1060 if (ring->d2h_buffers_only) {
1061 for (i = 0; i < ring->n_entries; ++i) {
1062 struct bcm4377_xfer_ring_entry *entry =
1063 ring->ring + i * sizeof(*entry);
1064 u16 raw_msgid = FIELD_PREP(BCM4377_MSGID_GENERATION,
1065 ring->generation);
1066 raw_msgid |= FIELD_PREP(BCM4377_MSGID_ID, i);
1067
1068 memset(entry, 0, sizeof(*entry));
1069 entry->id = cpu_to_le16(raw_msgid);
1070 entry->len = cpu_to_le16(ring->mapped_payload_size);
1071 entry->flags = BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED;
1072 entry->payload =
1073 cpu_to_le64(ring->payloads_dma +
1074 i * ring->mapped_payload_size);
1075 }
1076 }
1077
1078 /*
1079 * send some messages if this is a device->host ring to allow the device
1080 * to reply by acknowledging them in the completion ring
1081 */
1082 if (ring->virtual || ring->d2h_buffers_only) {
1083 bcm4377->ring_state->xfer_ring_head[ring->ring_id] =
1084 cpu_to_le16(0xf);
1085 bcm4377_ring_doorbell(bcm4377, ring->doorbell, 0xf);
1086 }
1087
1088 ring->enabled = true;
1089 spin_unlock_irqrestore(&ring->lock, spinlock_flags);
1090
1091 return ret;
1092}
1093
1094static int bcm4377_destroy_transfer_ring(struct bcm4377_data *bcm4377,
1095 struct bcm4377_transfer_ring *ring)
1096{
1097 struct bcm4377_destroy_transfer_ring_msg msg;
1098 int ret;
1099
1100 memset(&msg, 0, sizeof(msg));
1101 msg.msg_type = BCM4377_CONTROL_MSG_DESTROY_XFER_RING;
1102 msg.ring_id = cpu_to_le16(ring->ring_id);
1103
1104 ret = bcm4377_enqueue(bcm4377, &bcm4377->control_h2d_ring, &msg,
1105 sizeof(msg), true);
1106 if (ret)
1107 dev_warn(&bcm4377->pdev->dev,
1108 "failed to destroy transfer ring %d\n", ring->ring_id);
1109
1110 ring->enabled = false;
1111 return ret;
1112}
1113
1114static int __bcm4378_send_calibration_chunk(struct bcm4377_data *bcm4377,
1115 const void *data, size_t data_len,
1116 u16 blocks_left)
1117{
1118 struct bcm4378_hci_send_calibration_cmd cmd;
1119 struct sk_buff *skb;
1120
1121 if (data_len > sizeof(cmd.data))
1122 return -EINVAL;
1123
1124 memset(&cmd, 0, sizeof(cmd));
1125 cmd.unk = 0x03;
1126 cmd.blocks_left = cpu_to_le16(blocks_left);
1127 memcpy(cmd.data, data, data_len);
1128
1129 skb = __hci_cmd_sync(bcm4377->hdev, 0xfd97, sizeof(cmd), &cmd,
1130 HCI_INIT_TIMEOUT);
1131 if (IS_ERR(skb))
1132 return PTR_ERR(skb);
1133
1134 kfree_skb(skb);
1135 return 0;
1136}
1137
1138static int __bcm4378_send_calibration(struct bcm4377_data *bcm4377,
1139 const void *data, size_t data_size)
1140{
1141 int ret;
1142 size_t i, left, transfer_len;
1143 size_t blocks =
1144 DIV_ROUND_UP(data_size, (size_t)BCM4378_CALIBRATION_CHUNK_SIZE);
1145
1146 if (!data) {
1147 dev_err(&bcm4377->pdev->dev,
1148 "no calibration data available.\n");
1149 return -ENOENT;
1150 }
1151
1152 for (i = 0, left = data_size; i < blocks; ++i, left -= transfer_len) {
1153 transfer_len =
1154 min_t(size_t, left, BCM4378_CALIBRATION_CHUNK_SIZE);
1155
1156 ret = __bcm4378_send_calibration_chunk(
1157 bcm4377, data + i * BCM4378_CALIBRATION_CHUNK_SIZE,
1158 transfer_len, blocks - i - 1);
1159 if (ret) {
1160 dev_err(&bcm4377->pdev->dev,
1161 "send calibration chunk failed with %d\n", ret);
1162 return ret;
1163 }
1164 }
1165
1166 return 0;
1167}
1168
1169static int bcm4378_send_calibration(struct bcm4377_data *bcm4377)
1170{
1171 if ((strcmp(bcm4377->stepping, "b1") == 0) ||
1172 strcmp(bcm4377->stepping, "b3") == 0)
1173 return __bcm4378_send_calibration(
1174 bcm4377, bcm4377->taurus_beamforming_cal_blob,
1175 bcm4377->taurus_beamforming_cal_size);
1176 else
1177 return __bcm4378_send_calibration(bcm4377,
1178 bcm4377->taurus_cal_blob,
1179 bcm4377->taurus_cal_size);
1180}
1181
1182static int bcm4387_send_calibration(struct bcm4377_data *bcm4377)
1183{
1184 if (strcmp(bcm4377->stepping, "c2") == 0)
1185 return __bcm4378_send_calibration(
1186 bcm4377, bcm4377->taurus_beamforming_cal_blob,
1187 bcm4377->taurus_beamforming_cal_size);
1188 else
1189 return __bcm4378_send_calibration(bcm4377,
1190 bcm4377->taurus_cal_blob,
1191 bcm4377->taurus_cal_size);
1192}
1193
1194static const struct firmware *bcm4377_request_blob(struct bcm4377_data *bcm4377,
1195 const char *suffix)
1196{
1197 const struct firmware *fw;
1198 char name0[64], name1[64];
1199 int ret;
1200
1201 snprintf(name0, sizeof(name0), "brcm/brcmbt%04x%s-%s-%s.%s",
1202 bcm4377->hw->id, bcm4377->stepping, bcm4377->board_type,
1203 bcm4377->vendor, suffix);
1204 snprintf(name1, sizeof(name1), "brcm/brcmbt%04x%s-%s.%s",
1205 bcm4377->hw->id, bcm4377->stepping, bcm4377->board_type,
1206 suffix);
1207 dev_dbg(&bcm4377->pdev->dev, "Trying to load firmware: '%s' or '%s'\n",
1208 name0, name1);
1209
1210 ret = firmware_request_nowarn(&fw, name0, &bcm4377->pdev->dev);
1211 if (!ret)
1212 return fw;
1213 ret = firmware_request_nowarn(&fw, name1, &bcm4377->pdev->dev);
1214 if (!ret)
1215 return fw;
1216
1217 dev_err(&bcm4377->pdev->dev,
1218 "Unable to load firmware; tried '%s' and '%s'\n", name0, name1);
1219 return NULL;
1220}
1221
1222static int bcm4377_send_ptb(struct bcm4377_data *bcm4377,
1223 const struct firmware *fw)
1224{
1225 struct sk_buff *skb;
1226
1227 skb = __hci_cmd_sync(bcm4377->hdev, 0xfd98, fw->size, fw->data,
1228 HCI_INIT_TIMEOUT);
1229 /*
1230 * This command seems to always fail on more recent firmware versions
1231 * (even in traces taken from the macOS driver). It's unclear why this
1232 * happens but because the PTB file contains calibration and/or
1233 * regulatory data and may be required on older firmware we still try to
1234 * send it here just in case and just ignore if it fails.
1235 */
1236 if (!IS_ERR(skb))
1237 kfree_skb(skb);
1238 return 0;
1239}
1240
1241static int bcm4378_send_ptb_chunk(struct bcm4377_data *bcm4377,
1242 const void *data, size_t data_len,
1243 u16 blocks_left)
1244{
1245 struct bcm4378_hci_send_ptb_cmd cmd;
1246 struct sk_buff *skb;
1247
1248 if (data_len > BCM4378_PTB_CHUNK_SIZE)
1249 return -EINVAL;
1250
1251 memset(&cmd, 0, sizeof(cmd));
1252 cmd.blocks_left = cpu_to_le16(blocks_left);
1253 memcpy(cmd.data, data, data_len);
1254
1255 skb = __hci_cmd_sync(bcm4377->hdev, 0xfe0d, sizeof(cmd), &cmd,
1256 HCI_INIT_TIMEOUT);
1257 if (IS_ERR(skb))
1258 return PTR_ERR(skb);
1259
1260 kfree_skb(skb);
1261 return 0;
1262}
1263
1264static int bcm4378_send_ptb(struct bcm4377_data *bcm4377,
1265 const struct firmware *fw)
1266{
1267 size_t chunks = DIV_ROUND_UP(fw->size, (size_t)BCM4378_PTB_CHUNK_SIZE);
1268 size_t i, left, transfer_len;
1269 int ret;
1270
1271 for (i = 0, left = fw->size; i < chunks; ++i, left -= transfer_len) {
1272 transfer_len = min_t(size_t, left, BCM4378_PTB_CHUNK_SIZE);
1273
1274 dev_dbg(&bcm4377->pdev->dev, "sending ptb chunk %zu/%zu\n",
1275 i + 1, chunks);
1276 ret = bcm4378_send_ptb_chunk(
1277 bcm4377, fw->data + i * BCM4378_PTB_CHUNK_SIZE,
1278 transfer_len, chunks - i - 1);
1279 if (ret) {
1280 dev_err(&bcm4377->pdev->dev,
1281 "sending ptb chunk %zu failed (%d)", i, ret);
1282 return ret;
1283 }
1284 }
1285
1286 return 0;
1287}
1288
1289static int bcm4377_hci_open(struct hci_dev *hdev)
1290{
1291 struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1292 int ret;
1293
1294 dev_dbg(&bcm4377->pdev->dev, "creating rings\n");
1295
1296 ret = bcm4377_create_completion_ring(bcm4377,
1297 &bcm4377->hci_acl_ack_ring);
1298 if (ret)
1299 return ret;
1300 ret = bcm4377_create_completion_ring(bcm4377,
1301 &bcm4377->hci_acl_event_ring);
1302 if (ret)
1303 goto destroy_hci_acl_ack;
1304 ret = bcm4377_create_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
1305 if (ret)
1306 goto destroy_hci_acl_event;
1307 ret = bcm4377_create_completion_ring(bcm4377, &bcm4377->sco_event_ring);
1308 if (ret)
1309 goto destroy_sco_ack;
1310 dev_dbg(&bcm4377->pdev->dev,
1311 "all completion rings successfully created!\n");
1312
1313 ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring);
1314 if (ret)
1315 goto destroy_sco_event;
1316 ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring);
1317 if (ret)
1318 goto destroy_hci_h2d;
1319 ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring);
1320 if (ret)
1321 goto destroy_hci_d2h;
1322 ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring);
1323 if (ret)
1324 goto destroy_sco_h2d;
1325 ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring);
1326 if (ret)
1327 goto destroy_sco_d2h;
1328 ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->acl_d2h_ring);
1329 if (ret)
1330 goto destroy_acl_h2d;
1331 dev_dbg(&bcm4377->pdev->dev,
1332 "all transfer rings successfully created!\n");
1333
1334 return 0;
1335
1336destroy_acl_h2d:
1337 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring);
1338destroy_sco_d2h:
1339 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring);
1340destroy_sco_h2d:
1341 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring);
1342destroy_hci_d2h:
1343 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring);
1344destroy_hci_h2d:
1345 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring);
1346destroy_sco_event:
1347 bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_event_ring);
1348destroy_sco_ack:
1349 bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
1350destroy_hci_acl_event:
1351 bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_event_ring);
1352destroy_hci_acl_ack:
1353 bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_ack_ring);
1354
1355 dev_err(&bcm4377->pdev->dev, "Creating rings failed with %d\n", ret);
1356 return ret;
1357}
1358
1359static int bcm4377_hci_close(struct hci_dev *hdev)
1360{
1361 struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1362
1363 dev_dbg(&bcm4377->pdev->dev, "destroying rings in hci_close\n");
1364
1365 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->acl_d2h_ring);
1366 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring);
1367 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring);
1368 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring);
1369 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring);
1370 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring);
1371
1372 bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_event_ring);
1373 bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
1374 bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_event_ring);
1375 bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_ack_ring);
1376
1377 return 0;
1378}
1379
1380static bool bcm4377_is_valid_bdaddr(struct bcm4377_data *bcm4377,
1381 bdaddr_t *addr)
1382{
1383 if (addr->b[0] != 0x93)
1384 return true;
1385 if (addr->b[1] != 0x76)
1386 return true;
1387 if (addr->b[2] != 0x00)
1388 return true;
1389 if (addr->b[4] != (bcm4377->hw->id & 0xff))
1390 return true;
1391 if (addr->b[5] != (bcm4377->hw->id >> 8))
1392 return true;
1393 return false;
1394}
1395
1396static int bcm4377_check_bdaddr(struct bcm4377_data *bcm4377)
1397{
1398 struct hci_rp_read_bd_addr *bda;
1399 struct sk_buff *skb;
1400
1401 skb = __hci_cmd_sync(bcm4377->hdev, HCI_OP_READ_BD_ADDR, 0, NULL,
1402 HCI_INIT_TIMEOUT);
1403 if (IS_ERR(skb)) {
1404 int err = PTR_ERR(skb);
1405
1406 dev_err(&bcm4377->pdev->dev, "HCI_OP_READ_BD_ADDR failed (%d)",
1407 err);
1408 return err;
1409 }
1410
1411 if (skb->len != sizeof(*bda)) {
1412 dev_err(&bcm4377->pdev->dev,
1413 "HCI_OP_READ_BD_ADDR reply length invalid");
1414 kfree_skb(skb);
1415 return -EIO;
1416 }
1417
1418 bda = (struct hci_rp_read_bd_addr *)skb->data;
1419 if (!bcm4377_is_valid_bdaddr(bcm4377, &bda->bdaddr))
1420 set_bit(HCI_QUIRK_USE_BDADDR_PROPERTY, &bcm4377->hdev->quirks);
1421
1422 kfree_skb(skb);
1423 return 0;
1424}
1425
1426static int bcm4377_hci_setup(struct hci_dev *hdev)
1427{
1428 struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1429 const struct firmware *fw;
1430 int ret;
1431
1432 if (bcm4377->hw->send_calibration) {
1433 ret = bcm4377->hw->send_calibration(bcm4377);
1434 if (ret)
1435 return ret;
1436 }
1437
1438 fw = bcm4377_request_blob(bcm4377, "ptb");
1439 if (!fw) {
1440 dev_err(&bcm4377->pdev->dev, "failed to load PTB data");
1441 return -ENOENT;
1442 }
1443
1444 ret = bcm4377->hw->send_ptb(bcm4377, fw);
1445 release_firmware(fw);
1446 if (ret)
1447 return ret;
1448
1449 return bcm4377_check_bdaddr(bcm4377);
1450}
1451
1452static int bcm4377_hci_send_frame(struct hci_dev *hdev, struct sk_buff *skb)
1453{
1454 struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1455 struct bcm4377_transfer_ring *ring;
1456 int ret;
1457
1458 switch (hci_skb_pkt_type(skb)) {
1459 case HCI_COMMAND_PKT:
1460 hdev->stat.cmd_tx++;
1461 ring = &bcm4377->hci_h2d_ring;
1462 break;
1463
1464 case HCI_ACLDATA_PKT:
1465 hdev->stat.acl_tx++;
1466 ring = &bcm4377->acl_h2d_ring;
1467 break;
1468
1469 case HCI_SCODATA_PKT:
1470 hdev->stat.sco_tx++;
1471 ring = &bcm4377->sco_h2d_ring;
1472 break;
1473
1474 default:
1475 return -EILSEQ;
1476 }
1477
1478 ret = bcm4377_enqueue(bcm4377, ring, skb->data, skb->len, false);
1479 if (ret < 0) {
1480 hdev->stat.err_tx++;
1481 return ret;
1482 }
1483
1484 hdev->stat.byte_tx += skb->len;
1485 kfree_skb(skb);
1486 return ret;
1487}
1488
1489static int bcm4377_hci_set_bdaddr(struct hci_dev *hdev, const bdaddr_t *bdaddr)
1490{
1491 struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1492 struct sk_buff *skb;
1493 int err;
1494
1495 skb = __hci_cmd_sync(hdev, 0xfc01, 6, bdaddr, HCI_INIT_TIMEOUT);
1496 if (IS_ERR(skb)) {
1497 err = PTR_ERR(skb);
1498 dev_err(&bcm4377->pdev->dev,
1499 "Change address command failed (%d)", err);
1500 return err;
1501 }
1502 kfree_skb(skb);
1503
1504 return 0;
1505}
1506
1507static int bcm4377_alloc_transfer_ring(struct bcm4377_data *bcm4377,
1508 struct bcm4377_transfer_ring *ring)
1509{
1510 size_t entry_size;
1511
1512 spin_lock_init(&ring->lock);
1513 ring->payload_size = ALIGN(ring->payload_size, 4);
1514 ring->mapped_payload_size = ALIGN(ring->mapped_payload_size, 4);
1515
1516 if (ring->payload_size > BCM4377_XFER_RING_MAX_INPLACE_PAYLOAD_SIZE)
1517 return -EINVAL;
1518 if (ring->n_entries > BCM4377_MAX_RING_SIZE)
1519 return -EINVAL;
1520 if (ring->virtual && ring->allow_wait)
1521 return -EINVAL;
1522
1523 if (ring->d2h_buffers_only) {
1524 if (ring->virtual)
1525 return -EINVAL;
1526 if (ring->payload_size)
1527 return -EINVAL;
1528 if (!ring->mapped_payload_size)
1529 return -EINVAL;
1530 }
1531 if (ring->virtual)
1532 return 0;
1533
1534 entry_size =
1535 ring->payload_size + sizeof(struct bcm4377_xfer_ring_entry);
1536 ring->ring = dmam_alloc_coherent(&bcm4377->pdev->dev,
1537 ring->n_entries * entry_size,
1538 &ring->ring_dma, GFP_KERNEL);
1539 if (!ring->ring)
1540 return -ENOMEM;
1541
1542 if (ring->allow_wait) {
1543 ring->events = devm_kcalloc(&bcm4377->pdev->dev,
1544 ring->n_entries,
1545 sizeof(*ring->events), GFP_KERNEL);
1546 if (!ring->events)
1547 return -ENOMEM;
1548 }
1549
1550 if (ring->mapped_payload_size) {
1551 ring->payloads = dmam_alloc_coherent(
1552 &bcm4377->pdev->dev,
1553 ring->n_entries * ring->mapped_payload_size,
1554 &ring->payloads_dma, GFP_KERNEL);
1555 if (!ring->payloads)
1556 return -ENOMEM;
1557 }
1558
1559 return 0;
1560}
1561
1562static int bcm4377_alloc_completion_ring(struct bcm4377_data *bcm4377,
1563 struct bcm4377_completion_ring *ring)
1564{
1565 size_t entry_size;
1566
1567 ring->payload_size = ALIGN(ring->payload_size, 4);
1568 if (ring->payload_size > BCM4377_XFER_RING_MAX_INPLACE_PAYLOAD_SIZE)
1569 return -EINVAL;
1570 if (ring->n_entries > BCM4377_MAX_RING_SIZE)
1571 return -EINVAL;
1572
1573 entry_size = ring->payload_size +
1574 sizeof(struct bcm4377_completion_ring_entry);
1575
1576 ring->ring = dmam_alloc_coherent(&bcm4377->pdev->dev,
1577 ring->n_entries * entry_size,
1578 &ring->ring_dma, GFP_KERNEL);
1579 if (!ring->ring)
1580 return -ENOMEM;
1581 return 0;
1582}
1583
1584static int bcm4377_init_context(struct bcm4377_data *bcm4377)
1585{
1586 struct device *dev = &bcm4377->pdev->dev;
1587 dma_addr_t peripheral_info_dma;
1588
1589 bcm4377->ctx = dmam_alloc_coherent(dev, sizeof(*bcm4377->ctx),
1590 &bcm4377->ctx_dma, GFP_KERNEL);
1591 if (!bcm4377->ctx)
1592 return -ENOMEM;
1593 memset(bcm4377->ctx, 0, sizeof(*bcm4377->ctx));
1594
1595 bcm4377->ring_state =
1596 dmam_alloc_coherent(dev, sizeof(*bcm4377->ring_state),
1597 &bcm4377->ring_state_dma, GFP_KERNEL);
1598 if (!bcm4377->ring_state)
1599 return -ENOMEM;
1600 memset(bcm4377->ring_state, 0, sizeof(*bcm4377->ring_state));
1601
1602 bcm4377->ctx->version = cpu_to_le16(1);
1603 bcm4377->ctx->size = cpu_to_le16(sizeof(*bcm4377->ctx));
1604 bcm4377->ctx->enabled_caps = cpu_to_le32(2);
1605
1606 /*
1607 * The BT device will write 0x20 bytes of data to this buffer but
1608 * the exact contents are unknown. It only needs to exist for BT
1609 * to work such that we can just allocate and then ignore it.
1610 */
1611 if (!dmam_alloc_coherent(&bcm4377->pdev->dev, 0x20,
1612 &peripheral_info_dma, GFP_KERNEL))
1613 return -ENOMEM;
1614 bcm4377->ctx->peripheral_info_addr = cpu_to_le64(peripheral_info_dma);
1615
1616 bcm4377->ctx->xfer_ring_heads_addr = cpu_to_le64(
1617 bcm4377->ring_state_dma +
1618 offsetof(struct bcm4377_ring_state, xfer_ring_head));
1619 bcm4377->ctx->xfer_ring_tails_addr = cpu_to_le64(
1620 bcm4377->ring_state_dma +
1621 offsetof(struct bcm4377_ring_state, xfer_ring_tail));
1622 bcm4377->ctx->completion_ring_heads_addr = cpu_to_le64(
1623 bcm4377->ring_state_dma +
1624 offsetof(struct bcm4377_ring_state, completion_ring_head));
1625 bcm4377->ctx->completion_ring_tails_addr = cpu_to_le64(
1626 bcm4377->ring_state_dma +
1627 offsetof(struct bcm4377_ring_state, completion_ring_tail));
1628
1629 bcm4377->ctx->n_completion_rings =
1630 cpu_to_le16(BCM4377_N_COMPLETION_RINGS);
1631 bcm4377->ctx->n_xfer_rings = cpu_to_le16(BCM4377_N_TRANSFER_RINGS);
1632
1633 bcm4377->ctx->control_completion_ring_addr =
1634 cpu_to_le64(bcm4377->control_ack_ring.ring_dma);
1635 bcm4377->ctx->control_completion_ring_n_entries =
1636 cpu_to_le16(bcm4377->control_ack_ring.n_entries);
1637 bcm4377->ctx->control_completion_ring_doorbell = cpu_to_le16(0xffff);
1638 bcm4377->ctx->control_completion_ring_msi = 0;
1639 bcm4377->ctx->control_completion_ring_header_size = 0;
1640 bcm4377->ctx->control_completion_ring_footer_size = 0;
1641
1642 bcm4377->ctx->control_xfer_ring_addr =
1643 cpu_to_le64(bcm4377->control_h2d_ring.ring_dma);
1644 bcm4377->ctx->control_xfer_ring_n_entries =
1645 cpu_to_le16(bcm4377->control_h2d_ring.n_entries);
1646 bcm4377->ctx->control_xfer_ring_doorbell =
1647 cpu_to_le16(bcm4377->control_h2d_ring.doorbell);
1648 bcm4377->ctx->control_xfer_ring_msi = 0;
1649 bcm4377->ctx->control_xfer_ring_header_size = 0;
1650 bcm4377->ctx->control_xfer_ring_footer_size =
1651 bcm4377->control_h2d_ring.payload_size / 4;
1652
1653 dev_dbg(&bcm4377->pdev->dev, "context initialized at IOVA %pad",
1654 &bcm4377->ctx_dma);
1655
1656 return 0;
1657}
1658
1659static int bcm4377_prepare_rings(struct bcm4377_data *bcm4377)
1660{
1661 int ret;
1662
1663 /*
1664 * Even though many of these settings appear to be configurable
1665 * when sending the "create ring" messages most of these are
1666 * actually hardcoded in some (and quite possibly all) firmware versions
1667 * and changing them on the host has no effect.
1668 * Specifically, this applies to at least the doorbells, the transfer
1669 * and completion ring ids and their mapping (e.g. both HCI and ACL
1670 * entries will always be queued in completion rings 1 and 2 no matter
1671 * what we configure here).
1672 */
1673 bcm4377->control_ack_ring.ring_id = BCM4377_ACK_RING_CONTROL;
1674 bcm4377->control_ack_ring.n_entries = 32;
1675 bcm4377->control_ack_ring.transfer_rings =
1676 BIT(BCM4377_XFER_RING_CONTROL);
1677
1678 bcm4377->hci_acl_ack_ring.ring_id = BCM4377_ACK_RING_HCI_ACL;
1679 bcm4377->hci_acl_ack_ring.n_entries = 2 * BCM4377_RING_N_ENTRIES;
1680 bcm4377->hci_acl_ack_ring.transfer_rings =
1681 BIT(BCM4377_XFER_RING_HCI_H2D) | BIT(BCM4377_XFER_RING_ACL_H2D);
1682 bcm4377->hci_acl_ack_ring.delay = 1000;
1683
1684 /*
1685 * A payload size of MAX_EVENT_PAYLOAD_SIZE is enough here since large
1686 * ACL packets will be transmitted inside buffers mapped via
1687 * acl_d2h_ring anyway.
1688 */
1689 bcm4377->hci_acl_event_ring.ring_id = BCM4377_EVENT_RING_HCI_ACL;
1690 bcm4377->hci_acl_event_ring.payload_size = MAX_EVENT_PAYLOAD_SIZE;
1691 bcm4377->hci_acl_event_ring.n_entries = 2 * BCM4377_RING_N_ENTRIES;
1692 bcm4377->hci_acl_event_ring.transfer_rings =
1693 BIT(BCM4377_XFER_RING_HCI_D2H) | BIT(BCM4377_XFER_RING_ACL_D2H);
1694 bcm4377->hci_acl_event_ring.delay = 1000;
1695
1696 bcm4377->sco_ack_ring.ring_id = BCM4377_ACK_RING_SCO;
1697 bcm4377->sco_ack_ring.n_entries = BCM4377_RING_N_ENTRIES;
1698 bcm4377->sco_ack_ring.transfer_rings = BIT(BCM4377_XFER_RING_SCO_H2D);
1699
1700 bcm4377->sco_event_ring.ring_id = BCM4377_EVENT_RING_SCO;
1701 bcm4377->sco_event_ring.payload_size = MAX_SCO_PAYLOAD_SIZE;
1702 bcm4377->sco_event_ring.n_entries = BCM4377_RING_N_ENTRIES;
1703 bcm4377->sco_event_ring.transfer_rings = BIT(BCM4377_XFER_RING_SCO_D2H);
1704
1705 bcm4377->control_h2d_ring.ring_id = BCM4377_XFER_RING_CONTROL;
1706 bcm4377->control_h2d_ring.doorbell = BCM4377_DOORBELL_CONTROL;
1707 bcm4377->control_h2d_ring.payload_size = BCM4377_CONTROL_MSG_SIZE;
1708 bcm4377->control_h2d_ring.completion_ring = BCM4377_ACK_RING_CONTROL;
1709 bcm4377->control_h2d_ring.allow_wait = true;
1710 bcm4377->control_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1711
1712 bcm4377->hci_h2d_ring.ring_id = BCM4377_XFER_RING_HCI_H2D;
1713 bcm4377->hci_h2d_ring.doorbell = BCM4377_DOORBELL_HCI_H2D;
1714 bcm4377->hci_h2d_ring.payload_size = MAX_EVENT_PAYLOAD_SIZE;
1715 bcm4377->hci_h2d_ring.completion_ring = BCM4377_ACK_RING_HCI_ACL;
1716 bcm4377->hci_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1717
1718 bcm4377->hci_d2h_ring.ring_id = BCM4377_XFER_RING_HCI_D2H;
1719 bcm4377->hci_d2h_ring.doorbell = BCM4377_DOORBELL_HCI_D2H;
1720 bcm4377->hci_d2h_ring.completion_ring = BCM4377_EVENT_RING_HCI_ACL;
1721 bcm4377->hci_d2h_ring.virtual = true;
1722 bcm4377->hci_d2h_ring.n_entries = BCM4377_RING_N_ENTRIES;
1723
1724 bcm4377->sco_h2d_ring.ring_id = BCM4377_XFER_RING_SCO_H2D;
1725 bcm4377->sco_h2d_ring.doorbell = BCM4377_DOORBELL_SCO;
1726 bcm4377->sco_h2d_ring.payload_size = MAX_SCO_PAYLOAD_SIZE;
1727 bcm4377->sco_h2d_ring.completion_ring = BCM4377_ACK_RING_SCO;
1728 bcm4377->sco_h2d_ring.sync = true;
1729 bcm4377->sco_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1730
1731 bcm4377->sco_d2h_ring.ring_id = BCM4377_XFER_RING_SCO_D2H;
1732 bcm4377->sco_d2h_ring.doorbell = BCM4377_DOORBELL_SCO;
1733 bcm4377->sco_d2h_ring.completion_ring = BCM4377_EVENT_RING_SCO;
1734 bcm4377->sco_d2h_ring.virtual = true;
1735 bcm4377->sco_d2h_ring.sync = true;
1736 bcm4377->sco_d2h_ring.n_entries = BCM4377_RING_N_ENTRIES;
1737
1738 /*
1739 * This ring has to use mapped_payload_size because the largest ACL
1740 * packet doesn't fit inside the largest possible footer
1741 */
1742 bcm4377->acl_h2d_ring.ring_id = BCM4377_XFER_RING_ACL_H2D;
1743 bcm4377->acl_h2d_ring.doorbell = BCM4377_DOORBELL_ACL_H2D;
1744 bcm4377->acl_h2d_ring.mapped_payload_size = MAX_ACL_PAYLOAD_SIZE;
1745 bcm4377->acl_h2d_ring.completion_ring = BCM4377_ACK_RING_HCI_ACL;
1746 bcm4377->acl_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1747
1748 /*
1749 * This ring only contains empty buffers to be used by incoming
1750 * ACL packets that do not fit inside the footer of hci_acl_event_ring
1751 */
1752 bcm4377->acl_d2h_ring.ring_id = BCM4377_XFER_RING_ACL_D2H;
1753 bcm4377->acl_d2h_ring.doorbell = BCM4377_DOORBELL_ACL_D2H;
1754 bcm4377->acl_d2h_ring.completion_ring = BCM4377_EVENT_RING_HCI_ACL;
1755 bcm4377->acl_d2h_ring.d2h_buffers_only = true;
1756 bcm4377->acl_d2h_ring.mapped_payload_size = MAX_ACL_PAYLOAD_SIZE;
1757 bcm4377->acl_d2h_ring.n_entries = BCM4377_RING_N_ENTRIES;
1758
1759 /*
1760 * no need for any cleanup since this is only called from _probe
1761 * and only devres-managed allocations are used
1762 */
1763 ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->control_h2d_ring);
1764 if (ret)
1765 return ret;
1766 ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring);
1767 if (ret)
1768 return ret;
1769 ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring);
1770 if (ret)
1771 return ret;
1772 ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring);
1773 if (ret)
1774 return ret;
1775 ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring);
1776 if (ret)
1777 return ret;
1778 ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring);
1779 if (ret)
1780 return ret;
1781 ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->acl_d2h_ring);
1782 if (ret)
1783 return ret;
1784
1785 ret = bcm4377_alloc_completion_ring(bcm4377,
1786 &bcm4377->control_ack_ring);
1787 if (ret)
1788 return ret;
1789 ret = bcm4377_alloc_completion_ring(bcm4377,
1790 &bcm4377->hci_acl_ack_ring);
1791 if (ret)
1792 return ret;
1793 ret = bcm4377_alloc_completion_ring(bcm4377,
1794 &bcm4377->hci_acl_event_ring);
1795 if (ret)
1796 return ret;
1797 ret = bcm4377_alloc_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
1798 if (ret)
1799 return ret;
1800 ret = bcm4377_alloc_completion_ring(bcm4377, &bcm4377->sco_event_ring);
1801 if (ret)
1802 return ret;
1803
1804 dev_dbg(&bcm4377->pdev->dev, "all rings allocated and prepared\n");
1805
1806 return 0;
1807}
1808
1809static int bcm4377_boot(struct bcm4377_data *bcm4377)
1810{
1811 const struct firmware *fw;
1812 void *bfr;
1813 dma_addr_t fw_dma;
1814 int ret = 0;
1815 u32 bootstage, rti_status;
1816
1817 bootstage = ioread32(bcm4377->bar2 + BCM4377_BAR2_BOOTSTAGE);
1818 rti_status = ioread32(bcm4377->bar2 + BCM4377_BAR2_RTI_STATUS);
1819
1820 if (bootstage != 0) {
1821 dev_err(&bcm4377->pdev->dev, "bootstage is %d and not 0\n",
1822 bootstage);
1823 return -EINVAL;
1824 }
1825
1826 if (rti_status != 0) {
1827 dev_err(&bcm4377->pdev->dev, "RTI status is %d and not 0\n",
1828 rti_status);
1829 return -EINVAL;
1830 }
1831
1832 fw = bcm4377_request_blob(bcm4377, "bin");
1833 if (!fw) {
1834 dev_err(&bcm4377->pdev->dev, "Failed to load firmware\n");
1835 return -ENOENT;
1836 }
1837
1838 bfr = dma_alloc_coherent(&bcm4377->pdev->dev, fw->size, &fw_dma,
1839 GFP_KERNEL);
1840 if (!bfr) {
1841 ret = -ENOMEM;
1842 goto out_release_fw;
1843 }
1844
1845 memcpy(bfr, fw->data, fw->size);
1846
1847 iowrite32(0, bcm4377->bar0 + BCM4377_BAR0_HOST_WINDOW_LO);
1848 iowrite32(0, bcm4377->bar0 + BCM4377_BAR0_HOST_WINDOW_HI);
1849 iowrite32(BCM4377_DMA_MASK,
1850 bcm4377->bar0 + BCM4377_BAR0_HOST_WINDOW_SIZE);
1851
1852 iowrite32(lower_32_bits(fw_dma), bcm4377->bar2 + BCM4377_BAR2_FW_LO);
1853 iowrite32(upper_32_bits(fw_dma), bcm4377->bar2 + BCM4377_BAR2_FW_HI);
1854 iowrite32(fw->size, bcm4377->bar2 + BCM4377_BAR2_FW_SIZE);
1855 iowrite32(0, bcm4377->bar0 + BCM4377_BAR0_FW_DOORBELL);
1856
1857 dev_dbg(&bcm4377->pdev->dev, "waiting for firmware to boot\n");
1858
1859 ret = wait_for_completion_interruptible_timeout(&bcm4377->event,
1860 BCM4377_TIMEOUT);
1861 if (ret == 0) {
1862 ret = -ETIMEDOUT;
1863 goto out_dma_free;
1864 } else if (ret < 0) {
1865 goto out_dma_free;
1866 }
1867
1868 if (bcm4377->bootstage != 2) {
1869 dev_err(&bcm4377->pdev->dev, "boostage %d != 2\n",
1870 bcm4377->bootstage);
1871 ret = -ENXIO;
1872 goto out_dma_free;
1873 }
1874
1875 dev_dbg(&bcm4377->pdev->dev, "firmware has booted (stage = %x)\n",
1876 bcm4377->bootstage);
1877 ret = 0;
1878
1879out_dma_free:
1880 dma_free_coherent(&bcm4377->pdev->dev, fw->size, bfr, fw_dma);
1881out_release_fw:
1882 release_firmware(fw);
1883 return ret;
1884}
1885
1886static int bcm4377_setup_rti(struct bcm4377_data *bcm4377)
1887{
1888 int ret;
1889
1890 dev_dbg(&bcm4377->pdev->dev, "starting RTI\n");
1891 iowrite32(1, bcm4377->bar0 + BCM4377_BAR0_RTI_CONTROL);
1892
1893 ret = wait_for_completion_interruptible_timeout(&bcm4377->event,
1894 BCM4377_TIMEOUT);
1895 if (ret == 0) {
1896 dev_err(&bcm4377->pdev->dev,
1897 "timed out while waiting for RTI to transition to state 1");
1898 return -ETIMEDOUT;
1899 } else if (ret < 0) {
1900 return ret;
1901 }
1902
1903 if (bcm4377->rti_status != 1) {
1904 dev_err(&bcm4377->pdev->dev, "RTI did not ack state 1 (%d)\n",
1905 bcm4377->rti_status);
1906 return -ENODEV;
1907 }
1908 dev_dbg(&bcm4377->pdev->dev, "RTI is in state 1\n");
1909
1910 /* allow access to the entire IOVA space again */
1911 iowrite32(0, bcm4377->bar2 + BCM4377_BAR2_RTI_WINDOW_LO);
1912 iowrite32(0, bcm4377->bar2 + BCM4377_BAR2_RTI_WINDOW_HI);
1913 iowrite32(BCM4377_DMA_MASK,
1914 bcm4377->bar2 + BCM4377_BAR2_RTI_WINDOW_SIZE);
1915
1916 /* setup "Converged IPC" context */
1917 iowrite32(lower_32_bits(bcm4377->ctx_dma),
1918 bcm4377->bar2 + BCM4377_BAR2_CONTEXT_ADDR_LO);
1919 iowrite32(upper_32_bits(bcm4377->ctx_dma),
1920 bcm4377->bar2 + BCM4377_BAR2_CONTEXT_ADDR_HI);
1921 iowrite32(2, bcm4377->bar0 + BCM4377_BAR0_RTI_CONTROL);
1922
1923 ret = wait_for_completion_interruptible_timeout(&bcm4377->event,
1924 BCM4377_TIMEOUT);
1925 if (ret == 0) {
1926 dev_err(&bcm4377->pdev->dev,
1927 "timed out while waiting for RTI to transition to state 2");
1928 return -ETIMEDOUT;
1929 } else if (ret < 0) {
1930 return ret;
1931 }
1932
1933 if (bcm4377->rti_status != 2) {
1934 dev_err(&bcm4377->pdev->dev, "RTI did not ack state 2 (%d)\n",
1935 bcm4377->rti_status);
1936 return -ENODEV;
1937 }
1938
1939 dev_dbg(&bcm4377->pdev->dev,
1940 "RTI is in state 2; control ring is ready\n");
1941 bcm4377->control_ack_ring.enabled = true;
1942
1943 return 0;
1944}
1945
1946static int bcm4377_parse_otp_board_params(struct bcm4377_data *bcm4377,
1947 char tag, const char *val, size_t len)
1948{
1949 if (tag != 'V')
1950 return 0;
1951 if (len >= sizeof(bcm4377->vendor))
1952 return -EINVAL;
1953
1954 strscpy(bcm4377->vendor, val, len + 1);
1955 return 0;
1956}
1957
1958static int bcm4377_parse_otp_chip_params(struct bcm4377_data *bcm4377, char tag,
1959 const char *val, size_t len)
1960{
1961 size_t idx = 0;
1962
1963 if (tag != 's')
1964 return 0;
1965 if (len >= sizeof(bcm4377->stepping))
1966 return -EINVAL;
1967
1968 while (len != 0) {
1969 bcm4377->stepping[idx] = tolower(val[idx]);
1970 if (val[idx] == '\0')
1971 return 0;
1972
1973 idx++;
1974 len--;
1975 }
1976
1977 bcm4377->stepping[idx] = '\0';
1978 return 0;
1979}
1980
1981static int bcm4377_parse_otp_str(struct bcm4377_data *bcm4377, const u8 *str,
1982 enum bcm4377_otp_params_type type)
1983{
1984 const char *p;
1985 int ret;
1986
1987 p = skip_spaces(str);
1988 while (*p) {
1989 char tag = *p++;
1990 const char *end;
1991 size_t len;
1992
1993 if (*p++ != '=') /* implicit NUL check */
1994 return -EINVAL;
1995
1996 /* *p might be NUL here, if so end == p and len == 0 */
1997 end = strchrnul(p, ' ');
1998 len = end - p;
1999
2000 /* leave 1 byte for NUL in destination string */
2001 if (len > (BCM4377_OTP_MAX_PARAM_LEN - 1))
2002 return -EINVAL;
2003
2004 switch (type) {
2005 case BCM4377_OTP_BOARD_PARAMS:
2006 ret = bcm4377_parse_otp_board_params(bcm4377, tag, p,
2007 len);
2008 break;
2009 case BCM4377_OTP_CHIP_PARAMS:
2010 ret = bcm4377_parse_otp_chip_params(bcm4377, tag, p,
2011 len);
2012 break;
2013 default:
2014 ret = -EINVAL;
2015 break;
2016 }
2017
2018 if (ret)
2019 return ret;
2020
2021 /* Skip to next arg, if any */
2022 p = skip_spaces(end);
2023 }
2024
2025 return 0;
2026}
2027
2028static int bcm4377_parse_otp_sys_vendor(struct bcm4377_data *bcm4377, u8 *otp,
2029 size_t size)
2030{
2031 int idx = 4;
2032 const char *chip_params;
2033 const char *board_params;
2034 int ret;
2035
2036 /* 4-byte header and two empty strings */
2037 if (size < 6)
2038 return -EINVAL;
2039
2040 if (get_unaligned_le32(otp) != BCM4377_OTP_VENDOR_HDR)
2041 return -EINVAL;
2042
2043 chip_params = &otp[idx];
2044
2045 /* Skip first string, including terminator */
2046 idx += strnlen(chip_params, size - idx) + 1;
2047 if (idx >= size)
2048 return -EINVAL;
2049
2050 board_params = &otp[idx];
2051
2052 /* Skip to terminator of second string */
2053 idx += strnlen(board_params, size - idx);
2054 if (idx >= size)
2055 return -EINVAL;
2056
2057 /* At this point both strings are guaranteed NUL-terminated */
2058 dev_dbg(&bcm4377->pdev->dev,
2059 "OTP: chip_params='%s' board_params='%s'\n", chip_params,
2060 board_params);
2061
2062 ret = bcm4377_parse_otp_str(bcm4377, chip_params,
2063 BCM4377_OTP_CHIP_PARAMS);
2064 if (ret)
2065 return ret;
2066
2067 ret = bcm4377_parse_otp_str(bcm4377, board_params,
2068 BCM4377_OTP_BOARD_PARAMS);
2069 if (ret)
2070 return ret;
2071
2072 if (!bcm4377->stepping[0] || !bcm4377->vendor[0])
2073 return -EINVAL;
2074
2075 dev_dbg(&bcm4377->pdev->dev, "OTP: stepping=%s, vendor=%s\n",
2076 bcm4377->stepping, bcm4377->vendor);
2077 return 0;
2078}
2079
2080static int bcm4377_parse_otp(struct bcm4377_data *bcm4377)
2081{
2082 u8 *otp;
2083 int i;
2084 int ret = -ENOENT;
2085
2086 otp = kzalloc(BCM4377_OTP_SIZE, GFP_KERNEL);
2087 if (!otp)
2088 return -ENOMEM;
2089
2090 for (i = 0; i < BCM4377_OTP_SIZE; ++i)
2091 otp[i] = ioread8(bcm4377->bar0 + bcm4377->hw->otp_offset + i);
2092
2093 i = 0;
2094 while (i < (BCM4377_OTP_SIZE - 1)) {
2095 u8 type = otp[i];
2096 u8 length = otp[i + 1];
2097
2098 if (type == 0)
2099 break;
2100
2101 if ((i + 2 + length) > BCM4377_OTP_SIZE)
2102 break;
2103
2104 switch (type) {
2105 case BCM4377_OTP_SYS_VENDOR:
2106 dev_dbg(&bcm4377->pdev->dev,
2107 "OTP @ 0x%x (%d): SYS_VENDOR", i, length);
2108 ret = bcm4377_parse_otp_sys_vendor(bcm4377, &otp[i + 2],
2109 length);
2110 break;
2111 case BCM4377_OTP_CIS:
2112 dev_dbg(&bcm4377->pdev->dev, "OTP @ 0x%x (%d): CIS", i,
2113 length);
2114 break;
2115 default:
2116 dev_dbg(&bcm4377->pdev->dev, "OTP @ 0x%x (%d): unknown",
2117 i, length);
2118 break;
2119 }
2120
2121 i += 2 + length;
2122 }
2123
2124 kfree(otp);
2125 return ret;
2126}
2127
2128static int bcm4377_init_cfg(struct bcm4377_data *bcm4377)
2129{
2130 int ret;
2131 u32 ctrl;
2132
2133 ret = pci_write_config_dword(bcm4377->pdev,
2134 BCM4377_PCIECFG_BAR0_WINDOW1,
2135 bcm4377->hw->bar0_window1);
2136 if (ret)
2137 return ret;
2138
2139 ret = pci_write_config_dword(bcm4377->pdev,
2140 BCM4377_PCIECFG_BAR0_WINDOW2,
2141 bcm4377->hw->bar0_window2);
2142 if (ret)
2143 return ret;
2144
2145 ret = pci_write_config_dword(
2146 bcm4377->pdev, BCM4377_PCIECFG_BAR0_CORE2_WINDOW1,
2147 BCM4377_PCIECFG_BAR0_CORE2_WINDOW1_DEFAULT);
2148 if (ret)
2149 return ret;
2150
2151 if (bcm4377->hw->has_bar0_core2_window2) {
2152 ret = pci_write_config_dword(bcm4377->pdev,
2153 BCM4377_PCIECFG_BAR0_CORE2_WINDOW2,
2154 bcm4377->hw->bar0_core2_window2);
2155 if (ret)
2156 return ret;
2157 }
2158
2159 ret = pci_write_config_dword(bcm4377->pdev, BCM4377_PCIECFG_BAR2_WINDOW,
2160 BCM4377_PCIECFG_BAR2_WINDOW_DEFAULT);
2161 if (ret)
2162 return ret;
2163
2164 ret = pci_read_config_dword(bcm4377->pdev,
2165 BCM4377_PCIECFG_SUBSYSTEM_CTRL, &ctrl);
2166 if (ret)
2167 return ret;
2168
2169 if (bcm4377->hw->clear_pciecfg_subsystem_ctrl_bit19)
2170 ctrl &= ~BIT(19);
2171 ctrl |= BIT(16);
2172
2173 return pci_write_config_dword(bcm4377->pdev,
2174 BCM4377_PCIECFG_SUBSYSTEM_CTRL, ctrl);
2175}
2176
2177static int bcm4377_probe_dmi(struct bcm4377_data *bcm4377)
2178{
2179 const struct dmi_system_id *board_type_dmi_id;
2180
2181 board_type_dmi_id = dmi_first_match(bcm4377_dmi_board_table);
2182 if (board_type_dmi_id && board_type_dmi_id->driver_data) {
2183 bcm4377->board_type = board_type_dmi_id->driver_data;
2184 dev_dbg(&bcm4377->pdev->dev,
2185 "found board type via DMI match: %s\n",
2186 bcm4377->board_type);
2187 }
2188
2189 return 0;
2190}
2191
2192static int bcm4377_probe_of(struct bcm4377_data *bcm4377)
2193{
2194 struct device_node *np = bcm4377->pdev->dev.of_node;
2195 int ret;
2196
2197 if (!np)
2198 return 0;
2199
2200 ret = of_property_read_string(np, "brcm,board-type",
2201 &bcm4377->board_type);
2202 if (ret) {
2203 dev_err(&bcm4377->pdev->dev, "no brcm,board-type property\n");
2204 return ret;
2205 }
2206
2207 bcm4377->taurus_beamforming_cal_blob =
2208 of_get_property(np, "brcm,taurus-bf-cal-blob",
2209 &bcm4377->taurus_beamforming_cal_size);
2210 if (!bcm4377->taurus_beamforming_cal_blob) {
2211 dev_err(&bcm4377->pdev->dev,
2212 "no brcm,taurus-bf-cal-blob property\n");
2213 return -ENOENT;
2214 }
2215 bcm4377->taurus_cal_blob = of_get_property(np, "brcm,taurus-cal-blob",
2216 &bcm4377->taurus_cal_size);
2217 if (!bcm4377->taurus_cal_blob) {
2218 dev_err(&bcm4377->pdev->dev,
2219 "no brcm,taurus-cal-blob property\n");
2220 return -ENOENT;
2221 }
2222
2223 return 0;
2224}
2225
2226static void bcm4377_disable_aspm(struct bcm4377_data *bcm4377)
2227{
2228 pci_disable_link_state(bcm4377->pdev,
2229 PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1);
2230
2231 /*
2232 * pci_disable_link_state can fail if either CONFIG_PCIEASPM is disabled
2233 * or if the BIOS hasn't handed over control to us. We must *always*
2234 * disable ASPM for this device due to hardware errata though.
2235 */
2236 pcie_capability_clear_word(bcm4377->pdev, PCI_EXP_LNKCTL,
2237 PCI_EXP_LNKCTL_ASPMC);
2238}
2239
2240static void bcm4377_pci_free_irq_vectors(void *data)
2241{
2242 pci_free_irq_vectors(data);
2243}
2244
2245static void bcm4377_hci_free_dev(void *data)
2246{
2247 hci_free_dev(data);
2248}
2249
2250static void bcm4377_hci_unregister_dev(void *data)
2251{
2252 hci_unregister_dev(data);
2253}
2254
2255static int bcm4377_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2256{
2257 struct bcm4377_data *bcm4377;
2258 struct hci_dev *hdev;
2259 int ret, irq;
2260
2261 ret = dma_set_mask_and_coherent(&pdev->dev, BCM4377_DMA_MASK);
2262 if (ret)
2263 return ret;
2264
2265 bcm4377 = devm_kzalloc(&pdev->dev, sizeof(*bcm4377), GFP_KERNEL);
2266 if (!bcm4377)
2267 return -ENOMEM;
2268
2269 bcm4377->pdev = pdev;
2270 bcm4377->hw = &bcm4377_hw_variants[id->driver_data];
2271 init_completion(&bcm4377->event);
2272
2273 ret = bcm4377_prepare_rings(bcm4377);
2274 if (ret)
2275 return ret;
2276
2277 ret = bcm4377_init_context(bcm4377);
2278 if (ret)
2279 return ret;
2280
2281 ret = bcm4377_probe_dmi(bcm4377);
2282 if (ret)
2283 return ret;
2284 ret = bcm4377_probe_of(bcm4377);
2285 if (ret)
2286 return ret;
2287 if (!bcm4377->board_type) {
2288 dev_err(&pdev->dev, "unable to determine board type\n");
2289 return -ENODEV;
2290 }
2291
2292 if (bcm4377->hw->disable_aspm)
2293 bcm4377_disable_aspm(bcm4377);
2294
2295 ret = pci_reset_function_locked(pdev);
2296 if (ret)
2297 dev_warn(
2298 &pdev->dev,
2299 "function level reset failed with %d; trying to continue anyway\n",
2300 ret);
2301
2302 /*
2303 * If this number is too low and we try to access any BAR too
2304 * early the device will crash. Experiments have shown that
2305 * approximately 50 msec is the minimum amount we have to wait.
2306 * Let's double that to be safe.
2307 */
2308 msleep(100);
2309
2310 ret = pcim_enable_device(pdev);
2311 if (ret)
2312 return ret;
2313 pci_set_master(pdev);
2314
2315 ret = bcm4377_init_cfg(bcm4377);
2316 if (ret)
2317 return ret;
2318
2319 bcm4377->bar0 = pcim_iomap(pdev, 0, 0);
2320 if (!bcm4377->bar0)
2321 return -EBUSY;
2322 bcm4377->bar2 = pcim_iomap(pdev, 2, 0);
2323 if (!bcm4377->bar2)
2324 return -EBUSY;
2325
2326 ret = bcm4377_parse_otp(bcm4377);
2327 if (ret) {
2328 dev_err(&pdev->dev, "Reading OTP failed with %d\n", ret);
2329 return ret;
2330 }
2331
2332 /*
2333 * Legacy interrupts result in an IRQ storm because we don't know where
2334 * the interrupt mask and status registers for these chips are.
2335 * MSIs are acked automatically instead.
2336 */
2337 ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
2338 if (ret < 0)
2339 return -ENODEV;
2340 ret = devm_add_action_or_reset(&pdev->dev, bcm4377_pci_free_irq_vectors,
2341 pdev);
2342 if (ret)
2343 return ret;
2344
2345 irq = pci_irq_vector(pdev, 0);
2346 if (irq <= 0)
2347 return -ENODEV;
2348
2349 ret = devm_request_irq(&pdev->dev, irq, bcm4377_irq, 0, "bcm4377",
2350 bcm4377);
2351 if (ret)
2352 return ret;
2353
2354 hdev = hci_alloc_dev();
2355 if (!hdev)
2356 return -ENOMEM;
2357 ret = devm_add_action_or_reset(&pdev->dev, bcm4377_hci_free_dev, hdev);
2358 if (ret)
2359 return ret;
2360
2361 bcm4377->hdev = hdev;
2362
2363 hdev->bus = HCI_PCI;
2364 hdev->dev_type = HCI_PRIMARY;
2365 hdev->open = bcm4377_hci_open;
2366 hdev->close = bcm4377_hci_close;
2367 hdev->send = bcm4377_hci_send_frame;
2368 hdev->set_bdaddr = bcm4377_hci_set_bdaddr;
2369 hdev->setup = bcm4377_hci_setup;
2370
2371 if (bcm4377->hw->broken_mws_transport_config)
2372 set_bit(HCI_QUIRK_BROKEN_MWS_TRANSPORT_CONFIG, &hdev->quirks);
2373 if (bcm4377->hw->broken_ext_scan)
2374 set_bit(HCI_QUIRK_BROKEN_EXT_SCAN, &hdev->quirks);
2375 if (bcm4377->hw->broken_le_coded)
2376 set_bit(HCI_QUIRK_BROKEN_LE_CODED, &hdev->quirks);
2377
2378 pci_set_drvdata(pdev, bcm4377);
2379 hci_set_drvdata(hdev, bcm4377);
2380 SET_HCIDEV_DEV(hdev, &pdev->dev);
2381
2382 ret = bcm4377_boot(bcm4377);
2383 if (ret)
2384 return ret;
2385
2386 ret = bcm4377_setup_rti(bcm4377);
2387 if (ret)
2388 return ret;
2389
2390 ret = hci_register_dev(hdev);
2391 if (ret)
2392 return ret;
2393 return devm_add_action_or_reset(&pdev->dev, bcm4377_hci_unregister_dev,
2394 hdev);
2395}
2396
2397static int bcm4377_suspend(struct pci_dev *pdev, pm_message_t state)
2398{
2399 struct bcm4377_data *bcm4377 = pci_get_drvdata(pdev);
2400 int ret;
2401
2402 ret = hci_suspend_dev(bcm4377->hdev);
2403 if (ret)
2404 return ret;
2405
2406 iowrite32(BCM4377_BAR0_SLEEP_CONTROL_QUIESCE,
2407 bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL);
2408
2409 return 0;
2410}
2411
2412static int bcm4377_resume(struct pci_dev *pdev)
2413{
2414 struct bcm4377_data *bcm4377 = pci_get_drvdata(pdev);
2415
2416 iowrite32(BCM4377_BAR0_SLEEP_CONTROL_UNQUIESCE,
2417 bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL);
2418
2419 return hci_resume_dev(bcm4377->hdev);
2420}
2421
2422static const struct dmi_system_id bcm4377_dmi_board_table[] = {
2423 {
2424 .matches = {
2425 DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
2426 DMI_MATCH(DMI_PRODUCT_NAME, "MacBookAir9,1"),
2427 },
2428 .driver_data = "apple,formosa",
2429 },
2430 {
2431 .matches = {
2432 DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
2433 DMI_MATCH(DMI_PRODUCT_NAME, "MacBookPro15,4"),
2434 },
2435 .driver_data = "apple,formosa",
2436 },
2437 {
2438 .matches = {
2439 DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
2440 DMI_MATCH(DMI_PRODUCT_NAME, "MacBookPro16,3"),
2441 },
2442 .driver_data = "apple,formosa",
2443 },
2444 {}
2445};
2446
2447static const struct bcm4377_hw bcm4377_hw_variants[] = {
2448 [BCM4377] = {
2449 .id = 0x4377,
2450 .otp_offset = 0x4120,
2451 .bar0_window1 = 0x1800b000,
2452 .bar0_window2 = 0x1810c000,
2453 .disable_aspm = true,
2454 .broken_ext_scan = true,
2455 .send_ptb = bcm4377_send_ptb,
2456 },
2457
2458 [BCM4378] = {
2459 .id = 0x4378,
2460 .otp_offset = 0x4120,
2461 .bar0_window1 = 0x18002000,
2462 .bar0_window2 = 0x1810a000,
2463 .bar0_core2_window2 = 0x18107000,
2464 .has_bar0_core2_window2 = true,
2465 .broken_mws_transport_config = true,
2466 .broken_le_coded = true,
2467 .send_calibration = bcm4378_send_calibration,
2468 .send_ptb = bcm4378_send_ptb,
2469 },
2470
2471 [BCM4387] = {
2472 .id = 0x4387,
2473 .otp_offset = 0x413c,
2474 .bar0_window1 = 0x18002000,
2475 .bar0_window2 = 0x18109000,
2476 .bar0_core2_window2 = 0x18106000,
2477 .has_bar0_core2_window2 = true,
2478 .clear_pciecfg_subsystem_ctrl_bit19 = true,
2479 .broken_mws_transport_config = true,
2480 .broken_le_coded = true,
2481 .send_calibration = bcm4387_send_calibration,
2482 .send_ptb = bcm4378_send_ptb,
2483 },
2484};
2485
2486#define BCM4377_DEVID_ENTRY(id) \
2487 { \
2488 PCI_VENDOR_ID_BROADCOM, BCM##id##_DEVICE_ID, PCI_ANY_ID, \
2489 PCI_ANY_ID, PCI_CLASS_NETWORK_OTHER << 8, 0xffff00, \
2490 BCM##id \
2491 }
2492
2493static const struct pci_device_id bcm4377_devid_table[] = {
2494 BCM4377_DEVID_ENTRY(4377),
2495 BCM4377_DEVID_ENTRY(4378),
2496 BCM4377_DEVID_ENTRY(4387),
2497 {},
2498};
2499MODULE_DEVICE_TABLE(pci, bcm4377_devid_table);
2500
2501static struct pci_driver bcm4377_pci_driver = {
2502 .name = "hci_bcm4377",
2503 .id_table = bcm4377_devid_table,
2504 .probe = bcm4377_probe,
2505 .suspend = bcm4377_suspend,
2506 .resume = bcm4377_resume,
2507};
2508module_pci_driver(bcm4377_pci_driver);
2509
2510MODULE_AUTHOR("Sven Peter <sven@svenpeter.dev>");
2511MODULE_DESCRIPTION("Bluetooth support for Broadcom 4377/4378/4387 devices");
2512MODULE_LICENSE("Dual MIT/GPL");
2513MODULE_FIRMWARE("brcm/brcmbt4377*.bin");
2514MODULE_FIRMWARE("brcm/brcmbt4377*.ptb");
2515MODULE_FIRMWARE("brcm/brcmbt4378*.bin");
2516MODULE_FIRMWARE("brcm/brcmbt4378*.ptb");
2517MODULE_FIRMWARE("brcm/brcmbt4387*.bin");
2518MODULE_FIRMWARE("brcm/brcmbt4387*.ptb");
1// SPDX-License-Identifier: GPL-2.0-only OR MIT
2/*
3 * Bluetooth HCI driver for Broadcom 4377/4378/4387 devices attached via PCIe
4 *
5 * Copyright (C) The Asahi Linux Contributors
6 */
7
8#include <linux/async.h>
9#include <linux/bitfield.h>
10#include <linux/completion.h>
11#include <linux/dma-mapping.h>
12#include <linux/dmi.h>
13#include <linux/firmware.h>
14#include <linux/module.h>
15#include <linux/msi.h>
16#include <linux/of.h>
17#include <linux/pci.h>
18#include <linux/printk.h>
19
20#include <asm/unaligned.h>
21
22#include <net/bluetooth/bluetooth.h>
23#include <net/bluetooth/hci_core.h>
24
25enum bcm4377_chip {
26 BCM4377 = 0,
27 BCM4378,
28 BCM4387,
29};
30
31#define BCM4377_DEVICE_ID 0x5fa0
32#define BCM4378_DEVICE_ID 0x5f69
33#define BCM4387_DEVICE_ID 0x5f71
34
35#define BCM4377_TIMEOUT 1000
36
37/*
38 * These devices only support DMA transactions inside a 32bit window
39 * (possibly to avoid 64 bit arithmetic). The window size cannot exceed
40 * 0xffffffff but is always aligned down to the previous 0x200 byte boundary
41 * which effectively limits the window to [start, start+0xfffffe00].
42 * We just limit the DMA window to [0, 0xfffffe00] to make sure we don't
43 * run into this limitation.
44 */
45#define BCM4377_DMA_MASK 0xfffffe00
46
47#define BCM4377_PCIECFG_BAR0_WINDOW1 0x80
48#define BCM4377_PCIECFG_BAR0_WINDOW2 0x70
49#define BCM4377_PCIECFG_BAR0_CORE2_WINDOW1 0x74
50#define BCM4377_PCIECFG_BAR0_CORE2_WINDOW2 0x78
51#define BCM4377_PCIECFG_BAR2_WINDOW 0x84
52
53#define BCM4377_PCIECFG_BAR0_CORE2_WINDOW1_DEFAULT 0x18011000
54#define BCM4377_PCIECFG_BAR2_WINDOW_DEFAULT 0x19000000
55
56#define BCM4377_PCIECFG_SUBSYSTEM_CTRL 0x88
57
58#define BCM4377_BAR0_FW_DOORBELL 0x140
59#define BCM4377_BAR0_RTI_CONTROL 0x144
60
61#define BCM4377_BAR0_SLEEP_CONTROL 0x150
62#define BCM4377_BAR0_SLEEP_CONTROL_UNQUIESCE 0
63#define BCM4377_BAR0_SLEEP_CONTROL_AWAKE 2
64#define BCM4377_BAR0_SLEEP_CONTROL_QUIESCE 3
65
66#define BCM4377_BAR0_DOORBELL 0x174
67#define BCM4377_BAR0_DOORBELL_VALUE GENMASK(31, 16)
68#define BCM4377_BAR0_DOORBELL_IDX GENMASK(15, 8)
69#define BCM4377_BAR0_DOORBELL_RING BIT(5)
70
71#define BCM4377_BAR0_HOST_WINDOW_LO 0x590
72#define BCM4377_BAR0_HOST_WINDOW_HI 0x594
73#define BCM4377_BAR0_HOST_WINDOW_SIZE 0x598
74
75#define BCM4377_BAR2_BOOTSTAGE 0x200454
76
77#define BCM4377_BAR2_FW_LO 0x200478
78#define BCM4377_BAR2_FW_HI 0x20047c
79#define BCM4377_BAR2_FW_SIZE 0x200480
80
81#define BCM4377_BAR2_CONTEXT_ADDR_LO 0x20048c
82#define BCM4377_BAR2_CONTEXT_ADDR_HI 0x200450
83
84#define BCM4377_BAR2_RTI_STATUS 0x20045c
85#define BCM4377_BAR2_RTI_WINDOW_LO 0x200494
86#define BCM4377_BAR2_RTI_WINDOW_HI 0x200498
87#define BCM4377_BAR2_RTI_WINDOW_SIZE 0x20049c
88
89#define BCM4377_OTP_SIZE 0xe0
90#define BCM4377_OTP_SYS_VENDOR 0x15
91#define BCM4377_OTP_CIS 0x80
92#define BCM4377_OTP_VENDOR_HDR 0x00000008
93#define BCM4377_OTP_MAX_PARAM_LEN 16
94
95#define BCM4377_N_TRANSFER_RINGS 9
96#define BCM4377_N_COMPLETION_RINGS 6
97
98#define BCM4377_MAX_RING_SIZE 256
99
100#define BCM4377_MSGID_GENERATION GENMASK(15, 8)
101#define BCM4377_MSGID_ID GENMASK(7, 0)
102
103#define BCM4377_RING_N_ENTRIES 128
104
105#define BCM4377_CONTROL_MSG_SIZE 0x34
106#define BCM4377_XFER_RING_MAX_INPLACE_PAYLOAD_SIZE (4 * 0xff)
107
108#define MAX_ACL_PAYLOAD_SIZE (HCI_MAX_FRAME_SIZE + HCI_ACL_HDR_SIZE)
109#define MAX_SCO_PAYLOAD_SIZE (HCI_MAX_SCO_SIZE + HCI_SCO_HDR_SIZE)
110#define MAX_EVENT_PAYLOAD_SIZE (HCI_MAX_EVENT_SIZE + HCI_EVENT_HDR_SIZE)
111
112enum bcm4377_otp_params_type {
113 BCM4377_OTP_BOARD_PARAMS,
114 BCM4377_OTP_CHIP_PARAMS
115};
116
117enum bcm4377_transfer_ring_id {
118 BCM4377_XFER_RING_CONTROL = 0,
119 BCM4377_XFER_RING_HCI_H2D = 1,
120 BCM4377_XFER_RING_HCI_D2H = 2,
121 BCM4377_XFER_RING_SCO_H2D = 3,
122 BCM4377_XFER_RING_SCO_D2H = 4,
123 BCM4377_XFER_RING_ACL_H2D = 5,
124 BCM4377_XFER_RING_ACL_D2H = 6,
125};
126
127enum bcm4377_completion_ring_id {
128 BCM4377_ACK_RING_CONTROL = 0,
129 BCM4377_ACK_RING_HCI_ACL = 1,
130 BCM4377_EVENT_RING_HCI_ACL = 2,
131 BCM4377_ACK_RING_SCO = 3,
132 BCM4377_EVENT_RING_SCO = 4,
133};
134
135enum bcm4377_doorbell {
136 BCM4377_DOORBELL_CONTROL = 0,
137 BCM4377_DOORBELL_HCI_H2D = 1,
138 BCM4377_DOORBELL_HCI_D2H = 2,
139 BCM4377_DOORBELL_ACL_H2D = 3,
140 BCM4377_DOORBELL_ACL_D2H = 4,
141 BCM4377_DOORBELL_SCO = 6,
142};
143
144/*
145 * Transfer ring entry
146 *
147 * flags: Flags to indicate if the payload is appended or mapped
148 * len: Payload length
149 * payload: Optional payload DMA address
150 * id: Message id to recognize the answer in the completion ring entry
151 */
152struct bcm4377_xfer_ring_entry {
153#define BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED BIT(0)
154#define BCM4377_XFER_RING_FLAG_PAYLOAD_IN_FOOTER BIT(1)
155 u8 flags;
156 __le16 len;
157 u8 _unk0;
158 __le64 payload;
159 __le16 id;
160 u8 _unk1[2];
161} __packed;
162static_assert(sizeof(struct bcm4377_xfer_ring_entry) == 0x10);
163
164/*
165 * Completion ring entry
166 *
167 * flags: Flags to indicate if the payload is appended or mapped. If the payload
168 * is mapped it can be found in the buffer of the corresponding transfer
169 * ring message.
170 * ring_id: Transfer ring ID which required this message
171 * msg_id: Message ID specified in transfer ring entry
172 * len: Payload length
173 */
174struct bcm4377_completion_ring_entry {
175 u8 flags;
176 u8 _unk0;
177 __le16 ring_id;
178 __le16 msg_id;
179 __le32 len;
180 u8 _unk1[6];
181} __packed;
182static_assert(sizeof(struct bcm4377_completion_ring_entry) == 0x10);
183
184enum bcm4377_control_message_type {
185 BCM4377_CONTROL_MSG_CREATE_XFER_RING = 1,
186 BCM4377_CONTROL_MSG_CREATE_COMPLETION_RING = 2,
187 BCM4377_CONTROL_MSG_DESTROY_XFER_RING = 3,
188 BCM4377_CONTROL_MSG_DESTROY_COMPLETION_RING = 4,
189};
190
191/*
192 * Control message used to create a completion ring
193 *
194 * msg_type: Must be BCM4377_CONTROL_MSG_CREATE_COMPLETION_RING
195 * header_size: Unknown, but probably reserved space in front of the entry
196 * footer_size: Number of 32 bit words reserved for payloads after the entry
197 * id/id_again: Completion ring index
198 * ring_iova: DMA address of the ring buffer
199 * n_elements: Number of elements inside the ring buffer
200 * msi: MSI index, doesn't work for all rings though and should be zero
201 * intmod_delay: Unknown delay
202 * intmod_bytes: Unknown
203 */
204struct bcm4377_create_completion_ring_msg {
205 u8 msg_type;
206 u8 header_size;
207 u8 footer_size;
208 u8 _unk0;
209 __le16 id;
210 __le16 id_again;
211 __le64 ring_iova;
212 __le16 n_elements;
213 __le32 unk;
214 u8 _unk1[6];
215 __le16 msi;
216 __le16 intmod_delay;
217 __le32 intmod_bytes;
218 __le16 _unk2;
219 __le32 _unk3;
220 u8 _unk4[10];
221} __packed;
222static_assert(sizeof(struct bcm4377_create_completion_ring_msg) ==
223 BCM4377_CONTROL_MSG_SIZE);
224
225/*
226 * Control ring message used to destroy a completion ring
227 *
228 * msg_type: Must be BCM4377_CONTROL_MSG_DESTROY_COMPLETION_RING
229 * ring_id: Completion ring to be destroyed
230 */
231struct bcm4377_destroy_completion_ring_msg {
232 u8 msg_type;
233 u8 _pad0;
234 __le16 ring_id;
235 u8 _pad1[48];
236} __packed;
237static_assert(sizeof(struct bcm4377_destroy_completion_ring_msg) ==
238 BCM4377_CONTROL_MSG_SIZE);
239
240/*
241 * Control message used to create a transfer ring
242 *
243 * msg_type: Must be BCM4377_CONTROL_MSG_CREATE_XFER_RING
244 * header_size: Number of 32 bit words reserved for unknown content before the
245 * entry
246 * footer_size: Number of 32 bit words reserved for payloads after the entry
247 * ring_id/ring_id_again: Transfer ring index
248 * ring_iova: DMA address of the ring buffer
249 * n_elements: Number of elements inside the ring buffer
250 * completion_ring_id: Completion ring index for acknowledgements and events
251 * doorbell: Doorbell index used to notify device of new entries
252 * flags: Transfer ring flags
253 * - virtual: set if there is no associated shared memory and only the
254 * corresponding completion ring is used
255 * - sync: only set for the SCO rings
256 */
257struct bcm4377_create_transfer_ring_msg {
258 u8 msg_type;
259 u8 header_size;
260 u8 footer_size;
261 u8 _unk0;
262 __le16 ring_id;
263 __le16 ring_id_again;
264 __le64 ring_iova;
265 u8 _unk1[8];
266 __le16 n_elements;
267 __le16 completion_ring_id;
268 __le16 doorbell;
269#define BCM4377_XFER_RING_FLAG_VIRTUAL BIT(7)
270#define BCM4377_XFER_RING_FLAG_SYNC BIT(8)
271 __le16 flags;
272 u8 _unk2[20];
273} __packed;
274static_assert(sizeof(struct bcm4377_create_transfer_ring_msg) ==
275 BCM4377_CONTROL_MSG_SIZE);
276
277/*
278 * Control ring message used to destroy a transfer ring
279 *
280 * msg_type: Must be BCM4377_CONTROL_MSG_DESTROY_XFER_RING
281 * ring_id: Transfer ring to be destroyed
282 */
283struct bcm4377_destroy_transfer_ring_msg {
284 u8 msg_type;
285 u8 _pad0;
286 __le16 ring_id;
287 u8 _pad1[48];
288} __packed;
289static_assert(sizeof(struct bcm4377_destroy_transfer_ring_msg) ==
290 BCM4377_CONTROL_MSG_SIZE);
291
292/*
293 * "Converged IPC" context struct used to make the device aware of all other
294 * shared memory structures. A pointer to this structure is configured inside a
295 * MMIO register.
296 *
297 * version: Protocol version, must be 2.
298 * size: Size of this structure, must be 0x68.
299 * enabled_caps: Enabled capabilities. Unknown bitfield but should be 2.
300 * peripheral_info_addr: DMA address for a 0x20 buffer to which the device will
301 * write unknown contents
302 * {completion,xfer}_ring_{tails,heads}_addr: DMA pointers to ring heads/tails
303 * n_completion_rings: Number of completion rings, the firmware only works if
304 * this is set to BCM4377_N_COMPLETION_RINGS.
305 * n_xfer_rings: Number of transfer rings, the firmware only works if
306 * this is set to BCM4377_N_TRANSFER_RINGS.
307 * control_completion_ring_addr: Control completion ring buffer DMA address
308 * control_xfer_ring_addr: Control transfer ring buffer DMA address
309 * control_xfer_ring_n_entries: Number of control transfer ring entries
310 * control_completion_ring_n_entries: Number of control completion ring entries
311 * control_xfer_ring_doorbell: Control transfer ring doorbell
312 * control_completion_ring_doorbell: Control completion ring doorbell,
313 * must be set to 0xffff
314 * control_xfer_ring_msi: Control completion ring MSI index, must be 0
315 * control_completion_ring_msi: Control completion ring MSI index, must be 0.
316 * control_xfer_ring_header_size: Number of 32 bit words reserved in front of
317 * every control transfer ring entry
318 * control_xfer_ring_footer_size: Number of 32 bit words reserved after every
319 * control transfer ring entry
320 * control_completion_ring_header_size: Number of 32 bit words reserved in front
321 * of every control completion ring entry
322 * control_completion_ring_footer_size: Number of 32 bit words reserved after
323 * every control completion ring entry
324 * scratch_pad: Optional scratch pad DMA address
325 * scratch_pad_size: Scratch pad size
326 */
327struct bcm4377_context {
328 __le16 version;
329 __le16 size;
330 __le32 enabled_caps;
331
332 __le64 peripheral_info_addr;
333
334 /* ring heads and tails */
335 __le64 completion_ring_heads_addr;
336 __le64 xfer_ring_tails_addr;
337 __le64 completion_ring_tails_addr;
338 __le64 xfer_ring_heads_addr;
339 __le16 n_completion_rings;
340 __le16 n_xfer_rings;
341
342 /* control ring configuration */
343 __le64 control_completion_ring_addr;
344 __le64 control_xfer_ring_addr;
345 __le16 control_xfer_ring_n_entries;
346 __le16 control_completion_ring_n_entries;
347 __le16 control_xfer_ring_doorbell;
348 __le16 control_completion_ring_doorbell;
349 __le16 control_xfer_ring_msi;
350 __le16 control_completion_ring_msi;
351 u8 control_xfer_ring_header_size;
352 u8 control_xfer_ring_footer_size;
353 u8 control_completion_ring_header_size;
354 u8 control_completion_ring_footer_size;
355
356 __le16 _unk0;
357 __le16 _unk1;
358
359 __le64 scratch_pad;
360 __le32 scratch_pad_size;
361
362 __le32 _unk3;
363} __packed;
364static_assert(sizeof(struct bcm4377_context) == 0x68);
365
366#define BCM4378_CALIBRATION_CHUNK_SIZE 0xe6
367struct bcm4378_hci_send_calibration_cmd {
368 u8 unk;
369 __le16 blocks_left;
370 u8 data[BCM4378_CALIBRATION_CHUNK_SIZE];
371} __packed;
372
373#define BCM4378_PTB_CHUNK_SIZE 0xcf
374struct bcm4378_hci_send_ptb_cmd {
375 __le16 blocks_left;
376 u8 data[BCM4378_PTB_CHUNK_SIZE];
377} __packed;
378
379/*
380 * Shared memory structure used to store the ring head and tail pointers.
381 */
382struct bcm4377_ring_state {
383 __le16 completion_ring_head[BCM4377_N_COMPLETION_RINGS];
384 __le16 completion_ring_tail[BCM4377_N_COMPLETION_RINGS];
385 __le16 xfer_ring_head[BCM4377_N_TRANSFER_RINGS];
386 __le16 xfer_ring_tail[BCM4377_N_TRANSFER_RINGS];
387};
388
389/*
390 * A transfer ring can be used in two configurations:
391 * 1) Send control or HCI messages to the device which are then acknowledged
392 * in the corresponding completion ring
393 * 2) Receiving HCI frames from the devices. In this case the transfer ring
394 * itself contains empty messages that are acknowledged once data is
395 * available from the device. If the payloads fit inside the footers
396 * of the completion ring the transfer ring can be configured to be
397 * virtual such that it has no ring buffer.
398 *
399 * ring_id: ring index hardcoded in the firmware
400 * doorbell: doorbell index to notify device of new entries
401 * payload_size: optional in-place payload size
402 * mapped_payload_size: optional out-of-place payload size
403 * completion_ring: index of corresponding completion ring
404 * n_entries: number of entries inside this ring
405 * generation: ring generation; incremented on hci_open to detect stale messages
406 * sync: set to true for SCO rings
407 * virtual: set to true if this ring has no entries and is just required to
408 * setup a corresponding completion ring for device->host messages
409 * d2h_buffers_only: set to true if this ring is only used to provide large
410 * buffers used by device->host messages in the completion
411 * ring
412 * allow_wait: allow to wait for messages to be acknowledged
413 * enabled: true once the ring has been created and can be used
414 * ring: ring buffer for entries (struct bcm4377_xfer_ring_entry)
415 * ring_dma: DMA address for ring entry buffer
416 * payloads: payload buffer for mapped_payload_size payloads
417 * payloads_dma:DMA address for payload buffer
418 * events: pointer to array of completions if waiting is allowed
419 * msgids: bitmap to keep track of used message ids
420 * lock: Spinlock to protect access to ring structurs used in the irq handler
421 */
422struct bcm4377_transfer_ring {
423 enum bcm4377_transfer_ring_id ring_id;
424 enum bcm4377_doorbell doorbell;
425 size_t payload_size;
426 size_t mapped_payload_size;
427 u8 completion_ring;
428 u16 n_entries;
429 u8 generation;
430
431 bool sync;
432 bool virtual;
433 bool d2h_buffers_only;
434 bool allow_wait;
435 bool enabled;
436
437 void *ring;
438 dma_addr_t ring_dma;
439
440 void *payloads;
441 dma_addr_t payloads_dma;
442
443 struct completion **events;
444 DECLARE_BITMAP(msgids, BCM4377_MAX_RING_SIZE);
445 spinlock_t lock;
446};
447
448/*
449 * A completion ring can be either used to either acknowledge messages sent in
450 * the corresponding transfer ring or to receive messages associated with the
451 * transfer ring. When used to receive messages the transfer ring either
452 * has no ring buffer and is only advanced ("virtual transfer ring") or it
453 * only contains empty DMA buffers to be used for the payloads.
454 *
455 * ring_id: completion ring id, hardcoded in firmware
456 * payload_size: optional payload size after each entry
457 * delay: unknown delay
458 * n_entries: number of entries in this ring
459 * enabled: true once the ring has been created and can be used
460 * ring: ring buffer for entries (struct bcm4377_completion_ring_entry)
461 * ring_dma: DMA address of ring buffer
462 * transfer_rings: bitmap of corresponding transfer ring ids
463 */
464struct bcm4377_completion_ring {
465 enum bcm4377_completion_ring_id ring_id;
466 u16 payload_size;
467 u16 delay;
468 u16 n_entries;
469 bool enabled;
470
471 void *ring;
472 dma_addr_t ring_dma;
473
474 unsigned long transfer_rings;
475};
476
477struct bcm4377_data;
478
479/*
480 * Chip-specific configuration struct
481 *
482 * id: Chip id (e.g. 0x4377 for BCM4377)
483 * otp_offset: Offset to the start of the OTP inside BAR0
484 * bar0_window1: Backplane address mapped to the first window in BAR0
485 * bar0_window2: Backplane address mapped to the second window in BAR0
486 * bar0_core2_window2: Optional backplane address mapped to the second core's
487 * second window in BAR0
488 * has_bar0_core2_window2: Set to true if this chip requires the second core's
489 * second window to be configured
490 * clear_pciecfg_subsystem_ctrl_bit19: Set to true if bit 19 in the
491 * vendor-specific subsystem control
492 * register has to be cleared
493 * disable_aspm: Set to true if ASPM must be disabled due to hardware errata
494 * broken_ext_scan: Set to true if the chip erroneously claims to support
495 * extended scanning
496 * broken_mws_transport_config: Set to true if the chip erroneously claims to
497 * support MWS Transport Configuration
498 * send_calibration: Optional callback to send calibration data
499 * send_ptb: Callback to send "PTB" regulatory/calibration data
500 */
501struct bcm4377_hw {
502 unsigned int id;
503
504 u32 otp_offset;
505
506 u32 bar0_window1;
507 u32 bar0_window2;
508 u32 bar0_core2_window2;
509
510 unsigned long has_bar0_core2_window2 : 1;
511 unsigned long clear_pciecfg_subsystem_ctrl_bit19 : 1;
512 unsigned long disable_aspm : 1;
513 unsigned long broken_ext_scan : 1;
514 unsigned long broken_mws_transport_config : 1;
515
516 int (*send_calibration)(struct bcm4377_data *bcm4377);
517 int (*send_ptb)(struct bcm4377_data *bcm4377,
518 const struct firmware *fw);
519};
520
521static const struct bcm4377_hw bcm4377_hw_variants[];
522static const struct dmi_system_id bcm4377_dmi_board_table[];
523
524/*
525 * Private struct associated with each device containing global state
526 *
527 * pdev: Pointer to associated struct pci_dev
528 * hdev: Pointer to associated strucy hci_dev
529 * bar0: iomem pointing to BAR0
530 * bar1: iomem pointing to BAR2
531 * bootstage: Current value of the bootstage
532 * rti_status: Current "RTI" status value
533 * hw: Pointer to chip-specific struct bcm4377_hw
534 * taurus_cal_blob: "Taurus" calibration blob used for some chips
535 * taurus_cal_size: "Taurus" calibration blob size
536 * taurus_beamforming_cal_blob: "Taurus" beamforming calibration blob used for
537 * some chips
538 * taurus_beamforming_cal_size: "Taurus" beamforming calibration blob size
539 * stepping: Chip stepping read from OTP; used for firmware selection
540 * vendor: Antenna vendor read from OTP; used for firmware selection
541 * board_type: Board type from FDT or DMI match; used for firmware selection
542 * event: Event for changed bootstage or rti_status; used for booting firmware
543 * ctx: "Converged IPC" context
544 * ctx_dma: "Converged IPC" context DMA address
545 * ring_state: Shared memory buffer containing ring head and tail indexes
546 * ring_state_dma: DMA address for ring_state
547 * {control,hci_acl,sco}_ack_ring: Completion rings used to acknowledge messages
548 * {hci_acl,sco}_event_ring: Completion rings used for device->host messages
549 * control_h2d_ring: Transfer ring used for control messages
550 * {hci,sco,acl}_h2d_ring: Transfer ring used to transfer HCI frames
551 * {hci,sco,acl}_d2h_ring: Transfer ring used to receive HCI frames in the
552 * corresponding completion ring
553 */
554struct bcm4377_data {
555 struct pci_dev *pdev;
556 struct hci_dev *hdev;
557
558 void __iomem *bar0;
559 void __iomem *bar2;
560
561 u32 bootstage;
562 u32 rti_status;
563
564 const struct bcm4377_hw *hw;
565
566 const void *taurus_cal_blob;
567 int taurus_cal_size;
568 const void *taurus_beamforming_cal_blob;
569 int taurus_beamforming_cal_size;
570
571 char stepping[BCM4377_OTP_MAX_PARAM_LEN];
572 char vendor[BCM4377_OTP_MAX_PARAM_LEN];
573 const char *board_type;
574
575 struct completion event;
576
577 struct bcm4377_context *ctx;
578 dma_addr_t ctx_dma;
579
580 struct bcm4377_ring_state *ring_state;
581 dma_addr_t ring_state_dma;
582
583 /*
584 * The HCI and ACL rings have to be merged because this structure is
585 * hardcoded in the firmware.
586 */
587 struct bcm4377_completion_ring control_ack_ring;
588 struct bcm4377_completion_ring hci_acl_ack_ring;
589 struct bcm4377_completion_ring hci_acl_event_ring;
590 struct bcm4377_completion_ring sco_ack_ring;
591 struct bcm4377_completion_ring sco_event_ring;
592
593 struct bcm4377_transfer_ring control_h2d_ring;
594 struct bcm4377_transfer_ring hci_h2d_ring;
595 struct bcm4377_transfer_ring hci_d2h_ring;
596 struct bcm4377_transfer_ring sco_h2d_ring;
597 struct bcm4377_transfer_ring sco_d2h_ring;
598 struct bcm4377_transfer_ring acl_h2d_ring;
599 struct bcm4377_transfer_ring acl_d2h_ring;
600};
601
602static void bcm4377_ring_doorbell(struct bcm4377_data *bcm4377, u8 doorbell,
603 u16 val)
604{
605 u32 db = 0;
606
607 db |= FIELD_PREP(BCM4377_BAR0_DOORBELL_VALUE, val);
608 db |= FIELD_PREP(BCM4377_BAR0_DOORBELL_IDX, doorbell);
609 db |= BCM4377_BAR0_DOORBELL_RING;
610
611 dev_dbg(&bcm4377->pdev->dev, "write %d to doorbell #%d (0x%x)\n", val,
612 doorbell, db);
613 iowrite32(db, bcm4377->bar0 + BCM4377_BAR0_DOORBELL);
614}
615
616static int bcm4377_extract_msgid(struct bcm4377_data *bcm4377,
617 struct bcm4377_transfer_ring *ring,
618 u16 raw_msgid, u8 *msgid)
619{
620 u8 generation = FIELD_GET(BCM4377_MSGID_GENERATION, raw_msgid);
621 *msgid = FIELD_GET(BCM4377_MSGID_ID, raw_msgid);
622
623 if (generation != ring->generation) {
624 dev_warn(
625 &bcm4377->pdev->dev,
626 "invalid message generation %d should be %d in entry for ring %d\n",
627 generation, ring->generation, ring->ring_id);
628 return -EINVAL;
629 }
630
631 if (*msgid >= ring->n_entries) {
632 dev_warn(&bcm4377->pdev->dev,
633 "invalid message id in entry for ring %d: %d > %d\n",
634 ring->ring_id, *msgid, ring->n_entries);
635 return -EINVAL;
636 }
637
638 return 0;
639}
640
641static void bcm4377_handle_event(struct bcm4377_data *bcm4377,
642 struct bcm4377_transfer_ring *ring,
643 u16 raw_msgid, u8 entry_flags, u8 type,
644 void *payload, size_t len)
645{
646 struct sk_buff *skb;
647 u16 head;
648 u8 msgid;
649 unsigned long flags;
650
651 spin_lock_irqsave(&ring->lock, flags);
652 if (!ring->enabled) {
653 dev_warn(&bcm4377->pdev->dev,
654 "event for disabled transfer ring %d\n",
655 ring->ring_id);
656 goto out;
657 }
658
659 if (ring->d2h_buffers_only &&
660 entry_flags & BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED) {
661 if (bcm4377_extract_msgid(bcm4377, ring, raw_msgid, &msgid))
662 goto out;
663
664 if (len > ring->mapped_payload_size) {
665 dev_warn(
666 &bcm4377->pdev->dev,
667 "invalid payload len in event for ring %d: %zu > %zu\n",
668 ring->ring_id, len, ring->mapped_payload_size);
669 goto out;
670 }
671
672 payload = ring->payloads + msgid * ring->mapped_payload_size;
673 }
674
675 skb = bt_skb_alloc(len, GFP_ATOMIC);
676 if (!skb)
677 goto out;
678
679 memcpy(skb_put(skb, len), payload, len);
680 hci_skb_pkt_type(skb) = type;
681 hci_recv_frame(bcm4377->hdev, skb);
682
683out:
684 head = le16_to_cpu(bcm4377->ring_state->xfer_ring_head[ring->ring_id]);
685 head = (head + 1) % ring->n_entries;
686 bcm4377->ring_state->xfer_ring_head[ring->ring_id] = cpu_to_le16(head);
687
688 bcm4377_ring_doorbell(bcm4377, ring->doorbell, head);
689
690 spin_unlock_irqrestore(&ring->lock, flags);
691}
692
693static void bcm4377_handle_ack(struct bcm4377_data *bcm4377,
694 struct bcm4377_transfer_ring *ring,
695 u16 raw_msgid)
696{
697 unsigned long flags;
698 u8 msgid;
699
700 spin_lock_irqsave(&ring->lock, flags);
701
702 if (bcm4377_extract_msgid(bcm4377, ring, raw_msgid, &msgid))
703 goto unlock;
704
705 if (!test_bit(msgid, ring->msgids)) {
706 dev_warn(
707 &bcm4377->pdev->dev,
708 "invalid message id in ack for ring %d: %d is not used\n",
709 ring->ring_id, msgid);
710 goto unlock;
711 }
712
713 if (ring->allow_wait && ring->events[msgid]) {
714 complete(ring->events[msgid]);
715 ring->events[msgid] = NULL;
716 }
717
718 bitmap_release_region(ring->msgids, msgid, ring->n_entries);
719
720unlock:
721 spin_unlock_irqrestore(&ring->lock, flags);
722}
723
724static void bcm4377_handle_completion(struct bcm4377_data *bcm4377,
725 struct bcm4377_completion_ring *ring,
726 u16 pos)
727{
728 struct bcm4377_completion_ring_entry *entry;
729 u16 msg_id, transfer_ring;
730 size_t entry_size, data_len;
731 void *data;
732
733 if (pos >= ring->n_entries) {
734 dev_warn(&bcm4377->pdev->dev,
735 "invalid offset %d for completion ring %d\n", pos,
736 ring->ring_id);
737 return;
738 }
739
740 entry_size = sizeof(*entry) + ring->payload_size;
741 entry = ring->ring + pos * entry_size;
742 data = ring->ring + pos * entry_size + sizeof(*entry);
743 data_len = le32_to_cpu(entry->len);
744 msg_id = le16_to_cpu(entry->msg_id);
745 transfer_ring = le16_to_cpu(entry->ring_id);
746
747 if ((ring->transfer_rings & BIT(transfer_ring)) == 0) {
748 dev_warn(
749 &bcm4377->pdev->dev,
750 "invalid entry at offset %d for transfer ring %d in completion ring %d\n",
751 pos, transfer_ring, ring->ring_id);
752 return;
753 }
754
755 dev_dbg(&bcm4377->pdev->dev,
756 "entry in completion ring %d for transfer ring %d with msg_id %d\n",
757 ring->ring_id, transfer_ring, msg_id);
758
759 switch (transfer_ring) {
760 case BCM4377_XFER_RING_CONTROL:
761 bcm4377_handle_ack(bcm4377, &bcm4377->control_h2d_ring, msg_id);
762 break;
763 case BCM4377_XFER_RING_HCI_H2D:
764 bcm4377_handle_ack(bcm4377, &bcm4377->hci_h2d_ring, msg_id);
765 break;
766 case BCM4377_XFER_RING_SCO_H2D:
767 bcm4377_handle_ack(bcm4377, &bcm4377->sco_h2d_ring, msg_id);
768 break;
769 case BCM4377_XFER_RING_ACL_H2D:
770 bcm4377_handle_ack(bcm4377, &bcm4377->acl_h2d_ring, msg_id);
771 break;
772
773 case BCM4377_XFER_RING_HCI_D2H:
774 bcm4377_handle_event(bcm4377, &bcm4377->hci_d2h_ring, msg_id,
775 entry->flags, HCI_EVENT_PKT, data,
776 data_len);
777 break;
778 case BCM4377_XFER_RING_SCO_D2H:
779 bcm4377_handle_event(bcm4377, &bcm4377->sco_d2h_ring, msg_id,
780 entry->flags, HCI_SCODATA_PKT, data,
781 data_len);
782 break;
783 case BCM4377_XFER_RING_ACL_D2H:
784 bcm4377_handle_event(bcm4377, &bcm4377->acl_d2h_ring, msg_id,
785 entry->flags, HCI_ACLDATA_PKT, data,
786 data_len);
787 break;
788
789 default:
790 dev_warn(
791 &bcm4377->pdev->dev,
792 "entry in completion ring %d for unknown transfer ring %d with msg_id %d\n",
793 ring->ring_id, transfer_ring, msg_id);
794 }
795}
796
797static void bcm4377_poll_completion_ring(struct bcm4377_data *bcm4377,
798 struct bcm4377_completion_ring *ring)
799{
800 u16 tail;
801 __le16 *heads = bcm4377->ring_state->completion_ring_head;
802 __le16 *tails = bcm4377->ring_state->completion_ring_tail;
803
804 if (!ring->enabled)
805 return;
806
807 tail = le16_to_cpu(tails[ring->ring_id]);
808 dev_dbg(&bcm4377->pdev->dev,
809 "completion ring #%d: head: %d, tail: %d\n", ring->ring_id,
810 le16_to_cpu(heads[ring->ring_id]), tail);
811
812 while (tail != le16_to_cpu(READ_ONCE(heads[ring->ring_id]))) {
813 /*
814 * ensure the CPU doesn't speculate through the comparison.
815 * otherwise it might already read the (empty) queue entry
816 * before the updated head has been loaded and checked.
817 */
818 dma_rmb();
819
820 bcm4377_handle_completion(bcm4377, ring, tail);
821
822 tail = (tail + 1) % ring->n_entries;
823 tails[ring->ring_id] = cpu_to_le16(tail);
824 }
825}
826
827static irqreturn_t bcm4377_irq(int irq, void *data)
828{
829 struct bcm4377_data *bcm4377 = data;
830 u32 bootstage, rti_status;
831
832 bootstage = ioread32(bcm4377->bar2 + BCM4377_BAR2_BOOTSTAGE);
833 rti_status = ioread32(bcm4377->bar2 + BCM4377_BAR2_RTI_STATUS);
834
835 if (bootstage != bcm4377->bootstage ||
836 rti_status != bcm4377->rti_status) {
837 dev_dbg(&bcm4377->pdev->dev,
838 "bootstage = %d -> %d, rti state = %d -> %d\n",
839 bcm4377->bootstage, bootstage, bcm4377->rti_status,
840 rti_status);
841 complete(&bcm4377->event);
842 bcm4377->bootstage = bootstage;
843 bcm4377->rti_status = rti_status;
844 }
845
846 if (rti_status > 2)
847 dev_err(&bcm4377->pdev->dev, "RTI status is %d\n", rti_status);
848
849 bcm4377_poll_completion_ring(bcm4377, &bcm4377->control_ack_ring);
850 bcm4377_poll_completion_ring(bcm4377, &bcm4377->hci_acl_event_ring);
851 bcm4377_poll_completion_ring(bcm4377, &bcm4377->hci_acl_ack_ring);
852 bcm4377_poll_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
853 bcm4377_poll_completion_ring(bcm4377, &bcm4377->sco_event_ring);
854
855 return IRQ_HANDLED;
856}
857
858static int bcm4377_enqueue(struct bcm4377_data *bcm4377,
859 struct bcm4377_transfer_ring *ring, void *data,
860 size_t len, bool wait)
861{
862 unsigned long flags;
863 struct bcm4377_xfer_ring_entry *entry;
864 void *payload;
865 size_t offset;
866 u16 head, tail, new_head;
867 u16 raw_msgid;
868 int ret, msgid;
869 DECLARE_COMPLETION_ONSTACK(event);
870
871 if (len > ring->payload_size && len > ring->mapped_payload_size) {
872 dev_warn(
873 &bcm4377->pdev->dev,
874 "payload len %zu is too large for ring %d (max is %zu or %zu)\n",
875 len, ring->ring_id, ring->payload_size,
876 ring->mapped_payload_size);
877 return -EINVAL;
878 }
879 if (wait && !ring->allow_wait)
880 return -EINVAL;
881 if (ring->virtual)
882 return -EINVAL;
883
884 spin_lock_irqsave(&ring->lock, flags);
885
886 head = le16_to_cpu(bcm4377->ring_state->xfer_ring_head[ring->ring_id]);
887 tail = le16_to_cpu(bcm4377->ring_state->xfer_ring_tail[ring->ring_id]);
888
889 new_head = (head + 1) % ring->n_entries;
890
891 if (new_head == tail) {
892 dev_warn(&bcm4377->pdev->dev,
893 "can't send message because ring %d is full\n",
894 ring->ring_id);
895 ret = -EINVAL;
896 goto out;
897 }
898
899 msgid = bitmap_find_free_region(ring->msgids, ring->n_entries, 0);
900 if (msgid < 0) {
901 dev_warn(&bcm4377->pdev->dev,
902 "can't find message id for ring %d\n", ring->ring_id);
903 ret = -EINVAL;
904 goto out;
905 }
906
907 raw_msgid = FIELD_PREP(BCM4377_MSGID_GENERATION, ring->generation);
908 raw_msgid |= FIELD_PREP(BCM4377_MSGID_ID, msgid);
909
910 offset = head * (sizeof(*entry) + ring->payload_size);
911 entry = ring->ring + offset;
912
913 memset(entry, 0, sizeof(*entry));
914 entry->id = cpu_to_le16(raw_msgid);
915 entry->len = cpu_to_le16(len);
916
917 if (len <= ring->payload_size) {
918 entry->flags = BCM4377_XFER_RING_FLAG_PAYLOAD_IN_FOOTER;
919 payload = ring->ring + offset + sizeof(*entry);
920 } else {
921 entry->flags = BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED;
922 entry->payload = cpu_to_le64(ring->payloads_dma +
923 msgid * ring->mapped_payload_size);
924 payload = ring->payloads + msgid * ring->mapped_payload_size;
925 }
926
927 memcpy(payload, data, len);
928
929 if (wait)
930 ring->events[msgid] = &event;
931
932 /*
933 * The 4377 chips stop responding to any commands as soon as they
934 * have been idle for a while. Poking the sleep control register here
935 * makes them come alive again.
936 */
937 iowrite32(BCM4377_BAR0_SLEEP_CONTROL_AWAKE,
938 bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL);
939
940 dev_dbg(&bcm4377->pdev->dev,
941 "updating head for transfer queue #%d to %d\n", ring->ring_id,
942 new_head);
943 bcm4377->ring_state->xfer_ring_head[ring->ring_id] =
944 cpu_to_le16(new_head);
945
946 if (!ring->sync)
947 bcm4377_ring_doorbell(bcm4377, ring->doorbell, new_head);
948 ret = 0;
949
950out:
951 spin_unlock_irqrestore(&ring->lock, flags);
952
953 if (ret == 0 && wait) {
954 ret = wait_for_completion_interruptible_timeout(
955 &event, BCM4377_TIMEOUT);
956 if (ret == 0)
957 ret = -ETIMEDOUT;
958 else if (ret > 0)
959 ret = 0;
960
961 spin_lock_irqsave(&ring->lock, flags);
962 ring->events[msgid] = NULL;
963 spin_unlock_irqrestore(&ring->lock, flags);
964 }
965
966 return ret;
967}
968
969static int bcm4377_create_completion_ring(struct bcm4377_data *bcm4377,
970 struct bcm4377_completion_ring *ring)
971{
972 struct bcm4377_create_completion_ring_msg msg;
973 int ret;
974
975 if (ring->enabled) {
976 dev_warn(&bcm4377->pdev->dev,
977 "completion ring %d already enabled\n", ring->ring_id);
978 return 0;
979 }
980
981 memset(ring->ring, 0,
982 ring->n_entries * (sizeof(struct bcm4377_completion_ring_entry) +
983 ring->payload_size));
984 memset(&msg, 0, sizeof(msg));
985 msg.msg_type = BCM4377_CONTROL_MSG_CREATE_COMPLETION_RING;
986 msg.id = cpu_to_le16(ring->ring_id);
987 msg.id_again = cpu_to_le16(ring->ring_id);
988 msg.ring_iova = cpu_to_le64(ring->ring_dma);
989 msg.n_elements = cpu_to_le16(ring->n_entries);
990 msg.intmod_bytes = cpu_to_le32(0xffffffff);
991 msg.unk = cpu_to_le32(0xffffffff);
992 msg.intmod_delay = cpu_to_le16(ring->delay);
993 msg.footer_size = ring->payload_size / 4;
994
995 ret = bcm4377_enqueue(bcm4377, &bcm4377->control_h2d_ring, &msg,
996 sizeof(msg), true);
997 if (!ret)
998 ring->enabled = true;
999
1000 return ret;
1001}
1002
1003static int bcm4377_destroy_completion_ring(struct bcm4377_data *bcm4377,
1004 struct bcm4377_completion_ring *ring)
1005{
1006 struct bcm4377_destroy_completion_ring_msg msg;
1007 int ret;
1008
1009 memset(&msg, 0, sizeof(msg));
1010 msg.msg_type = BCM4377_CONTROL_MSG_DESTROY_COMPLETION_RING;
1011 msg.ring_id = cpu_to_le16(ring->ring_id);
1012
1013 ret = bcm4377_enqueue(bcm4377, &bcm4377->control_h2d_ring, &msg,
1014 sizeof(msg), true);
1015 if (ret)
1016 dev_warn(&bcm4377->pdev->dev,
1017 "failed to destroy completion ring %d\n",
1018 ring->ring_id);
1019
1020 ring->enabled = false;
1021 return ret;
1022}
1023
1024static int bcm4377_create_transfer_ring(struct bcm4377_data *bcm4377,
1025 struct bcm4377_transfer_ring *ring)
1026{
1027 struct bcm4377_create_transfer_ring_msg msg;
1028 u16 flags = 0;
1029 int ret, i;
1030 unsigned long spinlock_flags;
1031
1032 if (ring->virtual)
1033 flags |= BCM4377_XFER_RING_FLAG_VIRTUAL;
1034 if (ring->sync)
1035 flags |= BCM4377_XFER_RING_FLAG_SYNC;
1036
1037 spin_lock_irqsave(&ring->lock, spinlock_flags);
1038 memset(&msg, 0, sizeof(msg));
1039 msg.msg_type = BCM4377_CONTROL_MSG_CREATE_XFER_RING;
1040 msg.ring_id = cpu_to_le16(ring->ring_id);
1041 msg.ring_id_again = cpu_to_le16(ring->ring_id);
1042 msg.ring_iova = cpu_to_le64(ring->ring_dma);
1043 msg.n_elements = cpu_to_le16(ring->n_entries);
1044 msg.completion_ring_id = cpu_to_le16(ring->completion_ring);
1045 msg.doorbell = cpu_to_le16(ring->doorbell);
1046 msg.flags = cpu_to_le16(flags);
1047 msg.footer_size = ring->payload_size / 4;
1048
1049 bcm4377->ring_state->xfer_ring_head[ring->ring_id] = 0;
1050 bcm4377->ring_state->xfer_ring_tail[ring->ring_id] = 0;
1051 ring->generation++;
1052 spin_unlock_irqrestore(&ring->lock, spinlock_flags);
1053
1054 ret = bcm4377_enqueue(bcm4377, &bcm4377->control_h2d_ring, &msg,
1055 sizeof(msg), true);
1056
1057 spin_lock_irqsave(&ring->lock, spinlock_flags);
1058
1059 if (ring->d2h_buffers_only) {
1060 for (i = 0; i < ring->n_entries; ++i) {
1061 struct bcm4377_xfer_ring_entry *entry =
1062 ring->ring + i * sizeof(*entry);
1063 u16 raw_msgid = FIELD_PREP(BCM4377_MSGID_GENERATION,
1064 ring->generation);
1065 raw_msgid |= FIELD_PREP(BCM4377_MSGID_ID, i);
1066
1067 memset(entry, 0, sizeof(*entry));
1068 entry->id = cpu_to_le16(raw_msgid);
1069 entry->len = cpu_to_le16(ring->mapped_payload_size);
1070 entry->flags = BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED;
1071 entry->payload =
1072 cpu_to_le64(ring->payloads_dma +
1073 i * ring->mapped_payload_size);
1074 }
1075 }
1076
1077 /*
1078 * send some messages if this is a device->host ring to allow the device
1079 * to reply by acknowledging them in the completion ring
1080 */
1081 if (ring->virtual || ring->d2h_buffers_only) {
1082 bcm4377->ring_state->xfer_ring_head[ring->ring_id] =
1083 cpu_to_le16(0xf);
1084 bcm4377_ring_doorbell(bcm4377, ring->doorbell, 0xf);
1085 }
1086
1087 ring->enabled = true;
1088 spin_unlock_irqrestore(&ring->lock, spinlock_flags);
1089
1090 return ret;
1091}
1092
1093static int bcm4377_destroy_transfer_ring(struct bcm4377_data *bcm4377,
1094 struct bcm4377_transfer_ring *ring)
1095{
1096 struct bcm4377_destroy_transfer_ring_msg msg;
1097 int ret;
1098
1099 memset(&msg, 0, sizeof(msg));
1100 msg.msg_type = BCM4377_CONTROL_MSG_DESTROY_XFER_RING;
1101 msg.ring_id = cpu_to_le16(ring->ring_id);
1102
1103 ret = bcm4377_enqueue(bcm4377, &bcm4377->control_h2d_ring, &msg,
1104 sizeof(msg), true);
1105 if (ret)
1106 dev_warn(&bcm4377->pdev->dev,
1107 "failed to destroy transfer ring %d\n", ring->ring_id);
1108
1109 ring->enabled = false;
1110 return ret;
1111}
1112
1113static int __bcm4378_send_calibration_chunk(struct bcm4377_data *bcm4377,
1114 const void *data, size_t data_len,
1115 u16 blocks_left)
1116{
1117 struct bcm4378_hci_send_calibration_cmd cmd;
1118 struct sk_buff *skb;
1119
1120 if (data_len > sizeof(cmd.data))
1121 return -EINVAL;
1122
1123 memset(&cmd, 0, sizeof(cmd));
1124 cmd.unk = 0x03;
1125 cmd.blocks_left = cpu_to_le16(blocks_left);
1126 memcpy(cmd.data, data, data_len);
1127
1128 skb = __hci_cmd_sync(bcm4377->hdev, 0xfd97, sizeof(cmd), &cmd,
1129 HCI_INIT_TIMEOUT);
1130 if (IS_ERR(skb))
1131 return PTR_ERR(skb);
1132
1133 kfree_skb(skb);
1134 return 0;
1135}
1136
1137static int __bcm4378_send_calibration(struct bcm4377_data *bcm4377,
1138 const void *data, size_t data_size)
1139{
1140 int ret;
1141 size_t i, left, transfer_len;
1142 size_t blocks =
1143 DIV_ROUND_UP(data_size, (size_t)BCM4378_CALIBRATION_CHUNK_SIZE);
1144
1145 if (!data) {
1146 dev_err(&bcm4377->pdev->dev,
1147 "no calibration data available.\n");
1148 return -ENOENT;
1149 }
1150
1151 for (i = 0, left = data_size; i < blocks; ++i, left -= transfer_len) {
1152 transfer_len =
1153 min_t(size_t, left, BCM4378_CALIBRATION_CHUNK_SIZE);
1154
1155 ret = __bcm4378_send_calibration_chunk(
1156 bcm4377, data + i * BCM4378_CALIBRATION_CHUNK_SIZE,
1157 transfer_len, blocks - i - 1);
1158 if (ret) {
1159 dev_err(&bcm4377->pdev->dev,
1160 "send calibration chunk failed with %d\n", ret);
1161 return ret;
1162 }
1163 }
1164
1165 return 0;
1166}
1167
1168static int bcm4378_send_calibration(struct bcm4377_data *bcm4377)
1169{
1170 if ((strcmp(bcm4377->stepping, "b1") == 0) ||
1171 strcmp(bcm4377->stepping, "b3") == 0)
1172 return __bcm4378_send_calibration(
1173 bcm4377, bcm4377->taurus_beamforming_cal_blob,
1174 bcm4377->taurus_beamforming_cal_size);
1175 else
1176 return __bcm4378_send_calibration(bcm4377,
1177 bcm4377->taurus_cal_blob,
1178 bcm4377->taurus_cal_size);
1179}
1180
1181static int bcm4387_send_calibration(struct bcm4377_data *bcm4377)
1182{
1183 if (strcmp(bcm4377->stepping, "c2") == 0)
1184 return __bcm4378_send_calibration(
1185 bcm4377, bcm4377->taurus_beamforming_cal_blob,
1186 bcm4377->taurus_beamforming_cal_size);
1187 else
1188 return __bcm4378_send_calibration(bcm4377,
1189 bcm4377->taurus_cal_blob,
1190 bcm4377->taurus_cal_size);
1191}
1192
1193static const struct firmware *bcm4377_request_blob(struct bcm4377_data *bcm4377,
1194 const char *suffix)
1195{
1196 const struct firmware *fw;
1197 char name0[64], name1[64];
1198 int ret;
1199
1200 snprintf(name0, sizeof(name0), "brcm/brcmbt%04x%s-%s-%s.%s",
1201 bcm4377->hw->id, bcm4377->stepping, bcm4377->board_type,
1202 bcm4377->vendor, suffix);
1203 snprintf(name1, sizeof(name1), "brcm/brcmbt%04x%s-%s.%s",
1204 bcm4377->hw->id, bcm4377->stepping, bcm4377->board_type,
1205 suffix);
1206 dev_dbg(&bcm4377->pdev->dev, "Trying to load firmware: '%s' or '%s'\n",
1207 name0, name1);
1208
1209 ret = firmware_request_nowarn(&fw, name0, &bcm4377->pdev->dev);
1210 if (!ret)
1211 return fw;
1212 ret = firmware_request_nowarn(&fw, name1, &bcm4377->pdev->dev);
1213 if (!ret)
1214 return fw;
1215
1216 dev_err(&bcm4377->pdev->dev,
1217 "Unable to load firmware; tried '%s' and '%s'\n", name0, name1);
1218 return NULL;
1219}
1220
1221static int bcm4377_send_ptb(struct bcm4377_data *bcm4377,
1222 const struct firmware *fw)
1223{
1224 struct sk_buff *skb;
1225
1226 skb = __hci_cmd_sync(bcm4377->hdev, 0xfd98, fw->size, fw->data,
1227 HCI_INIT_TIMEOUT);
1228 /*
1229 * This command seems to always fail on more recent firmware versions
1230 * (even in traces taken from the macOS driver). It's unclear why this
1231 * happens but because the PTB file contains calibration and/or
1232 * regulatory data and may be required on older firmware we still try to
1233 * send it here just in case and just ignore if it fails.
1234 */
1235 if (!IS_ERR(skb))
1236 kfree_skb(skb);
1237 return 0;
1238}
1239
1240static int bcm4378_send_ptb_chunk(struct bcm4377_data *bcm4377,
1241 const void *data, size_t data_len,
1242 u16 blocks_left)
1243{
1244 struct bcm4378_hci_send_ptb_cmd cmd;
1245 struct sk_buff *skb;
1246
1247 if (data_len > BCM4378_PTB_CHUNK_SIZE)
1248 return -EINVAL;
1249
1250 memset(&cmd, 0, sizeof(cmd));
1251 cmd.blocks_left = cpu_to_le16(blocks_left);
1252 memcpy(cmd.data, data, data_len);
1253
1254 skb = __hci_cmd_sync(bcm4377->hdev, 0xfe0d, sizeof(cmd), &cmd,
1255 HCI_INIT_TIMEOUT);
1256 if (IS_ERR(skb))
1257 return PTR_ERR(skb);
1258
1259 kfree_skb(skb);
1260 return 0;
1261}
1262
1263static int bcm4378_send_ptb(struct bcm4377_data *bcm4377,
1264 const struct firmware *fw)
1265{
1266 size_t chunks = DIV_ROUND_UP(fw->size, (size_t)BCM4378_PTB_CHUNK_SIZE);
1267 size_t i, left, transfer_len;
1268 int ret;
1269
1270 for (i = 0, left = fw->size; i < chunks; ++i, left -= transfer_len) {
1271 transfer_len = min_t(size_t, left, BCM4378_PTB_CHUNK_SIZE);
1272
1273 dev_dbg(&bcm4377->pdev->dev, "sending ptb chunk %zu/%zu\n",
1274 i + 1, chunks);
1275 ret = bcm4378_send_ptb_chunk(
1276 bcm4377, fw->data + i * BCM4378_PTB_CHUNK_SIZE,
1277 transfer_len, chunks - i - 1);
1278 if (ret) {
1279 dev_err(&bcm4377->pdev->dev,
1280 "sending ptb chunk %zu failed (%d)", i, ret);
1281 return ret;
1282 }
1283 }
1284
1285 return 0;
1286}
1287
1288static int bcm4377_hci_open(struct hci_dev *hdev)
1289{
1290 struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1291 int ret;
1292
1293 dev_dbg(&bcm4377->pdev->dev, "creating rings\n");
1294
1295 ret = bcm4377_create_completion_ring(bcm4377,
1296 &bcm4377->hci_acl_ack_ring);
1297 if (ret)
1298 return ret;
1299 ret = bcm4377_create_completion_ring(bcm4377,
1300 &bcm4377->hci_acl_event_ring);
1301 if (ret)
1302 goto destroy_hci_acl_ack;
1303 ret = bcm4377_create_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
1304 if (ret)
1305 goto destroy_hci_acl_event;
1306 ret = bcm4377_create_completion_ring(bcm4377, &bcm4377->sco_event_ring);
1307 if (ret)
1308 goto destroy_sco_ack;
1309 dev_dbg(&bcm4377->pdev->dev,
1310 "all completion rings successfully created!\n");
1311
1312 ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring);
1313 if (ret)
1314 goto destroy_sco_event;
1315 ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring);
1316 if (ret)
1317 goto destroy_hci_h2d;
1318 ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring);
1319 if (ret)
1320 goto destroy_hci_d2h;
1321 ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring);
1322 if (ret)
1323 goto destroy_sco_h2d;
1324 ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring);
1325 if (ret)
1326 goto destroy_sco_d2h;
1327 ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->acl_d2h_ring);
1328 if (ret)
1329 goto destroy_acl_h2d;
1330 dev_dbg(&bcm4377->pdev->dev,
1331 "all transfer rings successfully created!\n");
1332
1333 return 0;
1334
1335destroy_acl_h2d:
1336 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring);
1337destroy_sco_d2h:
1338 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring);
1339destroy_sco_h2d:
1340 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring);
1341destroy_hci_d2h:
1342 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring);
1343destroy_hci_h2d:
1344 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring);
1345destroy_sco_event:
1346 bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_event_ring);
1347destroy_sco_ack:
1348 bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
1349destroy_hci_acl_event:
1350 bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_event_ring);
1351destroy_hci_acl_ack:
1352 bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_ack_ring);
1353
1354 dev_err(&bcm4377->pdev->dev, "Creating rings failed with %d\n", ret);
1355 return ret;
1356}
1357
1358static int bcm4377_hci_close(struct hci_dev *hdev)
1359{
1360 struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1361
1362 dev_dbg(&bcm4377->pdev->dev, "destroying rings in hci_close\n");
1363
1364 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->acl_d2h_ring);
1365 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring);
1366 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring);
1367 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring);
1368 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring);
1369 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring);
1370
1371 bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_event_ring);
1372 bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
1373 bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_event_ring);
1374 bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_ack_ring);
1375
1376 return 0;
1377}
1378
1379static bool bcm4377_is_valid_bdaddr(struct bcm4377_data *bcm4377,
1380 bdaddr_t *addr)
1381{
1382 if (addr->b[0] != 0x93)
1383 return true;
1384 if (addr->b[1] != 0x76)
1385 return true;
1386 if (addr->b[2] != 0x00)
1387 return true;
1388 if (addr->b[4] != (bcm4377->hw->id & 0xff))
1389 return true;
1390 if (addr->b[5] != (bcm4377->hw->id >> 8))
1391 return true;
1392 return false;
1393}
1394
1395static int bcm4377_check_bdaddr(struct bcm4377_data *bcm4377)
1396{
1397 struct hci_rp_read_bd_addr *bda;
1398 struct sk_buff *skb;
1399
1400 skb = __hci_cmd_sync(bcm4377->hdev, HCI_OP_READ_BD_ADDR, 0, NULL,
1401 HCI_INIT_TIMEOUT);
1402 if (IS_ERR(skb)) {
1403 int err = PTR_ERR(skb);
1404
1405 dev_err(&bcm4377->pdev->dev, "HCI_OP_READ_BD_ADDR failed (%d)",
1406 err);
1407 return err;
1408 }
1409
1410 if (skb->len != sizeof(*bda)) {
1411 dev_err(&bcm4377->pdev->dev,
1412 "HCI_OP_READ_BD_ADDR reply length invalid");
1413 kfree_skb(skb);
1414 return -EIO;
1415 }
1416
1417 bda = (struct hci_rp_read_bd_addr *)skb->data;
1418 if (!bcm4377_is_valid_bdaddr(bcm4377, &bda->bdaddr))
1419 set_bit(HCI_QUIRK_INVALID_BDADDR, &bcm4377->hdev->quirks);
1420
1421 kfree_skb(skb);
1422 return 0;
1423}
1424
1425static int bcm4377_hci_setup(struct hci_dev *hdev)
1426{
1427 struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1428 const struct firmware *fw;
1429 int ret;
1430
1431 if (bcm4377->hw->send_calibration) {
1432 ret = bcm4377->hw->send_calibration(bcm4377);
1433 if (ret)
1434 return ret;
1435 }
1436
1437 fw = bcm4377_request_blob(bcm4377, "ptb");
1438 if (!fw) {
1439 dev_err(&bcm4377->pdev->dev, "failed to load PTB data");
1440 return -ENOENT;
1441 }
1442
1443 ret = bcm4377->hw->send_ptb(bcm4377, fw);
1444 release_firmware(fw);
1445 if (ret)
1446 return ret;
1447
1448 return bcm4377_check_bdaddr(bcm4377);
1449}
1450
1451static int bcm4377_hci_send_frame(struct hci_dev *hdev, struct sk_buff *skb)
1452{
1453 struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1454 struct bcm4377_transfer_ring *ring;
1455 int ret;
1456
1457 switch (hci_skb_pkt_type(skb)) {
1458 case HCI_COMMAND_PKT:
1459 hdev->stat.cmd_tx++;
1460 ring = &bcm4377->hci_h2d_ring;
1461 break;
1462
1463 case HCI_ACLDATA_PKT:
1464 hdev->stat.acl_tx++;
1465 ring = &bcm4377->acl_h2d_ring;
1466 break;
1467
1468 case HCI_SCODATA_PKT:
1469 hdev->stat.sco_tx++;
1470 ring = &bcm4377->sco_h2d_ring;
1471 break;
1472
1473 default:
1474 return -EILSEQ;
1475 }
1476
1477 ret = bcm4377_enqueue(bcm4377, ring, skb->data, skb->len, false);
1478 if (ret < 0) {
1479 hdev->stat.err_tx++;
1480 return ret;
1481 }
1482
1483 hdev->stat.byte_tx += skb->len;
1484 kfree_skb(skb);
1485 return ret;
1486}
1487
1488static int bcm4377_hci_set_bdaddr(struct hci_dev *hdev, const bdaddr_t *bdaddr)
1489{
1490 struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1491 struct sk_buff *skb;
1492 int err;
1493
1494 skb = __hci_cmd_sync(hdev, 0xfc01, 6, bdaddr, HCI_INIT_TIMEOUT);
1495 if (IS_ERR(skb)) {
1496 err = PTR_ERR(skb);
1497 dev_err(&bcm4377->pdev->dev,
1498 "Change address command failed (%d)", err);
1499 return err;
1500 }
1501 kfree_skb(skb);
1502
1503 return 0;
1504}
1505
1506static int bcm4377_alloc_transfer_ring(struct bcm4377_data *bcm4377,
1507 struct bcm4377_transfer_ring *ring)
1508{
1509 size_t entry_size;
1510
1511 spin_lock_init(&ring->lock);
1512 ring->payload_size = ALIGN(ring->payload_size, 4);
1513 ring->mapped_payload_size = ALIGN(ring->mapped_payload_size, 4);
1514
1515 if (ring->payload_size > BCM4377_XFER_RING_MAX_INPLACE_PAYLOAD_SIZE)
1516 return -EINVAL;
1517 if (ring->n_entries > BCM4377_MAX_RING_SIZE)
1518 return -EINVAL;
1519 if (ring->virtual && ring->allow_wait)
1520 return -EINVAL;
1521
1522 if (ring->d2h_buffers_only) {
1523 if (ring->virtual)
1524 return -EINVAL;
1525 if (ring->payload_size)
1526 return -EINVAL;
1527 if (!ring->mapped_payload_size)
1528 return -EINVAL;
1529 }
1530 if (ring->virtual)
1531 return 0;
1532
1533 entry_size =
1534 ring->payload_size + sizeof(struct bcm4377_xfer_ring_entry);
1535 ring->ring = dmam_alloc_coherent(&bcm4377->pdev->dev,
1536 ring->n_entries * entry_size,
1537 &ring->ring_dma, GFP_KERNEL);
1538 if (!ring->ring)
1539 return -ENOMEM;
1540
1541 if (ring->allow_wait) {
1542 ring->events = devm_kcalloc(&bcm4377->pdev->dev,
1543 ring->n_entries,
1544 sizeof(*ring->events), GFP_KERNEL);
1545 if (!ring->events)
1546 return -ENOMEM;
1547 }
1548
1549 if (ring->mapped_payload_size) {
1550 ring->payloads = dmam_alloc_coherent(
1551 &bcm4377->pdev->dev,
1552 ring->n_entries * ring->mapped_payload_size,
1553 &ring->payloads_dma, GFP_KERNEL);
1554 if (!ring->payloads)
1555 return -ENOMEM;
1556 }
1557
1558 return 0;
1559}
1560
1561static int bcm4377_alloc_completion_ring(struct bcm4377_data *bcm4377,
1562 struct bcm4377_completion_ring *ring)
1563{
1564 size_t entry_size;
1565
1566 ring->payload_size = ALIGN(ring->payload_size, 4);
1567 if (ring->payload_size > BCM4377_XFER_RING_MAX_INPLACE_PAYLOAD_SIZE)
1568 return -EINVAL;
1569 if (ring->n_entries > BCM4377_MAX_RING_SIZE)
1570 return -EINVAL;
1571
1572 entry_size = ring->payload_size +
1573 sizeof(struct bcm4377_completion_ring_entry);
1574
1575 ring->ring = dmam_alloc_coherent(&bcm4377->pdev->dev,
1576 ring->n_entries * entry_size,
1577 &ring->ring_dma, GFP_KERNEL);
1578 if (!ring->ring)
1579 return -ENOMEM;
1580 return 0;
1581}
1582
1583static int bcm4377_init_context(struct bcm4377_data *bcm4377)
1584{
1585 struct device *dev = &bcm4377->pdev->dev;
1586 dma_addr_t peripheral_info_dma;
1587
1588 bcm4377->ctx = dmam_alloc_coherent(dev, sizeof(*bcm4377->ctx),
1589 &bcm4377->ctx_dma, GFP_KERNEL);
1590 if (!bcm4377->ctx)
1591 return -ENOMEM;
1592 memset(bcm4377->ctx, 0, sizeof(*bcm4377->ctx));
1593
1594 bcm4377->ring_state =
1595 dmam_alloc_coherent(dev, sizeof(*bcm4377->ring_state),
1596 &bcm4377->ring_state_dma, GFP_KERNEL);
1597 if (!bcm4377->ring_state)
1598 return -ENOMEM;
1599 memset(bcm4377->ring_state, 0, sizeof(*bcm4377->ring_state));
1600
1601 bcm4377->ctx->version = cpu_to_le16(1);
1602 bcm4377->ctx->size = cpu_to_le16(sizeof(*bcm4377->ctx));
1603 bcm4377->ctx->enabled_caps = cpu_to_le32(2);
1604
1605 /*
1606 * The BT device will write 0x20 bytes of data to this buffer but
1607 * the exact contents are unknown. It only needs to exist for BT
1608 * to work such that we can just allocate and then ignore it.
1609 */
1610 if (!dmam_alloc_coherent(&bcm4377->pdev->dev, 0x20,
1611 &peripheral_info_dma, GFP_KERNEL))
1612 return -ENOMEM;
1613 bcm4377->ctx->peripheral_info_addr = cpu_to_le64(peripheral_info_dma);
1614
1615 bcm4377->ctx->xfer_ring_heads_addr = cpu_to_le64(
1616 bcm4377->ring_state_dma +
1617 offsetof(struct bcm4377_ring_state, xfer_ring_head));
1618 bcm4377->ctx->xfer_ring_tails_addr = cpu_to_le64(
1619 bcm4377->ring_state_dma +
1620 offsetof(struct bcm4377_ring_state, xfer_ring_tail));
1621 bcm4377->ctx->completion_ring_heads_addr = cpu_to_le64(
1622 bcm4377->ring_state_dma +
1623 offsetof(struct bcm4377_ring_state, completion_ring_head));
1624 bcm4377->ctx->completion_ring_tails_addr = cpu_to_le64(
1625 bcm4377->ring_state_dma +
1626 offsetof(struct bcm4377_ring_state, completion_ring_tail));
1627
1628 bcm4377->ctx->n_completion_rings =
1629 cpu_to_le16(BCM4377_N_COMPLETION_RINGS);
1630 bcm4377->ctx->n_xfer_rings = cpu_to_le16(BCM4377_N_TRANSFER_RINGS);
1631
1632 bcm4377->ctx->control_completion_ring_addr =
1633 cpu_to_le64(bcm4377->control_ack_ring.ring_dma);
1634 bcm4377->ctx->control_completion_ring_n_entries =
1635 cpu_to_le16(bcm4377->control_ack_ring.n_entries);
1636 bcm4377->ctx->control_completion_ring_doorbell = cpu_to_le16(0xffff);
1637 bcm4377->ctx->control_completion_ring_msi = 0;
1638 bcm4377->ctx->control_completion_ring_header_size = 0;
1639 bcm4377->ctx->control_completion_ring_footer_size = 0;
1640
1641 bcm4377->ctx->control_xfer_ring_addr =
1642 cpu_to_le64(bcm4377->control_h2d_ring.ring_dma);
1643 bcm4377->ctx->control_xfer_ring_n_entries =
1644 cpu_to_le16(bcm4377->control_h2d_ring.n_entries);
1645 bcm4377->ctx->control_xfer_ring_doorbell =
1646 cpu_to_le16(bcm4377->control_h2d_ring.doorbell);
1647 bcm4377->ctx->control_xfer_ring_msi = 0;
1648 bcm4377->ctx->control_xfer_ring_header_size = 0;
1649 bcm4377->ctx->control_xfer_ring_footer_size =
1650 bcm4377->control_h2d_ring.payload_size / 4;
1651
1652 dev_dbg(&bcm4377->pdev->dev, "context initialized at IOVA %pad",
1653 &bcm4377->ctx_dma);
1654
1655 return 0;
1656}
1657
1658static int bcm4377_prepare_rings(struct bcm4377_data *bcm4377)
1659{
1660 int ret;
1661
1662 /*
1663 * Even though many of these settings appear to be configurable
1664 * when sending the "create ring" messages most of these are
1665 * actually hardcoded in some (and quite possibly all) firmware versions
1666 * and changing them on the host has no effect.
1667 * Specifically, this applies to at least the doorbells, the transfer
1668 * and completion ring ids and their mapping (e.g. both HCI and ACL
1669 * entries will always be queued in completion rings 1 and 2 no matter
1670 * what we configure here).
1671 */
1672 bcm4377->control_ack_ring.ring_id = BCM4377_ACK_RING_CONTROL;
1673 bcm4377->control_ack_ring.n_entries = 32;
1674 bcm4377->control_ack_ring.transfer_rings =
1675 BIT(BCM4377_XFER_RING_CONTROL);
1676
1677 bcm4377->hci_acl_ack_ring.ring_id = BCM4377_ACK_RING_HCI_ACL;
1678 bcm4377->hci_acl_ack_ring.n_entries = 2 * BCM4377_RING_N_ENTRIES;
1679 bcm4377->hci_acl_ack_ring.transfer_rings =
1680 BIT(BCM4377_XFER_RING_HCI_H2D) | BIT(BCM4377_XFER_RING_ACL_H2D);
1681 bcm4377->hci_acl_ack_ring.delay = 1000;
1682
1683 /*
1684 * A payload size of MAX_EVENT_PAYLOAD_SIZE is enough here since large
1685 * ACL packets will be transmitted inside buffers mapped via
1686 * acl_d2h_ring anyway.
1687 */
1688 bcm4377->hci_acl_event_ring.ring_id = BCM4377_EVENT_RING_HCI_ACL;
1689 bcm4377->hci_acl_event_ring.payload_size = MAX_EVENT_PAYLOAD_SIZE;
1690 bcm4377->hci_acl_event_ring.n_entries = 2 * BCM4377_RING_N_ENTRIES;
1691 bcm4377->hci_acl_event_ring.transfer_rings =
1692 BIT(BCM4377_XFER_RING_HCI_D2H) | BIT(BCM4377_XFER_RING_ACL_D2H);
1693 bcm4377->hci_acl_event_ring.delay = 1000;
1694
1695 bcm4377->sco_ack_ring.ring_id = BCM4377_ACK_RING_SCO;
1696 bcm4377->sco_ack_ring.n_entries = BCM4377_RING_N_ENTRIES;
1697 bcm4377->sco_ack_ring.transfer_rings = BIT(BCM4377_XFER_RING_SCO_H2D);
1698
1699 bcm4377->sco_event_ring.ring_id = BCM4377_EVENT_RING_SCO;
1700 bcm4377->sco_event_ring.payload_size = MAX_SCO_PAYLOAD_SIZE;
1701 bcm4377->sco_event_ring.n_entries = BCM4377_RING_N_ENTRIES;
1702 bcm4377->sco_event_ring.transfer_rings = BIT(BCM4377_XFER_RING_SCO_D2H);
1703
1704 bcm4377->control_h2d_ring.ring_id = BCM4377_XFER_RING_CONTROL;
1705 bcm4377->control_h2d_ring.doorbell = BCM4377_DOORBELL_CONTROL;
1706 bcm4377->control_h2d_ring.payload_size = BCM4377_CONTROL_MSG_SIZE;
1707 bcm4377->control_h2d_ring.completion_ring = BCM4377_ACK_RING_CONTROL;
1708 bcm4377->control_h2d_ring.allow_wait = true;
1709 bcm4377->control_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1710
1711 bcm4377->hci_h2d_ring.ring_id = BCM4377_XFER_RING_HCI_H2D;
1712 bcm4377->hci_h2d_ring.doorbell = BCM4377_DOORBELL_HCI_H2D;
1713 bcm4377->hci_h2d_ring.payload_size = MAX_EVENT_PAYLOAD_SIZE;
1714 bcm4377->hci_h2d_ring.completion_ring = BCM4377_ACK_RING_HCI_ACL;
1715 bcm4377->hci_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1716
1717 bcm4377->hci_d2h_ring.ring_id = BCM4377_XFER_RING_HCI_D2H;
1718 bcm4377->hci_d2h_ring.doorbell = BCM4377_DOORBELL_HCI_D2H;
1719 bcm4377->hci_d2h_ring.completion_ring = BCM4377_EVENT_RING_HCI_ACL;
1720 bcm4377->hci_d2h_ring.virtual = true;
1721 bcm4377->hci_d2h_ring.n_entries = BCM4377_RING_N_ENTRIES;
1722
1723 bcm4377->sco_h2d_ring.ring_id = BCM4377_XFER_RING_SCO_H2D;
1724 bcm4377->sco_h2d_ring.doorbell = BCM4377_DOORBELL_SCO;
1725 bcm4377->sco_h2d_ring.payload_size = MAX_SCO_PAYLOAD_SIZE;
1726 bcm4377->sco_h2d_ring.completion_ring = BCM4377_ACK_RING_SCO;
1727 bcm4377->sco_h2d_ring.sync = true;
1728 bcm4377->sco_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1729
1730 bcm4377->sco_d2h_ring.ring_id = BCM4377_XFER_RING_SCO_D2H;
1731 bcm4377->sco_d2h_ring.doorbell = BCM4377_DOORBELL_SCO;
1732 bcm4377->sco_d2h_ring.completion_ring = BCM4377_EVENT_RING_SCO;
1733 bcm4377->sco_d2h_ring.virtual = true;
1734 bcm4377->sco_d2h_ring.sync = true;
1735 bcm4377->sco_d2h_ring.n_entries = BCM4377_RING_N_ENTRIES;
1736
1737 /*
1738 * This ring has to use mapped_payload_size because the largest ACL
1739 * packet doesn't fit inside the largest possible footer
1740 */
1741 bcm4377->acl_h2d_ring.ring_id = BCM4377_XFER_RING_ACL_H2D;
1742 bcm4377->acl_h2d_ring.doorbell = BCM4377_DOORBELL_ACL_H2D;
1743 bcm4377->acl_h2d_ring.mapped_payload_size = MAX_ACL_PAYLOAD_SIZE;
1744 bcm4377->acl_h2d_ring.completion_ring = BCM4377_ACK_RING_HCI_ACL;
1745 bcm4377->acl_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1746
1747 /*
1748 * This ring only contains empty buffers to be used by incoming
1749 * ACL packets that do not fit inside the footer of hci_acl_event_ring
1750 */
1751 bcm4377->acl_d2h_ring.ring_id = BCM4377_XFER_RING_ACL_D2H;
1752 bcm4377->acl_d2h_ring.doorbell = BCM4377_DOORBELL_ACL_D2H;
1753 bcm4377->acl_d2h_ring.completion_ring = BCM4377_EVENT_RING_HCI_ACL;
1754 bcm4377->acl_d2h_ring.d2h_buffers_only = true;
1755 bcm4377->acl_d2h_ring.mapped_payload_size = MAX_ACL_PAYLOAD_SIZE;
1756 bcm4377->acl_d2h_ring.n_entries = BCM4377_RING_N_ENTRIES;
1757
1758 /*
1759 * no need for any cleanup since this is only called from _probe
1760 * and only devres-managed allocations are used
1761 */
1762 ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->control_h2d_ring);
1763 if (ret)
1764 return ret;
1765 ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring);
1766 if (ret)
1767 return ret;
1768 ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring);
1769 if (ret)
1770 return ret;
1771 ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring);
1772 if (ret)
1773 return ret;
1774 ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring);
1775 if (ret)
1776 return ret;
1777 ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring);
1778 if (ret)
1779 return ret;
1780 ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->acl_d2h_ring);
1781 if (ret)
1782 return ret;
1783
1784 ret = bcm4377_alloc_completion_ring(bcm4377,
1785 &bcm4377->control_ack_ring);
1786 if (ret)
1787 return ret;
1788 ret = bcm4377_alloc_completion_ring(bcm4377,
1789 &bcm4377->hci_acl_ack_ring);
1790 if (ret)
1791 return ret;
1792 ret = bcm4377_alloc_completion_ring(bcm4377,
1793 &bcm4377->hci_acl_event_ring);
1794 if (ret)
1795 return ret;
1796 ret = bcm4377_alloc_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
1797 if (ret)
1798 return ret;
1799 ret = bcm4377_alloc_completion_ring(bcm4377, &bcm4377->sco_event_ring);
1800 if (ret)
1801 return ret;
1802
1803 dev_dbg(&bcm4377->pdev->dev, "all rings allocated and prepared\n");
1804
1805 return 0;
1806}
1807
1808static int bcm4377_boot(struct bcm4377_data *bcm4377)
1809{
1810 const struct firmware *fw;
1811 void *bfr;
1812 dma_addr_t fw_dma;
1813 int ret = 0;
1814 u32 bootstage, rti_status;
1815
1816 bootstage = ioread32(bcm4377->bar2 + BCM4377_BAR2_BOOTSTAGE);
1817 rti_status = ioread32(bcm4377->bar2 + BCM4377_BAR2_RTI_STATUS);
1818
1819 if (bootstage != 0) {
1820 dev_err(&bcm4377->pdev->dev, "bootstage is %d and not 0\n",
1821 bootstage);
1822 return -EINVAL;
1823 }
1824
1825 if (rti_status != 0) {
1826 dev_err(&bcm4377->pdev->dev, "RTI status is %d and not 0\n",
1827 rti_status);
1828 return -EINVAL;
1829 }
1830
1831 fw = bcm4377_request_blob(bcm4377, "bin");
1832 if (!fw) {
1833 dev_err(&bcm4377->pdev->dev, "Failed to load firmware\n");
1834 return -ENOENT;
1835 }
1836
1837 bfr = dma_alloc_coherent(&bcm4377->pdev->dev, fw->size, &fw_dma,
1838 GFP_KERNEL);
1839 if (!bfr) {
1840 ret = -ENOMEM;
1841 goto out_release_fw;
1842 }
1843
1844 memcpy(bfr, fw->data, fw->size);
1845
1846 iowrite32(0, bcm4377->bar0 + BCM4377_BAR0_HOST_WINDOW_LO);
1847 iowrite32(0, bcm4377->bar0 + BCM4377_BAR0_HOST_WINDOW_HI);
1848 iowrite32(BCM4377_DMA_MASK,
1849 bcm4377->bar0 + BCM4377_BAR0_HOST_WINDOW_SIZE);
1850
1851 iowrite32(lower_32_bits(fw_dma), bcm4377->bar2 + BCM4377_BAR2_FW_LO);
1852 iowrite32(upper_32_bits(fw_dma), bcm4377->bar2 + BCM4377_BAR2_FW_HI);
1853 iowrite32(fw->size, bcm4377->bar2 + BCM4377_BAR2_FW_SIZE);
1854 iowrite32(0, bcm4377->bar0 + BCM4377_BAR0_FW_DOORBELL);
1855
1856 dev_dbg(&bcm4377->pdev->dev, "waiting for firmware to boot\n");
1857
1858 ret = wait_for_completion_interruptible_timeout(&bcm4377->event,
1859 BCM4377_TIMEOUT);
1860 if (ret == 0) {
1861 ret = -ETIMEDOUT;
1862 goto out_dma_free;
1863 } else if (ret < 0) {
1864 goto out_dma_free;
1865 }
1866
1867 if (bcm4377->bootstage != 2) {
1868 dev_err(&bcm4377->pdev->dev, "boostage %d != 2\n",
1869 bcm4377->bootstage);
1870 ret = -ENXIO;
1871 goto out_dma_free;
1872 }
1873
1874 dev_dbg(&bcm4377->pdev->dev, "firmware has booted (stage = %x)\n",
1875 bcm4377->bootstage);
1876 ret = 0;
1877
1878out_dma_free:
1879 dma_free_coherent(&bcm4377->pdev->dev, fw->size, bfr, fw_dma);
1880out_release_fw:
1881 release_firmware(fw);
1882 return ret;
1883}
1884
1885static int bcm4377_setup_rti(struct bcm4377_data *bcm4377)
1886{
1887 int ret;
1888
1889 dev_dbg(&bcm4377->pdev->dev, "starting RTI\n");
1890 iowrite32(1, bcm4377->bar0 + BCM4377_BAR0_RTI_CONTROL);
1891
1892 ret = wait_for_completion_interruptible_timeout(&bcm4377->event,
1893 BCM4377_TIMEOUT);
1894 if (ret == 0) {
1895 dev_err(&bcm4377->pdev->dev,
1896 "timed out while waiting for RTI to transition to state 1");
1897 return -ETIMEDOUT;
1898 } else if (ret < 0) {
1899 return ret;
1900 }
1901
1902 if (bcm4377->rti_status != 1) {
1903 dev_err(&bcm4377->pdev->dev, "RTI did not ack state 1 (%d)\n",
1904 bcm4377->rti_status);
1905 return -ENODEV;
1906 }
1907 dev_dbg(&bcm4377->pdev->dev, "RTI is in state 1\n");
1908
1909 /* allow access to the entire IOVA space again */
1910 iowrite32(0, bcm4377->bar2 + BCM4377_BAR2_RTI_WINDOW_LO);
1911 iowrite32(0, bcm4377->bar2 + BCM4377_BAR2_RTI_WINDOW_HI);
1912 iowrite32(BCM4377_DMA_MASK,
1913 bcm4377->bar2 + BCM4377_BAR2_RTI_WINDOW_SIZE);
1914
1915 /* setup "Converged IPC" context */
1916 iowrite32(lower_32_bits(bcm4377->ctx_dma),
1917 bcm4377->bar2 + BCM4377_BAR2_CONTEXT_ADDR_LO);
1918 iowrite32(upper_32_bits(bcm4377->ctx_dma),
1919 bcm4377->bar2 + BCM4377_BAR2_CONTEXT_ADDR_HI);
1920 iowrite32(2, bcm4377->bar0 + BCM4377_BAR0_RTI_CONTROL);
1921
1922 ret = wait_for_completion_interruptible_timeout(&bcm4377->event,
1923 BCM4377_TIMEOUT);
1924 if (ret == 0) {
1925 dev_err(&bcm4377->pdev->dev,
1926 "timed out while waiting for RTI to transition to state 2");
1927 return -ETIMEDOUT;
1928 } else if (ret < 0) {
1929 return ret;
1930 }
1931
1932 if (bcm4377->rti_status != 2) {
1933 dev_err(&bcm4377->pdev->dev, "RTI did not ack state 2 (%d)\n",
1934 bcm4377->rti_status);
1935 return -ENODEV;
1936 }
1937
1938 dev_dbg(&bcm4377->pdev->dev,
1939 "RTI is in state 2; control ring is ready\n");
1940 bcm4377->control_ack_ring.enabled = true;
1941
1942 return 0;
1943}
1944
1945static int bcm4377_parse_otp_board_params(struct bcm4377_data *bcm4377,
1946 char tag, const char *val, size_t len)
1947{
1948 if (tag != 'V')
1949 return 0;
1950 if (len >= sizeof(bcm4377->vendor))
1951 return -EINVAL;
1952
1953 strscpy(bcm4377->vendor, val, len + 1);
1954 return 0;
1955}
1956
1957static int bcm4377_parse_otp_chip_params(struct bcm4377_data *bcm4377, char tag,
1958 const char *val, size_t len)
1959{
1960 size_t idx = 0;
1961
1962 if (tag != 's')
1963 return 0;
1964 if (len >= sizeof(bcm4377->stepping))
1965 return -EINVAL;
1966
1967 while (len != 0) {
1968 bcm4377->stepping[idx] = tolower(val[idx]);
1969 if (val[idx] == '\0')
1970 return 0;
1971
1972 idx++;
1973 len--;
1974 }
1975
1976 bcm4377->stepping[idx] = '\0';
1977 return 0;
1978}
1979
1980static int bcm4377_parse_otp_str(struct bcm4377_data *bcm4377, const u8 *str,
1981 enum bcm4377_otp_params_type type)
1982{
1983 const char *p;
1984 int ret;
1985
1986 p = skip_spaces(str);
1987 while (*p) {
1988 char tag = *p++;
1989 const char *end;
1990 size_t len;
1991
1992 if (*p++ != '=') /* implicit NUL check */
1993 return -EINVAL;
1994
1995 /* *p might be NUL here, if so end == p and len == 0 */
1996 end = strchrnul(p, ' ');
1997 len = end - p;
1998
1999 /* leave 1 byte for NUL in destination string */
2000 if (len > (BCM4377_OTP_MAX_PARAM_LEN - 1))
2001 return -EINVAL;
2002
2003 switch (type) {
2004 case BCM4377_OTP_BOARD_PARAMS:
2005 ret = bcm4377_parse_otp_board_params(bcm4377, tag, p,
2006 len);
2007 break;
2008 case BCM4377_OTP_CHIP_PARAMS:
2009 ret = bcm4377_parse_otp_chip_params(bcm4377, tag, p,
2010 len);
2011 break;
2012 default:
2013 ret = -EINVAL;
2014 break;
2015 }
2016
2017 if (ret)
2018 return ret;
2019
2020 /* Skip to next arg, if any */
2021 p = skip_spaces(end);
2022 }
2023
2024 return 0;
2025}
2026
2027static int bcm4377_parse_otp_sys_vendor(struct bcm4377_data *bcm4377, u8 *otp,
2028 size_t size)
2029{
2030 int idx = 4;
2031 const char *chip_params;
2032 const char *board_params;
2033 int ret;
2034
2035 /* 4-byte header and two empty strings */
2036 if (size < 6)
2037 return -EINVAL;
2038
2039 if (get_unaligned_le32(otp) != BCM4377_OTP_VENDOR_HDR)
2040 return -EINVAL;
2041
2042 chip_params = &otp[idx];
2043
2044 /* Skip first string, including terminator */
2045 idx += strnlen(chip_params, size - idx) + 1;
2046 if (idx >= size)
2047 return -EINVAL;
2048
2049 board_params = &otp[idx];
2050
2051 /* Skip to terminator of second string */
2052 idx += strnlen(board_params, size - idx);
2053 if (idx >= size)
2054 return -EINVAL;
2055
2056 /* At this point both strings are guaranteed NUL-terminated */
2057 dev_dbg(&bcm4377->pdev->dev,
2058 "OTP: chip_params='%s' board_params='%s'\n", chip_params,
2059 board_params);
2060
2061 ret = bcm4377_parse_otp_str(bcm4377, chip_params,
2062 BCM4377_OTP_CHIP_PARAMS);
2063 if (ret)
2064 return ret;
2065
2066 ret = bcm4377_parse_otp_str(bcm4377, board_params,
2067 BCM4377_OTP_BOARD_PARAMS);
2068 if (ret)
2069 return ret;
2070
2071 if (!bcm4377->stepping[0] || !bcm4377->vendor[0])
2072 return -EINVAL;
2073
2074 dev_dbg(&bcm4377->pdev->dev, "OTP: stepping=%s, vendor=%s\n",
2075 bcm4377->stepping, bcm4377->vendor);
2076 return 0;
2077}
2078
2079static int bcm4377_parse_otp(struct bcm4377_data *bcm4377)
2080{
2081 u8 *otp;
2082 int i;
2083 int ret = -ENOENT;
2084
2085 otp = kzalloc(BCM4377_OTP_SIZE, GFP_KERNEL);
2086 if (!otp)
2087 return -ENOMEM;
2088
2089 for (i = 0; i < BCM4377_OTP_SIZE; ++i)
2090 otp[i] = ioread8(bcm4377->bar0 + bcm4377->hw->otp_offset + i);
2091
2092 i = 0;
2093 while (i < (BCM4377_OTP_SIZE - 1)) {
2094 u8 type = otp[i];
2095 u8 length = otp[i + 1];
2096
2097 if (type == 0)
2098 break;
2099
2100 if ((i + 2 + length) > BCM4377_OTP_SIZE)
2101 break;
2102
2103 switch (type) {
2104 case BCM4377_OTP_SYS_VENDOR:
2105 dev_dbg(&bcm4377->pdev->dev,
2106 "OTP @ 0x%x (%d): SYS_VENDOR", i, length);
2107 ret = bcm4377_parse_otp_sys_vendor(bcm4377, &otp[i + 2],
2108 length);
2109 break;
2110 case BCM4377_OTP_CIS:
2111 dev_dbg(&bcm4377->pdev->dev, "OTP @ 0x%x (%d): CIS", i,
2112 length);
2113 break;
2114 default:
2115 dev_dbg(&bcm4377->pdev->dev, "OTP @ 0x%x (%d): unknown",
2116 i, length);
2117 break;
2118 }
2119
2120 i += 2 + length;
2121 }
2122
2123 kfree(otp);
2124 return ret;
2125}
2126
2127static int bcm4377_init_cfg(struct bcm4377_data *bcm4377)
2128{
2129 int ret;
2130 u32 ctrl;
2131
2132 ret = pci_write_config_dword(bcm4377->pdev,
2133 BCM4377_PCIECFG_BAR0_WINDOW1,
2134 bcm4377->hw->bar0_window1);
2135 if (ret)
2136 return ret;
2137
2138 ret = pci_write_config_dword(bcm4377->pdev,
2139 BCM4377_PCIECFG_BAR0_WINDOW2,
2140 bcm4377->hw->bar0_window2);
2141 if (ret)
2142 return ret;
2143
2144 ret = pci_write_config_dword(
2145 bcm4377->pdev, BCM4377_PCIECFG_BAR0_CORE2_WINDOW1,
2146 BCM4377_PCIECFG_BAR0_CORE2_WINDOW1_DEFAULT);
2147 if (ret)
2148 return ret;
2149
2150 if (bcm4377->hw->has_bar0_core2_window2) {
2151 ret = pci_write_config_dword(bcm4377->pdev,
2152 BCM4377_PCIECFG_BAR0_CORE2_WINDOW2,
2153 bcm4377->hw->bar0_core2_window2);
2154 if (ret)
2155 return ret;
2156 }
2157
2158 ret = pci_write_config_dword(bcm4377->pdev, BCM4377_PCIECFG_BAR2_WINDOW,
2159 BCM4377_PCIECFG_BAR2_WINDOW_DEFAULT);
2160 if (ret)
2161 return ret;
2162
2163 ret = pci_read_config_dword(bcm4377->pdev,
2164 BCM4377_PCIECFG_SUBSYSTEM_CTRL, &ctrl);
2165 if (ret)
2166 return ret;
2167
2168 if (bcm4377->hw->clear_pciecfg_subsystem_ctrl_bit19)
2169 ctrl &= ~BIT(19);
2170 ctrl |= BIT(16);
2171
2172 return pci_write_config_dword(bcm4377->pdev,
2173 BCM4377_PCIECFG_SUBSYSTEM_CTRL, ctrl);
2174}
2175
2176static int bcm4377_probe_dmi(struct bcm4377_data *bcm4377)
2177{
2178 const struct dmi_system_id *board_type_dmi_id;
2179
2180 board_type_dmi_id = dmi_first_match(bcm4377_dmi_board_table);
2181 if (board_type_dmi_id && board_type_dmi_id->driver_data) {
2182 bcm4377->board_type = board_type_dmi_id->driver_data;
2183 dev_dbg(&bcm4377->pdev->dev,
2184 "found board type via DMI match: %s\n",
2185 bcm4377->board_type);
2186 }
2187
2188 return 0;
2189}
2190
2191static int bcm4377_probe_of(struct bcm4377_data *bcm4377)
2192{
2193 struct device_node *np = bcm4377->pdev->dev.of_node;
2194 int ret;
2195
2196 if (!np)
2197 return 0;
2198
2199 ret = of_property_read_string(np, "brcm,board-type",
2200 &bcm4377->board_type);
2201 if (ret) {
2202 dev_err(&bcm4377->pdev->dev, "no brcm,board-type property\n");
2203 return ret;
2204 }
2205
2206 bcm4377->taurus_beamforming_cal_blob =
2207 of_get_property(np, "brcm,taurus-bf-cal-blob",
2208 &bcm4377->taurus_beamforming_cal_size);
2209 if (!bcm4377->taurus_beamforming_cal_blob) {
2210 dev_err(&bcm4377->pdev->dev,
2211 "no brcm,taurus-bf-cal-blob property\n");
2212 return -ENOENT;
2213 }
2214 bcm4377->taurus_cal_blob = of_get_property(np, "brcm,taurus-cal-blob",
2215 &bcm4377->taurus_cal_size);
2216 if (!bcm4377->taurus_cal_blob) {
2217 dev_err(&bcm4377->pdev->dev,
2218 "no brcm,taurus-cal-blob property\n");
2219 return -ENOENT;
2220 }
2221
2222 return 0;
2223}
2224
2225static void bcm4377_disable_aspm(struct bcm4377_data *bcm4377)
2226{
2227 pci_disable_link_state(bcm4377->pdev,
2228 PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1);
2229
2230 /*
2231 * pci_disable_link_state can fail if either CONFIG_PCIEASPM is disabled
2232 * or if the BIOS hasn't handed over control to us. We must *always*
2233 * disable ASPM for this device due to hardware errata though.
2234 */
2235 pcie_capability_clear_word(bcm4377->pdev, PCI_EXP_LNKCTL,
2236 PCI_EXP_LNKCTL_ASPMC);
2237}
2238
2239static void bcm4377_pci_free_irq_vectors(void *data)
2240{
2241 pci_free_irq_vectors(data);
2242}
2243
2244static void bcm4377_hci_free_dev(void *data)
2245{
2246 hci_free_dev(data);
2247}
2248
2249static void bcm4377_hci_unregister_dev(void *data)
2250{
2251 hci_unregister_dev(data);
2252}
2253
2254static int bcm4377_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2255{
2256 struct bcm4377_data *bcm4377;
2257 struct hci_dev *hdev;
2258 int ret, irq;
2259
2260 ret = dma_set_mask_and_coherent(&pdev->dev, BCM4377_DMA_MASK);
2261 if (ret)
2262 return ret;
2263
2264 bcm4377 = devm_kzalloc(&pdev->dev, sizeof(*bcm4377), GFP_KERNEL);
2265 if (!bcm4377)
2266 return -ENOMEM;
2267
2268 bcm4377->pdev = pdev;
2269 bcm4377->hw = &bcm4377_hw_variants[id->driver_data];
2270 init_completion(&bcm4377->event);
2271
2272 ret = bcm4377_prepare_rings(bcm4377);
2273 if (ret)
2274 return ret;
2275
2276 ret = bcm4377_init_context(bcm4377);
2277 if (ret)
2278 return ret;
2279
2280 ret = bcm4377_probe_dmi(bcm4377);
2281 if (ret)
2282 return ret;
2283 ret = bcm4377_probe_of(bcm4377);
2284 if (ret)
2285 return ret;
2286 if (!bcm4377->board_type) {
2287 dev_err(&pdev->dev, "unable to determine board type\n");
2288 return -ENODEV;
2289 }
2290
2291 if (bcm4377->hw->disable_aspm)
2292 bcm4377_disable_aspm(bcm4377);
2293
2294 ret = pci_reset_function_locked(pdev);
2295 if (ret)
2296 dev_warn(
2297 &pdev->dev,
2298 "function level reset failed with %d; trying to continue anyway\n",
2299 ret);
2300
2301 /*
2302 * If this number is too low and we try to access any BAR too
2303 * early the device will crash. Experiments have shown that
2304 * approximately 50 msec is the minimum amount we have to wait.
2305 * Let's double that to be safe.
2306 */
2307 msleep(100);
2308
2309 ret = pcim_enable_device(pdev);
2310 if (ret)
2311 return ret;
2312 pci_set_master(pdev);
2313
2314 ret = bcm4377_init_cfg(bcm4377);
2315 if (ret)
2316 return ret;
2317
2318 bcm4377->bar0 = pcim_iomap(pdev, 0, 0);
2319 if (!bcm4377->bar0)
2320 return -EBUSY;
2321 bcm4377->bar2 = pcim_iomap(pdev, 2, 0);
2322 if (!bcm4377->bar2)
2323 return -EBUSY;
2324
2325 ret = bcm4377_parse_otp(bcm4377);
2326 if (ret) {
2327 dev_err(&pdev->dev, "Reading OTP failed with %d\n", ret);
2328 return ret;
2329 }
2330
2331 /*
2332 * Legacy interrupts result in an IRQ storm because we don't know where
2333 * the interrupt mask and status registers for these chips are.
2334 * MSIs are acked automatically instead.
2335 */
2336 ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
2337 if (ret < 0)
2338 return -ENODEV;
2339 ret = devm_add_action_or_reset(&pdev->dev, bcm4377_pci_free_irq_vectors,
2340 pdev);
2341 if (ret)
2342 return ret;
2343
2344 irq = pci_irq_vector(pdev, 0);
2345 if (irq <= 0)
2346 return -ENODEV;
2347
2348 ret = devm_request_irq(&pdev->dev, irq, bcm4377_irq, 0, "bcm4377",
2349 bcm4377);
2350 if (ret)
2351 return ret;
2352
2353 hdev = hci_alloc_dev();
2354 if (!hdev)
2355 return -ENOMEM;
2356 ret = devm_add_action_or_reset(&pdev->dev, bcm4377_hci_free_dev, hdev);
2357 if (ret)
2358 return ret;
2359
2360 bcm4377->hdev = hdev;
2361
2362 hdev->bus = HCI_PCI;
2363 hdev->dev_type = HCI_PRIMARY;
2364 hdev->open = bcm4377_hci_open;
2365 hdev->close = bcm4377_hci_close;
2366 hdev->send = bcm4377_hci_send_frame;
2367 hdev->set_bdaddr = bcm4377_hci_set_bdaddr;
2368 hdev->setup = bcm4377_hci_setup;
2369
2370 set_bit(HCI_QUIRK_USE_BDADDR_PROPERTY, &hdev->quirks);
2371 if (bcm4377->hw->broken_mws_transport_config)
2372 set_bit(HCI_QUIRK_BROKEN_MWS_TRANSPORT_CONFIG, &hdev->quirks);
2373 if (bcm4377->hw->broken_ext_scan)
2374 set_bit(HCI_QUIRK_BROKEN_EXT_SCAN, &hdev->quirks);
2375
2376 pci_set_drvdata(pdev, bcm4377);
2377 hci_set_drvdata(hdev, bcm4377);
2378 SET_HCIDEV_DEV(hdev, &pdev->dev);
2379
2380 ret = bcm4377_boot(bcm4377);
2381 if (ret)
2382 return ret;
2383
2384 ret = bcm4377_setup_rti(bcm4377);
2385 if (ret)
2386 return ret;
2387
2388 ret = hci_register_dev(hdev);
2389 if (ret)
2390 return ret;
2391 return devm_add_action_or_reset(&pdev->dev, bcm4377_hci_unregister_dev,
2392 hdev);
2393}
2394
2395static int bcm4377_suspend(struct pci_dev *pdev, pm_message_t state)
2396{
2397 struct bcm4377_data *bcm4377 = pci_get_drvdata(pdev);
2398 int ret;
2399
2400 ret = hci_suspend_dev(bcm4377->hdev);
2401 if (ret)
2402 return ret;
2403
2404 iowrite32(BCM4377_BAR0_SLEEP_CONTROL_QUIESCE,
2405 bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL);
2406
2407 return 0;
2408}
2409
2410static int bcm4377_resume(struct pci_dev *pdev)
2411{
2412 struct bcm4377_data *bcm4377 = pci_get_drvdata(pdev);
2413
2414 iowrite32(BCM4377_BAR0_SLEEP_CONTROL_UNQUIESCE,
2415 bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL);
2416
2417 return hci_resume_dev(bcm4377->hdev);
2418}
2419
2420static const struct dmi_system_id bcm4377_dmi_board_table[] = {
2421 {
2422 .matches = {
2423 DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
2424 DMI_MATCH(DMI_PRODUCT_NAME, "MacBookAir9,1"),
2425 },
2426 .driver_data = "apple,formosa",
2427 },
2428 {
2429 .matches = {
2430 DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
2431 DMI_MATCH(DMI_PRODUCT_NAME, "MacBookPro15,4"),
2432 },
2433 .driver_data = "apple,formosa",
2434 },
2435 {
2436 .matches = {
2437 DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
2438 DMI_MATCH(DMI_PRODUCT_NAME, "MacBookPro16,3"),
2439 },
2440 .driver_data = "apple,formosa",
2441 },
2442 {}
2443};
2444
2445static const struct bcm4377_hw bcm4377_hw_variants[] = {
2446 [BCM4377] = {
2447 .id = 0x4377,
2448 .otp_offset = 0x4120,
2449 .bar0_window1 = 0x1800b000,
2450 .bar0_window2 = 0x1810c000,
2451 .disable_aspm = true,
2452 .broken_ext_scan = true,
2453 .send_ptb = bcm4377_send_ptb,
2454 },
2455
2456 [BCM4378] = {
2457 .id = 0x4378,
2458 .otp_offset = 0x4120,
2459 .bar0_window1 = 0x18002000,
2460 .bar0_window2 = 0x1810a000,
2461 .bar0_core2_window2 = 0x18107000,
2462 .has_bar0_core2_window2 = true,
2463 .broken_mws_transport_config = true,
2464 .send_calibration = bcm4378_send_calibration,
2465 .send_ptb = bcm4378_send_ptb,
2466 },
2467
2468 [BCM4387] = {
2469 .id = 0x4387,
2470 .otp_offset = 0x413c,
2471 .bar0_window1 = 0x18002000,
2472 .bar0_window2 = 0x18109000,
2473 .bar0_core2_window2 = 0x18106000,
2474 .has_bar0_core2_window2 = true,
2475 .clear_pciecfg_subsystem_ctrl_bit19 = true,
2476 .broken_mws_transport_config = true,
2477 .send_calibration = bcm4387_send_calibration,
2478 .send_ptb = bcm4378_send_ptb,
2479 },
2480};
2481
2482#define BCM4377_DEVID_ENTRY(id) \
2483 { \
2484 PCI_VENDOR_ID_BROADCOM, BCM##id##_DEVICE_ID, PCI_ANY_ID, \
2485 PCI_ANY_ID, PCI_CLASS_NETWORK_OTHER << 8, 0xffff00, \
2486 BCM##id \
2487 }
2488
2489static const struct pci_device_id bcm4377_devid_table[] = {
2490 BCM4377_DEVID_ENTRY(4377),
2491 BCM4377_DEVID_ENTRY(4378),
2492 BCM4377_DEVID_ENTRY(4387),
2493 {},
2494};
2495MODULE_DEVICE_TABLE(pci, bcm4377_devid_table);
2496
2497static struct pci_driver bcm4377_pci_driver = {
2498 .name = "hci_bcm4377",
2499 .id_table = bcm4377_devid_table,
2500 .probe = bcm4377_probe,
2501 .suspend = bcm4377_suspend,
2502 .resume = bcm4377_resume,
2503};
2504module_pci_driver(bcm4377_pci_driver);
2505
2506MODULE_AUTHOR("Sven Peter <sven@svenpeter.dev>");
2507MODULE_DESCRIPTION("Bluetooth support for Broadcom 4377/4378/4387 devices");
2508MODULE_LICENSE("Dual MIT/GPL");
2509MODULE_FIRMWARE("brcm/brcmbt4377*.bin");
2510MODULE_FIRMWARE("brcm/brcmbt4377*.ptb");
2511MODULE_FIRMWARE("brcm/brcmbt4378*.bin");
2512MODULE_FIRMWARE("brcm/brcmbt4378*.ptb");
2513MODULE_FIRMWARE("brcm/brcmbt4387*.bin");
2514MODULE_FIRMWARE("brcm/brcmbt4387*.ptb");