Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Internal Thunderbolt Connection Manager. This is a firmware running on
4 * the Thunderbolt host controller performing most of the low-level
5 * handling.
6 *
7 * Copyright (C) 2017, Intel Corporation
8 * Authors: Michael Jamet <michael.jamet@intel.com>
9 * Mika Westerberg <mika.westerberg@linux.intel.com>
10 */
11
12#include <linux/delay.h>
13#include <linux/mutex.h>
14#include <linux/moduleparam.h>
15#include <linux/pci.h>
16#include <linux/pm_runtime.h>
17#include <linux/platform_data/x86/apple.h>
18#include <linux/sizes.h>
19#include <linux/slab.h>
20#include <linux/workqueue.h>
21
22#include "ctl.h"
23#include "nhi_regs.h"
24#include "tb.h"
25
26#define PCIE2CIO_CMD 0x30
27#define PCIE2CIO_CMD_TIMEOUT BIT(31)
28#define PCIE2CIO_CMD_START BIT(30)
29#define PCIE2CIO_CMD_WRITE BIT(21)
30#define PCIE2CIO_CMD_CS_MASK GENMASK(20, 19)
31#define PCIE2CIO_CMD_CS_SHIFT 19
32#define PCIE2CIO_CMD_PORT_MASK GENMASK(18, 13)
33#define PCIE2CIO_CMD_PORT_SHIFT 13
34
35#define PCIE2CIO_WRDATA 0x34
36#define PCIE2CIO_RDDATA 0x38
37
38#define PHY_PORT_CS1 0x37
39#define PHY_PORT_CS1_LINK_DISABLE BIT(14)
40#define PHY_PORT_CS1_LINK_STATE_MASK GENMASK(29, 26)
41#define PHY_PORT_CS1_LINK_STATE_SHIFT 26
42
43#define ICM_TIMEOUT 5000 /* ms */
44#define ICM_RETRIES 3
45#define ICM_APPROVE_TIMEOUT 10000 /* ms */
46#define ICM_MAX_LINK 4
47
48static bool start_icm;
49module_param(start_icm, bool, 0444);
50MODULE_PARM_DESC(start_icm, "start ICM firmware if it is not running (default: false)");
51
52/**
53 * struct usb4_switch_nvm_auth - Holds USB4 NVM_AUTH status
54 * @reply: Reply from ICM firmware is placed here
55 * @request: Request that is sent to ICM firmware
56 * @icm: Pointer to ICM private data
57 */
58struct usb4_switch_nvm_auth {
59 struct icm_usb4_switch_op_response reply;
60 struct icm_usb4_switch_op request;
61 struct icm *icm;
62};
63
64/**
65 * struct icm - Internal connection manager private data
66 * @request_lock: Makes sure only one message is send to ICM at time
67 * @rescan_work: Work used to rescan the surviving switches after resume
68 * @upstream_port: Pointer to the PCIe upstream port this host
69 * controller is connected. This is only set for systems
70 * where ICM needs to be started manually
71 * @vnd_cap: Vendor defined capability where PCIe2CIO mailbox resides
72 * (only set when @upstream_port is not %NULL)
73 * @safe_mode: ICM is in safe mode
74 * @max_boot_acl: Maximum number of preboot ACL entries (%0 if not supported)
75 * @rpm: Does the controller support runtime PM (RTD3)
76 * @can_upgrade_nvm: Can the NVM firmware be upgrade on this controller
77 * @proto_version: Firmware protocol version
78 * @last_nvm_auth: Last USB4 router NVM_AUTH result (or %NULL if not set)
79 * @veto: Is RTD3 veto in effect
80 * @is_supported: Checks if we can support ICM on this controller
81 * @cio_reset: Trigger CIO reset
82 * @get_mode: Read and return the ICM firmware mode (optional)
83 * @get_route: Find a route string for given switch
84 * @save_devices: Ask ICM to save devices to ACL when suspending (optional)
85 * @driver_ready: Send driver ready message to ICM
86 * @set_uuid: Set UUID for the root switch (optional)
87 * @device_connected: Handle device connected ICM message
88 * @device_disconnected: Handle device disconnected ICM message
89 * @xdomain_connected: Handle XDomain connected ICM message
90 * @xdomain_disconnected: Handle XDomain disconnected ICM message
91 * @rtd3_veto: Handle RTD3 veto notification ICM message
92 */
93struct icm {
94 struct mutex request_lock;
95 struct delayed_work rescan_work;
96 struct pci_dev *upstream_port;
97 int vnd_cap;
98 bool safe_mode;
99 size_t max_boot_acl;
100 bool rpm;
101 bool can_upgrade_nvm;
102 u8 proto_version;
103 struct usb4_switch_nvm_auth *last_nvm_auth;
104 bool veto;
105 bool (*is_supported)(struct tb *tb);
106 int (*cio_reset)(struct tb *tb);
107 int (*get_mode)(struct tb *tb);
108 int (*get_route)(struct tb *tb, u8 link, u8 depth, u64 *route);
109 void (*save_devices)(struct tb *tb);
110 int (*driver_ready)(struct tb *tb,
111 enum tb_security_level *security_level,
112 u8 *proto_version, size_t *nboot_acl, bool *rpm);
113 void (*set_uuid)(struct tb *tb);
114 void (*device_connected)(struct tb *tb,
115 const struct icm_pkg_header *hdr);
116 void (*device_disconnected)(struct tb *tb,
117 const struct icm_pkg_header *hdr);
118 void (*xdomain_connected)(struct tb *tb,
119 const struct icm_pkg_header *hdr);
120 void (*xdomain_disconnected)(struct tb *tb,
121 const struct icm_pkg_header *hdr);
122 void (*rtd3_veto)(struct tb *tb, const struct icm_pkg_header *hdr);
123};
124
125struct icm_notification {
126 struct work_struct work;
127 struct icm_pkg_header *pkg;
128 struct tb *tb;
129};
130
131struct ep_name_entry {
132 u8 len;
133 u8 type;
134 u8 data[];
135};
136
137#define EP_NAME_INTEL_VSS 0x10
138
139/* Intel Vendor specific structure */
140struct intel_vss {
141 u16 vendor;
142 u16 model;
143 u8 mc;
144 u8 flags;
145 u16 pci_devid;
146 u32 nvm_version;
147};
148
149#define INTEL_VSS_FLAGS_RTD3 BIT(0)
150
151static const struct intel_vss *parse_intel_vss(const void *ep_name, size_t size)
152{
153 const void *end = ep_name + size;
154
155 while (ep_name < end) {
156 const struct ep_name_entry *ep = ep_name;
157
158 if (!ep->len)
159 break;
160 if (ep_name + ep->len > end)
161 break;
162
163 if (ep->type == EP_NAME_INTEL_VSS)
164 return (const struct intel_vss *)ep->data;
165
166 ep_name += ep->len;
167 }
168
169 return NULL;
170}
171
172static bool intel_vss_is_rtd3(const void *ep_name, size_t size)
173{
174 const struct intel_vss *vss;
175
176 vss = parse_intel_vss(ep_name, size);
177 if (vss)
178 return !!(vss->flags & INTEL_VSS_FLAGS_RTD3);
179
180 return false;
181}
182
183static inline struct tb *icm_to_tb(struct icm *icm)
184{
185 return ((void *)icm - sizeof(struct tb));
186}
187
188static inline u8 phy_port_from_route(u64 route, u8 depth)
189{
190 u8 link;
191
192 link = depth ? route >> ((depth - 1) * 8) : route;
193 return tb_phy_port_from_link(link);
194}
195
196static inline u8 dual_link_from_link(u8 link)
197{
198 return link ? ((link - 1) ^ 0x01) + 1 : 0;
199}
200
201static inline u64 get_route(u32 route_hi, u32 route_lo)
202{
203 return (u64)route_hi << 32 | route_lo;
204}
205
206static inline u64 get_parent_route(u64 route)
207{
208 int depth = tb_route_length(route);
209 return depth ? route & ~(0xffULL << (depth - 1) * TB_ROUTE_SHIFT) : 0;
210}
211
212static int pci2cio_wait_completion(struct icm *icm, unsigned long timeout_msec)
213{
214 unsigned long end = jiffies + msecs_to_jiffies(timeout_msec);
215 u32 cmd;
216
217 do {
218 pci_read_config_dword(icm->upstream_port,
219 icm->vnd_cap + PCIE2CIO_CMD, &cmd);
220 if (!(cmd & PCIE2CIO_CMD_START)) {
221 if (cmd & PCIE2CIO_CMD_TIMEOUT)
222 break;
223 return 0;
224 }
225
226 msleep(50);
227 } while (time_before(jiffies, end));
228
229 return -ETIMEDOUT;
230}
231
232static int pcie2cio_read(struct icm *icm, enum tb_cfg_space cs,
233 unsigned int port, unsigned int index, u32 *data)
234{
235 struct pci_dev *pdev = icm->upstream_port;
236 int ret, vnd_cap = icm->vnd_cap;
237 u32 cmd;
238
239 cmd = index;
240 cmd |= (port << PCIE2CIO_CMD_PORT_SHIFT) & PCIE2CIO_CMD_PORT_MASK;
241 cmd |= (cs << PCIE2CIO_CMD_CS_SHIFT) & PCIE2CIO_CMD_CS_MASK;
242 cmd |= PCIE2CIO_CMD_START;
243 pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_CMD, cmd);
244
245 ret = pci2cio_wait_completion(icm, 5000);
246 if (ret)
247 return ret;
248
249 pci_read_config_dword(pdev, vnd_cap + PCIE2CIO_RDDATA, data);
250 return 0;
251}
252
253static int pcie2cio_write(struct icm *icm, enum tb_cfg_space cs,
254 unsigned int port, unsigned int index, u32 data)
255{
256 struct pci_dev *pdev = icm->upstream_port;
257 int vnd_cap = icm->vnd_cap;
258 u32 cmd;
259
260 pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_WRDATA, data);
261
262 cmd = index;
263 cmd |= (port << PCIE2CIO_CMD_PORT_SHIFT) & PCIE2CIO_CMD_PORT_MASK;
264 cmd |= (cs << PCIE2CIO_CMD_CS_SHIFT) & PCIE2CIO_CMD_CS_MASK;
265 cmd |= PCIE2CIO_CMD_WRITE | PCIE2CIO_CMD_START;
266 pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_CMD, cmd);
267
268 return pci2cio_wait_completion(icm, 5000);
269}
270
271static bool icm_match(const struct tb_cfg_request *req,
272 const struct ctl_pkg *pkg)
273{
274 const struct icm_pkg_header *res_hdr = pkg->buffer;
275 const struct icm_pkg_header *req_hdr = req->request;
276
277 if (pkg->frame.eof != req->response_type)
278 return false;
279 if (res_hdr->code != req_hdr->code)
280 return false;
281
282 return true;
283}
284
285static bool icm_copy(struct tb_cfg_request *req, const struct ctl_pkg *pkg)
286{
287 const struct icm_pkg_header *hdr = pkg->buffer;
288
289 if (hdr->packet_id < req->npackets) {
290 size_t offset = hdr->packet_id * req->response_size;
291
292 memcpy(req->response + offset, pkg->buffer, req->response_size);
293 }
294
295 return hdr->packet_id == hdr->total_packets - 1;
296}
297
298static int icm_request(struct tb *tb, const void *request, size_t request_size,
299 void *response, size_t response_size, size_t npackets,
300 int retries, unsigned int timeout_msec)
301{
302 struct icm *icm = tb_priv(tb);
303
304 do {
305 struct tb_cfg_request *req;
306 struct tb_cfg_result res;
307
308 req = tb_cfg_request_alloc();
309 if (!req)
310 return -ENOMEM;
311
312 req->match = icm_match;
313 req->copy = icm_copy;
314 req->request = request;
315 req->request_size = request_size;
316 req->request_type = TB_CFG_PKG_ICM_CMD;
317 req->response = response;
318 req->npackets = npackets;
319 req->response_size = response_size;
320 req->response_type = TB_CFG_PKG_ICM_RESP;
321
322 mutex_lock(&icm->request_lock);
323 res = tb_cfg_request_sync(tb->ctl, req, timeout_msec);
324 mutex_unlock(&icm->request_lock);
325
326 tb_cfg_request_put(req);
327
328 if (res.err != -ETIMEDOUT)
329 return res.err == 1 ? -EIO : res.err;
330
331 usleep_range(20, 50);
332 } while (retries--);
333
334 return -ETIMEDOUT;
335}
336
337/*
338 * If rescan is queued to run (we are resuming), postpone it to give the
339 * firmware some more time to send device connected notifications for next
340 * devices in the chain.
341 */
342static void icm_postpone_rescan(struct tb *tb)
343{
344 struct icm *icm = tb_priv(tb);
345
346 if (delayed_work_pending(&icm->rescan_work))
347 mod_delayed_work(tb->wq, &icm->rescan_work,
348 msecs_to_jiffies(500));
349}
350
351static void icm_veto_begin(struct tb *tb)
352{
353 struct icm *icm = tb_priv(tb);
354
355 if (!icm->veto) {
356 icm->veto = true;
357 /* Keep the domain powered while veto is in effect */
358 pm_runtime_get(&tb->dev);
359 }
360}
361
362static void icm_veto_end(struct tb *tb)
363{
364 struct icm *icm = tb_priv(tb);
365
366 if (icm->veto) {
367 icm->veto = false;
368 /* Allow the domain suspend now */
369 pm_runtime_mark_last_busy(&tb->dev);
370 pm_runtime_put_autosuspend(&tb->dev);
371 }
372}
373
374static bool icm_firmware_running(const struct tb_nhi *nhi)
375{
376 u32 val;
377
378 val = ioread32(nhi->iobase + REG_FW_STS);
379 return !!(val & REG_FW_STS_ICM_EN);
380}
381
382static bool icm_fr_is_supported(struct tb *tb)
383{
384 return !x86_apple_machine;
385}
386
387static inline int icm_fr_get_switch_index(u32 port)
388{
389 int index;
390
391 if ((port & ICM_PORT_TYPE_MASK) != TB_TYPE_PORT)
392 return 0;
393
394 index = port >> ICM_PORT_INDEX_SHIFT;
395 return index != 0xff ? index : 0;
396}
397
398static int icm_fr_get_route(struct tb *tb, u8 link, u8 depth, u64 *route)
399{
400 struct icm_fr_pkg_get_topology_response *switches, *sw;
401 struct icm_fr_pkg_get_topology request = {
402 .hdr = { .code = ICM_GET_TOPOLOGY },
403 };
404 size_t npackets = ICM_GET_TOPOLOGY_PACKETS;
405 int ret, index;
406 u8 i;
407
408 switches = kcalloc(npackets, sizeof(*switches), GFP_KERNEL);
409 if (!switches)
410 return -ENOMEM;
411
412 ret = icm_request(tb, &request, sizeof(request), switches,
413 sizeof(*switches), npackets, ICM_RETRIES, ICM_TIMEOUT);
414 if (ret)
415 goto err_free;
416
417 sw = &switches[0];
418 index = icm_fr_get_switch_index(sw->ports[link]);
419 if (!index) {
420 ret = -ENODEV;
421 goto err_free;
422 }
423
424 sw = &switches[index];
425 for (i = 1; i < depth; i++) {
426 unsigned int j;
427
428 if (!(sw->first_data & ICM_SWITCH_USED)) {
429 ret = -ENODEV;
430 goto err_free;
431 }
432
433 for (j = 0; j < ARRAY_SIZE(sw->ports); j++) {
434 index = icm_fr_get_switch_index(sw->ports[j]);
435 if (index > sw->switch_index) {
436 sw = &switches[index];
437 break;
438 }
439 }
440 }
441
442 *route = get_route(sw->route_hi, sw->route_lo);
443
444err_free:
445 kfree(switches);
446 return ret;
447}
448
449static void icm_fr_save_devices(struct tb *tb)
450{
451 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_SAVE_DEVS, 0);
452}
453
454static int
455icm_fr_driver_ready(struct tb *tb, enum tb_security_level *security_level,
456 u8 *proto_version, size_t *nboot_acl, bool *rpm)
457{
458 struct icm_fr_pkg_driver_ready_response reply;
459 struct icm_pkg_driver_ready request = {
460 .hdr.code = ICM_DRIVER_READY,
461 };
462 int ret;
463
464 memset(&reply, 0, sizeof(reply));
465 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
466 1, ICM_RETRIES, ICM_TIMEOUT);
467 if (ret)
468 return ret;
469
470 if (security_level)
471 *security_level = reply.security_level & ICM_FR_SLEVEL_MASK;
472
473 return 0;
474}
475
476static int icm_fr_approve_switch(struct tb *tb, struct tb_switch *sw)
477{
478 struct icm_fr_pkg_approve_device request;
479 struct icm_fr_pkg_approve_device reply;
480 int ret;
481
482 memset(&request, 0, sizeof(request));
483 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
484 request.hdr.code = ICM_APPROVE_DEVICE;
485 request.connection_id = sw->connection_id;
486 request.connection_key = sw->connection_key;
487
488 memset(&reply, 0, sizeof(reply));
489 /* Use larger timeout as establishing tunnels can take some time */
490 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
491 1, ICM_RETRIES, ICM_APPROVE_TIMEOUT);
492 if (ret)
493 return ret;
494
495 if (reply.hdr.flags & ICM_FLAGS_ERROR) {
496 tb_warn(tb, "PCIe tunnel creation failed\n");
497 return -EIO;
498 }
499
500 return 0;
501}
502
503static int icm_fr_add_switch_key(struct tb *tb, struct tb_switch *sw)
504{
505 struct icm_fr_pkg_add_device_key request;
506 struct icm_fr_pkg_add_device_key_response reply;
507 int ret;
508
509 memset(&request, 0, sizeof(request));
510 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
511 request.hdr.code = ICM_ADD_DEVICE_KEY;
512 request.connection_id = sw->connection_id;
513 request.connection_key = sw->connection_key;
514 memcpy(request.key, sw->key, TB_SWITCH_KEY_SIZE);
515
516 memset(&reply, 0, sizeof(reply));
517 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
518 1, ICM_RETRIES, ICM_TIMEOUT);
519 if (ret)
520 return ret;
521
522 if (reply.hdr.flags & ICM_FLAGS_ERROR) {
523 tb_warn(tb, "Adding key to switch failed\n");
524 return -EIO;
525 }
526
527 return 0;
528}
529
530static int icm_fr_challenge_switch_key(struct tb *tb, struct tb_switch *sw,
531 const u8 *challenge, u8 *response)
532{
533 struct icm_fr_pkg_challenge_device request;
534 struct icm_fr_pkg_challenge_device_response reply;
535 int ret;
536
537 memset(&request, 0, sizeof(request));
538 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
539 request.hdr.code = ICM_CHALLENGE_DEVICE;
540 request.connection_id = sw->connection_id;
541 request.connection_key = sw->connection_key;
542 memcpy(request.challenge, challenge, TB_SWITCH_KEY_SIZE);
543
544 memset(&reply, 0, sizeof(reply));
545 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
546 1, ICM_RETRIES, ICM_TIMEOUT);
547 if (ret)
548 return ret;
549
550 if (reply.hdr.flags & ICM_FLAGS_ERROR)
551 return -EKEYREJECTED;
552 if (reply.hdr.flags & ICM_FLAGS_NO_KEY)
553 return -ENOKEY;
554
555 memcpy(response, reply.response, TB_SWITCH_KEY_SIZE);
556
557 return 0;
558}
559
560static int icm_fr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
561 int transmit_path, int transmit_ring,
562 int receive_path, int receive_ring)
563{
564 struct icm_fr_pkg_approve_xdomain_response reply;
565 struct icm_fr_pkg_approve_xdomain request;
566 int ret;
567
568 memset(&request, 0, sizeof(request));
569 request.hdr.code = ICM_APPROVE_XDOMAIN;
570 request.link_info = xd->depth << ICM_LINK_INFO_DEPTH_SHIFT | xd->link;
571 memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
572
573 request.transmit_path = transmit_path;
574 request.transmit_ring = transmit_ring;
575 request.receive_path = receive_path;
576 request.receive_ring = receive_ring;
577
578 memset(&reply, 0, sizeof(reply));
579 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
580 1, ICM_RETRIES, ICM_TIMEOUT);
581 if (ret)
582 return ret;
583
584 if (reply.hdr.flags & ICM_FLAGS_ERROR)
585 return -EIO;
586
587 return 0;
588}
589
590static int icm_fr_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
591 int transmit_path, int transmit_ring,
592 int receive_path, int receive_ring)
593{
594 u8 phy_port;
595 u8 cmd;
596
597 phy_port = tb_phy_port_from_link(xd->link);
598 if (phy_port == 0)
599 cmd = NHI_MAILBOX_DISCONNECT_PA;
600 else
601 cmd = NHI_MAILBOX_DISCONNECT_PB;
602
603 nhi_mailbox_cmd(tb->nhi, cmd, 1);
604 usleep_range(10, 50);
605 nhi_mailbox_cmd(tb->nhi, cmd, 2);
606 return 0;
607}
608
609static struct tb_switch *alloc_switch(struct tb_switch *parent_sw, u64 route,
610 const uuid_t *uuid)
611{
612 struct tb *tb = parent_sw->tb;
613 struct tb_switch *sw;
614
615 sw = tb_switch_alloc(tb, &parent_sw->dev, route);
616 if (IS_ERR(sw)) {
617 tb_warn(tb, "failed to allocate switch at %llx\n", route);
618 return sw;
619 }
620
621 sw->uuid = kmemdup(uuid, sizeof(*uuid), GFP_KERNEL);
622 if (!sw->uuid) {
623 tb_switch_put(sw);
624 return ERR_PTR(-ENOMEM);
625 }
626
627 init_completion(&sw->rpm_complete);
628 return sw;
629}
630
631static int add_switch(struct tb_switch *parent_sw, struct tb_switch *sw)
632{
633 u64 route = tb_route(sw);
634 int ret;
635
636 /* Link the two switches now */
637 tb_port_at(route, parent_sw)->remote = tb_upstream_port(sw);
638 tb_upstream_port(sw)->remote = tb_port_at(route, parent_sw);
639
640 ret = tb_switch_add(sw);
641 if (ret)
642 tb_port_at(tb_route(sw), parent_sw)->remote = NULL;
643
644 return ret;
645}
646
647static void update_switch(struct tb_switch *sw, u64 route, u8 connection_id,
648 u8 connection_key, u8 link, u8 depth, bool boot)
649{
650 struct tb_switch *parent_sw = tb_switch_parent(sw);
651
652 /* Disconnect from parent */
653 tb_switch_downstream_port(sw)->remote = NULL;
654 /* Re-connect via updated port */
655 tb_port_at(route, parent_sw)->remote = tb_upstream_port(sw);
656
657 /* Update with the new addressing information */
658 sw->config.route_hi = upper_32_bits(route);
659 sw->config.route_lo = lower_32_bits(route);
660 sw->connection_id = connection_id;
661 sw->connection_key = connection_key;
662 sw->link = link;
663 sw->depth = depth;
664 sw->boot = boot;
665
666 /* This switch still exists */
667 sw->is_unplugged = false;
668
669 /* Runtime resume is now complete */
670 complete(&sw->rpm_complete);
671}
672
673static void remove_switch(struct tb_switch *sw)
674{
675 tb_switch_downstream_port(sw)->remote = NULL;
676 tb_switch_remove(sw);
677}
678
679static void add_xdomain(struct tb_switch *sw, u64 route,
680 const uuid_t *local_uuid, const uuid_t *remote_uuid,
681 u8 link, u8 depth)
682{
683 struct tb_xdomain *xd;
684
685 pm_runtime_get_sync(&sw->dev);
686
687 xd = tb_xdomain_alloc(sw->tb, &sw->dev, route, local_uuid, remote_uuid);
688 if (!xd)
689 goto out;
690
691 xd->link = link;
692 xd->depth = depth;
693
694 tb_port_at(route, sw)->xdomain = xd;
695
696 tb_xdomain_add(xd);
697
698out:
699 pm_runtime_mark_last_busy(&sw->dev);
700 pm_runtime_put_autosuspend(&sw->dev);
701}
702
703static void update_xdomain(struct tb_xdomain *xd, u64 route, u8 link)
704{
705 xd->link = link;
706 xd->route = route;
707 xd->is_unplugged = false;
708}
709
710static void remove_xdomain(struct tb_xdomain *xd)
711{
712 struct tb_switch *sw;
713
714 sw = tb_to_switch(xd->dev.parent);
715 tb_port_at(xd->route, sw)->xdomain = NULL;
716 tb_xdomain_remove(xd);
717}
718
719static void
720icm_fr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
721{
722 const struct icm_fr_event_device_connected *pkg =
723 (const struct icm_fr_event_device_connected *)hdr;
724 enum tb_security_level security_level;
725 struct tb_switch *sw, *parent_sw;
726 bool boot, dual_lane, speed_gen3;
727 struct icm *icm = tb_priv(tb);
728 bool authorized = false;
729 struct tb_xdomain *xd;
730 u8 link, depth;
731 u64 route;
732 int ret;
733
734 icm_postpone_rescan(tb);
735
736 link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
737 depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
738 ICM_LINK_INFO_DEPTH_SHIFT;
739 authorized = pkg->link_info & ICM_LINK_INFO_APPROVED;
740 security_level = (pkg->hdr.flags & ICM_FLAGS_SLEVEL_MASK) >>
741 ICM_FLAGS_SLEVEL_SHIFT;
742 boot = pkg->link_info & ICM_LINK_INFO_BOOT;
743 dual_lane = pkg->hdr.flags & ICM_FLAGS_DUAL_LANE;
744 speed_gen3 = pkg->hdr.flags & ICM_FLAGS_SPEED_GEN3;
745
746 if (pkg->link_info & ICM_LINK_INFO_REJECTED) {
747 tb_info(tb, "switch at %u.%u was rejected by ICM firmware because topology limit exceeded\n",
748 link, depth);
749 return;
750 }
751
752 sw = tb_switch_find_by_uuid(tb, &pkg->ep_uuid);
753 if (sw) {
754 u8 phy_port, sw_phy_port;
755
756 sw_phy_port = tb_phy_port_from_link(sw->link);
757 phy_port = tb_phy_port_from_link(link);
758
759 /*
760 * On resume ICM will send us connected events for the
761 * devices that still are present. However, that
762 * information might have changed for example by the
763 * fact that a switch on a dual-link connection might
764 * have been enumerated using the other link now. Make
765 * sure our book keeping matches that.
766 */
767 if (sw->depth == depth && sw_phy_port == phy_port &&
768 !!sw->authorized == authorized) {
769 /*
770 * It was enumerated through another link so update
771 * route string accordingly.
772 */
773 if (sw->link != link) {
774 ret = icm->get_route(tb, link, depth, &route);
775 if (ret) {
776 tb_err(tb, "failed to update route string for switch at %u.%u\n",
777 link, depth);
778 tb_switch_put(sw);
779 return;
780 }
781 } else {
782 route = tb_route(sw);
783 }
784
785 update_switch(sw, route, pkg->connection_id,
786 pkg->connection_key, link, depth, boot);
787 tb_switch_put(sw);
788 return;
789 }
790
791 /*
792 * User connected the same switch to another physical
793 * port or to another part of the topology. Remove the
794 * existing switch now before adding the new one.
795 */
796 remove_switch(sw);
797 tb_switch_put(sw);
798 }
799
800 /*
801 * If the switch was not found by UUID, look for a switch on
802 * same physical port (taking possible link aggregation into
803 * account) and depth. If we found one it is definitely a stale
804 * one so remove it first.
805 */
806 sw = tb_switch_find_by_link_depth(tb, link, depth);
807 if (!sw) {
808 u8 dual_link;
809
810 dual_link = dual_link_from_link(link);
811 if (dual_link)
812 sw = tb_switch_find_by_link_depth(tb, dual_link, depth);
813 }
814 if (sw) {
815 remove_switch(sw);
816 tb_switch_put(sw);
817 }
818
819 /* Remove existing XDomain connection if found */
820 xd = tb_xdomain_find_by_link_depth(tb, link, depth);
821 if (xd) {
822 remove_xdomain(xd);
823 tb_xdomain_put(xd);
824 }
825
826 parent_sw = tb_switch_find_by_link_depth(tb, link, depth - 1);
827 if (!parent_sw) {
828 tb_err(tb, "failed to find parent switch for %u.%u\n",
829 link, depth);
830 return;
831 }
832
833 ret = icm->get_route(tb, link, depth, &route);
834 if (ret) {
835 tb_err(tb, "failed to find route string for switch at %u.%u\n",
836 link, depth);
837 tb_switch_put(parent_sw);
838 return;
839 }
840
841 pm_runtime_get_sync(&parent_sw->dev);
842
843 sw = alloc_switch(parent_sw, route, &pkg->ep_uuid);
844 if (!IS_ERR(sw)) {
845 sw->connection_id = pkg->connection_id;
846 sw->connection_key = pkg->connection_key;
847 sw->link = link;
848 sw->depth = depth;
849 sw->authorized = authorized;
850 sw->security_level = security_level;
851 sw->boot = boot;
852 sw->link_speed = speed_gen3 ? 20 : 10;
853 sw->link_width = dual_lane ? TB_LINK_WIDTH_DUAL :
854 TB_LINK_WIDTH_SINGLE;
855 sw->rpm = intel_vss_is_rtd3(pkg->ep_name, sizeof(pkg->ep_name));
856
857 if (add_switch(parent_sw, sw))
858 tb_switch_put(sw);
859 }
860
861 pm_runtime_mark_last_busy(&parent_sw->dev);
862 pm_runtime_put_autosuspend(&parent_sw->dev);
863
864 tb_switch_put(parent_sw);
865}
866
867static void
868icm_fr_device_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
869{
870 const struct icm_fr_event_device_disconnected *pkg =
871 (const struct icm_fr_event_device_disconnected *)hdr;
872 struct tb_switch *sw;
873 u8 link, depth;
874
875 link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
876 depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
877 ICM_LINK_INFO_DEPTH_SHIFT;
878
879 if (link > ICM_MAX_LINK || depth > TB_SWITCH_MAX_DEPTH) {
880 tb_warn(tb, "invalid topology %u.%u, ignoring\n", link, depth);
881 return;
882 }
883
884 sw = tb_switch_find_by_link_depth(tb, link, depth);
885 if (!sw) {
886 tb_warn(tb, "no switch exists at %u.%u, ignoring\n", link,
887 depth);
888 return;
889 }
890
891 pm_runtime_get_sync(sw->dev.parent);
892
893 remove_switch(sw);
894
895 pm_runtime_mark_last_busy(sw->dev.parent);
896 pm_runtime_put_autosuspend(sw->dev.parent);
897
898 tb_switch_put(sw);
899}
900
901static void
902icm_fr_xdomain_connected(struct tb *tb, const struct icm_pkg_header *hdr)
903{
904 const struct icm_fr_event_xdomain_connected *pkg =
905 (const struct icm_fr_event_xdomain_connected *)hdr;
906 struct tb_xdomain *xd;
907 struct tb_switch *sw;
908 u8 link, depth;
909 u64 route;
910
911 link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
912 depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
913 ICM_LINK_INFO_DEPTH_SHIFT;
914
915 if (link > ICM_MAX_LINK || depth > TB_SWITCH_MAX_DEPTH) {
916 tb_warn(tb, "invalid topology %u.%u, ignoring\n", link, depth);
917 return;
918 }
919
920 route = get_route(pkg->local_route_hi, pkg->local_route_lo);
921
922 xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
923 if (xd) {
924 u8 xd_phy_port, phy_port;
925
926 xd_phy_port = phy_port_from_route(xd->route, xd->depth);
927 phy_port = phy_port_from_route(route, depth);
928
929 if (xd->depth == depth && xd_phy_port == phy_port) {
930 update_xdomain(xd, route, link);
931 tb_xdomain_put(xd);
932 return;
933 }
934
935 /*
936 * If we find an existing XDomain connection remove it
937 * now. We need to go through login handshake and
938 * everything anyway to be able to re-establish the
939 * connection.
940 */
941 remove_xdomain(xd);
942 tb_xdomain_put(xd);
943 }
944
945 /*
946 * Look if there already exists an XDomain in the same place
947 * than the new one and in that case remove it because it is
948 * most likely another host that got disconnected.
949 */
950 xd = tb_xdomain_find_by_link_depth(tb, link, depth);
951 if (!xd) {
952 u8 dual_link;
953
954 dual_link = dual_link_from_link(link);
955 if (dual_link)
956 xd = tb_xdomain_find_by_link_depth(tb, dual_link,
957 depth);
958 }
959 if (xd) {
960 remove_xdomain(xd);
961 tb_xdomain_put(xd);
962 }
963
964 /*
965 * If the user disconnected a switch during suspend and
966 * connected another host to the same port, remove the switch
967 * first.
968 */
969 sw = tb_switch_find_by_route(tb, route);
970 if (sw) {
971 remove_switch(sw);
972 tb_switch_put(sw);
973 }
974
975 sw = tb_switch_find_by_link_depth(tb, link, depth);
976 if (!sw) {
977 tb_warn(tb, "no switch exists at %u.%u, ignoring\n", link,
978 depth);
979 return;
980 }
981
982 add_xdomain(sw, route, &pkg->local_uuid, &pkg->remote_uuid, link,
983 depth);
984 tb_switch_put(sw);
985}
986
987static void
988icm_fr_xdomain_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
989{
990 const struct icm_fr_event_xdomain_disconnected *pkg =
991 (const struct icm_fr_event_xdomain_disconnected *)hdr;
992 struct tb_xdomain *xd;
993
994 /*
995 * If the connection is through one or multiple devices, the
996 * XDomain device is removed along with them so it is fine if we
997 * cannot find it here.
998 */
999 xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
1000 if (xd) {
1001 remove_xdomain(xd);
1002 tb_xdomain_put(xd);
1003 }
1004}
1005
1006static int icm_tr_cio_reset(struct tb *tb)
1007{
1008 return pcie2cio_write(tb_priv(tb), TB_CFG_SWITCH, 0, 0x777, BIT(1));
1009}
1010
1011static int
1012icm_tr_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1013 u8 *proto_version, size_t *nboot_acl, bool *rpm)
1014{
1015 struct icm_tr_pkg_driver_ready_response reply;
1016 struct icm_pkg_driver_ready request = {
1017 .hdr.code = ICM_DRIVER_READY,
1018 };
1019 int ret;
1020
1021 memset(&reply, 0, sizeof(reply));
1022 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1023 1, 10, 250);
1024 if (ret)
1025 return ret;
1026
1027 if (security_level)
1028 *security_level = reply.info & ICM_TR_INFO_SLEVEL_MASK;
1029 if (proto_version)
1030 *proto_version = (reply.info & ICM_TR_INFO_PROTO_VERSION_MASK) >>
1031 ICM_TR_INFO_PROTO_VERSION_SHIFT;
1032 if (nboot_acl)
1033 *nboot_acl = (reply.info & ICM_TR_INFO_BOOT_ACL_MASK) >>
1034 ICM_TR_INFO_BOOT_ACL_SHIFT;
1035 if (rpm)
1036 *rpm = !!(reply.hdr.flags & ICM_TR_FLAGS_RTD3);
1037
1038 return 0;
1039}
1040
1041static int icm_tr_approve_switch(struct tb *tb, struct tb_switch *sw)
1042{
1043 struct icm_tr_pkg_approve_device request;
1044 struct icm_tr_pkg_approve_device reply;
1045 int ret;
1046
1047 memset(&request, 0, sizeof(request));
1048 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1049 request.hdr.code = ICM_APPROVE_DEVICE;
1050 request.route_lo = sw->config.route_lo;
1051 request.route_hi = sw->config.route_hi;
1052 request.connection_id = sw->connection_id;
1053
1054 memset(&reply, 0, sizeof(reply));
1055 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1056 1, ICM_RETRIES, ICM_APPROVE_TIMEOUT);
1057 if (ret)
1058 return ret;
1059
1060 if (reply.hdr.flags & ICM_FLAGS_ERROR) {
1061 tb_warn(tb, "PCIe tunnel creation failed\n");
1062 return -EIO;
1063 }
1064
1065 return 0;
1066}
1067
1068static int icm_tr_add_switch_key(struct tb *tb, struct tb_switch *sw)
1069{
1070 struct icm_tr_pkg_add_device_key_response reply;
1071 struct icm_tr_pkg_add_device_key request;
1072 int ret;
1073
1074 memset(&request, 0, sizeof(request));
1075 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1076 request.hdr.code = ICM_ADD_DEVICE_KEY;
1077 request.route_lo = sw->config.route_lo;
1078 request.route_hi = sw->config.route_hi;
1079 request.connection_id = sw->connection_id;
1080 memcpy(request.key, sw->key, TB_SWITCH_KEY_SIZE);
1081
1082 memset(&reply, 0, sizeof(reply));
1083 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1084 1, ICM_RETRIES, ICM_TIMEOUT);
1085 if (ret)
1086 return ret;
1087
1088 if (reply.hdr.flags & ICM_FLAGS_ERROR) {
1089 tb_warn(tb, "Adding key to switch failed\n");
1090 return -EIO;
1091 }
1092
1093 return 0;
1094}
1095
1096static int icm_tr_challenge_switch_key(struct tb *tb, struct tb_switch *sw,
1097 const u8 *challenge, u8 *response)
1098{
1099 struct icm_tr_pkg_challenge_device_response reply;
1100 struct icm_tr_pkg_challenge_device request;
1101 int ret;
1102
1103 memset(&request, 0, sizeof(request));
1104 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1105 request.hdr.code = ICM_CHALLENGE_DEVICE;
1106 request.route_lo = sw->config.route_lo;
1107 request.route_hi = sw->config.route_hi;
1108 request.connection_id = sw->connection_id;
1109 memcpy(request.challenge, challenge, TB_SWITCH_KEY_SIZE);
1110
1111 memset(&reply, 0, sizeof(reply));
1112 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1113 1, ICM_RETRIES, ICM_TIMEOUT);
1114 if (ret)
1115 return ret;
1116
1117 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1118 return -EKEYREJECTED;
1119 if (reply.hdr.flags & ICM_FLAGS_NO_KEY)
1120 return -ENOKEY;
1121
1122 memcpy(response, reply.response, TB_SWITCH_KEY_SIZE);
1123
1124 return 0;
1125}
1126
1127static int icm_tr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
1128 int transmit_path, int transmit_ring,
1129 int receive_path, int receive_ring)
1130{
1131 struct icm_tr_pkg_approve_xdomain_response reply;
1132 struct icm_tr_pkg_approve_xdomain request;
1133 int ret;
1134
1135 memset(&request, 0, sizeof(request));
1136 request.hdr.code = ICM_APPROVE_XDOMAIN;
1137 request.route_hi = upper_32_bits(xd->route);
1138 request.route_lo = lower_32_bits(xd->route);
1139 request.transmit_path = transmit_path;
1140 request.transmit_ring = transmit_ring;
1141 request.receive_path = receive_path;
1142 request.receive_ring = receive_ring;
1143 memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
1144
1145 memset(&reply, 0, sizeof(reply));
1146 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1147 1, ICM_RETRIES, ICM_TIMEOUT);
1148 if (ret)
1149 return ret;
1150
1151 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1152 return -EIO;
1153
1154 return 0;
1155}
1156
1157static int icm_tr_xdomain_tear_down(struct tb *tb, struct tb_xdomain *xd,
1158 int stage)
1159{
1160 struct icm_tr_pkg_disconnect_xdomain_response reply;
1161 struct icm_tr_pkg_disconnect_xdomain request;
1162 int ret;
1163
1164 memset(&request, 0, sizeof(request));
1165 request.hdr.code = ICM_DISCONNECT_XDOMAIN;
1166 request.stage = stage;
1167 request.route_hi = upper_32_bits(xd->route);
1168 request.route_lo = lower_32_bits(xd->route);
1169 memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
1170
1171 memset(&reply, 0, sizeof(reply));
1172 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1173 1, ICM_RETRIES, ICM_TIMEOUT);
1174 if (ret)
1175 return ret;
1176
1177 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1178 return -EIO;
1179
1180 return 0;
1181}
1182
1183static int icm_tr_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
1184 int transmit_path, int transmit_ring,
1185 int receive_path, int receive_ring)
1186{
1187 int ret;
1188
1189 ret = icm_tr_xdomain_tear_down(tb, xd, 1);
1190 if (ret)
1191 return ret;
1192
1193 usleep_range(10, 50);
1194 return icm_tr_xdomain_tear_down(tb, xd, 2);
1195}
1196
1197static void
1198__icm_tr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr,
1199 bool force_rtd3)
1200{
1201 const struct icm_tr_event_device_connected *pkg =
1202 (const struct icm_tr_event_device_connected *)hdr;
1203 bool authorized, boot, dual_lane, speed_gen3;
1204 enum tb_security_level security_level;
1205 struct tb_switch *sw, *parent_sw;
1206 struct tb_xdomain *xd;
1207 u64 route;
1208
1209 icm_postpone_rescan(tb);
1210
1211 /*
1212 * Currently we don't use the QoS information coming with the
1213 * device connected message so simply just ignore that extra
1214 * packet for now.
1215 */
1216 if (pkg->hdr.packet_id)
1217 return;
1218
1219 route = get_route(pkg->route_hi, pkg->route_lo);
1220 authorized = pkg->link_info & ICM_LINK_INFO_APPROVED;
1221 security_level = (pkg->hdr.flags & ICM_FLAGS_SLEVEL_MASK) >>
1222 ICM_FLAGS_SLEVEL_SHIFT;
1223 boot = pkg->link_info & ICM_LINK_INFO_BOOT;
1224 dual_lane = pkg->hdr.flags & ICM_FLAGS_DUAL_LANE;
1225 speed_gen3 = pkg->hdr.flags & ICM_FLAGS_SPEED_GEN3;
1226
1227 if (pkg->link_info & ICM_LINK_INFO_REJECTED) {
1228 tb_info(tb, "switch at %llx was rejected by ICM firmware because topology limit exceeded\n",
1229 route);
1230 return;
1231 }
1232
1233 sw = tb_switch_find_by_uuid(tb, &pkg->ep_uuid);
1234 if (sw) {
1235 /* Update the switch if it is still in the same place */
1236 if (tb_route(sw) == route && !!sw->authorized == authorized) {
1237 update_switch(sw, route, pkg->connection_id, 0, 0, 0,
1238 boot);
1239 tb_switch_put(sw);
1240 return;
1241 }
1242
1243 remove_switch(sw);
1244 tb_switch_put(sw);
1245 }
1246
1247 /* Another switch with the same address */
1248 sw = tb_switch_find_by_route(tb, route);
1249 if (sw) {
1250 remove_switch(sw);
1251 tb_switch_put(sw);
1252 }
1253
1254 /* XDomain connection with the same address */
1255 xd = tb_xdomain_find_by_route(tb, route);
1256 if (xd) {
1257 remove_xdomain(xd);
1258 tb_xdomain_put(xd);
1259 }
1260
1261 parent_sw = tb_switch_find_by_route(tb, get_parent_route(route));
1262 if (!parent_sw) {
1263 tb_err(tb, "failed to find parent switch for %llx\n", route);
1264 return;
1265 }
1266
1267 pm_runtime_get_sync(&parent_sw->dev);
1268
1269 sw = alloc_switch(parent_sw, route, &pkg->ep_uuid);
1270 if (!IS_ERR(sw)) {
1271 sw->connection_id = pkg->connection_id;
1272 sw->authorized = authorized;
1273 sw->security_level = security_level;
1274 sw->boot = boot;
1275 sw->link_speed = speed_gen3 ? 20 : 10;
1276 sw->link_width = dual_lane ? TB_LINK_WIDTH_DUAL :
1277 TB_LINK_WIDTH_SINGLE;
1278 sw->rpm = force_rtd3;
1279 if (!sw->rpm)
1280 sw->rpm = intel_vss_is_rtd3(pkg->ep_name,
1281 sizeof(pkg->ep_name));
1282
1283 if (add_switch(parent_sw, sw))
1284 tb_switch_put(sw);
1285 }
1286
1287 pm_runtime_mark_last_busy(&parent_sw->dev);
1288 pm_runtime_put_autosuspend(&parent_sw->dev);
1289
1290 tb_switch_put(parent_sw);
1291}
1292
1293static void
1294icm_tr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1295{
1296 __icm_tr_device_connected(tb, hdr, false);
1297}
1298
1299static void
1300icm_tr_device_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
1301{
1302 const struct icm_tr_event_device_disconnected *pkg =
1303 (const struct icm_tr_event_device_disconnected *)hdr;
1304 struct tb_switch *sw;
1305 u64 route;
1306
1307 route = get_route(pkg->route_hi, pkg->route_lo);
1308
1309 sw = tb_switch_find_by_route(tb, route);
1310 if (!sw) {
1311 tb_warn(tb, "no switch exists at %llx, ignoring\n", route);
1312 return;
1313 }
1314 pm_runtime_get_sync(sw->dev.parent);
1315
1316 remove_switch(sw);
1317
1318 pm_runtime_mark_last_busy(sw->dev.parent);
1319 pm_runtime_put_autosuspend(sw->dev.parent);
1320
1321 tb_switch_put(sw);
1322}
1323
1324static void
1325icm_tr_xdomain_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1326{
1327 const struct icm_tr_event_xdomain_connected *pkg =
1328 (const struct icm_tr_event_xdomain_connected *)hdr;
1329 struct tb_xdomain *xd;
1330 struct tb_switch *sw;
1331 u64 route;
1332
1333 if (!tb->root_switch)
1334 return;
1335
1336 route = get_route(pkg->local_route_hi, pkg->local_route_lo);
1337
1338 xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
1339 if (xd) {
1340 if (xd->route == route) {
1341 update_xdomain(xd, route, 0);
1342 tb_xdomain_put(xd);
1343 return;
1344 }
1345
1346 remove_xdomain(xd);
1347 tb_xdomain_put(xd);
1348 }
1349
1350 /* An existing xdomain with the same address */
1351 xd = tb_xdomain_find_by_route(tb, route);
1352 if (xd) {
1353 remove_xdomain(xd);
1354 tb_xdomain_put(xd);
1355 }
1356
1357 /*
1358 * If the user disconnected a switch during suspend and
1359 * connected another host to the same port, remove the switch
1360 * first.
1361 */
1362 sw = tb_switch_find_by_route(tb, route);
1363 if (sw) {
1364 remove_switch(sw);
1365 tb_switch_put(sw);
1366 }
1367
1368 sw = tb_switch_find_by_route(tb, get_parent_route(route));
1369 if (!sw) {
1370 tb_warn(tb, "no switch exists at %llx, ignoring\n", route);
1371 return;
1372 }
1373
1374 add_xdomain(sw, route, &pkg->local_uuid, &pkg->remote_uuid, 0, 0);
1375 tb_switch_put(sw);
1376}
1377
1378static void
1379icm_tr_xdomain_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
1380{
1381 const struct icm_tr_event_xdomain_disconnected *pkg =
1382 (const struct icm_tr_event_xdomain_disconnected *)hdr;
1383 struct tb_xdomain *xd;
1384 u64 route;
1385
1386 route = get_route(pkg->route_hi, pkg->route_lo);
1387
1388 xd = tb_xdomain_find_by_route(tb, route);
1389 if (xd) {
1390 remove_xdomain(xd);
1391 tb_xdomain_put(xd);
1392 }
1393}
1394
1395static struct pci_dev *get_upstream_port(struct pci_dev *pdev)
1396{
1397 struct pci_dev *parent;
1398
1399 parent = pci_upstream_bridge(pdev);
1400 while (parent) {
1401 if (!pci_is_pcie(parent))
1402 return NULL;
1403 if (pci_pcie_type(parent) == PCI_EXP_TYPE_UPSTREAM)
1404 break;
1405 parent = pci_upstream_bridge(parent);
1406 }
1407
1408 if (!parent)
1409 return NULL;
1410
1411 switch (parent->device) {
1412 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_BRIDGE:
1413 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_BRIDGE:
1414 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_BRIDGE:
1415 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_BRIDGE:
1416 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_BRIDGE:
1417 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_BRIDGE:
1418 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_BRIDGE:
1419 return parent;
1420 }
1421
1422 return NULL;
1423}
1424
1425static bool icm_ar_is_supported(struct tb *tb)
1426{
1427 struct pci_dev *upstream_port;
1428 struct icm *icm = tb_priv(tb);
1429
1430 /*
1431 * Starting from Alpine Ridge we can use ICM on Apple machines
1432 * as well. We just need to reset and re-enable it first.
1433 * However, only start it if explicitly asked by the user.
1434 */
1435 if (icm_firmware_running(tb->nhi))
1436 return true;
1437 if (!start_icm)
1438 return false;
1439
1440 /*
1441 * Find the upstream PCIe port in case we need to do reset
1442 * through its vendor specific registers.
1443 */
1444 upstream_port = get_upstream_port(tb->nhi->pdev);
1445 if (upstream_port) {
1446 int cap;
1447
1448 cap = pci_find_ext_capability(upstream_port,
1449 PCI_EXT_CAP_ID_VNDR);
1450 if (cap > 0) {
1451 icm->upstream_port = upstream_port;
1452 icm->vnd_cap = cap;
1453
1454 return true;
1455 }
1456 }
1457
1458 return false;
1459}
1460
1461static int icm_ar_cio_reset(struct tb *tb)
1462{
1463 return pcie2cio_write(tb_priv(tb), TB_CFG_SWITCH, 0, 0x50, BIT(9));
1464}
1465
1466static int icm_ar_get_mode(struct tb *tb)
1467{
1468 struct tb_nhi *nhi = tb->nhi;
1469 int retries = 60;
1470 u32 val;
1471
1472 do {
1473 val = ioread32(nhi->iobase + REG_FW_STS);
1474 if (val & REG_FW_STS_NVM_AUTH_DONE)
1475 break;
1476 msleep(50);
1477 } while (--retries);
1478
1479 if (!retries) {
1480 dev_err(&nhi->pdev->dev, "ICM firmware not authenticated\n");
1481 return -ENODEV;
1482 }
1483
1484 return nhi_mailbox_mode(nhi);
1485}
1486
1487static int
1488icm_ar_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1489 u8 *proto_version, size_t *nboot_acl, bool *rpm)
1490{
1491 struct icm_ar_pkg_driver_ready_response reply;
1492 struct icm_pkg_driver_ready request = {
1493 .hdr.code = ICM_DRIVER_READY,
1494 };
1495 int ret;
1496
1497 memset(&reply, 0, sizeof(reply));
1498 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1499 1, ICM_RETRIES, ICM_TIMEOUT);
1500 if (ret)
1501 return ret;
1502
1503 if (security_level)
1504 *security_level = reply.info & ICM_AR_INFO_SLEVEL_MASK;
1505 if (nboot_acl && (reply.info & ICM_AR_INFO_BOOT_ACL_SUPPORTED))
1506 *nboot_acl = (reply.info & ICM_AR_INFO_BOOT_ACL_MASK) >>
1507 ICM_AR_INFO_BOOT_ACL_SHIFT;
1508 if (rpm)
1509 *rpm = !!(reply.hdr.flags & ICM_AR_FLAGS_RTD3);
1510
1511 return 0;
1512}
1513
1514static int icm_ar_get_route(struct tb *tb, u8 link, u8 depth, u64 *route)
1515{
1516 struct icm_ar_pkg_get_route_response reply;
1517 struct icm_ar_pkg_get_route request = {
1518 .hdr = { .code = ICM_GET_ROUTE },
1519 .link_info = depth << ICM_LINK_INFO_DEPTH_SHIFT | link,
1520 };
1521 int ret;
1522
1523 memset(&reply, 0, sizeof(reply));
1524 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1525 1, ICM_RETRIES, ICM_TIMEOUT);
1526 if (ret)
1527 return ret;
1528
1529 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1530 return -EIO;
1531
1532 *route = get_route(reply.route_hi, reply.route_lo);
1533 return 0;
1534}
1535
1536static int icm_ar_get_boot_acl(struct tb *tb, uuid_t *uuids, size_t nuuids)
1537{
1538 struct icm_ar_pkg_preboot_acl_response reply;
1539 struct icm_ar_pkg_preboot_acl request = {
1540 .hdr = { .code = ICM_PREBOOT_ACL },
1541 };
1542 int ret, i;
1543
1544 memset(&reply, 0, sizeof(reply));
1545 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1546 1, ICM_RETRIES, ICM_TIMEOUT);
1547 if (ret)
1548 return ret;
1549
1550 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1551 return -EIO;
1552
1553 for (i = 0; i < nuuids; i++) {
1554 u32 *uuid = (u32 *)&uuids[i];
1555
1556 uuid[0] = reply.acl[i].uuid_lo;
1557 uuid[1] = reply.acl[i].uuid_hi;
1558
1559 if (uuid[0] == 0xffffffff && uuid[1] == 0xffffffff) {
1560 /* Map empty entries to null UUID */
1561 uuid[0] = 0;
1562 uuid[1] = 0;
1563 } else if (uuid[0] != 0 || uuid[1] != 0) {
1564 /* Upper two DWs are always one's */
1565 uuid[2] = 0xffffffff;
1566 uuid[3] = 0xffffffff;
1567 }
1568 }
1569
1570 return ret;
1571}
1572
1573static int icm_ar_set_boot_acl(struct tb *tb, const uuid_t *uuids,
1574 size_t nuuids)
1575{
1576 struct icm_ar_pkg_preboot_acl_response reply;
1577 struct icm_ar_pkg_preboot_acl request = {
1578 .hdr = {
1579 .code = ICM_PREBOOT_ACL,
1580 .flags = ICM_FLAGS_WRITE,
1581 },
1582 };
1583 int ret, i;
1584
1585 for (i = 0; i < nuuids; i++) {
1586 const u32 *uuid = (const u32 *)&uuids[i];
1587
1588 if (uuid_is_null(&uuids[i])) {
1589 /*
1590 * Map null UUID to the empty (all one) entries
1591 * for ICM.
1592 */
1593 request.acl[i].uuid_lo = 0xffffffff;
1594 request.acl[i].uuid_hi = 0xffffffff;
1595 } else {
1596 /* Two high DWs need to be set to all one */
1597 if (uuid[2] != 0xffffffff || uuid[3] != 0xffffffff)
1598 return -EINVAL;
1599
1600 request.acl[i].uuid_lo = uuid[0];
1601 request.acl[i].uuid_hi = uuid[1];
1602 }
1603 }
1604
1605 memset(&reply, 0, sizeof(reply));
1606 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1607 1, ICM_RETRIES, ICM_TIMEOUT);
1608 if (ret)
1609 return ret;
1610
1611 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1612 return -EIO;
1613
1614 return 0;
1615}
1616
1617static int
1618icm_icl_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1619 u8 *proto_version, size_t *nboot_acl, bool *rpm)
1620{
1621 struct icm_tr_pkg_driver_ready_response reply;
1622 struct icm_pkg_driver_ready request = {
1623 .hdr.code = ICM_DRIVER_READY,
1624 };
1625 int ret;
1626
1627 memset(&reply, 0, sizeof(reply));
1628 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1629 1, ICM_RETRIES, 20000);
1630 if (ret)
1631 return ret;
1632
1633 if (proto_version)
1634 *proto_version = (reply.info & ICM_TR_INFO_PROTO_VERSION_MASK) >>
1635 ICM_TR_INFO_PROTO_VERSION_SHIFT;
1636
1637 /* Ice Lake always supports RTD3 */
1638 if (rpm)
1639 *rpm = true;
1640
1641 return 0;
1642}
1643
1644static void icm_icl_set_uuid(struct tb *tb)
1645{
1646 struct tb_nhi *nhi = tb->nhi;
1647 u32 uuid[4];
1648
1649 pci_read_config_dword(nhi->pdev, VS_CAP_10, &uuid[0]);
1650 pci_read_config_dword(nhi->pdev, VS_CAP_11, &uuid[1]);
1651 uuid[2] = 0xffffffff;
1652 uuid[3] = 0xffffffff;
1653
1654 tb->root_switch->uuid = kmemdup(uuid, sizeof(uuid), GFP_KERNEL);
1655}
1656
1657static void
1658icm_icl_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1659{
1660 __icm_tr_device_connected(tb, hdr, true);
1661}
1662
1663static void icm_icl_rtd3_veto(struct tb *tb, const struct icm_pkg_header *hdr)
1664{
1665 const struct icm_icl_event_rtd3_veto *pkg =
1666 (const struct icm_icl_event_rtd3_veto *)hdr;
1667
1668 tb_dbg(tb, "ICM rtd3 veto=0x%08x\n", pkg->veto_reason);
1669
1670 if (pkg->veto_reason)
1671 icm_veto_begin(tb);
1672 else
1673 icm_veto_end(tb);
1674}
1675
1676static bool icm_tgl_is_supported(struct tb *tb)
1677{
1678 unsigned long end = jiffies + msecs_to_jiffies(10);
1679
1680 do {
1681 u32 val;
1682
1683 val = ioread32(tb->nhi->iobase + REG_FW_STS);
1684 if (val & REG_FW_STS_NVM_AUTH_DONE)
1685 return true;
1686 usleep_range(100, 500);
1687 } while (time_before(jiffies, end));
1688
1689 return false;
1690}
1691
1692static void icm_handle_notification(struct work_struct *work)
1693{
1694 struct icm_notification *n = container_of(work, typeof(*n), work);
1695 struct tb *tb = n->tb;
1696 struct icm *icm = tb_priv(tb);
1697
1698 mutex_lock(&tb->lock);
1699
1700 /*
1701 * When the domain is stopped we flush its workqueue but before
1702 * that the root switch is removed. In that case we should treat
1703 * the queued events as being canceled.
1704 */
1705 if (tb->root_switch) {
1706 switch (n->pkg->code) {
1707 case ICM_EVENT_DEVICE_CONNECTED:
1708 icm->device_connected(tb, n->pkg);
1709 break;
1710 case ICM_EVENT_DEVICE_DISCONNECTED:
1711 icm->device_disconnected(tb, n->pkg);
1712 break;
1713 case ICM_EVENT_XDOMAIN_CONNECTED:
1714 if (tb_is_xdomain_enabled())
1715 icm->xdomain_connected(tb, n->pkg);
1716 break;
1717 case ICM_EVENT_XDOMAIN_DISCONNECTED:
1718 if (tb_is_xdomain_enabled())
1719 icm->xdomain_disconnected(tb, n->pkg);
1720 break;
1721 case ICM_EVENT_RTD3_VETO:
1722 icm->rtd3_veto(tb, n->pkg);
1723 break;
1724 }
1725 }
1726
1727 mutex_unlock(&tb->lock);
1728
1729 kfree(n->pkg);
1730 kfree(n);
1731}
1732
1733static void icm_handle_event(struct tb *tb, enum tb_cfg_pkg_type type,
1734 const void *buf, size_t size)
1735{
1736 struct icm_notification *n;
1737
1738 n = kmalloc(sizeof(*n), GFP_KERNEL);
1739 if (!n)
1740 return;
1741
1742 n->pkg = kmemdup(buf, size, GFP_KERNEL);
1743 if (!n->pkg) {
1744 kfree(n);
1745 return;
1746 }
1747
1748 INIT_WORK(&n->work, icm_handle_notification);
1749 n->tb = tb;
1750
1751 queue_work(tb->wq, &n->work);
1752}
1753
1754static int
1755__icm_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1756 u8 *proto_version, size_t *nboot_acl, bool *rpm)
1757{
1758 struct icm *icm = tb_priv(tb);
1759 unsigned int retries = 50;
1760 int ret;
1761
1762 ret = icm->driver_ready(tb, security_level, proto_version, nboot_acl,
1763 rpm);
1764 if (ret) {
1765 tb_err(tb, "failed to send driver ready to ICM\n");
1766 return ret;
1767 }
1768
1769 /*
1770 * Hold on here until the switch config space is accessible so
1771 * that we can read root switch config successfully.
1772 */
1773 do {
1774 struct tb_cfg_result res;
1775 u32 tmp;
1776
1777 res = tb_cfg_read_raw(tb->ctl, &tmp, 0, 0, TB_CFG_SWITCH,
1778 0, 1, 100);
1779 if (!res.err)
1780 return 0;
1781
1782 msleep(50);
1783 } while (--retries);
1784
1785 tb_err(tb, "failed to read root switch config space, giving up\n");
1786 return -ETIMEDOUT;
1787}
1788
1789static int icm_firmware_reset(struct tb *tb, struct tb_nhi *nhi)
1790{
1791 struct icm *icm = tb_priv(tb);
1792 u32 val;
1793
1794 if (!icm->upstream_port)
1795 return -ENODEV;
1796
1797 /* Put ARC to wait for CIO reset event to happen */
1798 val = ioread32(nhi->iobase + REG_FW_STS);
1799 val |= REG_FW_STS_CIO_RESET_REQ;
1800 iowrite32(val, nhi->iobase + REG_FW_STS);
1801
1802 /* Re-start ARC */
1803 val = ioread32(nhi->iobase + REG_FW_STS);
1804 val |= REG_FW_STS_ICM_EN_INVERT;
1805 val |= REG_FW_STS_ICM_EN_CPU;
1806 iowrite32(val, nhi->iobase + REG_FW_STS);
1807
1808 /* Trigger CIO reset now */
1809 return icm->cio_reset(tb);
1810}
1811
1812static int icm_firmware_start(struct tb *tb, struct tb_nhi *nhi)
1813{
1814 unsigned int retries = 10;
1815 int ret;
1816 u32 val;
1817
1818 /* Check if the ICM firmware is already running */
1819 if (icm_firmware_running(nhi))
1820 return 0;
1821
1822 dev_dbg(&nhi->pdev->dev, "starting ICM firmware\n");
1823
1824 ret = icm_firmware_reset(tb, nhi);
1825 if (ret)
1826 return ret;
1827
1828 /* Wait until the ICM firmware tells us it is up and running */
1829 do {
1830 /* Check that the ICM firmware is running */
1831 val = ioread32(nhi->iobase + REG_FW_STS);
1832 if (val & REG_FW_STS_NVM_AUTH_DONE)
1833 return 0;
1834
1835 msleep(300);
1836 } while (--retries);
1837
1838 return -ETIMEDOUT;
1839}
1840
1841static int icm_reset_phy_port(struct tb *tb, int phy_port)
1842{
1843 struct icm *icm = tb_priv(tb);
1844 u32 state0, state1;
1845 int port0, port1;
1846 u32 val0, val1;
1847 int ret;
1848
1849 if (!icm->upstream_port)
1850 return 0;
1851
1852 if (phy_port) {
1853 port0 = 3;
1854 port1 = 4;
1855 } else {
1856 port0 = 1;
1857 port1 = 2;
1858 }
1859
1860 /*
1861 * Read link status of both null ports belonging to a single
1862 * physical port.
1863 */
1864 ret = pcie2cio_read(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, &val0);
1865 if (ret)
1866 return ret;
1867 ret = pcie2cio_read(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, &val1);
1868 if (ret)
1869 return ret;
1870
1871 state0 = val0 & PHY_PORT_CS1_LINK_STATE_MASK;
1872 state0 >>= PHY_PORT_CS1_LINK_STATE_SHIFT;
1873 state1 = val1 & PHY_PORT_CS1_LINK_STATE_MASK;
1874 state1 >>= PHY_PORT_CS1_LINK_STATE_SHIFT;
1875
1876 /* If they are both up we need to reset them now */
1877 if (state0 != TB_PORT_UP || state1 != TB_PORT_UP)
1878 return 0;
1879
1880 val0 |= PHY_PORT_CS1_LINK_DISABLE;
1881 ret = pcie2cio_write(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, val0);
1882 if (ret)
1883 return ret;
1884
1885 val1 |= PHY_PORT_CS1_LINK_DISABLE;
1886 ret = pcie2cio_write(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, val1);
1887 if (ret)
1888 return ret;
1889
1890 /* Wait a bit and then re-enable both ports */
1891 usleep_range(10, 100);
1892
1893 ret = pcie2cio_read(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, &val0);
1894 if (ret)
1895 return ret;
1896 ret = pcie2cio_read(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, &val1);
1897 if (ret)
1898 return ret;
1899
1900 val0 &= ~PHY_PORT_CS1_LINK_DISABLE;
1901 ret = pcie2cio_write(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, val0);
1902 if (ret)
1903 return ret;
1904
1905 val1 &= ~PHY_PORT_CS1_LINK_DISABLE;
1906 return pcie2cio_write(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, val1);
1907}
1908
1909static int icm_firmware_init(struct tb *tb)
1910{
1911 struct icm *icm = tb_priv(tb);
1912 struct tb_nhi *nhi = tb->nhi;
1913 int ret;
1914
1915 ret = icm_firmware_start(tb, nhi);
1916 if (ret) {
1917 dev_err(&nhi->pdev->dev, "could not start ICM firmware\n");
1918 return ret;
1919 }
1920
1921 if (icm->get_mode) {
1922 ret = icm->get_mode(tb);
1923
1924 switch (ret) {
1925 case NHI_FW_SAFE_MODE:
1926 icm->safe_mode = true;
1927 break;
1928
1929 case NHI_FW_CM_MODE:
1930 /* Ask ICM to accept all Thunderbolt devices */
1931 nhi_mailbox_cmd(nhi, NHI_MAILBOX_ALLOW_ALL_DEVS, 0);
1932 break;
1933
1934 default:
1935 if (ret < 0)
1936 return ret;
1937
1938 tb_err(tb, "ICM firmware is in wrong mode: %u\n", ret);
1939 return -ENODEV;
1940 }
1941 }
1942
1943 /*
1944 * Reset both physical ports if there is anything connected to
1945 * them already.
1946 */
1947 ret = icm_reset_phy_port(tb, 0);
1948 if (ret)
1949 dev_warn(&nhi->pdev->dev, "failed to reset links on port0\n");
1950 ret = icm_reset_phy_port(tb, 1);
1951 if (ret)
1952 dev_warn(&nhi->pdev->dev, "failed to reset links on port1\n");
1953
1954 return 0;
1955}
1956
1957static int icm_driver_ready(struct tb *tb)
1958{
1959 struct icm *icm = tb_priv(tb);
1960 int ret;
1961
1962 ret = icm_firmware_init(tb);
1963 if (ret)
1964 return ret;
1965
1966 if (icm->safe_mode) {
1967 tb_info(tb, "Thunderbolt host controller is in safe mode.\n");
1968 tb_info(tb, "You need to update NVM firmware of the controller before it can be used.\n");
1969 tb_info(tb, "For latest updates check https://thunderbolttechnology.net/updates.\n");
1970 return 0;
1971 }
1972
1973 ret = __icm_driver_ready(tb, &tb->security_level, &icm->proto_version,
1974 &tb->nboot_acl, &icm->rpm);
1975 if (ret)
1976 return ret;
1977
1978 /*
1979 * Make sure the number of supported preboot ACL matches what we
1980 * expect or disable the whole feature.
1981 */
1982 if (tb->nboot_acl > icm->max_boot_acl)
1983 tb->nboot_acl = 0;
1984
1985 if (icm->proto_version >= 3)
1986 tb_dbg(tb, "USB4 proxy operations supported\n");
1987
1988 return 0;
1989}
1990
1991static int icm_suspend(struct tb *tb)
1992{
1993 struct icm *icm = tb_priv(tb);
1994
1995 if (icm->save_devices)
1996 icm->save_devices(tb);
1997
1998 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
1999 return 0;
2000}
2001
2002/*
2003 * Mark all switches (except root switch) below this one unplugged. ICM
2004 * firmware will send us an updated list of switches after we have send
2005 * it driver ready command. If a switch is not in that list it will be
2006 * removed when we perform rescan.
2007 */
2008static void icm_unplug_children(struct tb_switch *sw)
2009{
2010 struct tb_port *port;
2011
2012 if (tb_route(sw))
2013 sw->is_unplugged = true;
2014
2015 tb_switch_for_each_port(sw, port) {
2016 if (port->xdomain)
2017 port->xdomain->is_unplugged = true;
2018 else if (tb_port_has_remote(port))
2019 icm_unplug_children(port->remote->sw);
2020 }
2021}
2022
2023static int complete_rpm(struct device *dev, void *data)
2024{
2025 struct tb_switch *sw = tb_to_switch(dev);
2026
2027 if (sw)
2028 complete(&sw->rpm_complete);
2029 return 0;
2030}
2031
2032static void remove_unplugged_switch(struct tb_switch *sw)
2033{
2034 struct device *parent = get_device(sw->dev.parent);
2035
2036 pm_runtime_get_sync(parent);
2037
2038 /*
2039 * Signal this and switches below for rpm_complete because
2040 * tb_switch_remove() calls pm_runtime_get_sync() that then waits
2041 * for it.
2042 */
2043 complete_rpm(&sw->dev, NULL);
2044 bus_for_each_dev(&tb_bus_type, &sw->dev, NULL, complete_rpm);
2045 tb_switch_remove(sw);
2046
2047 pm_runtime_mark_last_busy(parent);
2048 pm_runtime_put_autosuspend(parent);
2049
2050 put_device(parent);
2051}
2052
2053static void icm_free_unplugged_children(struct tb_switch *sw)
2054{
2055 struct tb_port *port;
2056
2057 tb_switch_for_each_port(sw, port) {
2058 if (port->xdomain && port->xdomain->is_unplugged) {
2059 tb_xdomain_remove(port->xdomain);
2060 port->xdomain = NULL;
2061 } else if (tb_port_has_remote(port)) {
2062 if (port->remote->sw->is_unplugged) {
2063 remove_unplugged_switch(port->remote->sw);
2064 port->remote = NULL;
2065 } else {
2066 icm_free_unplugged_children(port->remote->sw);
2067 }
2068 }
2069 }
2070}
2071
2072static void icm_rescan_work(struct work_struct *work)
2073{
2074 struct icm *icm = container_of(work, struct icm, rescan_work.work);
2075 struct tb *tb = icm_to_tb(icm);
2076
2077 mutex_lock(&tb->lock);
2078 if (tb->root_switch)
2079 icm_free_unplugged_children(tb->root_switch);
2080 mutex_unlock(&tb->lock);
2081}
2082
2083static void icm_complete(struct tb *tb)
2084{
2085 struct icm *icm = tb_priv(tb);
2086
2087 if (tb->nhi->going_away)
2088 return;
2089
2090 /*
2091 * If RTD3 was vetoed before we entered system suspend allow it
2092 * again now before driver ready is sent. Firmware sends a new RTD3
2093 * veto if it is still the case after we have sent it driver ready
2094 * command.
2095 */
2096 icm_veto_end(tb);
2097 icm_unplug_children(tb->root_switch);
2098
2099 /*
2100 * Now all existing children should be resumed, start events
2101 * from ICM to get updated status.
2102 */
2103 __icm_driver_ready(tb, NULL, NULL, NULL, NULL);
2104
2105 /*
2106 * We do not get notifications of devices that have been
2107 * unplugged during suspend so schedule rescan to clean them up
2108 * if any.
2109 */
2110 queue_delayed_work(tb->wq, &icm->rescan_work, msecs_to_jiffies(500));
2111}
2112
2113static int icm_runtime_suspend(struct tb *tb)
2114{
2115 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
2116 return 0;
2117}
2118
2119static int icm_runtime_suspend_switch(struct tb_switch *sw)
2120{
2121 if (tb_route(sw))
2122 reinit_completion(&sw->rpm_complete);
2123 return 0;
2124}
2125
2126static int icm_runtime_resume_switch(struct tb_switch *sw)
2127{
2128 if (tb_route(sw)) {
2129 if (!wait_for_completion_timeout(&sw->rpm_complete,
2130 msecs_to_jiffies(500))) {
2131 dev_dbg(&sw->dev, "runtime resuming timed out\n");
2132 }
2133 }
2134 return 0;
2135}
2136
2137static int icm_runtime_resume(struct tb *tb)
2138{
2139 /*
2140 * We can reuse the same resume functionality than with system
2141 * suspend.
2142 */
2143 icm_complete(tb);
2144 return 0;
2145}
2146
2147static int icm_start(struct tb *tb, bool not_used)
2148{
2149 struct icm *icm = tb_priv(tb);
2150 int ret;
2151
2152 if (icm->safe_mode)
2153 tb->root_switch = tb_switch_alloc_safe_mode(tb, &tb->dev, 0);
2154 else
2155 tb->root_switch = tb_switch_alloc(tb, &tb->dev, 0);
2156 if (IS_ERR(tb->root_switch))
2157 return PTR_ERR(tb->root_switch);
2158
2159 tb->root_switch->no_nvm_upgrade = !icm->can_upgrade_nvm;
2160 tb->root_switch->rpm = icm->rpm;
2161
2162 if (icm->set_uuid)
2163 icm->set_uuid(tb);
2164
2165 ret = tb_switch_add(tb->root_switch);
2166 if (ret) {
2167 tb_switch_put(tb->root_switch);
2168 tb->root_switch = NULL;
2169 }
2170
2171 return ret;
2172}
2173
2174static void icm_stop(struct tb *tb)
2175{
2176 struct icm *icm = tb_priv(tb);
2177
2178 cancel_delayed_work(&icm->rescan_work);
2179 tb_switch_remove(tb->root_switch);
2180 tb->root_switch = NULL;
2181 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
2182 kfree(icm->last_nvm_auth);
2183 icm->last_nvm_auth = NULL;
2184}
2185
2186static int icm_disconnect_pcie_paths(struct tb *tb)
2187{
2188 return nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DISCONNECT_PCIE_PATHS, 0);
2189}
2190
2191static void icm_usb4_switch_nvm_auth_complete(void *data)
2192{
2193 struct usb4_switch_nvm_auth *auth = data;
2194 struct icm *icm = auth->icm;
2195 struct tb *tb = icm_to_tb(icm);
2196
2197 tb_dbg(tb, "NVM_AUTH response for %llx flags %#x status %#x\n",
2198 get_route(auth->reply.route_hi, auth->reply.route_lo),
2199 auth->reply.hdr.flags, auth->reply.status);
2200
2201 mutex_lock(&tb->lock);
2202 if (WARN_ON(icm->last_nvm_auth))
2203 kfree(icm->last_nvm_auth);
2204 icm->last_nvm_auth = auth;
2205 mutex_unlock(&tb->lock);
2206}
2207
2208static int icm_usb4_switch_nvm_authenticate(struct tb *tb, u64 route)
2209{
2210 struct usb4_switch_nvm_auth *auth;
2211 struct icm *icm = tb_priv(tb);
2212 struct tb_cfg_request *req;
2213 int ret;
2214
2215 auth = kzalloc(sizeof(*auth), GFP_KERNEL);
2216 if (!auth)
2217 return -ENOMEM;
2218
2219 auth->icm = icm;
2220 auth->request.hdr.code = ICM_USB4_SWITCH_OP;
2221 auth->request.route_hi = upper_32_bits(route);
2222 auth->request.route_lo = lower_32_bits(route);
2223 auth->request.opcode = USB4_SWITCH_OP_NVM_AUTH;
2224
2225 req = tb_cfg_request_alloc();
2226 if (!req) {
2227 ret = -ENOMEM;
2228 goto err_free_auth;
2229 }
2230
2231 req->match = icm_match;
2232 req->copy = icm_copy;
2233 req->request = &auth->request;
2234 req->request_size = sizeof(auth->request);
2235 req->request_type = TB_CFG_PKG_ICM_CMD;
2236 req->response = &auth->reply;
2237 req->npackets = 1;
2238 req->response_size = sizeof(auth->reply);
2239 req->response_type = TB_CFG_PKG_ICM_RESP;
2240
2241 tb_dbg(tb, "NVM_AUTH request for %llx\n", route);
2242
2243 mutex_lock(&icm->request_lock);
2244 ret = tb_cfg_request(tb->ctl, req, icm_usb4_switch_nvm_auth_complete,
2245 auth);
2246 mutex_unlock(&icm->request_lock);
2247
2248 tb_cfg_request_put(req);
2249 if (ret)
2250 goto err_free_auth;
2251 return 0;
2252
2253err_free_auth:
2254 kfree(auth);
2255 return ret;
2256}
2257
2258static int icm_usb4_switch_op(struct tb_switch *sw, u16 opcode, u32 *metadata,
2259 u8 *status, const void *tx_data, size_t tx_data_len,
2260 void *rx_data, size_t rx_data_len)
2261{
2262 struct icm_usb4_switch_op_response reply;
2263 struct icm_usb4_switch_op request;
2264 struct tb *tb = sw->tb;
2265 struct icm *icm = tb_priv(tb);
2266 u64 route = tb_route(sw);
2267 int ret;
2268
2269 /*
2270 * USB4 router operation proxy is supported in firmware if the
2271 * protocol version is 3 or higher.
2272 */
2273 if (icm->proto_version < 3)
2274 return -EOPNOTSUPP;
2275
2276 /*
2277 * NVM_AUTH is a special USB4 proxy operation that does not
2278 * return immediately so handle it separately.
2279 */
2280 if (opcode == USB4_SWITCH_OP_NVM_AUTH)
2281 return icm_usb4_switch_nvm_authenticate(tb, route);
2282
2283 memset(&request, 0, sizeof(request));
2284 request.hdr.code = ICM_USB4_SWITCH_OP;
2285 request.route_hi = upper_32_bits(route);
2286 request.route_lo = lower_32_bits(route);
2287 request.opcode = opcode;
2288 if (metadata)
2289 request.metadata = *metadata;
2290
2291 if (tx_data_len) {
2292 request.data_len_valid |= ICM_USB4_SWITCH_DATA_VALID;
2293 if (tx_data_len < ARRAY_SIZE(request.data))
2294 request.data_len_valid =
2295 tx_data_len & ICM_USB4_SWITCH_DATA_LEN_MASK;
2296 memcpy(request.data, tx_data, tx_data_len * sizeof(u32));
2297 }
2298
2299 memset(&reply, 0, sizeof(reply));
2300 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
2301 1, ICM_RETRIES, ICM_TIMEOUT);
2302 if (ret)
2303 return ret;
2304
2305 if (reply.hdr.flags & ICM_FLAGS_ERROR)
2306 return -EIO;
2307
2308 if (status)
2309 *status = reply.status;
2310
2311 if (metadata)
2312 *metadata = reply.metadata;
2313
2314 if (rx_data_len)
2315 memcpy(rx_data, reply.data, rx_data_len * sizeof(u32));
2316
2317 return 0;
2318}
2319
2320static int icm_usb4_switch_nvm_authenticate_status(struct tb_switch *sw,
2321 u32 *status)
2322{
2323 struct usb4_switch_nvm_auth *auth;
2324 struct tb *tb = sw->tb;
2325 struct icm *icm = tb_priv(tb);
2326 int ret = 0;
2327
2328 if (icm->proto_version < 3)
2329 return -EOPNOTSUPP;
2330
2331 auth = icm->last_nvm_auth;
2332 icm->last_nvm_auth = NULL;
2333
2334 if (auth && auth->reply.route_hi == sw->config.route_hi &&
2335 auth->reply.route_lo == sw->config.route_lo) {
2336 tb_dbg(tb, "NVM_AUTH found for %llx flags %#x status %#x\n",
2337 tb_route(sw), auth->reply.hdr.flags, auth->reply.status);
2338 if (auth->reply.hdr.flags & ICM_FLAGS_ERROR)
2339 ret = -EIO;
2340 else
2341 *status = auth->reply.status;
2342 } else {
2343 *status = 0;
2344 }
2345
2346 kfree(auth);
2347 return ret;
2348}
2349
2350/* Falcon Ridge */
2351static const struct tb_cm_ops icm_fr_ops = {
2352 .driver_ready = icm_driver_ready,
2353 .start = icm_start,
2354 .stop = icm_stop,
2355 .suspend = icm_suspend,
2356 .complete = icm_complete,
2357 .handle_event = icm_handle_event,
2358 .approve_switch = icm_fr_approve_switch,
2359 .add_switch_key = icm_fr_add_switch_key,
2360 .challenge_switch_key = icm_fr_challenge_switch_key,
2361 .disconnect_pcie_paths = icm_disconnect_pcie_paths,
2362 .approve_xdomain_paths = icm_fr_approve_xdomain_paths,
2363 .disconnect_xdomain_paths = icm_fr_disconnect_xdomain_paths,
2364};
2365
2366/* Alpine Ridge */
2367static const struct tb_cm_ops icm_ar_ops = {
2368 .driver_ready = icm_driver_ready,
2369 .start = icm_start,
2370 .stop = icm_stop,
2371 .suspend = icm_suspend,
2372 .complete = icm_complete,
2373 .runtime_suspend = icm_runtime_suspend,
2374 .runtime_resume = icm_runtime_resume,
2375 .runtime_suspend_switch = icm_runtime_suspend_switch,
2376 .runtime_resume_switch = icm_runtime_resume_switch,
2377 .handle_event = icm_handle_event,
2378 .get_boot_acl = icm_ar_get_boot_acl,
2379 .set_boot_acl = icm_ar_set_boot_acl,
2380 .approve_switch = icm_fr_approve_switch,
2381 .add_switch_key = icm_fr_add_switch_key,
2382 .challenge_switch_key = icm_fr_challenge_switch_key,
2383 .disconnect_pcie_paths = icm_disconnect_pcie_paths,
2384 .approve_xdomain_paths = icm_fr_approve_xdomain_paths,
2385 .disconnect_xdomain_paths = icm_fr_disconnect_xdomain_paths,
2386};
2387
2388/* Titan Ridge */
2389static const struct tb_cm_ops icm_tr_ops = {
2390 .driver_ready = icm_driver_ready,
2391 .start = icm_start,
2392 .stop = icm_stop,
2393 .suspend = icm_suspend,
2394 .complete = icm_complete,
2395 .runtime_suspend = icm_runtime_suspend,
2396 .runtime_resume = icm_runtime_resume,
2397 .runtime_suspend_switch = icm_runtime_suspend_switch,
2398 .runtime_resume_switch = icm_runtime_resume_switch,
2399 .handle_event = icm_handle_event,
2400 .get_boot_acl = icm_ar_get_boot_acl,
2401 .set_boot_acl = icm_ar_set_boot_acl,
2402 .approve_switch = icm_tr_approve_switch,
2403 .add_switch_key = icm_tr_add_switch_key,
2404 .challenge_switch_key = icm_tr_challenge_switch_key,
2405 .disconnect_pcie_paths = icm_disconnect_pcie_paths,
2406 .approve_xdomain_paths = icm_tr_approve_xdomain_paths,
2407 .disconnect_xdomain_paths = icm_tr_disconnect_xdomain_paths,
2408 .usb4_switch_op = icm_usb4_switch_op,
2409 .usb4_switch_nvm_authenticate_status =
2410 icm_usb4_switch_nvm_authenticate_status,
2411};
2412
2413/* Ice Lake */
2414static const struct tb_cm_ops icm_icl_ops = {
2415 .driver_ready = icm_driver_ready,
2416 .start = icm_start,
2417 .stop = icm_stop,
2418 .complete = icm_complete,
2419 .runtime_suspend = icm_runtime_suspend,
2420 .runtime_resume = icm_runtime_resume,
2421 .handle_event = icm_handle_event,
2422 .approve_xdomain_paths = icm_tr_approve_xdomain_paths,
2423 .disconnect_xdomain_paths = icm_tr_disconnect_xdomain_paths,
2424 .usb4_switch_op = icm_usb4_switch_op,
2425 .usb4_switch_nvm_authenticate_status =
2426 icm_usb4_switch_nvm_authenticate_status,
2427};
2428
2429struct tb *icm_probe(struct tb_nhi *nhi)
2430{
2431 struct icm *icm;
2432 struct tb *tb;
2433
2434 tb = tb_domain_alloc(nhi, ICM_TIMEOUT, sizeof(struct icm));
2435 if (!tb)
2436 return NULL;
2437
2438 icm = tb_priv(tb);
2439 INIT_DELAYED_WORK(&icm->rescan_work, icm_rescan_work);
2440 mutex_init(&icm->request_lock);
2441
2442 switch (nhi->pdev->device) {
2443 case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI:
2444 case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI:
2445 icm->can_upgrade_nvm = true;
2446 icm->is_supported = icm_fr_is_supported;
2447 icm->get_route = icm_fr_get_route;
2448 icm->save_devices = icm_fr_save_devices;
2449 icm->driver_ready = icm_fr_driver_ready;
2450 icm->device_connected = icm_fr_device_connected;
2451 icm->device_disconnected = icm_fr_device_disconnected;
2452 icm->xdomain_connected = icm_fr_xdomain_connected;
2453 icm->xdomain_disconnected = icm_fr_xdomain_disconnected;
2454 tb->cm_ops = &icm_fr_ops;
2455 break;
2456
2457 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_NHI:
2458 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_NHI:
2459 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_NHI:
2460 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_NHI:
2461 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_NHI:
2462 icm->max_boot_acl = ICM_AR_PREBOOT_ACL_ENTRIES;
2463 /*
2464 * NVM upgrade has not been tested on Apple systems and
2465 * they don't provide images publicly either. To be on
2466 * the safe side prevent root switch NVM upgrade on Macs
2467 * for now.
2468 */
2469 icm->can_upgrade_nvm = !x86_apple_machine;
2470 icm->is_supported = icm_ar_is_supported;
2471 icm->cio_reset = icm_ar_cio_reset;
2472 icm->get_mode = icm_ar_get_mode;
2473 icm->get_route = icm_ar_get_route;
2474 icm->save_devices = icm_fr_save_devices;
2475 icm->driver_ready = icm_ar_driver_ready;
2476 icm->device_connected = icm_fr_device_connected;
2477 icm->device_disconnected = icm_fr_device_disconnected;
2478 icm->xdomain_connected = icm_fr_xdomain_connected;
2479 icm->xdomain_disconnected = icm_fr_xdomain_disconnected;
2480 tb->cm_ops = &icm_ar_ops;
2481 break;
2482
2483 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_NHI:
2484 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_NHI:
2485 icm->max_boot_acl = ICM_AR_PREBOOT_ACL_ENTRIES;
2486 icm->can_upgrade_nvm = !x86_apple_machine;
2487 icm->is_supported = icm_ar_is_supported;
2488 icm->cio_reset = icm_tr_cio_reset;
2489 icm->get_mode = icm_ar_get_mode;
2490 icm->driver_ready = icm_tr_driver_ready;
2491 icm->device_connected = icm_tr_device_connected;
2492 icm->device_disconnected = icm_tr_device_disconnected;
2493 icm->xdomain_connected = icm_tr_xdomain_connected;
2494 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2495 tb->cm_ops = &icm_tr_ops;
2496 break;
2497
2498 case PCI_DEVICE_ID_INTEL_ICL_NHI0:
2499 case PCI_DEVICE_ID_INTEL_ICL_NHI1:
2500 icm->is_supported = icm_fr_is_supported;
2501 icm->driver_ready = icm_icl_driver_ready;
2502 icm->set_uuid = icm_icl_set_uuid;
2503 icm->device_connected = icm_icl_device_connected;
2504 icm->device_disconnected = icm_tr_device_disconnected;
2505 icm->xdomain_connected = icm_tr_xdomain_connected;
2506 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2507 icm->rtd3_veto = icm_icl_rtd3_veto;
2508 tb->cm_ops = &icm_icl_ops;
2509 break;
2510
2511 case PCI_DEVICE_ID_INTEL_TGL_NHI0:
2512 case PCI_DEVICE_ID_INTEL_TGL_NHI1:
2513 case PCI_DEVICE_ID_INTEL_TGL_H_NHI0:
2514 case PCI_DEVICE_ID_INTEL_TGL_H_NHI1:
2515 case PCI_DEVICE_ID_INTEL_ADL_NHI0:
2516 case PCI_DEVICE_ID_INTEL_ADL_NHI1:
2517 case PCI_DEVICE_ID_INTEL_RPL_NHI0:
2518 case PCI_DEVICE_ID_INTEL_RPL_NHI1:
2519 case PCI_DEVICE_ID_INTEL_MTL_M_NHI0:
2520 case PCI_DEVICE_ID_INTEL_MTL_P_NHI0:
2521 case PCI_DEVICE_ID_INTEL_MTL_P_NHI1:
2522 icm->is_supported = icm_tgl_is_supported;
2523 icm->driver_ready = icm_icl_driver_ready;
2524 icm->set_uuid = icm_icl_set_uuid;
2525 icm->device_connected = icm_icl_device_connected;
2526 icm->device_disconnected = icm_tr_device_disconnected;
2527 icm->xdomain_connected = icm_tr_xdomain_connected;
2528 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2529 icm->rtd3_veto = icm_icl_rtd3_veto;
2530 tb->cm_ops = &icm_icl_ops;
2531 break;
2532
2533 case PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_2C_NHI:
2534 case PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_4C_NHI:
2535 icm->is_supported = icm_tgl_is_supported;
2536 icm->get_mode = icm_ar_get_mode;
2537 icm->driver_ready = icm_tr_driver_ready;
2538 icm->device_connected = icm_tr_device_connected;
2539 icm->device_disconnected = icm_tr_device_disconnected;
2540 icm->xdomain_connected = icm_tr_xdomain_connected;
2541 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2542 tb->cm_ops = &icm_tr_ops;
2543 break;
2544 }
2545
2546 if (!icm->is_supported || !icm->is_supported(tb)) {
2547 dev_dbg(&nhi->pdev->dev, "ICM not supported on this controller\n");
2548 tb_domain_put(tb);
2549 return NULL;
2550 }
2551
2552 tb_dbg(tb, "using firmware connection manager\n");
2553
2554 return tb;
2555}