Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Internal Thunderbolt Connection Manager. This is a firmware running on
4 * the Thunderbolt host controller performing most of the low-level
5 * handling.
6 *
7 * Copyright (C) 2017, Intel Corporation
8 * Authors: Michael Jamet <michael.jamet@intel.com>
9 * Mika Westerberg <mika.westerberg@linux.intel.com>
10 */
11
12#include <linux/delay.h>
13#include <linux/mutex.h>
14#include <linux/moduleparam.h>
15#include <linux/pci.h>
16#include <linux/pm_runtime.h>
17#include <linux/platform_data/x86/apple.h>
18#include <linux/sizes.h>
19#include <linux/slab.h>
20#include <linux/workqueue.h>
21
22#include "ctl.h"
23#include "nhi_regs.h"
24#include "tb.h"
25
26#define PCIE2CIO_CMD 0x30
27#define PCIE2CIO_CMD_TIMEOUT BIT(31)
28#define PCIE2CIO_CMD_START BIT(30)
29#define PCIE2CIO_CMD_WRITE BIT(21)
30#define PCIE2CIO_CMD_CS_MASK GENMASK(20, 19)
31#define PCIE2CIO_CMD_CS_SHIFT 19
32#define PCIE2CIO_CMD_PORT_MASK GENMASK(18, 13)
33#define PCIE2CIO_CMD_PORT_SHIFT 13
34
35#define PCIE2CIO_WRDATA 0x34
36#define PCIE2CIO_RDDATA 0x38
37
38#define PHY_PORT_CS1 0x37
39#define PHY_PORT_CS1_LINK_DISABLE BIT(14)
40#define PHY_PORT_CS1_LINK_STATE_MASK GENMASK(29, 26)
41#define PHY_PORT_CS1_LINK_STATE_SHIFT 26
42
43#define ICM_TIMEOUT 5000 /* ms */
44#define ICM_APPROVE_TIMEOUT 10000 /* ms */
45#define ICM_MAX_LINK 4
46
47static bool start_icm;
48module_param(start_icm, bool, 0444);
49MODULE_PARM_DESC(start_icm, "start ICM firmware if it is not running (default: false)");
50
51/**
52 * struct usb4_switch_nvm_auth - Holds USB4 NVM_AUTH status
53 * @reply: Reply from ICM firmware is placed here
54 * @request: Request that is sent to ICM firmware
55 * @icm: Pointer to ICM private data
56 */
57struct usb4_switch_nvm_auth {
58 struct icm_usb4_switch_op_response reply;
59 struct icm_usb4_switch_op request;
60 struct icm *icm;
61};
62
63/**
64 * struct icm - Internal connection manager private data
65 * @request_lock: Makes sure only one message is send to ICM at time
66 * @rescan_work: Work used to rescan the surviving switches after resume
67 * @upstream_port: Pointer to the PCIe upstream port this host
68 * controller is connected. This is only set for systems
69 * where ICM needs to be started manually
70 * @vnd_cap: Vendor defined capability where PCIe2CIO mailbox resides
71 * (only set when @upstream_port is not %NULL)
72 * @safe_mode: ICM is in safe mode
73 * @max_boot_acl: Maximum number of preboot ACL entries (%0 if not supported)
74 * @rpm: Does the controller support runtime PM (RTD3)
75 * @can_upgrade_nvm: Can the NVM firmware be upgrade on this controller
76 * @proto_version: Firmware protocol version
77 * @last_nvm_auth: Last USB4 router NVM_AUTH result (or %NULL if not set)
78 * @veto: Is RTD3 veto in effect
79 * @is_supported: Checks if we can support ICM on this controller
80 * @cio_reset: Trigger CIO reset
81 * @get_mode: Read and return the ICM firmware mode (optional)
82 * @get_route: Find a route string for given switch
83 * @save_devices: Ask ICM to save devices to ACL when suspending (optional)
84 * @driver_ready: Send driver ready message to ICM
85 * @set_uuid: Set UUID for the root switch (optional)
86 * @device_connected: Handle device connected ICM message
87 * @device_disconnected: Handle device disconnected ICM message
88 * @xdomain_connected: Handle XDomain connected ICM message
89 * @xdomain_disconnected: Handle XDomain disconnected ICM message
90 * @rtd3_veto: Handle RTD3 veto notification ICM message
91 */
92struct icm {
93 struct mutex request_lock;
94 struct delayed_work rescan_work;
95 struct pci_dev *upstream_port;
96 int vnd_cap;
97 bool safe_mode;
98 size_t max_boot_acl;
99 bool rpm;
100 bool can_upgrade_nvm;
101 u8 proto_version;
102 struct usb4_switch_nvm_auth *last_nvm_auth;
103 bool veto;
104 bool (*is_supported)(struct tb *tb);
105 int (*cio_reset)(struct tb *tb);
106 int (*get_mode)(struct tb *tb);
107 int (*get_route)(struct tb *tb, u8 link, u8 depth, u64 *route);
108 void (*save_devices)(struct tb *tb);
109 int (*driver_ready)(struct tb *tb,
110 enum tb_security_level *security_level,
111 u8 *proto_version, size_t *nboot_acl, bool *rpm);
112 void (*set_uuid)(struct tb *tb);
113 void (*device_connected)(struct tb *tb,
114 const struct icm_pkg_header *hdr);
115 void (*device_disconnected)(struct tb *tb,
116 const struct icm_pkg_header *hdr);
117 void (*xdomain_connected)(struct tb *tb,
118 const struct icm_pkg_header *hdr);
119 void (*xdomain_disconnected)(struct tb *tb,
120 const struct icm_pkg_header *hdr);
121 void (*rtd3_veto)(struct tb *tb, const struct icm_pkg_header *hdr);
122};
123
124struct icm_notification {
125 struct work_struct work;
126 struct icm_pkg_header *pkg;
127 struct tb *tb;
128};
129
130struct ep_name_entry {
131 u8 len;
132 u8 type;
133 u8 data[];
134};
135
136#define EP_NAME_INTEL_VSS 0x10
137
138/* Intel Vendor specific structure */
139struct intel_vss {
140 u16 vendor;
141 u16 model;
142 u8 mc;
143 u8 flags;
144 u16 pci_devid;
145 u32 nvm_version;
146};
147
148#define INTEL_VSS_FLAGS_RTD3 BIT(0)
149
150static const struct intel_vss *parse_intel_vss(const void *ep_name, size_t size)
151{
152 const void *end = ep_name + size;
153
154 while (ep_name < end) {
155 const struct ep_name_entry *ep = ep_name;
156
157 if (!ep->len)
158 break;
159 if (ep_name + ep->len > end)
160 break;
161
162 if (ep->type == EP_NAME_INTEL_VSS)
163 return (const struct intel_vss *)ep->data;
164
165 ep_name += ep->len;
166 }
167
168 return NULL;
169}
170
171static bool intel_vss_is_rtd3(const void *ep_name, size_t size)
172{
173 const struct intel_vss *vss;
174
175 vss = parse_intel_vss(ep_name, size);
176 if (vss)
177 return !!(vss->flags & INTEL_VSS_FLAGS_RTD3);
178
179 return false;
180}
181
182static inline struct tb *icm_to_tb(struct icm *icm)
183{
184 return ((void *)icm - sizeof(struct tb));
185}
186
187static inline u8 phy_port_from_route(u64 route, u8 depth)
188{
189 u8 link;
190
191 link = depth ? route >> ((depth - 1) * 8) : route;
192 return tb_phy_port_from_link(link);
193}
194
195static inline u8 dual_link_from_link(u8 link)
196{
197 return link ? ((link - 1) ^ 0x01) + 1 : 0;
198}
199
200static inline u64 get_route(u32 route_hi, u32 route_lo)
201{
202 return (u64)route_hi << 32 | route_lo;
203}
204
205static inline u64 get_parent_route(u64 route)
206{
207 int depth = tb_route_length(route);
208 return depth ? route & ~(0xffULL << (depth - 1) * TB_ROUTE_SHIFT) : 0;
209}
210
211static int pci2cio_wait_completion(struct icm *icm, unsigned long timeout_msec)
212{
213 unsigned long end = jiffies + msecs_to_jiffies(timeout_msec);
214 u32 cmd;
215
216 do {
217 pci_read_config_dword(icm->upstream_port,
218 icm->vnd_cap + PCIE2CIO_CMD, &cmd);
219 if (!(cmd & PCIE2CIO_CMD_START)) {
220 if (cmd & PCIE2CIO_CMD_TIMEOUT)
221 break;
222 return 0;
223 }
224
225 msleep(50);
226 } while (time_before(jiffies, end));
227
228 return -ETIMEDOUT;
229}
230
231static int pcie2cio_read(struct icm *icm, enum tb_cfg_space cs,
232 unsigned int port, unsigned int index, u32 *data)
233{
234 struct pci_dev *pdev = icm->upstream_port;
235 int ret, vnd_cap = icm->vnd_cap;
236 u32 cmd;
237
238 cmd = index;
239 cmd |= (port << PCIE2CIO_CMD_PORT_SHIFT) & PCIE2CIO_CMD_PORT_MASK;
240 cmd |= (cs << PCIE2CIO_CMD_CS_SHIFT) & PCIE2CIO_CMD_CS_MASK;
241 cmd |= PCIE2CIO_CMD_START;
242 pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_CMD, cmd);
243
244 ret = pci2cio_wait_completion(icm, 5000);
245 if (ret)
246 return ret;
247
248 pci_read_config_dword(pdev, vnd_cap + PCIE2CIO_RDDATA, data);
249 return 0;
250}
251
252static int pcie2cio_write(struct icm *icm, enum tb_cfg_space cs,
253 unsigned int port, unsigned int index, u32 data)
254{
255 struct pci_dev *pdev = icm->upstream_port;
256 int vnd_cap = icm->vnd_cap;
257 u32 cmd;
258
259 pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_WRDATA, data);
260
261 cmd = index;
262 cmd |= (port << PCIE2CIO_CMD_PORT_SHIFT) & PCIE2CIO_CMD_PORT_MASK;
263 cmd |= (cs << PCIE2CIO_CMD_CS_SHIFT) & PCIE2CIO_CMD_CS_MASK;
264 cmd |= PCIE2CIO_CMD_WRITE | PCIE2CIO_CMD_START;
265 pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_CMD, cmd);
266
267 return pci2cio_wait_completion(icm, 5000);
268}
269
270static bool icm_match(const struct tb_cfg_request *req,
271 const struct ctl_pkg *pkg)
272{
273 const struct icm_pkg_header *res_hdr = pkg->buffer;
274 const struct icm_pkg_header *req_hdr = req->request;
275
276 if (pkg->frame.eof != req->response_type)
277 return false;
278 if (res_hdr->code != req_hdr->code)
279 return false;
280
281 return true;
282}
283
284static bool icm_copy(struct tb_cfg_request *req, const struct ctl_pkg *pkg)
285{
286 const struct icm_pkg_header *hdr = pkg->buffer;
287
288 if (hdr->packet_id < req->npackets) {
289 size_t offset = hdr->packet_id * req->response_size;
290
291 memcpy(req->response + offset, pkg->buffer, req->response_size);
292 }
293
294 return hdr->packet_id == hdr->total_packets - 1;
295}
296
297static int icm_request(struct tb *tb, const void *request, size_t request_size,
298 void *response, size_t response_size, size_t npackets,
299 unsigned int timeout_msec)
300{
301 struct icm *icm = tb_priv(tb);
302 int retries = 3;
303
304 do {
305 struct tb_cfg_request *req;
306 struct tb_cfg_result res;
307
308 req = tb_cfg_request_alloc();
309 if (!req)
310 return -ENOMEM;
311
312 req->match = icm_match;
313 req->copy = icm_copy;
314 req->request = request;
315 req->request_size = request_size;
316 req->request_type = TB_CFG_PKG_ICM_CMD;
317 req->response = response;
318 req->npackets = npackets;
319 req->response_size = response_size;
320 req->response_type = TB_CFG_PKG_ICM_RESP;
321
322 mutex_lock(&icm->request_lock);
323 res = tb_cfg_request_sync(tb->ctl, req, timeout_msec);
324 mutex_unlock(&icm->request_lock);
325
326 tb_cfg_request_put(req);
327
328 if (res.err != -ETIMEDOUT)
329 return res.err == 1 ? -EIO : res.err;
330
331 usleep_range(20, 50);
332 } while (retries--);
333
334 return -ETIMEDOUT;
335}
336
337/*
338 * If rescan is queued to run (we are resuming), postpone it to give the
339 * firmware some more time to send device connected notifications for next
340 * devices in the chain.
341 */
342static void icm_postpone_rescan(struct tb *tb)
343{
344 struct icm *icm = tb_priv(tb);
345
346 if (delayed_work_pending(&icm->rescan_work))
347 mod_delayed_work(tb->wq, &icm->rescan_work,
348 msecs_to_jiffies(500));
349}
350
351static void icm_veto_begin(struct tb *tb)
352{
353 struct icm *icm = tb_priv(tb);
354
355 if (!icm->veto) {
356 icm->veto = true;
357 /* Keep the domain powered while veto is in effect */
358 pm_runtime_get(&tb->dev);
359 }
360}
361
362static void icm_veto_end(struct tb *tb)
363{
364 struct icm *icm = tb_priv(tb);
365
366 if (icm->veto) {
367 icm->veto = false;
368 /* Allow the domain suspend now */
369 pm_runtime_mark_last_busy(&tb->dev);
370 pm_runtime_put_autosuspend(&tb->dev);
371 }
372}
373
374static bool icm_firmware_running(const struct tb_nhi *nhi)
375{
376 u32 val;
377
378 val = ioread32(nhi->iobase + REG_FW_STS);
379 return !!(val & REG_FW_STS_ICM_EN);
380}
381
382static bool icm_fr_is_supported(struct tb *tb)
383{
384 return !x86_apple_machine;
385}
386
387static inline int icm_fr_get_switch_index(u32 port)
388{
389 int index;
390
391 if ((port & ICM_PORT_TYPE_MASK) != TB_TYPE_PORT)
392 return 0;
393
394 index = port >> ICM_PORT_INDEX_SHIFT;
395 return index != 0xff ? index : 0;
396}
397
398static int icm_fr_get_route(struct tb *tb, u8 link, u8 depth, u64 *route)
399{
400 struct icm_fr_pkg_get_topology_response *switches, *sw;
401 struct icm_fr_pkg_get_topology request = {
402 .hdr = { .code = ICM_GET_TOPOLOGY },
403 };
404 size_t npackets = ICM_GET_TOPOLOGY_PACKETS;
405 int ret, index;
406 u8 i;
407
408 switches = kcalloc(npackets, sizeof(*switches), GFP_KERNEL);
409 if (!switches)
410 return -ENOMEM;
411
412 ret = icm_request(tb, &request, sizeof(request), switches,
413 sizeof(*switches), npackets, ICM_TIMEOUT);
414 if (ret)
415 goto err_free;
416
417 sw = &switches[0];
418 index = icm_fr_get_switch_index(sw->ports[link]);
419 if (!index) {
420 ret = -ENODEV;
421 goto err_free;
422 }
423
424 sw = &switches[index];
425 for (i = 1; i < depth; i++) {
426 unsigned int j;
427
428 if (!(sw->first_data & ICM_SWITCH_USED)) {
429 ret = -ENODEV;
430 goto err_free;
431 }
432
433 for (j = 0; j < ARRAY_SIZE(sw->ports); j++) {
434 index = icm_fr_get_switch_index(sw->ports[j]);
435 if (index > sw->switch_index) {
436 sw = &switches[index];
437 break;
438 }
439 }
440 }
441
442 *route = get_route(sw->route_hi, sw->route_lo);
443
444err_free:
445 kfree(switches);
446 return ret;
447}
448
449static void icm_fr_save_devices(struct tb *tb)
450{
451 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_SAVE_DEVS, 0);
452}
453
454static int
455icm_fr_driver_ready(struct tb *tb, enum tb_security_level *security_level,
456 u8 *proto_version, size_t *nboot_acl, bool *rpm)
457{
458 struct icm_fr_pkg_driver_ready_response reply;
459 struct icm_pkg_driver_ready request = {
460 .hdr.code = ICM_DRIVER_READY,
461 };
462 int ret;
463
464 memset(&reply, 0, sizeof(reply));
465 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
466 1, ICM_TIMEOUT);
467 if (ret)
468 return ret;
469
470 if (security_level)
471 *security_level = reply.security_level & ICM_FR_SLEVEL_MASK;
472
473 return 0;
474}
475
476static int icm_fr_approve_switch(struct tb *tb, struct tb_switch *sw)
477{
478 struct icm_fr_pkg_approve_device request;
479 struct icm_fr_pkg_approve_device reply;
480 int ret;
481
482 memset(&request, 0, sizeof(request));
483 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
484 request.hdr.code = ICM_APPROVE_DEVICE;
485 request.connection_id = sw->connection_id;
486 request.connection_key = sw->connection_key;
487
488 memset(&reply, 0, sizeof(reply));
489 /* Use larger timeout as establishing tunnels can take some time */
490 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
491 1, ICM_APPROVE_TIMEOUT);
492 if (ret)
493 return ret;
494
495 if (reply.hdr.flags & ICM_FLAGS_ERROR) {
496 tb_warn(tb, "PCIe tunnel creation failed\n");
497 return -EIO;
498 }
499
500 return 0;
501}
502
503static int icm_fr_add_switch_key(struct tb *tb, struct tb_switch *sw)
504{
505 struct icm_fr_pkg_add_device_key request;
506 struct icm_fr_pkg_add_device_key_response reply;
507 int ret;
508
509 memset(&request, 0, sizeof(request));
510 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
511 request.hdr.code = ICM_ADD_DEVICE_KEY;
512 request.connection_id = sw->connection_id;
513 request.connection_key = sw->connection_key;
514 memcpy(request.key, sw->key, TB_SWITCH_KEY_SIZE);
515
516 memset(&reply, 0, sizeof(reply));
517 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
518 1, ICM_TIMEOUT);
519 if (ret)
520 return ret;
521
522 if (reply.hdr.flags & ICM_FLAGS_ERROR) {
523 tb_warn(tb, "Adding key to switch failed\n");
524 return -EIO;
525 }
526
527 return 0;
528}
529
530static int icm_fr_challenge_switch_key(struct tb *tb, struct tb_switch *sw,
531 const u8 *challenge, u8 *response)
532{
533 struct icm_fr_pkg_challenge_device request;
534 struct icm_fr_pkg_challenge_device_response reply;
535 int ret;
536
537 memset(&request, 0, sizeof(request));
538 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
539 request.hdr.code = ICM_CHALLENGE_DEVICE;
540 request.connection_id = sw->connection_id;
541 request.connection_key = sw->connection_key;
542 memcpy(request.challenge, challenge, TB_SWITCH_KEY_SIZE);
543
544 memset(&reply, 0, sizeof(reply));
545 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
546 1, ICM_TIMEOUT);
547 if (ret)
548 return ret;
549
550 if (reply.hdr.flags & ICM_FLAGS_ERROR)
551 return -EKEYREJECTED;
552 if (reply.hdr.flags & ICM_FLAGS_NO_KEY)
553 return -ENOKEY;
554
555 memcpy(response, reply.response, TB_SWITCH_KEY_SIZE);
556
557 return 0;
558}
559
560static int icm_fr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
561 int transmit_path, int transmit_ring,
562 int receive_path, int receive_ring)
563{
564 struct icm_fr_pkg_approve_xdomain_response reply;
565 struct icm_fr_pkg_approve_xdomain request;
566 int ret;
567
568 memset(&request, 0, sizeof(request));
569 request.hdr.code = ICM_APPROVE_XDOMAIN;
570 request.link_info = xd->depth << ICM_LINK_INFO_DEPTH_SHIFT | xd->link;
571 memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
572
573 request.transmit_path = transmit_path;
574 request.transmit_ring = transmit_ring;
575 request.receive_path = receive_path;
576 request.receive_ring = receive_ring;
577
578 memset(&reply, 0, sizeof(reply));
579 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
580 1, ICM_TIMEOUT);
581 if (ret)
582 return ret;
583
584 if (reply.hdr.flags & ICM_FLAGS_ERROR)
585 return -EIO;
586
587 return 0;
588}
589
590static int icm_fr_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
591 int transmit_path, int transmit_ring,
592 int receive_path, int receive_ring)
593{
594 u8 phy_port;
595 u8 cmd;
596
597 phy_port = tb_phy_port_from_link(xd->link);
598 if (phy_port == 0)
599 cmd = NHI_MAILBOX_DISCONNECT_PA;
600 else
601 cmd = NHI_MAILBOX_DISCONNECT_PB;
602
603 nhi_mailbox_cmd(tb->nhi, cmd, 1);
604 usleep_range(10, 50);
605 nhi_mailbox_cmd(tb->nhi, cmd, 2);
606 return 0;
607}
608
609static struct tb_switch *alloc_switch(struct tb_switch *parent_sw, u64 route,
610 const uuid_t *uuid)
611{
612 struct tb *tb = parent_sw->tb;
613 struct tb_switch *sw;
614
615 sw = tb_switch_alloc(tb, &parent_sw->dev, route);
616 if (IS_ERR(sw)) {
617 tb_warn(tb, "failed to allocate switch at %llx\n", route);
618 return sw;
619 }
620
621 sw->uuid = kmemdup(uuid, sizeof(*uuid), GFP_KERNEL);
622 if (!sw->uuid) {
623 tb_switch_put(sw);
624 return ERR_PTR(-ENOMEM);
625 }
626
627 init_completion(&sw->rpm_complete);
628 return sw;
629}
630
631static int add_switch(struct tb_switch *parent_sw, struct tb_switch *sw)
632{
633 u64 route = tb_route(sw);
634 int ret;
635
636 /* Link the two switches now */
637 tb_port_at(route, parent_sw)->remote = tb_upstream_port(sw);
638 tb_upstream_port(sw)->remote = tb_port_at(route, parent_sw);
639
640 ret = tb_switch_add(sw);
641 if (ret)
642 tb_port_at(tb_route(sw), parent_sw)->remote = NULL;
643
644 return ret;
645}
646
647static void update_switch(struct tb_switch *parent_sw, struct tb_switch *sw,
648 u64 route, u8 connection_id, u8 connection_key,
649 u8 link, u8 depth, bool boot)
650{
651 /* Disconnect from parent */
652 tb_port_at(tb_route(sw), parent_sw)->remote = NULL;
653 /* Re-connect via updated port*/
654 tb_port_at(route, parent_sw)->remote = tb_upstream_port(sw);
655
656 /* Update with the new addressing information */
657 sw->config.route_hi = upper_32_bits(route);
658 sw->config.route_lo = lower_32_bits(route);
659 sw->connection_id = connection_id;
660 sw->connection_key = connection_key;
661 sw->link = link;
662 sw->depth = depth;
663 sw->boot = boot;
664
665 /* This switch still exists */
666 sw->is_unplugged = false;
667
668 /* Runtime resume is now complete */
669 complete(&sw->rpm_complete);
670}
671
672static void remove_switch(struct tb_switch *sw)
673{
674 struct tb_switch *parent_sw;
675
676 parent_sw = tb_to_switch(sw->dev.parent);
677 tb_port_at(tb_route(sw), parent_sw)->remote = NULL;
678 tb_switch_remove(sw);
679}
680
681static void add_xdomain(struct tb_switch *sw, u64 route,
682 const uuid_t *local_uuid, const uuid_t *remote_uuid,
683 u8 link, u8 depth)
684{
685 struct tb_xdomain *xd;
686
687 pm_runtime_get_sync(&sw->dev);
688
689 xd = tb_xdomain_alloc(sw->tb, &sw->dev, route, local_uuid, remote_uuid);
690 if (!xd)
691 goto out;
692
693 xd->link = link;
694 xd->depth = depth;
695
696 tb_port_at(route, sw)->xdomain = xd;
697
698 tb_xdomain_add(xd);
699
700out:
701 pm_runtime_mark_last_busy(&sw->dev);
702 pm_runtime_put_autosuspend(&sw->dev);
703}
704
705static void update_xdomain(struct tb_xdomain *xd, u64 route, u8 link)
706{
707 xd->link = link;
708 xd->route = route;
709 xd->is_unplugged = false;
710}
711
712static void remove_xdomain(struct tb_xdomain *xd)
713{
714 struct tb_switch *sw;
715
716 sw = tb_to_switch(xd->dev.parent);
717 tb_port_at(xd->route, sw)->xdomain = NULL;
718 tb_xdomain_remove(xd);
719}
720
721static void
722icm_fr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
723{
724 const struct icm_fr_event_device_connected *pkg =
725 (const struct icm_fr_event_device_connected *)hdr;
726 enum tb_security_level security_level;
727 struct tb_switch *sw, *parent_sw;
728 bool boot, dual_lane, speed_gen3;
729 struct icm *icm = tb_priv(tb);
730 bool authorized = false;
731 struct tb_xdomain *xd;
732 u8 link, depth;
733 u64 route;
734 int ret;
735
736 icm_postpone_rescan(tb);
737
738 link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
739 depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
740 ICM_LINK_INFO_DEPTH_SHIFT;
741 authorized = pkg->link_info & ICM_LINK_INFO_APPROVED;
742 security_level = (pkg->hdr.flags & ICM_FLAGS_SLEVEL_MASK) >>
743 ICM_FLAGS_SLEVEL_SHIFT;
744 boot = pkg->link_info & ICM_LINK_INFO_BOOT;
745 dual_lane = pkg->hdr.flags & ICM_FLAGS_DUAL_LANE;
746 speed_gen3 = pkg->hdr.flags & ICM_FLAGS_SPEED_GEN3;
747
748 if (pkg->link_info & ICM_LINK_INFO_REJECTED) {
749 tb_info(tb, "switch at %u.%u was rejected by ICM firmware because topology limit exceeded\n",
750 link, depth);
751 return;
752 }
753
754 sw = tb_switch_find_by_uuid(tb, &pkg->ep_uuid);
755 if (sw) {
756 u8 phy_port, sw_phy_port;
757
758 parent_sw = tb_to_switch(sw->dev.parent);
759 sw_phy_port = tb_phy_port_from_link(sw->link);
760 phy_port = tb_phy_port_from_link(link);
761
762 /*
763 * On resume ICM will send us connected events for the
764 * devices that still are present. However, that
765 * information might have changed for example by the
766 * fact that a switch on a dual-link connection might
767 * have been enumerated using the other link now. Make
768 * sure our book keeping matches that.
769 */
770 if (sw->depth == depth && sw_phy_port == phy_port &&
771 !!sw->authorized == authorized) {
772 /*
773 * It was enumerated through another link so update
774 * route string accordingly.
775 */
776 if (sw->link != link) {
777 ret = icm->get_route(tb, link, depth, &route);
778 if (ret) {
779 tb_err(tb, "failed to update route string for switch at %u.%u\n",
780 link, depth);
781 tb_switch_put(sw);
782 return;
783 }
784 } else {
785 route = tb_route(sw);
786 }
787
788 update_switch(parent_sw, sw, route, pkg->connection_id,
789 pkg->connection_key, link, depth, boot);
790 tb_switch_put(sw);
791 return;
792 }
793
794 /*
795 * User connected the same switch to another physical
796 * port or to another part of the topology. Remove the
797 * existing switch now before adding the new one.
798 */
799 remove_switch(sw);
800 tb_switch_put(sw);
801 }
802
803 /*
804 * If the switch was not found by UUID, look for a switch on
805 * same physical port (taking possible link aggregation into
806 * account) and depth. If we found one it is definitely a stale
807 * one so remove it first.
808 */
809 sw = tb_switch_find_by_link_depth(tb, link, depth);
810 if (!sw) {
811 u8 dual_link;
812
813 dual_link = dual_link_from_link(link);
814 if (dual_link)
815 sw = tb_switch_find_by_link_depth(tb, dual_link, depth);
816 }
817 if (sw) {
818 remove_switch(sw);
819 tb_switch_put(sw);
820 }
821
822 /* Remove existing XDomain connection if found */
823 xd = tb_xdomain_find_by_link_depth(tb, link, depth);
824 if (xd) {
825 remove_xdomain(xd);
826 tb_xdomain_put(xd);
827 }
828
829 parent_sw = tb_switch_find_by_link_depth(tb, link, depth - 1);
830 if (!parent_sw) {
831 tb_err(tb, "failed to find parent switch for %u.%u\n",
832 link, depth);
833 return;
834 }
835
836 ret = icm->get_route(tb, link, depth, &route);
837 if (ret) {
838 tb_err(tb, "failed to find route string for switch at %u.%u\n",
839 link, depth);
840 tb_switch_put(parent_sw);
841 return;
842 }
843
844 pm_runtime_get_sync(&parent_sw->dev);
845
846 sw = alloc_switch(parent_sw, route, &pkg->ep_uuid);
847 if (!IS_ERR(sw)) {
848 sw->connection_id = pkg->connection_id;
849 sw->connection_key = pkg->connection_key;
850 sw->link = link;
851 sw->depth = depth;
852 sw->authorized = authorized;
853 sw->security_level = security_level;
854 sw->boot = boot;
855 sw->link_speed = speed_gen3 ? 20 : 10;
856 sw->link_width = dual_lane ? 2 : 1;
857 sw->rpm = intel_vss_is_rtd3(pkg->ep_name, sizeof(pkg->ep_name));
858
859 if (add_switch(parent_sw, sw))
860 tb_switch_put(sw);
861 }
862
863 pm_runtime_mark_last_busy(&parent_sw->dev);
864 pm_runtime_put_autosuspend(&parent_sw->dev);
865
866 tb_switch_put(parent_sw);
867}
868
869static void
870icm_fr_device_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
871{
872 const struct icm_fr_event_device_disconnected *pkg =
873 (const struct icm_fr_event_device_disconnected *)hdr;
874 struct tb_switch *sw;
875 u8 link, depth;
876
877 link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
878 depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
879 ICM_LINK_INFO_DEPTH_SHIFT;
880
881 if (link > ICM_MAX_LINK || depth > TB_SWITCH_MAX_DEPTH) {
882 tb_warn(tb, "invalid topology %u.%u, ignoring\n", link, depth);
883 return;
884 }
885
886 sw = tb_switch_find_by_link_depth(tb, link, depth);
887 if (!sw) {
888 tb_warn(tb, "no switch exists at %u.%u, ignoring\n", link,
889 depth);
890 return;
891 }
892
893 pm_runtime_get_sync(sw->dev.parent);
894
895 remove_switch(sw);
896
897 pm_runtime_mark_last_busy(sw->dev.parent);
898 pm_runtime_put_autosuspend(sw->dev.parent);
899
900 tb_switch_put(sw);
901}
902
903static void
904icm_fr_xdomain_connected(struct tb *tb, const struct icm_pkg_header *hdr)
905{
906 const struct icm_fr_event_xdomain_connected *pkg =
907 (const struct icm_fr_event_xdomain_connected *)hdr;
908 struct tb_xdomain *xd;
909 struct tb_switch *sw;
910 u8 link, depth;
911 u64 route;
912
913 link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
914 depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
915 ICM_LINK_INFO_DEPTH_SHIFT;
916
917 if (link > ICM_MAX_LINK || depth > TB_SWITCH_MAX_DEPTH) {
918 tb_warn(tb, "invalid topology %u.%u, ignoring\n", link, depth);
919 return;
920 }
921
922 route = get_route(pkg->local_route_hi, pkg->local_route_lo);
923
924 xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
925 if (xd) {
926 u8 xd_phy_port, phy_port;
927
928 xd_phy_port = phy_port_from_route(xd->route, xd->depth);
929 phy_port = phy_port_from_route(route, depth);
930
931 if (xd->depth == depth && xd_phy_port == phy_port) {
932 update_xdomain(xd, route, link);
933 tb_xdomain_put(xd);
934 return;
935 }
936
937 /*
938 * If we find an existing XDomain connection remove it
939 * now. We need to go through login handshake and
940 * everything anyway to be able to re-establish the
941 * connection.
942 */
943 remove_xdomain(xd);
944 tb_xdomain_put(xd);
945 }
946
947 /*
948 * Look if there already exists an XDomain in the same place
949 * than the new one and in that case remove it because it is
950 * most likely another host that got disconnected.
951 */
952 xd = tb_xdomain_find_by_link_depth(tb, link, depth);
953 if (!xd) {
954 u8 dual_link;
955
956 dual_link = dual_link_from_link(link);
957 if (dual_link)
958 xd = tb_xdomain_find_by_link_depth(tb, dual_link,
959 depth);
960 }
961 if (xd) {
962 remove_xdomain(xd);
963 tb_xdomain_put(xd);
964 }
965
966 /*
967 * If the user disconnected a switch during suspend and
968 * connected another host to the same port, remove the switch
969 * first.
970 */
971 sw = tb_switch_find_by_route(tb, route);
972 if (sw) {
973 remove_switch(sw);
974 tb_switch_put(sw);
975 }
976
977 sw = tb_switch_find_by_link_depth(tb, link, depth);
978 if (!sw) {
979 tb_warn(tb, "no switch exists at %u.%u, ignoring\n", link,
980 depth);
981 return;
982 }
983
984 add_xdomain(sw, route, &pkg->local_uuid, &pkg->remote_uuid, link,
985 depth);
986 tb_switch_put(sw);
987}
988
989static void
990icm_fr_xdomain_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
991{
992 const struct icm_fr_event_xdomain_disconnected *pkg =
993 (const struct icm_fr_event_xdomain_disconnected *)hdr;
994 struct tb_xdomain *xd;
995
996 /*
997 * If the connection is through one or multiple devices, the
998 * XDomain device is removed along with them so it is fine if we
999 * cannot find it here.
1000 */
1001 xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
1002 if (xd) {
1003 remove_xdomain(xd);
1004 tb_xdomain_put(xd);
1005 }
1006}
1007
1008static int icm_tr_cio_reset(struct tb *tb)
1009{
1010 return pcie2cio_write(tb_priv(tb), TB_CFG_SWITCH, 0, 0x777, BIT(1));
1011}
1012
1013static int
1014icm_tr_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1015 u8 *proto_version, size_t *nboot_acl, bool *rpm)
1016{
1017 struct icm_tr_pkg_driver_ready_response reply;
1018 struct icm_pkg_driver_ready request = {
1019 .hdr.code = ICM_DRIVER_READY,
1020 };
1021 int ret;
1022
1023 memset(&reply, 0, sizeof(reply));
1024 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1025 1, 20000);
1026 if (ret)
1027 return ret;
1028
1029 if (security_level)
1030 *security_level = reply.info & ICM_TR_INFO_SLEVEL_MASK;
1031 if (proto_version)
1032 *proto_version = (reply.info & ICM_TR_INFO_PROTO_VERSION_MASK) >>
1033 ICM_TR_INFO_PROTO_VERSION_SHIFT;
1034 if (nboot_acl)
1035 *nboot_acl = (reply.info & ICM_TR_INFO_BOOT_ACL_MASK) >>
1036 ICM_TR_INFO_BOOT_ACL_SHIFT;
1037 if (rpm)
1038 *rpm = !!(reply.hdr.flags & ICM_TR_FLAGS_RTD3);
1039
1040 return 0;
1041}
1042
1043static int icm_tr_approve_switch(struct tb *tb, struct tb_switch *sw)
1044{
1045 struct icm_tr_pkg_approve_device request;
1046 struct icm_tr_pkg_approve_device reply;
1047 int ret;
1048
1049 memset(&request, 0, sizeof(request));
1050 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1051 request.hdr.code = ICM_APPROVE_DEVICE;
1052 request.route_lo = sw->config.route_lo;
1053 request.route_hi = sw->config.route_hi;
1054 request.connection_id = sw->connection_id;
1055
1056 memset(&reply, 0, sizeof(reply));
1057 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1058 1, ICM_APPROVE_TIMEOUT);
1059 if (ret)
1060 return ret;
1061
1062 if (reply.hdr.flags & ICM_FLAGS_ERROR) {
1063 tb_warn(tb, "PCIe tunnel creation failed\n");
1064 return -EIO;
1065 }
1066
1067 return 0;
1068}
1069
1070static int icm_tr_add_switch_key(struct tb *tb, struct tb_switch *sw)
1071{
1072 struct icm_tr_pkg_add_device_key_response reply;
1073 struct icm_tr_pkg_add_device_key request;
1074 int ret;
1075
1076 memset(&request, 0, sizeof(request));
1077 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1078 request.hdr.code = ICM_ADD_DEVICE_KEY;
1079 request.route_lo = sw->config.route_lo;
1080 request.route_hi = sw->config.route_hi;
1081 request.connection_id = sw->connection_id;
1082 memcpy(request.key, sw->key, TB_SWITCH_KEY_SIZE);
1083
1084 memset(&reply, 0, sizeof(reply));
1085 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1086 1, ICM_TIMEOUT);
1087 if (ret)
1088 return ret;
1089
1090 if (reply.hdr.flags & ICM_FLAGS_ERROR) {
1091 tb_warn(tb, "Adding key to switch failed\n");
1092 return -EIO;
1093 }
1094
1095 return 0;
1096}
1097
1098static int icm_tr_challenge_switch_key(struct tb *tb, struct tb_switch *sw,
1099 const u8 *challenge, u8 *response)
1100{
1101 struct icm_tr_pkg_challenge_device_response reply;
1102 struct icm_tr_pkg_challenge_device request;
1103 int ret;
1104
1105 memset(&request, 0, sizeof(request));
1106 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1107 request.hdr.code = ICM_CHALLENGE_DEVICE;
1108 request.route_lo = sw->config.route_lo;
1109 request.route_hi = sw->config.route_hi;
1110 request.connection_id = sw->connection_id;
1111 memcpy(request.challenge, challenge, TB_SWITCH_KEY_SIZE);
1112
1113 memset(&reply, 0, sizeof(reply));
1114 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1115 1, ICM_TIMEOUT);
1116 if (ret)
1117 return ret;
1118
1119 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1120 return -EKEYREJECTED;
1121 if (reply.hdr.flags & ICM_FLAGS_NO_KEY)
1122 return -ENOKEY;
1123
1124 memcpy(response, reply.response, TB_SWITCH_KEY_SIZE);
1125
1126 return 0;
1127}
1128
1129static int icm_tr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
1130 int transmit_path, int transmit_ring,
1131 int receive_path, int receive_ring)
1132{
1133 struct icm_tr_pkg_approve_xdomain_response reply;
1134 struct icm_tr_pkg_approve_xdomain request;
1135 int ret;
1136
1137 memset(&request, 0, sizeof(request));
1138 request.hdr.code = ICM_APPROVE_XDOMAIN;
1139 request.route_hi = upper_32_bits(xd->route);
1140 request.route_lo = lower_32_bits(xd->route);
1141 request.transmit_path = transmit_path;
1142 request.transmit_ring = transmit_ring;
1143 request.receive_path = receive_path;
1144 request.receive_ring = receive_ring;
1145 memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
1146
1147 memset(&reply, 0, sizeof(reply));
1148 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1149 1, ICM_TIMEOUT);
1150 if (ret)
1151 return ret;
1152
1153 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1154 return -EIO;
1155
1156 return 0;
1157}
1158
1159static int icm_tr_xdomain_tear_down(struct tb *tb, struct tb_xdomain *xd,
1160 int stage)
1161{
1162 struct icm_tr_pkg_disconnect_xdomain_response reply;
1163 struct icm_tr_pkg_disconnect_xdomain request;
1164 int ret;
1165
1166 memset(&request, 0, sizeof(request));
1167 request.hdr.code = ICM_DISCONNECT_XDOMAIN;
1168 request.stage = stage;
1169 request.route_hi = upper_32_bits(xd->route);
1170 request.route_lo = lower_32_bits(xd->route);
1171 memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
1172
1173 memset(&reply, 0, sizeof(reply));
1174 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1175 1, ICM_TIMEOUT);
1176 if (ret)
1177 return ret;
1178
1179 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1180 return -EIO;
1181
1182 return 0;
1183}
1184
1185static int icm_tr_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
1186 int transmit_path, int transmit_ring,
1187 int receive_path, int receive_ring)
1188{
1189 int ret;
1190
1191 ret = icm_tr_xdomain_tear_down(tb, xd, 1);
1192 if (ret)
1193 return ret;
1194
1195 usleep_range(10, 50);
1196 return icm_tr_xdomain_tear_down(tb, xd, 2);
1197}
1198
1199static void
1200__icm_tr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr,
1201 bool force_rtd3)
1202{
1203 const struct icm_tr_event_device_connected *pkg =
1204 (const struct icm_tr_event_device_connected *)hdr;
1205 bool authorized, boot, dual_lane, speed_gen3;
1206 enum tb_security_level security_level;
1207 struct tb_switch *sw, *parent_sw;
1208 struct tb_xdomain *xd;
1209 u64 route;
1210
1211 icm_postpone_rescan(tb);
1212
1213 /*
1214 * Currently we don't use the QoS information coming with the
1215 * device connected message so simply just ignore that extra
1216 * packet for now.
1217 */
1218 if (pkg->hdr.packet_id)
1219 return;
1220
1221 route = get_route(pkg->route_hi, pkg->route_lo);
1222 authorized = pkg->link_info & ICM_LINK_INFO_APPROVED;
1223 security_level = (pkg->hdr.flags & ICM_FLAGS_SLEVEL_MASK) >>
1224 ICM_FLAGS_SLEVEL_SHIFT;
1225 boot = pkg->link_info & ICM_LINK_INFO_BOOT;
1226 dual_lane = pkg->hdr.flags & ICM_FLAGS_DUAL_LANE;
1227 speed_gen3 = pkg->hdr.flags & ICM_FLAGS_SPEED_GEN3;
1228
1229 if (pkg->link_info & ICM_LINK_INFO_REJECTED) {
1230 tb_info(tb, "switch at %llx was rejected by ICM firmware because topology limit exceeded\n",
1231 route);
1232 return;
1233 }
1234
1235 sw = tb_switch_find_by_uuid(tb, &pkg->ep_uuid);
1236 if (sw) {
1237 /* Update the switch if it is still in the same place */
1238 if (tb_route(sw) == route && !!sw->authorized == authorized) {
1239 parent_sw = tb_to_switch(sw->dev.parent);
1240 update_switch(parent_sw, sw, route, pkg->connection_id,
1241 0, 0, 0, boot);
1242 tb_switch_put(sw);
1243 return;
1244 }
1245
1246 remove_switch(sw);
1247 tb_switch_put(sw);
1248 }
1249
1250 /* Another switch with the same address */
1251 sw = tb_switch_find_by_route(tb, route);
1252 if (sw) {
1253 remove_switch(sw);
1254 tb_switch_put(sw);
1255 }
1256
1257 /* XDomain connection with the same address */
1258 xd = tb_xdomain_find_by_route(tb, route);
1259 if (xd) {
1260 remove_xdomain(xd);
1261 tb_xdomain_put(xd);
1262 }
1263
1264 parent_sw = tb_switch_find_by_route(tb, get_parent_route(route));
1265 if (!parent_sw) {
1266 tb_err(tb, "failed to find parent switch for %llx\n", route);
1267 return;
1268 }
1269
1270 pm_runtime_get_sync(&parent_sw->dev);
1271
1272 sw = alloc_switch(parent_sw, route, &pkg->ep_uuid);
1273 if (!IS_ERR(sw)) {
1274 sw->connection_id = pkg->connection_id;
1275 sw->authorized = authorized;
1276 sw->security_level = security_level;
1277 sw->boot = boot;
1278 sw->link_speed = speed_gen3 ? 20 : 10;
1279 sw->link_width = dual_lane ? 2 : 1;
1280 sw->rpm = force_rtd3;
1281 if (!sw->rpm)
1282 sw->rpm = intel_vss_is_rtd3(pkg->ep_name,
1283 sizeof(pkg->ep_name));
1284
1285 if (add_switch(parent_sw, sw))
1286 tb_switch_put(sw);
1287 }
1288
1289 pm_runtime_mark_last_busy(&parent_sw->dev);
1290 pm_runtime_put_autosuspend(&parent_sw->dev);
1291
1292 tb_switch_put(parent_sw);
1293}
1294
1295static void
1296icm_tr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1297{
1298 __icm_tr_device_connected(tb, hdr, false);
1299}
1300
1301static void
1302icm_tr_device_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
1303{
1304 const struct icm_tr_event_device_disconnected *pkg =
1305 (const struct icm_tr_event_device_disconnected *)hdr;
1306 struct tb_switch *sw;
1307 u64 route;
1308
1309 route = get_route(pkg->route_hi, pkg->route_lo);
1310
1311 sw = tb_switch_find_by_route(tb, route);
1312 if (!sw) {
1313 tb_warn(tb, "no switch exists at %llx, ignoring\n", route);
1314 return;
1315 }
1316 pm_runtime_get_sync(sw->dev.parent);
1317
1318 remove_switch(sw);
1319
1320 pm_runtime_mark_last_busy(sw->dev.parent);
1321 pm_runtime_put_autosuspend(sw->dev.parent);
1322
1323 tb_switch_put(sw);
1324}
1325
1326static void
1327icm_tr_xdomain_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1328{
1329 const struct icm_tr_event_xdomain_connected *pkg =
1330 (const struct icm_tr_event_xdomain_connected *)hdr;
1331 struct tb_xdomain *xd;
1332 struct tb_switch *sw;
1333 u64 route;
1334
1335 if (!tb->root_switch)
1336 return;
1337
1338 route = get_route(pkg->local_route_hi, pkg->local_route_lo);
1339
1340 xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
1341 if (xd) {
1342 if (xd->route == route) {
1343 update_xdomain(xd, route, 0);
1344 tb_xdomain_put(xd);
1345 return;
1346 }
1347
1348 remove_xdomain(xd);
1349 tb_xdomain_put(xd);
1350 }
1351
1352 /* An existing xdomain with the same address */
1353 xd = tb_xdomain_find_by_route(tb, route);
1354 if (xd) {
1355 remove_xdomain(xd);
1356 tb_xdomain_put(xd);
1357 }
1358
1359 /*
1360 * If the user disconnected a switch during suspend and
1361 * connected another host to the same port, remove the switch
1362 * first.
1363 */
1364 sw = tb_switch_find_by_route(tb, route);
1365 if (sw) {
1366 remove_switch(sw);
1367 tb_switch_put(sw);
1368 }
1369
1370 sw = tb_switch_find_by_route(tb, get_parent_route(route));
1371 if (!sw) {
1372 tb_warn(tb, "no switch exists at %llx, ignoring\n", route);
1373 return;
1374 }
1375
1376 add_xdomain(sw, route, &pkg->local_uuid, &pkg->remote_uuid, 0, 0);
1377 tb_switch_put(sw);
1378}
1379
1380static void
1381icm_tr_xdomain_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
1382{
1383 const struct icm_tr_event_xdomain_disconnected *pkg =
1384 (const struct icm_tr_event_xdomain_disconnected *)hdr;
1385 struct tb_xdomain *xd;
1386 u64 route;
1387
1388 route = get_route(pkg->route_hi, pkg->route_lo);
1389
1390 xd = tb_xdomain_find_by_route(tb, route);
1391 if (xd) {
1392 remove_xdomain(xd);
1393 tb_xdomain_put(xd);
1394 }
1395}
1396
1397static struct pci_dev *get_upstream_port(struct pci_dev *pdev)
1398{
1399 struct pci_dev *parent;
1400
1401 parent = pci_upstream_bridge(pdev);
1402 while (parent) {
1403 if (!pci_is_pcie(parent))
1404 return NULL;
1405 if (pci_pcie_type(parent) == PCI_EXP_TYPE_UPSTREAM)
1406 break;
1407 parent = pci_upstream_bridge(parent);
1408 }
1409
1410 if (!parent)
1411 return NULL;
1412
1413 switch (parent->device) {
1414 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_BRIDGE:
1415 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_BRIDGE:
1416 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_BRIDGE:
1417 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_BRIDGE:
1418 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_BRIDGE:
1419 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_BRIDGE:
1420 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_BRIDGE:
1421 return parent;
1422 }
1423
1424 return NULL;
1425}
1426
1427static bool icm_ar_is_supported(struct tb *tb)
1428{
1429 struct pci_dev *upstream_port;
1430 struct icm *icm = tb_priv(tb);
1431
1432 /*
1433 * Starting from Alpine Ridge we can use ICM on Apple machines
1434 * as well. We just need to reset and re-enable it first.
1435 * However, only start it if explicitly asked by the user.
1436 */
1437 if (icm_firmware_running(tb->nhi))
1438 return true;
1439 if (!start_icm)
1440 return false;
1441
1442 /*
1443 * Find the upstream PCIe port in case we need to do reset
1444 * through its vendor specific registers.
1445 */
1446 upstream_port = get_upstream_port(tb->nhi->pdev);
1447 if (upstream_port) {
1448 int cap;
1449
1450 cap = pci_find_ext_capability(upstream_port,
1451 PCI_EXT_CAP_ID_VNDR);
1452 if (cap > 0) {
1453 icm->upstream_port = upstream_port;
1454 icm->vnd_cap = cap;
1455
1456 return true;
1457 }
1458 }
1459
1460 return false;
1461}
1462
1463static int icm_ar_cio_reset(struct tb *tb)
1464{
1465 return pcie2cio_write(tb_priv(tb), TB_CFG_SWITCH, 0, 0x50, BIT(9));
1466}
1467
1468static int icm_ar_get_mode(struct tb *tb)
1469{
1470 struct tb_nhi *nhi = tb->nhi;
1471 int retries = 60;
1472 u32 val;
1473
1474 do {
1475 val = ioread32(nhi->iobase + REG_FW_STS);
1476 if (val & REG_FW_STS_NVM_AUTH_DONE)
1477 break;
1478 msleep(50);
1479 } while (--retries);
1480
1481 if (!retries) {
1482 dev_err(&nhi->pdev->dev, "ICM firmware not authenticated\n");
1483 return -ENODEV;
1484 }
1485
1486 return nhi_mailbox_mode(nhi);
1487}
1488
1489static int
1490icm_ar_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1491 u8 *proto_version, size_t *nboot_acl, bool *rpm)
1492{
1493 struct icm_ar_pkg_driver_ready_response reply;
1494 struct icm_pkg_driver_ready request = {
1495 .hdr.code = ICM_DRIVER_READY,
1496 };
1497 int ret;
1498
1499 memset(&reply, 0, sizeof(reply));
1500 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1501 1, ICM_TIMEOUT);
1502 if (ret)
1503 return ret;
1504
1505 if (security_level)
1506 *security_level = reply.info & ICM_AR_INFO_SLEVEL_MASK;
1507 if (nboot_acl && (reply.info & ICM_AR_INFO_BOOT_ACL_SUPPORTED))
1508 *nboot_acl = (reply.info & ICM_AR_INFO_BOOT_ACL_MASK) >>
1509 ICM_AR_INFO_BOOT_ACL_SHIFT;
1510 if (rpm)
1511 *rpm = !!(reply.hdr.flags & ICM_AR_FLAGS_RTD3);
1512
1513 return 0;
1514}
1515
1516static int icm_ar_get_route(struct tb *tb, u8 link, u8 depth, u64 *route)
1517{
1518 struct icm_ar_pkg_get_route_response reply;
1519 struct icm_ar_pkg_get_route request = {
1520 .hdr = { .code = ICM_GET_ROUTE },
1521 .link_info = depth << ICM_LINK_INFO_DEPTH_SHIFT | link,
1522 };
1523 int ret;
1524
1525 memset(&reply, 0, sizeof(reply));
1526 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1527 1, ICM_TIMEOUT);
1528 if (ret)
1529 return ret;
1530
1531 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1532 return -EIO;
1533
1534 *route = get_route(reply.route_hi, reply.route_lo);
1535 return 0;
1536}
1537
1538static int icm_ar_get_boot_acl(struct tb *tb, uuid_t *uuids, size_t nuuids)
1539{
1540 struct icm_ar_pkg_preboot_acl_response reply;
1541 struct icm_ar_pkg_preboot_acl request = {
1542 .hdr = { .code = ICM_PREBOOT_ACL },
1543 };
1544 int ret, i;
1545
1546 memset(&reply, 0, sizeof(reply));
1547 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1548 1, ICM_TIMEOUT);
1549 if (ret)
1550 return ret;
1551
1552 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1553 return -EIO;
1554
1555 for (i = 0; i < nuuids; i++) {
1556 u32 *uuid = (u32 *)&uuids[i];
1557
1558 uuid[0] = reply.acl[i].uuid_lo;
1559 uuid[1] = reply.acl[i].uuid_hi;
1560
1561 if (uuid[0] == 0xffffffff && uuid[1] == 0xffffffff) {
1562 /* Map empty entries to null UUID */
1563 uuid[0] = 0;
1564 uuid[1] = 0;
1565 } else if (uuid[0] != 0 || uuid[1] != 0) {
1566 /* Upper two DWs are always one's */
1567 uuid[2] = 0xffffffff;
1568 uuid[3] = 0xffffffff;
1569 }
1570 }
1571
1572 return ret;
1573}
1574
1575static int icm_ar_set_boot_acl(struct tb *tb, const uuid_t *uuids,
1576 size_t nuuids)
1577{
1578 struct icm_ar_pkg_preboot_acl_response reply;
1579 struct icm_ar_pkg_preboot_acl request = {
1580 .hdr = {
1581 .code = ICM_PREBOOT_ACL,
1582 .flags = ICM_FLAGS_WRITE,
1583 },
1584 };
1585 int ret, i;
1586
1587 for (i = 0; i < nuuids; i++) {
1588 const u32 *uuid = (const u32 *)&uuids[i];
1589
1590 if (uuid_is_null(&uuids[i])) {
1591 /*
1592 * Map null UUID to the empty (all one) entries
1593 * for ICM.
1594 */
1595 request.acl[i].uuid_lo = 0xffffffff;
1596 request.acl[i].uuid_hi = 0xffffffff;
1597 } else {
1598 /* Two high DWs need to be set to all one */
1599 if (uuid[2] != 0xffffffff || uuid[3] != 0xffffffff)
1600 return -EINVAL;
1601
1602 request.acl[i].uuid_lo = uuid[0];
1603 request.acl[i].uuid_hi = uuid[1];
1604 }
1605 }
1606
1607 memset(&reply, 0, sizeof(reply));
1608 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1609 1, ICM_TIMEOUT);
1610 if (ret)
1611 return ret;
1612
1613 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1614 return -EIO;
1615
1616 return 0;
1617}
1618
1619static int
1620icm_icl_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1621 u8 *proto_version, size_t *nboot_acl, bool *rpm)
1622{
1623 struct icm_tr_pkg_driver_ready_response reply;
1624 struct icm_pkg_driver_ready request = {
1625 .hdr.code = ICM_DRIVER_READY,
1626 };
1627 int ret;
1628
1629 memset(&reply, 0, sizeof(reply));
1630 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1631 1, 20000);
1632 if (ret)
1633 return ret;
1634
1635 if (proto_version)
1636 *proto_version = (reply.info & ICM_TR_INFO_PROTO_VERSION_MASK) >>
1637 ICM_TR_INFO_PROTO_VERSION_SHIFT;
1638
1639 /* Ice Lake always supports RTD3 */
1640 if (rpm)
1641 *rpm = true;
1642
1643 return 0;
1644}
1645
1646static void icm_icl_set_uuid(struct tb *tb)
1647{
1648 struct tb_nhi *nhi = tb->nhi;
1649 u32 uuid[4];
1650
1651 pci_read_config_dword(nhi->pdev, VS_CAP_10, &uuid[0]);
1652 pci_read_config_dword(nhi->pdev, VS_CAP_11, &uuid[1]);
1653 uuid[2] = 0xffffffff;
1654 uuid[3] = 0xffffffff;
1655
1656 tb->root_switch->uuid = kmemdup(uuid, sizeof(uuid), GFP_KERNEL);
1657}
1658
1659static void
1660icm_icl_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1661{
1662 __icm_tr_device_connected(tb, hdr, true);
1663}
1664
1665static void icm_icl_rtd3_veto(struct tb *tb, const struct icm_pkg_header *hdr)
1666{
1667 const struct icm_icl_event_rtd3_veto *pkg =
1668 (const struct icm_icl_event_rtd3_veto *)hdr;
1669
1670 tb_dbg(tb, "ICM rtd3 veto=0x%08x\n", pkg->veto_reason);
1671
1672 if (pkg->veto_reason)
1673 icm_veto_begin(tb);
1674 else
1675 icm_veto_end(tb);
1676}
1677
1678static bool icm_tgl_is_supported(struct tb *tb)
1679{
1680 unsigned long end = jiffies + msecs_to_jiffies(10);
1681
1682 do {
1683 u32 val;
1684
1685 val = ioread32(tb->nhi->iobase + REG_FW_STS);
1686 if (val & REG_FW_STS_NVM_AUTH_DONE)
1687 return true;
1688 usleep_range(100, 500);
1689 } while (time_before(jiffies, end));
1690
1691 return false;
1692}
1693
1694static void icm_handle_notification(struct work_struct *work)
1695{
1696 struct icm_notification *n = container_of(work, typeof(*n), work);
1697 struct tb *tb = n->tb;
1698 struct icm *icm = tb_priv(tb);
1699
1700 mutex_lock(&tb->lock);
1701
1702 /*
1703 * When the domain is stopped we flush its workqueue but before
1704 * that the root switch is removed. In that case we should treat
1705 * the queued events as being canceled.
1706 */
1707 if (tb->root_switch) {
1708 switch (n->pkg->code) {
1709 case ICM_EVENT_DEVICE_CONNECTED:
1710 icm->device_connected(tb, n->pkg);
1711 break;
1712 case ICM_EVENT_DEVICE_DISCONNECTED:
1713 icm->device_disconnected(tb, n->pkg);
1714 break;
1715 case ICM_EVENT_XDOMAIN_CONNECTED:
1716 if (tb_is_xdomain_enabled())
1717 icm->xdomain_connected(tb, n->pkg);
1718 break;
1719 case ICM_EVENT_XDOMAIN_DISCONNECTED:
1720 if (tb_is_xdomain_enabled())
1721 icm->xdomain_disconnected(tb, n->pkg);
1722 break;
1723 case ICM_EVENT_RTD3_VETO:
1724 icm->rtd3_veto(tb, n->pkg);
1725 break;
1726 }
1727 }
1728
1729 mutex_unlock(&tb->lock);
1730
1731 kfree(n->pkg);
1732 kfree(n);
1733}
1734
1735static void icm_handle_event(struct tb *tb, enum tb_cfg_pkg_type type,
1736 const void *buf, size_t size)
1737{
1738 struct icm_notification *n;
1739
1740 n = kmalloc(sizeof(*n), GFP_KERNEL);
1741 if (!n)
1742 return;
1743
1744 n->pkg = kmemdup(buf, size, GFP_KERNEL);
1745 if (!n->pkg) {
1746 kfree(n);
1747 return;
1748 }
1749
1750 INIT_WORK(&n->work, icm_handle_notification);
1751 n->tb = tb;
1752
1753 queue_work(tb->wq, &n->work);
1754}
1755
1756static int
1757__icm_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1758 u8 *proto_version, size_t *nboot_acl, bool *rpm)
1759{
1760 struct icm *icm = tb_priv(tb);
1761 unsigned int retries = 50;
1762 int ret;
1763
1764 ret = icm->driver_ready(tb, security_level, proto_version, nboot_acl,
1765 rpm);
1766 if (ret) {
1767 tb_err(tb, "failed to send driver ready to ICM\n");
1768 return ret;
1769 }
1770
1771 /*
1772 * Hold on here until the switch config space is accessible so
1773 * that we can read root switch config successfully.
1774 */
1775 do {
1776 struct tb_cfg_result res;
1777 u32 tmp;
1778
1779 res = tb_cfg_read_raw(tb->ctl, &tmp, 0, 0, TB_CFG_SWITCH,
1780 0, 1, 100);
1781 if (!res.err)
1782 return 0;
1783
1784 msleep(50);
1785 } while (--retries);
1786
1787 tb_err(tb, "failed to read root switch config space, giving up\n");
1788 return -ETIMEDOUT;
1789}
1790
1791static int icm_firmware_reset(struct tb *tb, struct tb_nhi *nhi)
1792{
1793 struct icm *icm = tb_priv(tb);
1794 u32 val;
1795
1796 if (!icm->upstream_port)
1797 return -ENODEV;
1798
1799 /* Put ARC to wait for CIO reset event to happen */
1800 val = ioread32(nhi->iobase + REG_FW_STS);
1801 val |= REG_FW_STS_CIO_RESET_REQ;
1802 iowrite32(val, nhi->iobase + REG_FW_STS);
1803
1804 /* Re-start ARC */
1805 val = ioread32(nhi->iobase + REG_FW_STS);
1806 val |= REG_FW_STS_ICM_EN_INVERT;
1807 val |= REG_FW_STS_ICM_EN_CPU;
1808 iowrite32(val, nhi->iobase + REG_FW_STS);
1809
1810 /* Trigger CIO reset now */
1811 return icm->cio_reset(tb);
1812}
1813
1814static int icm_firmware_start(struct tb *tb, struct tb_nhi *nhi)
1815{
1816 unsigned int retries = 10;
1817 int ret;
1818 u32 val;
1819
1820 /* Check if the ICM firmware is already running */
1821 if (icm_firmware_running(nhi))
1822 return 0;
1823
1824 dev_dbg(&nhi->pdev->dev, "starting ICM firmware\n");
1825
1826 ret = icm_firmware_reset(tb, nhi);
1827 if (ret)
1828 return ret;
1829
1830 /* Wait until the ICM firmware tells us it is up and running */
1831 do {
1832 /* Check that the ICM firmware is running */
1833 val = ioread32(nhi->iobase + REG_FW_STS);
1834 if (val & REG_FW_STS_NVM_AUTH_DONE)
1835 return 0;
1836
1837 msleep(300);
1838 } while (--retries);
1839
1840 return -ETIMEDOUT;
1841}
1842
1843static int icm_reset_phy_port(struct tb *tb, int phy_port)
1844{
1845 struct icm *icm = tb_priv(tb);
1846 u32 state0, state1;
1847 int port0, port1;
1848 u32 val0, val1;
1849 int ret;
1850
1851 if (!icm->upstream_port)
1852 return 0;
1853
1854 if (phy_port) {
1855 port0 = 3;
1856 port1 = 4;
1857 } else {
1858 port0 = 1;
1859 port1 = 2;
1860 }
1861
1862 /*
1863 * Read link status of both null ports belonging to a single
1864 * physical port.
1865 */
1866 ret = pcie2cio_read(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, &val0);
1867 if (ret)
1868 return ret;
1869 ret = pcie2cio_read(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, &val1);
1870 if (ret)
1871 return ret;
1872
1873 state0 = val0 & PHY_PORT_CS1_LINK_STATE_MASK;
1874 state0 >>= PHY_PORT_CS1_LINK_STATE_SHIFT;
1875 state1 = val1 & PHY_PORT_CS1_LINK_STATE_MASK;
1876 state1 >>= PHY_PORT_CS1_LINK_STATE_SHIFT;
1877
1878 /* If they are both up we need to reset them now */
1879 if (state0 != TB_PORT_UP || state1 != TB_PORT_UP)
1880 return 0;
1881
1882 val0 |= PHY_PORT_CS1_LINK_DISABLE;
1883 ret = pcie2cio_write(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, val0);
1884 if (ret)
1885 return ret;
1886
1887 val1 |= PHY_PORT_CS1_LINK_DISABLE;
1888 ret = pcie2cio_write(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, val1);
1889 if (ret)
1890 return ret;
1891
1892 /* Wait a bit and then re-enable both ports */
1893 usleep_range(10, 100);
1894
1895 ret = pcie2cio_read(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, &val0);
1896 if (ret)
1897 return ret;
1898 ret = pcie2cio_read(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, &val1);
1899 if (ret)
1900 return ret;
1901
1902 val0 &= ~PHY_PORT_CS1_LINK_DISABLE;
1903 ret = pcie2cio_write(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, val0);
1904 if (ret)
1905 return ret;
1906
1907 val1 &= ~PHY_PORT_CS1_LINK_DISABLE;
1908 return pcie2cio_write(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, val1);
1909}
1910
1911static int icm_firmware_init(struct tb *tb)
1912{
1913 struct icm *icm = tb_priv(tb);
1914 struct tb_nhi *nhi = tb->nhi;
1915 int ret;
1916
1917 ret = icm_firmware_start(tb, nhi);
1918 if (ret) {
1919 dev_err(&nhi->pdev->dev, "could not start ICM firmware\n");
1920 return ret;
1921 }
1922
1923 if (icm->get_mode) {
1924 ret = icm->get_mode(tb);
1925
1926 switch (ret) {
1927 case NHI_FW_SAFE_MODE:
1928 icm->safe_mode = true;
1929 break;
1930
1931 case NHI_FW_CM_MODE:
1932 /* Ask ICM to accept all Thunderbolt devices */
1933 nhi_mailbox_cmd(nhi, NHI_MAILBOX_ALLOW_ALL_DEVS, 0);
1934 break;
1935
1936 default:
1937 if (ret < 0)
1938 return ret;
1939
1940 tb_err(tb, "ICM firmware is in wrong mode: %u\n", ret);
1941 return -ENODEV;
1942 }
1943 }
1944
1945 /*
1946 * Reset both physical ports if there is anything connected to
1947 * them already.
1948 */
1949 ret = icm_reset_phy_port(tb, 0);
1950 if (ret)
1951 dev_warn(&nhi->pdev->dev, "failed to reset links on port0\n");
1952 ret = icm_reset_phy_port(tb, 1);
1953 if (ret)
1954 dev_warn(&nhi->pdev->dev, "failed to reset links on port1\n");
1955
1956 return 0;
1957}
1958
1959static int icm_driver_ready(struct tb *tb)
1960{
1961 struct icm *icm = tb_priv(tb);
1962 int ret;
1963
1964 ret = icm_firmware_init(tb);
1965 if (ret)
1966 return ret;
1967
1968 if (icm->safe_mode) {
1969 tb_info(tb, "Thunderbolt host controller is in safe mode.\n");
1970 tb_info(tb, "You need to update NVM firmware of the controller before it can be used.\n");
1971 tb_info(tb, "For latest updates check https://thunderbolttechnology.net/updates.\n");
1972 return 0;
1973 }
1974
1975 ret = __icm_driver_ready(tb, &tb->security_level, &icm->proto_version,
1976 &tb->nboot_acl, &icm->rpm);
1977 if (ret)
1978 return ret;
1979
1980 /*
1981 * Make sure the number of supported preboot ACL matches what we
1982 * expect or disable the whole feature.
1983 */
1984 if (tb->nboot_acl > icm->max_boot_acl)
1985 tb->nboot_acl = 0;
1986
1987 if (icm->proto_version >= 3)
1988 tb_dbg(tb, "USB4 proxy operations supported\n");
1989
1990 return 0;
1991}
1992
1993static int icm_suspend(struct tb *tb)
1994{
1995 struct icm *icm = tb_priv(tb);
1996
1997 if (icm->save_devices)
1998 icm->save_devices(tb);
1999
2000 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
2001 return 0;
2002}
2003
2004/*
2005 * Mark all switches (except root switch) below this one unplugged. ICM
2006 * firmware will send us an updated list of switches after we have send
2007 * it driver ready command. If a switch is not in that list it will be
2008 * removed when we perform rescan.
2009 */
2010static void icm_unplug_children(struct tb_switch *sw)
2011{
2012 struct tb_port *port;
2013
2014 if (tb_route(sw))
2015 sw->is_unplugged = true;
2016
2017 tb_switch_for_each_port(sw, port) {
2018 if (port->xdomain)
2019 port->xdomain->is_unplugged = true;
2020 else if (tb_port_has_remote(port))
2021 icm_unplug_children(port->remote->sw);
2022 }
2023}
2024
2025static int complete_rpm(struct device *dev, void *data)
2026{
2027 struct tb_switch *sw = tb_to_switch(dev);
2028
2029 if (sw)
2030 complete(&sw->rpm_complete);
2031 return 0;
2032}
2033
2034static void remove_unplugged_switch(struct tb_switch *sw)
2035{
2036 struct device *parent = get_device(sw->dev.parent);
2037
2038 pm_runtime_get_sync(parent);
2039
2040 /*
2041 * Signal this and switches below for rpm_complete because
2042 * tb_switch_remove() calls pm_runtime_get_sync() that then waits
2043 * for it.
2044 */
2045 complete_rpm(&sw->dev, NULL);
2046 bus_for_each_dev(&tb_bus_type, &sw->dev, NULL, complete_rpm);
2047 tb_switch_remove(sw);
2048
2049 pm_runtime_mark_last_busy(parent);
2050 pm_runtime_put_autosuspend(parent);
2051
2052 put_device(parent);
2053}
2054
2055static void icm_free_unplugged_children(struct tb_switch *sw)
2056{
2057 struct tb_port *port;
2058
2059 tb_switch_for_each_port(sw, port) {
2060 if (port->xdomain && port->xdomain->is_unplugged) {
2061 tb_xdomain_remove(port->xdomain);
2062 port->xdomain = NULL;
2063 } else if (tb_port_has_remote(port)) {
2064 if (port->remote->sw->is_unplugged) {
2065 remove_unplugged_switch(port->remote->sw);
2066 port->remote = NULL;
2067 } else {
2068 icm_free_unplugged_children(port->remote->sw);
2069 }
2070 }
2071 }
2072}
2073
2074static void icm_rescan_work(struct work_struct *work)
2075{
2076 struct icm *icm = container_of(work, struct icm, rescan_work.work);
2077 struct tb *tb = icm_to_tb(icm);
2078
2079 mutex_lock(&tb->lock);
2080 if (tb->root_switch)
2081 icm_free_unplugged_children(tb->root_switch);
2082 mutex_unlock(&tb->lock);
2083}
2084
2085static void icm_complete(struct tb *tb)
2086{
2087 struct icm *icm = tb_priv(tb);
2088
2089 if (tb->nhi->going_away)
2090 return;
2091
2092 /*
2093 * If RTD3 was vetoed before we entered system suspend allow it
2094 * again now before driver ready is sent. Firmware sends a new RTD3
2095 * veto if it is still the case after we have sent it driver ready
2096 * command.
2097 */
2098 icm_veto_end(tb);
2099 icm_unplug_children(tb->root_switch);
2100
2101 /*
2102 * Now all existing children should be resumed, start events
2103 * from ICM to get updated status.
2104 */
2105 __icm_driver_ready(tb, NULL, NULL, NULL, NULL);
2106
2107 /*
2108 * We do not get notifications of devices that have been
2109 * unplugged during suspend so schedule rescan to clean them up
2110 * if any.
2111 */
2112 queue_delayed_work(tb->wq, &icm->rescan_work, msecs_to_jiffies(500));
2113}
2114
2115static int icm_runtime_suspend(struct tb *tb)
2116{
2117 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
2118 return 0;
2119}
2120
2121static int icm_runtime_suspend_switch(struct tb_switch *sw)
2122{
2123 if (tb_route(sw))
2124 reinit_completion(&sw->rpm_complete);
2125 return 0;
2126}
2127
2128static int icm_runtime_resume_switch(struct tb_switch *sw)
2129{
2130 if (tb_route(sw)) {
2131 if (!wait_for_completion_timeout(&sw->rpm_complete,
2132 msecs_to_jiffies(500))) {
2133 dev_dbg(&sw->dev, "runtime resuming timed out\n");
2134 }
2135 }
2136 return 0;
2137}
2138
2139static int icm_runtime_resume(struct tb *tb)
2140{
2141 /*
2142 * We can reuse the same resume functionality than with system
2143 * suspend.
2144 */
2145 icm_complete(tb);
2146 return 0;
2147}
2148
2149static int icm_start(struct tb *tb)
2150{
2151 struct icm *icm = tb_priv(tb);
2152 int ret;
2153
2154 if (icm->safe_mode)
2155 tb->root_switch = tb_switch_alloc_safe_mode(tb, &tb->dev, 0);
2156 else
2157 tb->root_switch = tb_switch_alloc(tb, &tb->dev, 0);
2158 if (IS_ERR(tb->root_switch))
2159 return PTR_ERR(tb->root_switch);
2160
2161 tb->root_switch->no_nvm_upgrade = !icm->can_upgrade_nvm;
2162 tb->root_switch->rpm = icm->rpm;
2163
2164 if (icm->set_uuid)
2165 icm->set_uuid(tb);
2166
2167 ret = tb_switch_add(tb->root_switch);
2168 if (ret) {
2169 tb_switch_put(tb->root_switch);
2170 tb->root_switch = NULL;
2171 }
2172
2173 return ret;
2174}
2175
2176static void icm_stop(struct tb *tb)
2177{
2178 struct icm *icm = tb_priv(tb);
2179
2180 cancel_delayed_work(&icm->rescan_work);
2181 tb_switch_remove(tb->root_switch);
2182 tb->root_switch = NULL;
2183 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
2184 kfree(icm->last_nvm_auth);
2185 icm->last_nvm_auth = NULL;
2186}
2187
2188static int icm_disconnect_pcie_paths(struct tb *tb)
2189{
2190 return nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DISCONNECT_PCIE_PATHS, 0);
2191}
2192
2193static void icm_usb4_switch_nvm_auth_complete(void *data)
2194{
2195 struct usb4_switch_nvm_auth *auth = data;
2196 struct icm *icm = auth->icm;
2197 struct tb *tb = icm_to_tb(icm);
2198
2199 tb_dbg(tb, "NVM_AUTH response for %llx flags %#x status %#x\n",
2200 get_route(auth->reply.route_hi, auth->reply.route_lo),
2201 auth->reply.hdr.flags, auth->reply.status);
2202
2203 mutex_lock(&tb->lock);
2204 if (WARN_ON(icm->last_nvm_auth))
2205 kfree(icm->last_nvm_auth);
2206 icm->last_nvm_auth = auth;
2207 mutex_unlock(&tb->lock);
2208}
2209
2210static int icm_usb4_switch_nvm_authenticate(struct tb *tb, u64 route)
2211{
2212 struct usb4_switch_nvm_auth *auth;
2213 struct icm *icm = tb_priv(tb);
2214 struct tb_cfg_request *req;
2215 int ret;
2216
2217 auth = kzalloc(sizeof(*auth), GFP_KERNEL);
2218 if (!auth)
2219 return -ENOMEM;
2220
2221 auth->icm = icm;
2222 auth->request.hdr.code = ICM_USB4_SWITCH_OP;
2223 auth->request.route_hi = upper_32_bits(route);
2224 auth->request.route_lo = lower_32_bits(route);
2225 auth->request.opcode = USB4_SWITCH_OP_NVM_AUTH;
2226
2227 req = tb_cfg_request_alloc();
2228 if (!req) {
2229 ret = -ENOMEM;
2230 goto err_free_auth;
2231 }
2232
2233 req->match = icm_match;
2234 req->copy = icm_copy;
2235 req->request = &auth->request;
2236 req->request_size = sizeof(auth->request);
2237 req->request_type = TB_CFG_PKG_ICM_CMD;
2238 req->response = &auth->reply;
2239 req->npackets = 1;
2240 req->response_size = sizeof(auth->reply);
2241 req->response_type = TB_CFG_PKG_ICM_RESP;
2242
2243 tb_dbg(tb, "NVM_AUTH request for %llx\n", route);
2244
2245 mutex_lock(&icm->request_lock);
2246 ret = tb_cfg_request(tb->ctl, req, icm_usb4_switch_nvm_auth_complete,
2247 auth);
2248 mutex_unlock(&icm->request_lock);
2249
2250 tb_cfg_request_put(req);
2251 if (ret)
2252 goto err_free_auth;
2253 return 0;
2254
2255err_free_auth:
2256 kfree(auth);
2257 return ret;
2258}
2259
2260static int icm_usb4_switch_op(struct tb_switch *sw, u16 opcode, u32 *metadata,
2261 u8 *status, const void *tx_data, size_t tx_data_len,
2262 void *rx_data, size_t rx_data_len)
2263{
2264 struct icm_usb4_switch_op_response reply;
2265 struct icm_usb4_switch_op request;
2266 struct tb *tb = sw->tb;
2267 struct icm *icm = tb_priv(tb);
2268 u64 route = tb_route(sw);
2269 int ret;
2270
2271 /*
2272 * USB4 router operation proxy is supported in firmware if the
2273 * protocol version is 3 or higher.
2274 */
2275 if (icm->proto_version < 3)
2276 return -EOPNOTSUPP;
2277
2278 /*
2279 * NVM_AUTH is a special USB4 proxy operation that does not
2280 * return immediately so handle it separately.
2281 */
2282 if (opcode == USB4_SWITCH_OP_NVM_AUTH)
2283 return icm_usb4_switch_nvm_authenticate(tb, route);
2284
2285 memset(&request, 0, sizeof(request));
2286 request.hdr.code = ICM_USB4_SWITCH_OP;
2287 request.route_hi = upper_32_bits(route);
2288 request.route_lo = lower_32_bits(route);
2289 request.opcode = opcode;
2290 if (metadata)
2291 request.metadata = *metadata;
2292
2293 if (tx_data_len) {
2294 request.data_len_valid |= ICM_USB4_SWITCH_DATA_VALID;
2295 if (tx_data_len < ARRAY_SIZE(request.data))
2296 request.data_len_valid =
2297 tx_data_len & ICM_USB4_SWITCH_DATA_LEN_MASK;
2298 memcpy(request.data, tx_data, tx_data_len * sizeof(u32));
2299 }
2300
2301 memset(&reply, 0, sizeof(reply));
2302 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
2303 1, ICM_TIMEOUT);
2304 if (ret)
2305 return ret;
2306
2307 if (reply.hdr.flags & ICM_FLAGS_ERROR)
2308 return -EIO;
2309
2310 if (status)
2311 *status = reply.status;
2312
2313 if (metadata)
2314 *metadata = reply.metadata;
2315
2316 if (rx_data_len)
2317 memcpy(rx_data, reply.data, rx_data_len * sizeof(u32));
2318
2319 return 0;
2320}
2321
2322static int icm_usb4_switch_nvm_authenticate_status(struct tb_switch *sw,
2323 u32 *status)
2324{
2325 struct usb4_switch_nvm_auth *auth;
2326 struct tb *tb = sw->tb;
2327 struct icm *icm = tb_priv(tb);
2328 int ret = 0;
2329
2330 if (icm->proto_version < 3)
2331 return -EOPNOTSUPP;
2332
2333 auth = icm->last_nvm_auth;
2334 icm->last_nvm_auth = NULL;
2335
2336 if (auth && auth->reply.route_hi == sw->config.route_hi &&
2337 auth->reply.route_lo == sw->config.route_lo) {
2338 tb_dbg(tb, "NVM_AUTH found for %llx flags %#x status %#x\n",
2339 tb_route(sw), auth->reply.hdr.flags, auth->reply.status);
2340 if (auth->reply.hdr.flags & ICM_FLAGS_ERROR)
2341 ret = -EIO;
2342 else
2343 *status = auth->reply.status;
2344 } else {
2345 *status = 0;
2346 }
2347
2348 kfree(auth);
2349 return ret;
2350}
2351
2352/* Falcon Ridge */
2353static const struct tb_cm_ops icm_fr_ops = {
2354 .driver_ready = icm_driver_ready,
2355 .start = icm_start,
2356 .stop = icm_stop,
2357 .suspend = icm_suspend,
2358 .complete = icm_complete,
2359 .handle_event = icm_handle_event,
2360 .approve_switch = icm_fr_approve_switch,
2361 .add_switch_key = icm_fr_add_switch_key,
2362 .challenge_switch_key = icm_fr_challenge_switch_key,
2363 .disconnect_pcie_paths = icm_disconnect_pcie_paths,
2364 .approve_xdomain_paths = icm_fr_approve_xdomain_paths,
2365 .disconnect_xdomain_paths = icm_fr_disconnect_xdomain_paths,
2366};
2367
2368/* Alpine Ridge */
2369static const struct tb_cm_ops icm_ar_ops = {
2370 .driver_ready = icm_driver_ready,
2371 .start = icm_start,
2372 .stop = icm_stop,
2373 .suspend = icm_suspend,
2374 .complete = icm_complete,
2375 .runtime_suspend = icm_runtime_suspend,
2376 .runtime_resume = icm_runtime_resume,
2377 .runtime_suspend_switch = icm_runtime_suspend_switch,
2378 .runtime_resume_switch = icm_runtime_resume_switch,
2379 .handle_event = icm_handle_event,
2380 .get_boot_acl = icm_ar_get_boot_acl,
2381 .set_boot_acl = icm_ar_set_boot_acl,
2382 .approve_switch = icm_fr_approve_switch,
2383 .add_switch_key = icm_fr_add_switch_key,
2384 .challenge_switch_key = icm_fr_challenge_switch_key,
2385 .disconnect_pcie_paths = icm_disconnect_pcie_paths,
2386 .approve_xdomain_paths = icm_fr_approve_xdomain_paths,
2387 .disconnect_xdomain_paths = icm_fr_disconnect_xdomain_paths,
2388};
2389
2390/* Titan Ridge */
2391static const struct tb_cm_ops icm_tr_ops = {
2392 .driver_ready = icm_driver_ready,
2393 .start = icm_start,
2394 .stop = icm_stop,
2395 .suspend = icm_suspend,
2396 .complete = icm_complete,
2397 .runtime_suspend = icm_runtime_suspend,
2398 .runtime_resume = icm_runtime_resume,
2399 .runtime_suspend_switch = icm_runtime_suspend_switch,
2400 .runtime_resume_switch = icm_runtime_resume_switch,
2401 .handle_event = icm_handle_event,
2402 .get_boot_acl = icm_ar_get_boot_acl,
2403 .set_boot_acl = icm_ar_set_boot_acl,
2404 .approve_switch = icm_tr_approve_switch,
2405 .add_switch_key = icm_tr_add_switch_key,
2406 .challenge_switch_key = icm_tr_challenge_switch_key,
2407 .disconnect_pcie_paths = icm_disconnect_pcie_paths,
2408 .approve_xdomain_paths = icm_tr_approve_xdomain_paths,
2409 .disconnect_xdomain_paths = icm_tr_disconnect_xdomain_paths,
2410 .usb4_switch_op = icm_usb4_switch_op,
2411 .usb4_switch_nvm_authenticate_status =
2412 icm_usb4_switch_nvm_authenticate_status,
2413};
2414
2415/* Ice Lake */
2416static const struct tb_cm_ops icm_icl_ops = {
2417 .driver_ready = icm_driver_ready,
2418 .start = icm_start,
2419 .stop = icm_stop,
2420 .complete = icm_complete,
2421 .runtime_suspend = icm_runtime_suspend,
2422 .runtime_resume = icm_runtime_resume,
2423 .handle_event = icm_handle_event,
2424 .approve_xdomain_paths = icm_tr_approve_xdomain_paths,
2425 .disconnect_xdomain_paths = icm_tr_disconnect_xdomain_paths,
2426 .usb4_switch_op = icm_usb4_switch_op,
2427 .usb4_switch_nvm_authenticate_status =
2428 icm_usb4_switch_nvm_authenticate_status,
2429};
2430
2431struct tb *icm_probe(struct tb_nhi *nhi)
2432{
2433 struct icm *icm;
2434 struct tb *tb;
2435
2436 tb = tb_domain_alloc(nhi, ICM_TIMEOUT, sizeof(struct icm));
2437 if (!tb)
2438 return NULL;
2439
2440 icm = tb_priv(tb);
2441 INIT_DELAYED_WORK(&icm->rescan_work, icm_rescan_work);
2442 mutex_init(&icm->request_lock);
2443
2444 switch (nhi->pdev->device) {
2445 case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI:
2446 case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI:
2447 icm->can_upgrade_nvm = true;
2448 icm->is_supported = icm_fr_is_supported;
2449 icm->get_route = icm_fr_get_route;
2450 icm->save_devices = icm_fr_save_devices;
2451 icm->driver_ready = icm_fr_driver_ready;
2452 icm->device_connected = icm_fr_device_connected;
2453 icm->device_disconnected = icm_fr_device_disconnected;
2454 icm->xdomain_connected = icm_fr_xdomain_connected;
2455 icm->xdomain_disconnected = icm_fr_xdomain_disconnected;
2456 tb->cm_ops = &icm_fr_ops;
2457 break;
2458
2459 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_NHI:
2460 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_NHI:
2461 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_NHI:
2462 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_NHI:
2463 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_NHI:
2464 icm->max_boot_acl = ICM_AR_PREBOOT_ACL_ENTRIES;
2465 /*
2466 * NVM upgrade has not been tested on Apple systems and
2467 * they don't provide images publicly either. To be on
2468 * the safe side prevent root switch NVM upgrade on Macs
2469 * for now.
2470 */
2471 icm->can_upgrade_nvm = !x86_apple_machine;
2472 icm->is_supported = icm_ar_is_supported;
2473 icm->cio_reset = icm_ar_cio_reset;
2474 icm->get_mode = icm_ar_get_mode;
2475 icm->get_route = icm_ar_get_route;
2476 icm->save_devices = icm_fr_save_devices;
2477 icm->driver_ready = icm_ar_driver_ready;
2478 icm->device_connected = icm_fr_device_connected;
2479 icm->device_disconnected = icm_fr_device_disconnected;
2480 icm->xdomain_connected = icm_fr_xdomain_connected;
2481 icm->xdomain_disconnected = icm_fr_xdomain_disconnected;
2482 tb->cm_ops = &icm_ar_ops;
2483 break;
2484
2485 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_NHI:
2486 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_NHI:
2487 icm->max_boot_acl = ICM_AR_PREBOOT_ACL_ENTRIES;
2488 icm->can_upgrade_nvm = !x86_apple_machine;
2489 icm->is_supported = icm_ar_is_supported;
2490 icm->cio_reset = icm_tr_cio_reset;
2491 icm->get_mode = icm_ar_get_mode;
2492 icm->driver_ready = icm_tr_driver_ready;
2493 icm->device_connected = icm_tr_device_connected;
2494 icm->device_disconnected = icm_tr_device_disconnected;
2495 icm->xdomain_connected = icm_tr_xdomain_connected;
2496 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2497 tb->cm_ops = &icm_tr_ops;
2498 break;
2499
2500 case PCI_DEVICE_ID_INTEL_ICL_NHI0:
2501 case PCI_DEVICE_ID_INTEL_ICL_NHI1:
2502 icm->is_supported = icm_fr_is_supported;
2503 icm->driver_ready = icm_icl_driver_ready;
2504 icm->set_uuid = icm_icl_set_uuid;
2505 icm->device_connected = icm_icl_device_connected;
2506 icm->device_disconnected = icm_tr_device_disconnected;
2507 icm->xdomain_connected = icm_tr_xdomain_connected;
2508 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2509 icm->rtd3_veto = icm_icl_rtd3_veto;
2510 tb->cm_ops = &icm_icl_ops;
2511 break;
2512
2513 case PCI_DEVICE_ID_INTEL_TGL_NHI0:
2514 case PCI_DEVICE_ID_INTEL_TGL_NHI1:
2515 case PCI_DEVICE_ID_INTEL_TGL_H_NHI0:
2516 case PCI_DEVICE_ID_INTEL_TGL_H_NHI1:
2517 case PCI_DEVICE_ID_INTEL_ADL_NHI0:
2518 case PCI_DEVICE_ID_INTEL_ADL_NHI1:
2519 case PCI_DEVICE_ID_INTEL_RPL_NHI0:
2520 case PCI_DEVICE_ID_INTEL_RPL_NHI1:
2521 case PCI_DEVICE_ID_INTEL_MTL_M_NHI0:
2522 case PCI_DEVICE_ID_INTEL_MTL_P_NHI0:
2523 case PCI_DEVICE_ID_INTEL_MTL_P_NHI1:
2524 icm->is_supported = icm_tgl_is_supported;
2525 icm->driver_ready = icm_icl_driver_ready;
2526 icm->set_uuid = icm_icl_set_uuid;
2527 icm->device_connected = icm_icl_device_connected;
2528 icm->device_disconnected = icm_tr_device_disconnected;
2529 icm->xdomain_connected = icm_tr_xdomain_connected;
2530 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2531 icm->rtd3_veto = icm_icl_rtd3_veto;
2532 tb->cm_ops = &icm_icl_ops;
2533 break;
2534
2535 case PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_2C_NHI:
2536 case PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_4C_NHI:
2537 icm->is_supported = icm_tgl_is_supported;
2538 icm->get_mode = icm_ar_get_mode;
2539 icm->driver_ready = icm_tr_driver_ready;
2540 icm->device_connected = icm_tr_device_connected;
2541 icm->device_disconnected = icm_tr_device_disconnected;
2542 icm->xdomain_connected = icm_tr_xdomain_connected;
2543 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2544 tb->cm_ops = &icm_tr_ops;
2545 break;
2546 }
2547
2548 if (!icm->is_supported || !icm->is_supported(tb)) {
2549 dev_dbg(&nhi->pdev->dev, "ICM not supported on this controller\n");
2550 tb_domain_put(tb);
2551 return NULL;
2552 }
2553
2554 tb_dbg(tb, "using firmware connection manager\n");
2555
2556 return tb;
2557}