xref: /openbmc/linux/drivers/thunderbolt/icm.c (revision d088d6b6)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Internal Thunderbolt Connection Manager. This is a firmware running on
4  * the Thunderbolt host controller performing most of the low-level
5  * handling.
6  *
7  * Copyright (C) 2017, Intel Corporation
8  * Authors: Michael Jamet <michael.jamet@intel.com>
9  *          Mika Westerberg <mika.westerberg@linux.intel.com>
10  */
11 
12 #include <linux/delay.h>
13 #include <linux/mutex.h>
14 #include <linux/moduleparam.h>
15 #include <linux/pci.h>
16 #include <linux/pm_runtime.h>
17 #include <linux/platform_data/x86/apple.h>
18 #include <linux/sizes.h>
19 #include <linux/slab.h>
20 #include <linux/workqueue.h>
21 
22 #include "ctl.h"
23 #include "nhi_regs.h"
24 #include "tb.h"
25 
26 #define PCIE2CIO_CMD			0x30
27 #define PCIE2CIO_CMD_TIMEOUT		BIT(31)
28 #define PCIE2CIO_CMD_START		BIT(30)
29 #define PCIE2CIO_CMD_WRITE		BIT(21)
30 #define PCIE2CIO_CMD_CS_MASK		GENMASK(20, 19)
31 #define PCIE2CIO_CMD_CS_SHIFT		19
32 #define PCIE2CIO_CMD_PORT_MASK		GENMASK(18, 13)
33 #define PCIE2CIO_CMD_PORT_SHIFT		13
34 
35 #define PCIE2CIO_WRDATA			0x34
36 #define PCIE2CIO_RDDATA			0x38
37 
38 #define PHY_PORT_CS1			0x37
39 #define PHY_PORT_CS1_LINK_DISABLE	BIT(14)
40 #define PHY_PORT_CS1_LINK_STATE_MASK	GENMASK(29, 26)
41 #define PHY_PORT_CS1_LINK_STATE_SHIFT	26
42 
43 #define ICM_TIMEOUT			5000	/* ms */
44 #define ICM_APPROVE_TIMEOUT		10000	/* ms */
45 #define ICM_MAX_LINK			4
46 
47 static bool start_icm;
48 module_param(start_icm, bool, 0444);
49 MODULE_PARM_DESC(start_icm, "start ICM firmware if it is not running (default: false)");
50 
51 /**
52  * struct usb4_switch_nvm_auth - Holds USB4 NVM_AUTH status
53  * @reply: Reply from ICM firmware is placed here
54  * @request: Request that is sent to ICM firmware
55  * @icm: Pointer to ICM private data
56  */
57 struct usb4_switch_nvm_auth {
58 	struct icm_usb4_switch_op_response reply;
59 	struct icm_usb4_switch_op request;
60 	struct icm *icm;
61 };
62 
63 /**
64  * struct icm - Internal connection manager private data
65  * @request_lock: Makes sure only one message is send to ICM at time
66  * @rescan_work: Work used to rescan the surviving switches after resume
67  * @upstream_port: Pointer to the PCIe upstream port this host
68  *		   controller is connected. This is only set for systems
69  *		   where ICM needs to be started manually
70  * @vnd_cap: Vendor defined capability where PCIe2CIO mailbox resides
71  *	     (only set when @upstream_port is not %NULL)
72  * @safe_mode: ICM is in safe mode
73  * @max_boot_acl: Maximum number of preboot ACL entries (%0 if not supported)
74  * @rpm: Does the controller support runtime PM (RTD3)
75  * @can_upgrade_nvm: Can the NVM firmware be upgrade on this controller
76  * @proto_version: Firmware protocol version
77  * @last_nvm_auth: Last USB4 router NVM_AUTH result (or %NULL if not set)
78  * @veto: Is RTD3 veto in effect
79  * @is_supported: Checks if we can support ICM on this controller
80  * @cio_reset: Trigger CIO reset
81  * @get_mode: Read and return the ICM firmware mode (optional)
82  * @get_route: Find a route string for given switch
83  * @save_devices: Ask ICM to save devices to ACL when suspending (optional)
84  * @driver_ready: Send driver ready message to ICM
85  * @set_uuid: Set UUID for the root switch (optional)
86  * @device_connected: Handle device connected ICM message
87  * @device_disconnected: Handle device disconnected ICM message
88  * @xdomain_connected: Handle XDomain connected ICM message
89  * @xdomain_disconnected: Handle XDomain disconnected ICM message
90  * @rtd3_veto: Handle RTD3 veto notification ICM message
91  */
92 struct icm {
93 	struct mutex request_lock;
94 	struct delayed_work rescan_work;
95 	struct pci_dev *upstream_port;
96 	int vnd_cap;
97 	bool safe_mode;
98 	size_t max_boot_acl;
99 	bool rpm;
100 	bool can_upgrade_nvm;
101 	u8 proto_version;
102 	struct usb4_switch_nvm_auth *last_nvm_auth;
103 	bool veto;
104 	bool (*is_supported)(struct tb *tb);
105 	int (*cio_reset)(struct tb *tb);
106 	int (*get_mode)(struct tb *tb);
107 	int (*get_route)(struct tb *tb, u8 link, u8 depth, u64 *route);
108 	void (*save_devices)(struct tb *tb);
109 	int (*driver_ready)(struct tb *tb,
110 			    enum tb_security_level *security_level,
111 			    u8 *proto_version, size_t *nboot_acl, bool *rpm);
112 	void (*set_uuid)(struct tb *tb);
113 	void (*device_connected)(struct tb *tb,
114 				 const struct icm_pkg_header *hdr);
115 	void (*device_disconnected)(struct tb *tb,
116 				    const struct icm_pkg_header *hdr);
117 	void (*xdomain_connected)(struct tb *tb,
118 				  const struct icm_pkg_header *hdr);
119 	void (*xdomain_disconnected)(struct tb *tb,
120 				     const struct icm_pkg_header *hdr);
121 	void (*rtd3_veto)(struct tb *tb, const struct icm_pkg_header *hdr);
122 };
123 
124 struct icm_notification {
125 	struct work_struct work;
126 	struct icm_pkg_header *pkg;
127 	struct tb *tb;
128 };
129 
130 struct ep_name_entry {
131 	u8 len;
132 	u8 type;
133 	u8 data[];
134 };
135 
136 #define EP_NAME_INTEL_VSS	0x10
137 
138 /* Intel Vendor specific structure */
139 struct intel_vss {
140 	u16 vendor;
141 	u16 model;
142 	u8 mc;
143 	u8 flags;
144 	u16 pci_devid;
145 	u32 nvm_version;
146 };
147 
148 #define INTEL_VSS_FLAGS_RTD3	BIT(0)
149 
150 static const struct intel_vss *parse_intel_vss(const void *ep_name, size_t size)
151 {
152 	const void *end = ep_name + size;
153 
154 	while (ep_name < end) {
155 		const struct ep_name_entry *ep = ep_name;
156 
157 		if (!ep->len)
158 			break;
159 		if (ep_name + ep->len > end)
160 			break;
161 
162 		if (ep->type == EP_NAME_INTEL_VSS)
163 			return (const struct intel_vss *)ep->data;
164 
165 		ep_name += ep->len;
166 	}
167 
168 	return NULL;
169 }
170 
171 static bool intel_vss_is_rtd3(const void *ep_name, size_t size)
172 {
173 	const struct intel_vss *vss;
174 
175 	vss = parse_intel_vss(ep_name, size);
176 	if (vss)
177 		return !!(vss->flags & INTEL_VSS_FLAGS_RTD3);
178 
179 	return false;
180 }
181 
182 static inline struct tb *icm_to_tb(struct icm *icm)
183 {
184 	return ((void *)icm - sizeof(struct tb));
185 }
186 
187 static inline u8 phy_port_from_route(u64 route, u8 depth)
188 {
189 	u8 link;
190 
191 	link = depth ? route >> ((depth - 1) * 8) : route;
192 	return tb_phy_port_from_link(link);
193 }
194 
195 static inline u8 dual_link_from_link(u8 link)
196 {
197 	return link ? ((link - 1) ^ 0x01) + 1 : 0;
198 }
199 
200 static inline u64 get_route(u32 route_hi, u32 route_lo)
201 {
202 	return (u64)route_hi << 32 | route_lo;
203 }
204 
205 static inline u64 get_parent_route(u64 route)
206 {
207 	int depth = tb_route_length(route);
208 	return depth ? route & ~(0xffULL << (depth - 1) * TB_ROUTE_SHIFT) : 0;
209 }
210 
211 static int pci2cio_wait_completion(struct icm *icm, unsigned long timeout_msec)
212 {
213 	unsigned long end = jiffies + msecs_to_jiffies(timeout_msec);
214 	u32 cmd;
215 
216 	do {
217 		pci_read_config_dword(icm->upstream_port,
218 				      icm->vnd_cap + PCIE2CIO_CMD, &cmd);
219 		if (!(cmd & PCIE2CIO_CMD_START)) {
220 			if (cmd & PCIE2CIO_CMD_TIMEOUT)
221 				break;
222 			return 0;
223 		}
224 
225 		msleep(50);
226 	} while (time_before(jiffies, end));
227 
228 	return -ETIMEDOUT;
229 }
230 
231 static int pcie2cio_read(struct icm *icm, enum tb_cfg_space cs,
232 			 unsigned int port, unsigned int index, u32 *data)
233 {
234 	struct pci_dev *pdev = icm->upstream_port;
235 	int ret, vnd_cap = icm->vnd_cap;
236 	u32 cmd;
237 
238 	cmd = index;
239 	cmd |= (port << PCIE2CIO_CMD_PORT_SHIFT) & PCIE2CIO_CMD_PORT_MASK;
240 	cmd |= (cs << PCIE2CIO_CMD_CS_SHIFT) & PCIE2CIO_CMD_CS_MASK;
241 	cmd |= PCIE2CIO_CMD_START;
242 	pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_CMD, cmd);
243 
244 	ret = pci2cio_wait_completion(icm, 5000);
245 	if (ret)
246 		return ret;
247 
248 	pci_read_config_dword(pdev, vnd_cap + PCIE2CIO_RDDATA, data);
249 	return 0;
250 }
251 
252 static int pcie2cio_write(struct icm *icm, enum tb_cfg_space cs,
253 			  unsigned int port, unsigned int index, u32 data)
254 {
255 	struct pci_dev *pdev = icm->upstream_port;
256 	int vnd_cap = icm->vnd_cap;
257 	u32 cmd;
258 
259 	pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_WRDATA, data);
260 
261 	cmd = index;
262 	cmd |= (port << PCIE2CIO_CMD_PORT_SHIFT) & PCIE2CIO_CMD_PORT_MASK;
263 	cmd |= (cs << PCIE2CIO_CMD_CS_SHIFT) & PCIE2CIO_CMD_CS_MASK;
264 	cmd |= PCIE2CIO_CMD_WRITE | PCIE2CIO_CMD_START;
265 	pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_CMD, cmd);
266 
267 	return pci2cio_wait_completion(icm, 5000);
268 }
269 
270 static bool icm_match(const struct tb_cfg_request *req,
271 		      const struct ctl_pkg *pkg)
272 {
273 	const struct icm_pkg_header *res_hdr = pkg->buffer;
274 	const struct icm_pkg_header *req_hdr = req->request;
275 
276 	if (pkg->frame.eof != req->response_type)
277 		return false;
278 	if (res_hdr->code != req_hdr->code)
279 		return false;
280 
281 	return true;
282 }
283 
284 static bool icm_copy(struct tb_cfg_request *req, const struct ctl_pkg *pkg)
285 {
286 	const struct icm_pkg_header *hdr = pkg->buffer;
287 
288 	if (hdr->packet_id < req->npackets) {
289 		size_t offset = hdr->packet_id * req->response_size;
290 
291 		memcpy(req->response + offset, pkg->buffer, req->response_size);
292 	}
293 
294 	return hdr->packet_id == hdr->total_packets - 1;
295 }
296 
297 static int icm_request(struct tb *tb, const void *request, size_t request_size,
298 		       void *response, size_t response_size, size_t npackets,
299 		       unsigned int timeout_msec)
300 {
301 	struct icm *icm = tb_priv(tb);
302 	int retries = 3;
303 
304 	do {
305 		struct tb_cfg_request *req;
306 		struct tb_cfg_result res;
307 
308 		req = tb_cfg_request_alloc();
309 		if (!req)
310 			return -ENOMEM;
311 
312 		req->match = icm_match;
313 		req->copy = icm_copy;
314 		req->request = request;
315 		req->request_size = request_size;
316 		req->request_type = TB_CFG_PKG_ICM_CMD;
317 		req->response = response;
318 		req->npackets = npackets;
319 		req->response_size = response_size;
320 		req->response_type = TB_CFG_PKG_ICM_RESP;
321 
322 		mutex_lock(&icm->request_lock);
323 		res = tb_cfg_request_sync(tb->ctl, req, timeout_msec);
324 		mutex_unlock(&icm->request_lock);
325 
326 		tb_cfg_request_put(req);
327 
328 		if (res.err != -ETIMEDOUT)
329 			return res.err == 1 ? -EIO : res.err;
330 
331 		usleep_range(20, 50);
332 	} while (retries--);
333 
334 	return -ETIMEDOUT;
335 }
336 
337 /*
338  * If rescan is queued to run (we are resuming), postpone it to give the
339  * firmware some more time to send device connected notifications for next
340  * devices in the chain.
341  */
342 static void icm_postpone_rescan(struct tb *tb)
343 {
344 	struct icm *icm = tb_priv(tb);
345 
346 	if (delayed_work_pending(&icm->rescan_work))
347 		mod_delayed_work(tb->wq, &icm->rescan_work,
348 				 msecs_to_jiffies(500));
349 }
350 
351 static void icm_veto_begin(struct tb *tb)
352 {
353 	struct icm *icm = tb_priv(tb);
354 
355 	if (!icm->veto) {
356 		icm->veto = true;
357 		/* Keep the domain powered while veto is in effect */
358 		pm_runtime_get(&tb->dev);
359 	}
360 }
361 
362 static void icm_veto_end(struct tb *tb)
363 {
364 	struct icm *icm = tb_priv(tb);
365 
366 	if (icm->veto) {
367 		icm->veto = false;
368 		/* Allow the domain suspend now */
369 		pm_runtime_mark_last_busy(&tb->dev);
370 		pm_runtime_put_autosuspend(&tb->dev);
371 	}
372 }
373 
374 static bool icm_firmware_running(const struct tb_nhi *nhi)
375 {
376 	u32 val;
377 
378 	val = ioread32(nhi->iobase + REG_FW_STS);
379 	return !!(val & REG_FW_STS_ICM_EN);
380 }
381 
382 static bool icm_fr_is_supported(struct tb *tb)
383 {
384 	return !x86_apple_machine;
385 }
386 
387 static inline int icm_fr_get_switch_index(u32 port)
388 {
389 	int index;
390 
391 	if ((port & ICM_PORT_TYPE_MASK) != TB_TYPE_PORT)
392 		return 0;
393 
394 	index = port >> ICM_PORT_INDEX_SHIFT;
395 	return index != 0xff ? index : 0;
396 }
397 
398 static int icm_fr_get_route(struct tb *tb, u8 link, u8 depth, u64 *route)
399 {
400 	struct icm_fr_pkg_get_topology_response *switches, *sw;
401 	struct icm_fr_pkg_get_topology request = {
402 		.hdr = { .code = ICM_GET_TOPOLOGY },
403 	};
404 	size_t npackets = ICM_GET_TOPOLOGY_PACKETS;
405 	int ret, index;
406 	u8 i;
407 
408 	switches = kcalloc(npackets, sizeof(*switches), GFP_KERNEL);
409 	if (!switches)
410 		return -ENOMEM;
411 
412 	ret = icm_request(tb, &request, sizeof(request), switches,
413 			  sizeof(*switches), npackets, ICM_TIMEOUT);
414 	if (ret)
415 		goto err_free;
416 
417 	sw = &switches[0];
418 	index = icm_fr_get_switch_index(sw->ports[link]);
419 	if (!index) {
420 		ret = -ENODEV;
421 		goto err_free;
422 	}
423 
424 	sw = &switches[index];
425 	for (i = 1; i < depth; i++) {
426 		unsigned int j;
427 
428 		if (!(sw->first_data & ICM_SWITCH_USED)) {
429 			ret = -ENODEV;
430 			goto err_free;
431 		}
432 
433 		for (j = 0; j < ARRAY_SIZE(sw->ports); j++) {
434 			index = icm_fr_get_switch_index(sw->ports[j]);
435 			if (index > sw->switch_index) {
436 				sw = &switches[index];
437 				break;
438 			}
439 		}
440 	}
441 
442 	*route = get_route(sw->route_hi, sw->route_lo);
443 
444 err_free:
445 	kfree(switches);
446 	return ret;
447 }
448 
449 static void icm_fr_save_devices(struct tb *tb)
450 {
451 	nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_SAVE_DEVS, 0);
452 }
453 
454 static int
455 icm_fr_driver_ready(struct tb *tb, enum tb_security_level *security_level,
456 		    u8 *proto_version, size_t *nboot_acl, bool *rpm)
457 {
458 	struct icm_fr_pkg_driver_ready_response reply;
459 	struct icm_pkg_driver_ready request = {
460 		.hdr.code = ICM_DRIVER_READY,
461 	};
462 	int ret;
463 
464 	memset(&reply, 0, sizeof(reply));
465 	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
466 			  1, ICM_TIMEOUT);
467 	if (ret)
468 		return ret;
469 
470 	if (security_level)
471 		*security_level = reply.security_level & ICM_FR_SLEVEL_MASK;
472 
473 	return 0;
474 }
475 
476 static int icm_fr_approve_switch(struct tb *tb, struct tb_switch *sw)
477 {
478 	struct icm_fr_pkg_approve_device request;
479 	struct icm_fr_pkg_approve_device reply;
480 	int ret;
481 
482 	memset(&request, 0, sizeof(request));
483 	memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
484 	request.hdr.code = ICM_APPROVE_DEVICE;
485 	request.connection_id = sw->connection_id;
486 	request.connection_key = sw->connection_key;
487 
488 	memset(&reply, 0, sizeof(reply));
489 	/* Use larger timeout as establishing tunnels can take some time */
490 	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
491 			  1, ICM_APPROVE_TIMEOUT);
492 	if (ret)
493 		return ret;
494 
495 	if (reply.hdr.flags & ICM_FLAGS_ERROR) {
496 		tb_warn(tb, "PCIe tunnel creation failed\n");
497 		return -EIO;
498 	}
499 
500 	return 0;
501 }
502 
503 static int icm_fr_add_switch_key(struct tb *tb, struct tb_switch *sw)
504 {
505 	struct icm_fr_pkg_add_device_key request;
506 	struct icm_fr_pkg_add_device_key_response reply;
507 	int ret;
508 
509 	memset(&request, 0, sizeof(request));
510 	memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
511 	request.hdr.code = ICM_ADD_DEVICE_KEY;
512 	request.connection_id = sw->connection_id;
513 	request.connection_key = sw->connection_key;
514 	memcpy(request.key, sw->key, TB_SWITCH_KEY_SIZE);
515 
516 	memset(&reply, 0, sizeof(reply));
517 	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
518 			  1, ICM_TIMEOUT);
519 	if (ret)
520 		return ret;
521 
522 	if (reply.hdr.flags & ICM_FLAGS_ERROR) {
523 		tb_warn(tb, "Adding key to switch failed\n");
524 		return -EIO;
525 	}
526 
527 	return 0;
528 }
529 
530 static int icm_fr_challenge_switch_key(struct tb *tb, struct tb_switch *sw,
531 				       const u8 *challenge, u8 *response)
532 {
533 	struct icm_fr_pkg_challenge_device request;
534 	struct icm_fr_pkg_challenge_device_response reply;
535 	int ret;
536 
537 	memset(&request, 0, sizeof(request));
538 	memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
539 	request.hdr.code = ICM_CHALLENGE_DEVICE;
540 	request.connection_id = sw->connection_id;
541 	request.connection_key = sw->connection_key;
542 	memcpy(request.challenge, challenge, TB_SWITCH_KEY_SIZE);
543 
544 	memset(&reply, 0, sizeof(reply));
545 	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
546 			  1, ICM_TIMEOUT);
547 	if (ret)
548 		return ret;
549 
550 	if (reply.hdr.flags & ICM_FLAGS_ERROR)
551 		return -EKEYREJECTED;
552 	if (reply.hdr.flags & ICM_FLAGS_NO_KEY)
553 		return -ENOKEY;
554 
555 	memcpy(response, reply.response, TB_SWITCH_KEY_SIZE);
556 
557 	return 0;
558 }
559 
560 static int icm_fr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
561 					int transmit_path, int transmit_ring,
562 					int receive_path, int receive_ring)
563 {
564 	struct icm_fr_pkg_approve_xdomain_response reply;
565 	struct icm_fr_pkg_approve_xdomain request;
566 	int ret;
567 
568 	memset(&request, 0, sizeof(request));
569 	request.hdr.code = ICM_APPROVE_XDOMAIN;
570 	request.link_info = xd->depth << ICM_LINK_INFO_DEPTH_SHIFT | xd->link;
571 	memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
572 
573 	request.transmit_path = transmit_path;
574 	request.transmit_ring = transmit_ring;
575 	request.receive_path = receive_path;
576 	request.receive_ring = receive_ring;
577 
578 	memset(&reply, 0, sizeof(reply));
579 	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
580 			  1, ICM_TIMEOUT);
581 	if (ret)
582 		return ret;
583 
584 	if (reply.hdr.flags & ICM_FLAGS_ERROR)
585 		return -EIO;
586 
587 	return 0;
588 }
589 
590 static int icm_fr_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
591 					   int transmit_path, int transmit_ring,
592 					   int receive_path, int receive_ring)
593 {
594 	u8 phy_port;
595 	u8 cmd;
596 
597 	phy_port = tb_phy_port_from_link(xd->link);
598 	if (phy_port == 0)
599 		cmd = NHI_MAILBOX_DISCONNECT_PA;
600 	else
601 		cmd = NHI_MAILBOX_DISCONNECT_PB;
602 
603 	nhi_mailbox_cmd(tb->nhi, cmd, 1);
604 	usleep_range(10, 50);
605 	nhi_mailbox_cmd(tb->nhi, cmd, 2);
606 	return 0;
607 }
608 
609 static struct tb_switch *alloc_switch(struct tb_switch *parent_sw, u64 route,
610 				      const uuid_t *uuid)
611 {
612 	struct tb *tb = parent_sw->tb;
613 	struct tb_switch *sw;
614 
615 	sw = tb_switch_alloc(tb, &parent_sw->dev, route);
616 	if (IS_ERR(sw)) {
617 		tb_warn(tb, "failed to allocate switch at %llx\n", route);
618 		return sw;
619 	}
620 
621 	sw->uuid = kmemdup(uuid, sizeof(*uuid), GFP_KERNEL);
622 	if (!sw->uuid) {
623 		tb_switch_put(sw);
624 		return ERR_PTR(-ENOMEM);
625 	}
626 
627 	init_completion(&sw->rpm_complete);
628 	return sw;
629 }
630 
631 static int add_switch(struct tb_switch *parent_sw, struct tb_switch *sw)
632 {
633 	u64 route = tb_route(sw);
634 	int ret;
635 
636 	/* Link the two switches now */
637 	tb_port_at(route, parent_sw)->remote = tb_upstream_port(sw);
638 	tb_upstream_port(sw)->remote = tb_port_at(route, parent_sw);
639 
640 	ret = tb_switch_add(sw);
641 	if (ret)
642 		tb_port_at(tb_route(sw), parent_sw)->remote = NULL;
643 
644 	return ret;
645 }
646 
647 static void update_switch(struct tb_switch *sw, u64 route, u8 connection_id,
648 			  u8 connection_key, u8 link, u8 depth, bool boot)
649 {
650 	struct tb_switch *parent_sw = tb_switch_parent(sw);
651 
652 	/* Disconnect from parent */
653 	tb_switch_downstream_port(sw)->remote = NULL;
654 	/* Re-connect via updated port */
655 	tb_port_at(route, parent_sw)->remote = tb_upstream_port(sw);
656 
657 	/* Update with the new addressing information */
658 	sw->config.route_hi = upper_32_bits(route);
659 	sw->config.route_lo = lower_32_bits(route);
660 	sw->connection_id = connection_id;
661 	sw->connection_key = connection_key;
662 	sw->link = link;
663 	sw->depth = depth;
664 	sw->boot = boot;
665 
666 	/* This switch still exists */
667 	sw->is_unplugged = false;
668 
669 	/* Runtime resume is now complete */
670 	complete(&sw->rpm_complete);
671 }
672 
673 static void remove_switch(struct tb_switch *sw)
674 {
675 	tb_switch_downstream_port(sw)->remote = NULL;
676 	tb_switch_remove(sw);
677 }
678 
679 static void add_xdomain(struct tb_switch *sw, u64 route,
680 			const uuid_t *local_uuid, const uuid_t *remote_uuid,
681 			u8 link, u8 depth)
682 {
683 	struct tb_xdomain *xd;
684 
685 	pm_runtime_get_sync(&sw->dev);
686 
687 	xd = tb_xdomain_alloc(sw->tb, &sw->dev, route, local_uuid, remote_uuid);
688 	if (!xd)
689 		goto out;
690 
691 	xd->link = link;
692 	xd->depth = depth;
693 
694 	tb_port_at(route, sw)->xdomain = xd;
695 
696 	tb_xdomain_add(xd);
697 
698 out:
699 	pm_runtime_mark_last_busy(&sw->dev);
700 	pm_runtime_put_autosuspend(&sw->dev);
701 }
702 
703 static void update_xdomain(struct tb_xdomain *xd, u64 route, u8 link)
704 {
705 	xd->link = link;
706 	xd->route = route;
707 	xd->is_unplugged = false;
708 }
709 
710 static void remove_xdomain(struct tb_xdomain *xd)
711 {
712 	struct tb_switch *sw;
713 
714 	sw = tb_to_switch(xd->dev.parent);
715 	tb_port_at(xd->route, sw)->xdomain = NULL;
716 	tb_xdomain_remove(xd);
717 }
718 
719 static void
720 icm_fr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
721 {
722 	const struct icm_fr_event_device_connected *pkg =
723 		(const struct icm_fr_event_device_connected *)hdr;
724 	enum tb_security_level security_level;
725 	struct tb_switch *sw, *parent_sw;
726 	bool boot, dual_lane, speed_gen3;
727 	struct icm *icm = tb_priv(tb);
728 	bool authorized = false;
729 	struct tb_xdomain *xd;
730 	u8 link, depth;
731 	u64 route;
732 	int ret;
733 
734 	icm_postpone_rescan(tb);
735 
736 	link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
737 	depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
738 		ICM_LINK_INFO_DEPTH_SHIFT;
739 	authorized = pkg->link_info & ICM_LINK_INFO_APPROVED;
740 	security_level = (pkg->hdr.flags & ICM_FLAGS_SLEVEL_MASK) >>
741 			 ICM_FLAGS_SLEVEL_SHIFT;
742 	boot = pkg->link_info & ICM_LINK_INFO_BOOT;
743 	dual_lane = pkg->hdr.flags & ICM_FLAGS_DUAL_LANE;
744 	speed_gen3 = pkg->hdr.flags & ICM_FLAGS_SPEED_GEN3;
745 
746 	if (pkg->link_info & ICM_LINK_INFO_REJECTED) {
747 		tb_info(tb, "switch at %u.%u was rejected by ICM firmware because topology limit exceeded\n",
748 			link, depth);
749 		return;
750 	}
751 
752 	sw = tb_switch_find_by_uuid(tb, &pkg->ep_uuid);
753 	if (sw) {
754 		u8 phy_port, sw_phy_port;
755 
756 		sw_phy_port = tb_phy_port_from_link(sw->link);
757 		phy_port = tb_phy_port_from_link(link);
758 
759 		/*
760 		 * On resume ICM will send us connected events for the
761 		 * devices that still are present. However, that
762 		 * information might have changed for example by the
763 		 * fact that a switch on a dual-link connection might
764 		 * have been enumerated using the other link now. Make
765 		 * sure our book keeping matches that.
766 		 */
767 		if (sw->depth == depth && sw_phy_port == phy_port &&
768 		    !!sw->authorized == authorized) {
769 			/*
770 			 * It was enumerated through another link so update
771 			 * route string accordingly.
772 			 */
773 			if (sw->link != link) {
774 				ret = icm->get_route(tb, link, depth, &route);
775 				if (ret) {
776 					tb_err(tb, "failed to update route string for switch at %u.%u\n",
777 					       link, depth);
778 					tb_switch_put(sw);
779 					return;
780 				}
781 			} else {
782 				route = tb_route(sw);
783 			}
784 
785 			update_switch(sw, route, pkg->connection_id,
786 				      pkg->connection_key, link, depth, boot);
787 			tb_switch_put(sw);
788 			return;
789 		}
790 
791 		/*
792 		 * User connected the same switch to another physical
793 		 * port or to another part of the topology. Remove the
794 		 * existing switch now before adding the new one.
795 		 */
796 		remove_switch(sw);
797 		tb_switch_put(sw);
798 	}
799 
800 	/*
801 	 * If the switch was not found by UUID, look for a switch on
802 	 * same physical port (taking possible link aggregation into
803 	 * account) and depth. If we found one it is definitely a stale
804 	 * one so remove it first.
805 	 */
806 	sw = tb_switch_find_by_link_depth(tb, link, depth);
807 	if (!sw) {
808 		u8 dual_link;
809 
810 		dual_link = dual_link_from_link(link);
811 		if (dual_link)
812 			sw = tb_switch_find_by_link_depth(tb, dual_link, depth);
813 	}
814 	if (sw) {
815 		remove_switch(sw);
816 		tb_switch_put(sw);
817 	}
818 
819 	/* Remove existing XDomain connection if found */
820 	xd = tb_xdomain_find_by_link_depth(tb, link, depth);
821 	if (xd) {
822 		remove_xdomain(xd);
823 		tb_xdomain_put(xd);
824 	}
825 
826 	parent_sw = tb_switch_find_by_link_depth(tb, link, depth - 1);
827 	if (!parent_sw) {
828 		tb_err(tb, "failed to find parent switch for %u.%u\n",
829 		       link, depth);
830 		return;
831 	}
832 
833 	ret = icm->get_route(tb, link, depth, &route);
834 	if (ret) {
835 		tb_err(tb, "failed to find route string for switch at %u.%u\n",
836 		       link, depth);
837 		tb_switch_put(parent_sw);
838 		return;
839 	}
840 
841 	pm_runtime_get_sync(&parent_sw->dev);
842 
843 	sw = alloc_switch(parent_sw, route, &pkg->ep_uuid);
844 	if (!IS_ERR(sw)) {
845 		sw->connection_id = pkg->connection_id;
846 		sw->connection_key = pkg->connection_key;
847 		sw->link = link;
848 		sw->depth = depth;
849 		sw->authorized = authorized;
850 		sw->security_level = security_level;
851 		sw->boot = boot;
852 		sw->link_speed = speed_gen3 ? 20 : 10;
853 		sw->link_width = dual_lane ? TB_LINK_WIDTH_DUAL :
854 					     TB_LINK_WIDTH_SINGLE;
855 		sw->rpm = intel_vss_is_rtd3(pkg->ep_name, sizeof(pkg->ep_name));
856 
857 		if (add_switch(parent_sw, sw))
858 			tb_switch_put(sw);
859 	}
860 
861 	pm_runtime_mark_last_busy(&parent_sw->dev);
862 	pm_runtime_put_autosuspend(&parent_sw->dev);
863 
864 	tb_switch_put(parent_sw);
865 }
866 
867 static void
868 icm_fr_device_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
869 {
870 	const struct icm_fr_event_device_disconnected *pkg =
871 		(const struct icm_fr_event_device_disconnected *)hdr;
872 	struct tb_switch *sw;
873 	u8 link, depth;
874 
875 	link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
876 	depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
877 		ICM_LINK_INFO_DEPTH_SHIFT;
878 
879 	if (link > ICM_MAX_LINK || depth > TB_SWITCH_MAX_DEPTH) {
880 		tb_warn(tb, "invalid topology %u.%u, ignoring\n", link, depth);
881 		return;
882 	}
883 
884 	sw = tb_switch_find_by_link_depth(tb, link, depth);
885 	if (!sw) {
886 		tb_warn(tb, "no switch exists at %u.%u, ignoring\n", link,
887 			depth);
888 		return;
889 	}
890 
891 	pm_runtime_get_sync(sw->dev.parent);
892 
893 	remove_switch(sw);
894 
895 	pm_runtime_mark_last_busy(sw->dev.parent);
896 	pm_runtime_put_autosuspend(sw->dev.parent);
897 
898 	tb_switch_put(sw);
899 }
900 
901 static void
902 icm_fr_xdomain_connected(struct tb *tb, const struct icm_pkg_header *hdr)
903 {
904 	const struct icm_fr_event_xdomain_connected *pkg =
905 		(const struct icm_fr_event_xdomain_connected *)hdr;
906 	struct tb_xdomain *xd;
907 	struct tb_switch *sw;
908 	u8 link, depth;
909 	u64 route;
910 
911 	link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
912 	depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
913 		ICM_LINK_INFO_DEPTH_SHIFT;
914 
915 	if (link > ICM_MAX_LINK || depth > TB_SWITCH_MAX_DEPTH) {
916 		tb_warn(tb, "invalid topology %u.%u, ignoring\n", link, depth);
917 		return;
918 	}
919 
920 	route = get_route(pkg->local_route_hi, pkg->local_route_lo);
921 
922 	xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
923 	if (xd) {
924 		u8 xd_phy_port, phy_port;
925 
926 		xd_phy_port = phy_port_from_route(xd->route, xd->depth);
927 		phy_port = phy_port_from_route(route, depth);
928 
929 		if (xd->depth == depth && xd_phy_port == phy_port) {
930 			update_xdomain(xd, route, link);
931 			tb_xdomain_put(xd);
932 			return;
933 		}
934 
935 		/*
936 		 * If we find an existing XDomain connection remove it
937 		 * now. We need to go through login handshake and
938 		 * everything anyway to be able to re-establish the
939 		 * connection.
940 		 */
941 		remove_xdomain(xd);
942 		tb_xdomain_put(xd);
943 	}
944 
945 	/*
946 	 * Look if there already exists an XDomain in the same place
947 	 * than the new one and in that case remove it because it is
948 	 * most likely another host that got disconnected.
949 	 */
950 	xd = tb_xdomain_find_by_link_depth(tb, link, depth);
951 	if (!xd) {
952 		u8 dual_link;
953 
954 		dual_link = dual_link_from_link(link);
955 		if (dual_link)
956 			xd = tb_xdomain_find_by_link_depth(tb, dual_link,
957 							   depth);
958 	}
959 	if (xd) {
960 		remove_xdomain(xd);
961 		tb_xdomain_put(xd);
962 	}
963 
964 	/*
965 	 * If the user disconnected a switch during suspend and
966 	 * connected another host to the same port, remove the switch
967 	 * first.
968 	 */
969 	sw = tb_switch_find_by_route(tb, route);
970 	if (sw) {
971 		remove_switch(sw);
972 		tb_switch_put(sw);
973 	}
974 
975 	sw = tb_switch_find_by_link_depth(tb, link, depth);
976 	if (!sw) {
977 		tb_warn(tb, "no switch exists at %u.%u, ignoring\n", link,
978 			depth);
979 		return;
980 	}
981 
982 	add_xdomain(sw, route, &pkg->local_uuid, &pkg->remote_uuid, link,
983 		    depth);
984 	tb_switch_put(sw);
985 }
986 
987 static void
988 icm_fr_xdomain_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
989 {
990 	const struct icm_fr_event_xdomain_disconnected *pkg =
991 		(const struct icm_fr_event_xdomain_disconnected *)hdr;
992 	struct tb_xdomain *xd;
993 
994 	/*
995 	 * If the connection is through one or multiple devices, the
996 	 * XDomain device is removed along with them so it is fine if we
997 	 * cannot find it here.
998 	 */
999 	xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
1000 	if (xd) {
1001 		remove_xdomain(xd);
1002 		tb_xdomain_put(xd);
1003 	}
1004 }
1005 
1006 static int icm_tr_cio_reset(struct tb *tb)
1007 {
1008 	return pcie2cio_write(tb_priv(tb), TB_CFG_SWITCH, 0, 0x777, BIT(1));
1009 }
1010 
1011 static int
1012 icm_tr_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1013 		    u8 *proto_version, size_t *nboot_acl, bool *rpm)
1014 {
1015 	struct icm_tr_pkg_driver_ready_response reply;
1016 	struct icm_pkg_driver_ready request = {
1017 		.hdr.code = ICM_DRIVER_READY,
1018 	};
1019 	int ret;
1020 
1021 	memset(&reply, 0, sizeof(reply));
1022 	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1023 			  1, 20000);
1024 	if (ret)
1025 		return ret;
1026 
1027 	if (security_level)
1028 		*security_level = reply.info & ICM_TR_INFO_SLEVEL_MASK;
1029 	if (proto_version)
1030 		*proto_version = (reply.info & ICM_TR_INFO_PROTO_VERSION_MASK) >>
1031 				ICM_TR_INFO_PROTO_VERSION_SHIFT;
1032 	if (nboot_acl)
1033 		*nboot_acl = (reply.info & ICM_TR_INFO_BOOT_ACL_MASK) >>
1034 				ICM_TR_INFO_BOOT_ACL_SHIFT;
1035 	if (rpm)
1036 		*rpm = !!(reply.hdr.flags & ICM_TR_FLAGS_RTD3);
1037 
1038 	return 0;
1039 }
1040 
1041 static int icm_tr_approve_switch(struct tb *tb, struct tb_switch *sw)
1042 {
1043 	struct icm_tr_pkg_approve_device request;
1044 	struct icm_tr_pkg_approve_device reply;
1045 	int ret;
1046 
1047 	memset(&request, 0, sizeof(request));
1048 	memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1049 	request.hdr.code = ICM_APPROVE_DEVICE;
1050 	request.route_lo = sw->config.route_lo;
1051 	request.route_hi = sw->config.route_hi;
1052 	request.connection_id = sw->connection_id;
1053 
1054 	memset(&reply, 0, sizeof(reply));
1055 	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1056 			  1, ICM_APPROVE_TIMEOUT);
1057 	if (ret)
1058 		return ret;
1059 
1060 	if (reply.hdr.flags & ICM_FLAGS_ERROR) {
1061 		tb_warn(tb, "PCIe tunnel creation failed\n");
1062 		return -EIO;
1063 	}
1064 
1065 	return 0;
1066 }
1067 
1068 static int icm_tr_add_switch_key(struct tb *tb, struct tb_switch *sw)
1069 {
1070 	struct icm_tr_pkg_add_device_key_response reply;
1071 	struct icm_tr_pkg_add_device_key request;
1072 	int ret;
1073 
1074 	memset(&request, 0, sizeof(request));
1075 	memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1076 	request.hdr.code = ICM_ADD_DEVICE_KEY;
1077 	request.route_lo = sw->config.route_lo;
1078 	request.route_hi = sw->config.route_hi;
1079 	request.connection_id = sw->connection_id;
1080 	memcpy(request.key, sw->key, TB_SWITCH_KEY_SIZE);
1081 
1082 	memset(&reply, 0, sizeof(reply));
1083 	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1084 			  1, ICM_TIMEOUT);
1085 	if (ret)
1086 		return ret;
1087 
1088 	if (reply.hdr.flags & ICM_FLAGS_ERROR) {
1089 		tb_warn(tb, "Adding key to switch failed\n");
1090 		return -EIO;
1091 	}
1092 
1093 	return 0;
1094 }
1095 
1096 static int icm_tr_challenge_switch_key(struct tb *tb, struct tb_switch *sw,
1097 				       const u8 *challenge, u8 *response)
1098 {
1099 	struct icm_tr_pkg_challenge_device_response reply;
1100 	struct icm_tr_pkg_challenge_device request;
1101 	int ret;
1102 
1103 	memset(&request, 0, sizeof(request));
1104 	memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1105 	request.hdr.code = ICM_CHALLENGE_DEVICE;
1106 	request.route_lo = sw->config.route_lo;
1107 	request.route_hi = sw->config.route_hi;
1108 	request.connection_id = sw->connection_id;
1109 	memcpy(request.challenge, challenge, TB_SWITCH_KEY_SIZE);
1110 
1111 	memset(&reply, 0, sizeof(reply));
1112 	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1113 			  1, ICM_TIMEOUT);
1114 	if (ret)
1115 		return ret;
1116 
1117 	if (reply.hdr.flags & ICM_FLAGS_ERROR)
1118 		return -EKEYREJECTED;
1119 	if (reply.hdr.flags & ICM_FLAGS_NO_KEY)
1120 		return -ENOKEY;
1121 
1122 	memcpy(response, reply.response, TB_SWITCH_KEY_SIZE);
1123 
1124 	return 0;
1125 }
1126 
1127 static int icm_tr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
1128 					int transmit_path, int transmit_ring,
1129 					int receive_path, int receive_ring)
1130 {
1131 	struct icm_tr_pkg_approve_xdomain_response reply;
1132 	struct icm_tr_pkg_approve_xdomain request;
1133 	int ret;
1134 
1135 	memset(&request, 0, sizeof(request));
1136 	request.hdr.code = ICM_APPROVE_XDOMAIN;
1137 	request.route_hi = upper_32_bits(xd->route);
1138 	request.route_lo = lower_32_bits(xd->route);
1139 	request.transmit_path = transmit_path;
1140 	request.transmit_ring = transmit_ring;
1141 	request.receive_path = receive_path;
1142 	request.receive_ring = receive_ring;
1143 	memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
1144 
1145 	memset(&reply, 0, sizeof(reply));
1146 	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1147 			  1, ICM_TIMEOUT);
1148 	if (ret)
1149 		return ret;
1150 
1151 	if (reply.hdr.flags & ICM_FLAGS_ERROR)
1152 		return -EIO;
1153 
1154 	return 0;
1155 }
1156 
1157 static int icm_tr_xdomain_tear_down(struct tb *tb, struct tb_xdomain *xd,
1158 				    int stage)
1159 {
1160 	struct icm_tr_pkg_disconnect_xdomain_response reply;
1161 	struct icm_tr_pkg_disconnect_xdomain request;
1162 	int ret;
1163 
1164 	memset(&request, 0, sizeof(request));
1165 	request.hdr.code = ICM_DISCONNECT_XDOMAIN;
1166 	request.stage = stage;
1167 	request.route_hi = upper_32_bits(xd->route);
1168 	request.route_lo = lower_32_bits(xd->route);
1169 	memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
1170 
1171 	memset(&reply, 0, sizeof(reply));
1172 	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1173 			  1, ICM_TIMEOUT);
1174 	if (ret)
1175 		return ret;
1176 
1177 	if (reply.hdr.flags & ICM_FLAGS_ERROR)
1178 		return -EIO;
1179 
1180 	return 0;
1181 }
1182 
1183 static int icm_tr_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
1184 					   int transmit_path, int transmit_ring,
1185 					   int receive_path, int receive_ring)
1186 {
1187 	int ret;
1188 
1189 	ret = icm_tr_xdomain_tear_down(tb, xd, 1);
1190 	if (ret)
1191 		return ret;
1192 
1193 	usleep_range(10, 50);
1194 	return icm_tr_xdomain_tear_down(tb, xd, 2);
1195 }
1196 
1197 static void
1198 __icm_tr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr,
1199 			  bool force_rtd3)
1200 {
1201 	const struct icm_tr_event_device_connected *pkg =
1202 		(const struct icm_tr_event_device_connected *)hdr;
1203 	bool authorized, boot, dual_lane, speed_gen3;
1204 	enum tb_security_level security_level;
1205 	struct tb_switch *sw, *parent_sw;
1206 	struct tb_xdomain *xd;
1207 	u64 route;
1208 
1209 	icm_postpone_rescan(tb);
1210 
1211 	/*
1212 	 * Currently we don't use the QoS information coming with the
1213 	 * device connected message so simply just ignore that extra
1214 	 * packet for now.
1215 	 */
1216 	if (pkg->hdr.packet_id)
1217 		return;
1218 
1219 	route = get_route(pkg->route_hi, pkg->route_lo);
1220 	authorized = pkg->link_info & ICM_LINK_INFO_APPROVED;
1221 	security_level = (pkg->hdr.flags & ICM_FLAGS_SLEVEL_MASK) >>
1222 			 ICM_FLAGS_SLEVEL_SHIFT;
1223 	boot = pkg->link_info & ICM_LINK_INFO_BOOT;
1224 	dual_lane = pkg->hdr.flags & ICM_FLAGS_DUAL_LANE;
1225 	speed_gen3 = pkg->hdr.flags & ICM_FLAGS_SPEED_GEN3;
1226 
1227 	if (pkg->link_info & ICM_LINK_INFO_REJECTED) {
1228 		tb_info(tb, "switch at %llx was rejected by ICM firmware because topology limit exceeded\n",
1229 			route);
1230 		return;
1231 	}
1232 
1233 	sw = tb_switch_find_by_uuid(tb, &pkg->ep_uuid);
1234 	if (sw) {
1235 		/* Update the switch if it is still in the same place */
1236 		if (tb_route(sw) == route && !!sw->authorized == authorized) {
1237 			update_switch(sw, route, pkg->connection_id, 0, 0, 0,
1238 				      boot);
1239 			tb_switch_put(sw);
1240 			return;
1241 		}
1242 
1243 		remove_switch(sw);
1244 		tb_switch_put(sw);
1245 	}
1246 
1247 	/* Another switch with the same address */
1248 	sw = tb_switch_find_by_route(tb, route);
1249 	if (sw) {
1250 		remove_switch(sw);
1251 		tb_switch_put(sw);
1252 	}
1253 
1254 	/* XDomain connection with the same address */
1255 	xd = tb_xdomain_find_by_route(tb, route);
1256 	if (xd) {
1257 		remove_xdomain(xd);
1258 		tb_xdomain_put(xd);
1259 	}
1260 
1261 	parent_sw = tb_switch_find_by_route(tb, get_parent_route(route));
1262 	if (!parent_sw) {
1263 		tb_err(tb, "failed to find parent switch for %llx\n", route);
1264 		return;
1265 	}
1266 
1267 	pm_runtime_get_sync(&parent_sw->dev);
1268 
1269 	sw = alloc_switch(parent_sw, route, &pkg->ep_uuid);
1270 	if (!IS_ERR(sw)) {
1271 		sw->connection_id = pkg->connection_id;
1272 		sw->authorized = authorized;
1273 		sw->security_level = security_level;
1274 		sw->boot = boot;
1275 		sw->link_speed = speed_gen3 ? 20 : 10;
1276 		sw->link_width = dual_lane ? TB_LINK_WIDTH_DUAL :
1277 					     TB_LINK_WIDTH_SINGLE;
1278 		sw->rpm = force_rtd3;
1279 		if (!sw->rpm)
1280 			sw->rpm = intel_vss_is_rtd3(pkg->ep_name,
1281 						    sizeof(pkg->ep_name));
1282 
1283 		if (add_switch(parent_sw, sw))
1284 			tb_switch_put(sw);
1285 	}
1286 
1287 	pm_runtime_mark_last_busy(&parent_sw->dev);
1288 	pm_runtime_put_autosuspend(&parent_sw->dev);
1289 
1290 	tb_switch_put(parent_sw);
1291 }
1292 
1293 static void
1294 icm_tr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1295 {
1296 	__icm_tr_device_connected(tb, hdr, false);
1297 }
1298 
1299 static void
1300 icm_tr_device_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
1301 {
1302 	const struct icm_tr_event_device_disconnected *pkg =
1303 		(const struct icm_tr_event_device_disconnected *)hdr;
1304 	struct tb_switch *sw;
1305 	u64 route;
1306 
1307 	route = get_route(pkg->route_hi, pkg->route_lo);
1308 
1309 	sw = tb_switch_find_by_route(tb, route);
1310 	if (!sw) {
1311 		tb_warn(tb, "no switch exists at %llx, ignoring\n", route);
1312 		return;
1313 	}
1314 	pm_runtime_get_sync(sw->dev.parent);
1315 
1316 	remove_switch(sw);
1317 
1318 	pm_runtime_mark_last_busy(sw->dev.parent);
1319 	pm_runtime_put_autosuspend(sw->dev.parent);
1320 
1321 	tb_switch_put(sw);
1322 }
1323 
1324 static void
1325 icm_tr_xdomain_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1326 {
1327 	const struct icm_tr_event_xdomain_connected *pkg =
1328 		(const struct icm_tr_event_xdomain_connected *)hdr;
1329 	struct tb_xdomain *xd;
1330 	struct tb_switch *sw;
1331 	u64 route;
1332 
1333 	if (!tb->root_switch)
1334 		return;
1335 
1336 	route = get_route(pkg->local_route_hi, pkg->local_route_lo);
1337 
1338 	xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
1339 	if (xd) {
1340 		if (xd->route == route) {
1341 			update_xdomain(xd, route, 0);
1342 			tb_xdomain_put(xd);
1343 			return;
1344 		}
1345 
1346 		remove_xdomain(xd);
1347 		tb_xdomain_put(xd);
1348 	}
1349 
1350 	/* An existing xdomain with the same address */
1351 	xd = tb_xdomain_find_by_route(tb, route);
1352 	if (xd) {
1353 		remove_xdomain(xd);
1354 		tb_xdomain_put(xd);
1355 	}
1356 
1357 	/*
1358 	 * If the user disconnected a switch during suspend and
1359 	 * connected another host to the same port, remove the switch
1360 	 * first.
1361 	 */
1362 	sw = tb_switch_find_by_route(tb, route);
1363 	if (sw) {
1364 		remove_switch(sw);
1365 		tb_switch_put(sw);
1366 	}
1367 
1368 	sw = tb_switch_find_by_route(tb, get_parent_route(route));
1369 	if (!sw) {
1370 		tb_warn(tb, "no switch exists at %llx, ignoring\n", route);
1371 		return;
1372 	}
1373 
1374 	add_xdomain(sw, route, &pkg->local_uuid, &pkg->remote_uuid, 0, 0);
1375 	tb_switch_put(sw);
1376 }
1377 
1378 static void
1379 icm_tr_xdomain_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
1380 {
1381 	const struct icm_tr_event_xdomain_disconnected *pkg =
1382 		(const struct icm_tr_event_xdomain_disconnected *)hdr;
1383 	struct tb_xdomain *xd;
1384 	u64 route;
1385 
1386 	route = get_route(pkg->route_hi, pkg->route_lo);
1387 
1388 	xd = tb_xdomain_find_by_route(tb, route);
1389 	if (xd) {
1390 		remove_xdomain(xd);
1391 		tb_xdomain_put(xd);
1392 	}
1393 }
1394 
1395 static struct pci_dev *get_upstream_port(struct pci_dev *pdev)
1396 {
1397 	struct pci_dev *parent;
1398 
1399 	parent = pci_upstream_bridge(pdev);
1400 	while (parent) {
1401 		if (!pci_is_pcie(parent))
1402 			return NULL;
1403 		if (pci_pcie_type(parent) == PCI_EXP_TYPE_UPSTREAM)
1404 			break;
1405 		parent = pci_upstream_bridge(parent);
1406 	}
1407 
1408 	if (!parent)
1409 		return NULL;
1410 
1411 	switch (parent->device) {
1412 	case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_BRIDGE:
1413 	case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_BRIDGE:
1414 	case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_BRIDGE:
1415 	case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_BRIDGE:
1416 	case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_BRIDGE:
1417 	case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_BRIDGE:
1418 	case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_BRIDGE:
1419 		return parent;
1420 	}
1421 
1422 	return NULL;
1423 }
1424 
1425 static bool icm_ar_is_supported(struct tb *tb)
1426 {
1427 	struct pci_dev *upstream_port;
1428 	struct icm *icm = tb_priv(tb);
1429 
1430 	/*
1431 	 * Starting from Alpine Ridge we can use ICM on Apple machines
1432 	 * as well. We just need to reset and re-enable it first.
1433 	 * However, only start it if explicitly asked by the user.
1434 	 */
1435 	if (icm_firmware_running(tb->nhi))
1436 		return true;
1437 	if (!start_icm)
1438 		return false;
1439 
1440 	/*
1441 	 * Find the upstream PCIe port in case we need to do reset
1442 	 * through its vendor specific registers.
1443 	 */
1444 	upstream_port = get_upstream_port(tb->nhi->pdev);
1445 	if (upstream_port) {
1446 		int cap;
1447 
1448 		cap = pci_find_ext_capability(upstream_port,
1449 					      PCI_EXT_CAP_ID_VNDR);
1450 		if (cap > 0) {
1451 			icm->upstream_port = upstream_port;
1452 			icm->vnd_cap = cap;
1453 
1454 			return true;
1455 		}
1456 	}
1457 
1458 	return false;
1459 }
1460 
1461 static int icm_ar_cio_reset(struct tb *tb)
1462 {
1463 	return pcie2cio_write(tb_priv(tb), TB_CFG_SWITCH, 0, 0x50, BIT(9));
1464 }
1465 
1466 static int icm_ar_get_mode(struct tb *tb)
1467 {
1468 	struct tb_nhi *nhi = tb->nhi;
1469 	int retries = 60;
1470 	u32 val;
1471 
1472 	do {
1473 		val = ioread32(nhi->iobase + REG_FW_STS);
1474 		if (val & REG_FW_STS_NVM_AUTH_DONE)
1475 			break;
1476 		msleep(50);
1477 	} while (--retries);
1478 
1479 	if (!retries) {
1480 		dev_err(&nhi->pdev->dev, "ICM firmware not authenticated\n");
1481 		return -ENODEV;
1482 	}
1483 
1484 	return nhi_mailbox_mode(nhi);
1485 }
1486 
1487 static int
1488 icm_ar_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1489 		    u8 *proto_version, size_t *nboot_acl, bool *rpm)
1490 {
1491 	struct icm_ar_pkg_driver_ready_response reply;
1492 	struct icm_pkg_driver_ready request = {
1493 		.hdr.code = ICM_DRIVER_READY,
1494 	};
1495 	int ret;
1496 
1497 	memset(&reply, 0, sizeof(reply));
1498 	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1499 			  1, ICM_TIMEOUT);
1500 	if (ret)
1501 		return ret;
1502 
1503 	if (security_level)
1504 		*security_level = reply.info & ICM_AR_INFO_SLEVEL_MASK;
1505 	if (nboot_acl && (reply.info & ICM_AR_INFO_BOOT_ACL_SUPPORTED))
1506 		*nboot_acl = (reply.info & ICM_AR_INFO_BOOT_ACL_MASK) >>
1507 				ICM_AR_INFO_BOOT_ACL_SHIFT;
1508 	if (rpm)
1509 		*rpm = !!(reply.hdr.flags & ICM_AR_FLAGS_RTD3);
1510 
1511 	return 0;
1512 }
1513 
1514 static int icm_ar_get_route(struct tb *tb, u8 link, u8 depth, u64 *route)
1515 {
1516 	struct icm_ar_pkg_get_route_response reply;
1517 	struct icm_ar_pkg_get_route request = {
1518 		.hdr = { .code = ICM_GET_ROUTE },
1519 		.link_info = depth << ICM_LINK_INFO_DEPTH_SHIFT | link,
1520 	};
1521 	int ret;
1522 
1523 	memset(&reply, 0, sizeof(reply));
1524 	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1525 			  1, ICM_TIMEOUT);
1526 	if (ret)
1527 		return ret;
1528 
1529 	if (reply.hdr.flags & ICM_FLAGS_ERROR)
1530 		return -EIO;
1531 
1532 	*route = get_route(reply.route_hi, reply.route_lo);
1533 	return 0;
1534 }
1535 
1536 static int icm_ar_get_boot_acl(struct tb *tb, uuid_t *uuids, size_t nuuids)
1537 {
1538 	struct icm_ar_pkg_preboot_acl_response reply;
1539 	struct icm_ar_pkg_preboot_acl request = {
1540 		.hdr = { .code = ICM_PREBOOT_ACL },
1541 	};
1542 	int ret, i;
1543 
1544 	memset(&reply, 0, sizeof(reply));
1545 	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1546 			  1, ICM_TIMEOUT);
1547 	if (ret)
1548 		return ret;
1549 
1550 	if (reply.hdr.flags & ICM_FLAGS_ERROR)
1551 		return -EIO;
1552 
1553 	for (i = 0; i < nuuids; i++) {
1554 		u32 *uuid = (u32 *)&uuids[i];
1555 
1556 		uuid[0] = reply.acl[i].uuid_lo;
1557 		uuid[1] = reply.acl[i].uuid_hi;
1558 
1559 		if (uuid[0] == 0xffffffff && uuid[1] == 0xffffffff) {
1560 			/* Map empty entries to null UUID */
1561 			uuid[0] = 0;
1562 			uuid[1] = 0;
1563 		} else if (uuid[0] != 0 || uuid[1] != 0) {
1564 			/* Upper two DWs are always one's */
1565 			uuid[2] = 0xffffffff;
1566 			uuid[3] = 0xffffffff;
1567 		}
1568 	}
1569 
1570 	return ret;
1571 }
1572 
1573 static int icm_ar_set_boot_acl(struct tb *tb, const uuid_t *uuids,
1574 			       size_t nuuids)
1575 {
1576 	struct icm_ar_pkg_preboot_acl_response reply;
1577 	struct icm_ar_pkg_preboot_acl request = {
1578 		.hdr = {
1579 			.code = ICM_PREBOOT_ACL,
1580 			.flags = ICM_FLAGS_WRITE,
1581 		},
1582 	};
1583 	int ret, i;
1584 
1585 	for (i = 0; i < nuuids; i++) {
1586 		const u32 *uuid = (const u32 *)&uuids[i];
1587 
1588 		if (uuid_is_null(&uuids[i])) {
1589 			/*
1590 			 * Map null UUID to the empty (all one) entries
1591 			 * for ICM.
1592 			 */
1593 			request.acl[i].uuid_lo = 0xffffffff;
1594 			request.acl[i].uuid_hi = 0xffffffff;
1595 		} else {
1596 			/* Two high DWs need to be set to all one */
1597 			if (uuid[2] != 0xffffffff || uuid[3] != 0xffffffff)
1598 				return -EINVAL;
1599 
1600 			request.acl[i].uuid_lo = uuid[0];
1601 			request.acl[i].uuid_hi = uuid[1];
1602 		}
1603 	}
1604 
1605 	memset(&reply, 0, sizeof(reply));
1606 	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1607 			  1, ICM_TIMEOUT);
1608 	if (ret)
1609 		return ret;
1610 
1611 	if (reply.hdr.flags & ICM_FLAGS_ERROR)
1612 		return -EIO;
1613 
1614 	return 0;
1615 }
1616 
1617 static int
1618 icm_icl_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1619 		     u8 *proto_version, size_t *nboot_acl, bool *rpm)
1620 {
1621 	struct icm_tr_pkg_driver_ready_response reply;
1622 	struct icm_pkg_driver_ready request = {
1623 		.hdr.code = ICM_DRIVER_READY,
1624 	};
1625 	int ret;
1626 
1627 	memset(&reply, 0, sizeof(reply));
1628 	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1629 			  1, 20000);
1630 	if (ret)
1631 		return ret;
1632 
1633 	if (proto_version)
1634 		*proto_version = (reply.info & ICM_TR_INFO_PROTO_VERSION_MASK) >>
1635 				ICM_TR_INFO_PROTO_VERSION_SHIFT;
1636 
1637 	/* Ice Lake always supports RTD3 */
1638 	if (rpm)
1639 		*rpm = true;
1640 
1641 	return 0;
1642 }
1643 
1644 static void icm_icl_set_uuid(struct tb *tb)
1645 {
1646 	struct tb_nhi *nhi = tb->nhi;
1647 	u32 uuid[4];
1648 
1649 	pci_read_config_dword(nhi->pdev, VS_CAP_10, &uuid[0]);
1650 	pci_read_config_dword(nhi->pdev, VS_CAP_11, &uuid[1]);
1651 	uuid[2] = 0xffffffff;
1652 	uuid[3] = 0xffffffff;
1653 
1654 	tb->root_switch->uuid = kmemdup(uuid, sizeof(uuid), GFP_KERNEL);
1655 }
1656 
1657 static void
1658 icm_icl_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1659 {
1660 	__icm_tr_device_connected(tb, hdr, true);
1661 }
1662 
1663 static void icm_icl_rtd3_veto(struct tb *tb, const struct icm_pkg_header *hdr)
1664 {
1665 	const struct icm_icl_event_rtd3_veto *pkg =
1666 		(const struct icm_icl_event_rtd3_veto *)hdr;
1667 
1668 	tb_dbg(tb, "ICM rtd3 veto=0x%08x\n", pkg->veto_reason);
1669 
1670 	if (pkg->veto_reason)
1671 		icm_veto_begin(tb);
1672 	else
1673 		icm_veto_end(tb);
1674 }
1675 
1676 static bool icm_tgl_is_supported(struct tb *tb)
1677 {
1678 	unsigned long end = jiffies + msecs_to_jiffies(10);
1679 
1680 	do {
1681 		u32 val;
1682 
1683 		val = ioread32(tb->nhi->iobase + REG_FW_STS);
1684 		if (val & REG_FW_STS_NVM_AUTH_DONE)
1685 			return true;
1686 		usleep_range(100, 500);
1687 	} while (time_before(jiffies, end));
1688 
1689 	return false;
1690 }
1691 
1692 static void icm_handle_notification(struct work_struct *work)
1693 {
1694 	struct icm_notification *n = container_of(work, typeof(*n), work);
1695 	struct tb *tb = n->tb;
1696 	struct icm *icm = tb_priv(tb);
1697 
1698 	mutex_lock(&tb->lock);
1699 
1700 	/*
1701 	 * When the domain is stopped we flush its workqueue but before
1702 	 * that the root switch is removed. In that case we should treat
1703 	 * the queued events as being canceled.
1704 	 */
1705 	if (tb->root_switch) {
1706 		switch (n->pkg->code) {
1707 		case ICM_EVENT_DEVICE_CONNECTED:
1708 			icm->device_connected(tb, n->pkg);
1709 			break;
1710 		case ICM_EVENT_DEVICE_DISCONNECTED:
1711 			icm->device_disconnected(tb, n->pkg);
1712 			break;
1713 		case ICM_EVENT_XDOMAIN_CONNECTED:
1714 			if (tb_is_xdomain_enabled())
1715 				icm->xdomain_connected(tb, n->pkg);
1716 			break;
1717 		case ICM_EVENT_XDOMAIN_DISCONNECTED:
1718 			if (tb_is_xdomain_enabled())
1719 				icm->xdomain_disconnected(tb, n->pkg);
1720 			break;
1721 		case ICM_EVENT_RTD3_VETO:
1722 			icm->rtd3_veto(tb, n->pkg);
1723 			break;
1724 		}
1725 	}
1726 
1727 	mutex_unlock(&tb->lock);
1728 
1729 	kfree(n->pkg);
1730 	kfree(n);
1731 }
1732 
1733 static void icm_handle_event(struct tb *tb, enum tb_cfg_pkg_type type,
1734 			     const void *buf, size_t size)
1735 {
1736 	struct icm_notification *n;
1737 
1738 	n = kmalloc(sizeof(*n), GFP_KERNEL);
1739 	if (!n)
1740 		return;
1741 
1742 	n->pkg = kmemdup(buf, size, GFP_KERNEL);
1743 	if (!n->pkg) {
1744 		kfree(n);
1745 		return;
1746 	}
1747 
1748 	INIT_WORK(&n->work, icm_handle_notification);
1749 	n->tb = tb;
1750 
1751 	queue_work(tb->wq, &n->work);
1752 }
1753 
1754 static int
1755 __icm_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1756 		   u8 *proto_version, size_t *nboot_acl, bool *rpm)
1757 {
1758 	struct icm *icm = tb_priv(tb);
1759 	unsigned int retries = 50;
1760 	int ret;
1761 
1762 	ret = icm->driver_ready(tb, security_level, proto_version, nboot_acl,
1763 				rpm);
1764 	if (ret) {
1765 		tb_err(tb, "failed to send driver ready to ICM\n");
1766 		return ret;
1767 	}
1768 
1769 	/*
1770 	 * Hold on here until the switch config space is accessible so
1771 	 * that we can read root switch config successfully.
1772 	 */
1773 	do {
1774 		struct tb_cfg_result res;
1775 		u32 tmp;
1776 
1777 		res = tb_cfg_read_raw(tb->ctl, &tmp, 0, 0, TB_CFG_SWITCH,
1778 				      0, 1, 100);
1779 		if (!res.err)
1780 			return 0;
1781 
1782 		msleep(50);
1783 	} while (--retries);
1784 
1785 	tb_err(tb, "failed to read root switch config space, giving up\n");
1786 	return -ETIMEDOUT;
1787 }
1788 
1789 static int icm_firmware_reset(struct tb *tb, struct tb_nhi *nhi)
1790 {
1791 	struct icm *icm = tb_priv(tb);
1792 	u32 val;
1793 
1794 	if (!icm->upstream_port)
1795 		return -ENODEV;
1796 
1797 	/* Put ARC to wait for CIO reset event to happen */
1798 	val = ioread32(nhi->iobase + REG_FW_STS);
1799 	val |= REG_FW_STS_CIO_RESET_REQ;
1800 	iowrite32(val, nhi->iobase + REG_FW_STS);
1801 
1802 	/* Re-start ARC */
1803 	val = ioread32(nhi->iobase + REG_FW_STS);
1804 	val |= REG_FW_STS_ICM_EN_INVERT;
1805 	val |= REG_FW_STS_ICM_EN_CPU;
1806 	iowrite32(val, nhi->iobase + REG_FW_STS);
1807 
1808 	/* Trigger CIO reset now */
1809 	return icm->cio_reset(tb);
1810 }
1811 
1812 static int icm_firmware_start(struct tb *tb, struct tb_nhi *nhi)
1813 {
1814 	unsigned int retries = 10;
1815 	int ret;
1816 	u32 val;
1817 
1818 	/* Check if the ICM firmware is already running */
1819 	if (icm_firmware_running(nhi))
1820 		return 0;
1821 
1822 	dev_dbg(&nhi->pdev->dev, "starting ICM firmware\n");
1823 
1824 	ret = icm_firmware_reset(tb, nhi);
1825 	if (ret)
1826 		return ret;
1827 
1828 	/* Wait until the ICM firmware tells us it is up and running */
1829 	do {
1830 		/* Check that the ICM firmware is running */
1831 		val = ioread32(nhi->iobase + REG_FW_STS);
1832 		if (val & REG_FW_STS_NVM_AUTH_DONE)
1833 			return 0;
1834 
1835 		msleep(300);
1836 	} while (--retries);
1837 
1838 	return -ETIMEDOUT;
1839 }
1840 
1841 static int icm_reset_phy_port(struct tb *tb, int phy_port)
1842 {
1843 	struct icm *icm = tb_priv(tb);
1844 	u32 state0, state1;
1845 	int port0, port1;
1846 	u32 val0, val1;
1847 	int ret;
1848 
1849 	if (!icm->upstream_port)
1850 		return 0;
1851 
1852 	if (phy_port) {
1853 		port0 = 3;
1854 		port1 = 4;
1855 	} else {
1856 		port0 = 1;
1857 		port1 = 2;
1858 	}
1859 
1860 	/*
1861 	 * Read link status of both null ports belonging to a single
1862 	 * physical port.
1863 	 */
1864 	ret = pcie2cio_read(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, &val0);
1865 	if (ret)
1866 		return ret;
1867 	ret = pcie2cio_read(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, &val1);
1868 	if (ret)
1869 		return ret;
1870 
1871 	state0 = val0 & PHY_PORT_CS1_LINK_STATE_MASK;
1872 	state0 >>= PHY_PORT_CS1_LINK_STATE_SHIFT;
1873 	state1 = val1 & PHY_PORT_CS1_LINK_STATE_MASK;
1874 	state1 >>= PHY_PORT_CS1_LINK_STATE_SHIFT;
1875 
1876 	/* If they are both up we need to reset them now */
1877 	if (state0 != TB_PORT_UP || state1 != TB_PORT_UP)
1878 		return 0;
1879 
1880 	val0 |= PHY_PORT_CS1_LINK_DISABLE;
1881 	ret = pcie2cio_write(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, val0);
1882 	if (ret)
1883 		return ret;
1884 
1885 	val1 |= PHY_PORT_CS1_LINK_DISABLE;
1886 	ret = pcie2cio_write(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, val1);
1887 	if (ret)
1888 		return ret;
1889 
1890 	/* Wait a bit and then re-enable both ports */
1891 	usleep_range(10, 100);
1892 
1893 	ret = pcie2cio_read(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, &val0);
1894 	if (ret)
1895 		return ret;
1896 	ret = pcie2cio_read(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, &val1);
1897 	if (ret)
1898 		return ret;
1899 
1900 	val0 &= ~PHY_PORT_CS1_LINK_DISABLE;
1901 	ret = pcie2cio_write(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, val0);
1902 	if (ret)
1903 		return ret;
1904 
1905 	val1 &= ~PHY_PORT_CS1_LINK_DISABLE;
1906 	return pcie2cio_write(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, val1);
1907 }
1908 
1909 static int icm_firmware_init(struct tb *tb)
1910 {
1911 	struct icm *icm = tb_priv(tb);
1912 	struct tb_nhi *nhi = tb->nhi;
1913 	int ret;
1914 
1915 	ret = icm_firmware_start(tb, nhi);
1916 	if (ret) {
1917 		dev_err(&nhi->pdev->dev, "could not start ICM firmware\n");
1918 		return ret;
1919 	}
1920 
1921 	if (icm->get_mode) {
1922 		ret = icm->get_mode(tb);
1923 
1924 		switch (ret) {
1925 		case NHI_FW_SAFE_MODE:
1926 			icm->safe_mode = true;
1927 			break;
1928 
1929 		case NHI_FW_CM_MODE:
1930 			/* Ask ICM to accept all Thunderbolt devices */
1931 			nhi_mailbox_cmd(nhi, NHI_MAILBOX_ALLOW_ALL_DEVS, 0);
1932 			break;
1933 
1934 		default:
1935 			if (ret < 0)
1936 				return ret;
1937 
1938 			tb_err(tb, "ICM firmware is in wrong mode: %u\n", ret);
1939 			return -ENODEV;
1940 		}
1941 	}
1942 
1943 	/*
1944 	 * Reset both physical ports if there is anything connected to
1945 	 * them already.
1946 	 */
1947 	ret = icm_reset_phy_port(tb, 0);
1948 	if (ret)
1949 		dev_warn(&nhi->pdev->dev, "failed to reset links on port0\n");
1950 	ret = icm_reset_phy_port(tb, 1);
1951 	if (ret)
1952 		dev_warn(&nhi->pdev->dev, "failed to reset links on port1\n");
1953 
1954 	return 0;
1955 }
1956 
1957 static int icm_driver_ready(struct tb *tb)
1958 {
1959 	struct icm *icm = tb_priv(tb);
1960 	int ret;
1961 
1962 	ret = icm_firmware_init(tb);
1963 	if (ret)
1964 		return ret;
1965 
1966 	if (icm->safe_mode) {
1967 		tb_info(tb, "Thunderbolt host controller is in safe mode.\n");
1968 		tb_info(tb, "You need to update NVM firmware of the controller before it can be used.\n");
1969 		tb_info(tb, "For latest updates check https://thunderbolttechnology.net/updates.\n");
1970 		return 0;
1971 	}
1972 
1973 	ret = __icm_driver_ready(tb, &tb->security_level, &icm->proto_version,
1974 				 &tb->nboot_acl, &icm->rpm);
1975 	if (ret)
1976 		return ret;
1977 
1978 	/*
1979 	 * Make sure the number of supported preboot ACL matches what we
1980 	 * expect or disable the whole feature.
1981 	 */
1982 	if (tb->nboot_acl > icm->max_boot_acl)
1983 		tb->nboot_acl = 0;
1984 
1985 	if (icm->proto_version >= 3)
1986 		tb_dbg(tb, "USB4 proxy operations supported\n");
1987 
1988 	return 0;
1989 }
1990 
1991 static int icm_suspend(struct tb *tb)
1992 {
1993 	struct icm *icm = tb_priv(tb);
1994 
1995 	if (icm->save_devices)
1996 		icm->save_devices(tb);
1997 
1998 	nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
1999 	return 0;
2000 }
2001 
2002 /*
2003  * Mark all switches (except root switch) below this one unplugged. ICM
2004  * firmware will send us an updated list of switches after we have send
2005  * it driver ready command. If a switch is not in that list it will be
2006  * removed when we perform rescan.
2007  */
2008 static void icm_unplug_children(struct tb_switch *sw)
2009 {
2010 	struct tb_port *port;
2011 
2012 	if (tb_route(sw))
2013 		sw->is_unplugged = true;
2014 
2015 	tb_switch_for_each_port(sw, port) {
2016 		if (port->xdomain)
2017 			port->xdomain->is_unplugged = true;
2018 		else if (tb_port_has_remote(port))
2019 			icm_unplug_children(port->remote->sw);
2020 	}
2021 }
2022 
2023 static int complete_rpm(struct device *dev, void *data)
2024 {
2025 	struct tb_switch *sw = tb_to_switch(dev);
2026 
2027 	if (sw)
2028 		complete(&sw->rpm_complete);
2029 	return 0;
2030 }
2031 
2032 static void remove_unplugged_switch(struct tb_switch *sw)
2033 {
2034 	struct device *parent = get_device(sw->dev.parent);
2035 
2036 	pm_runtime_get_sync(parent);
2037 
2038 	/*
2039 	 * Signal this and switches below for rpm_complete because
2040 	 * tb_switch_remove() calls pm_runtime_get_sync() that then waits
2041 	 * for it.
2042 	 */
2043 	complete_rpm(&sw->dev, NULL);
2044 	bus_for_each_dev(&tb_bus_type, &sw->dev, NULL, complete_rpm);
2045 	tb_switch_remove(sw);
2046 
2047 	pm_runtime_mark_last_busy(parent);
2048 	pm_runtime_put_autosuspend(parent);
2049 
2050 	put_device(parent);
2051 }
2052 
2053 static void icm_free_unplugged_children(struct tb_switch *sw)
2054 {
2055 	struct tb_port *port;
2056 
2057 	tb_switch_for_each_port(sw, port) {
2058 		if (port->xdomain && port->xdomain->is_unplugged) {
2059 			tb_xdomain_remove(port->xdomain);
2060 			port->xdomain = NULL;
2061 		} else if (tb_port_has_remote(port)) {
2062 			if (port->remote->sw->is_unplugged) {
2063 				remove_unplugged_switch(port->remote->sw);
2064 				port->remote = NULL;
2065 			} else {
2066 				icm_free_unplugged_children(port->remote->sw);
2067 			}
2068 		}
2069 	}
2070 }
2071 
2072 static void icm_rescan_work(struct work_struct *work)
2073 {
2074 	struct icm *icm = container_of(work, struct icm, rescan_work.work);
2075 	struct tb *tb = icm_to_tb(icm);
2076 
2077 	mutex_lock(&tb->lock);
2078 	if (tb->root_switch)
2079 		icm_free_unplugged_children(tb->root_switch);
2080 	mutex_unlock(&tb->lock);
2081 }
2082 
2083 static void icm_complete(struct tb *tb)
2084 {
2085 	struct icm *icm = tb_priv(tb);
2086 
2087 	if (tb->nhi->going_away)
2088 		return;
2089 
2090 	/*
2091 	 * If RTD3 was vetoed before we entered system suspend allow it
2092 	 * again now before driver ready is sent. Firmware sends a new RTD3
2093 	 * veto if it is still the case after we have sent it driver ready
2094 	 * command.
2095 	 */
2096 	icm_veto_end(tb);
2097 	icm_unplug_children(tb->root_switch);
2098 
2099 	/*
2100 	 * Now all existing children should be resumed, start events
2101 	 * from ICM to get updated status.
2102 	 */
2103 	__icm_driver_ready(tb, NULL, NULL, NULL, NULL);
2104 
2105 	/*
2106 	 * We do not get notifications of devices that have been
2107 	 * unplugged during suspend so schedule rescan to clean them up
2108 	 * if any.
2109 	 */
2110 	queue_delayed_work(tb->wq, &icm->rescan_work, msecs_to_jiffies(500));
2111 }
2112 
2113 static int icm_runtime_suspend(struct tb *tb)
2114 {
2115 	nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
2116 	return 0;
2117 }
2118 
2119 static int icm_runtime_suspend_switch(struct tb_switch *sw)
2120 {
2121 	if (tb_route(sw))
2122 		reinit_completion(&sw->rpm_complete);
2123 	return 0;
2124 }
2125 
2126 static int icm_runtime_resume_switch(struct tb_switch *sw)
2127 {
2128 	if (tb_route(sw)) {
2129 		if (!wait_for_completion_timeout(&sw->rpm_complete,
2130 						 msecs_to_jiffies(500))) {
2131 			dev_dbg(&sw->dev, "runtime resuming timed out\n");
2132 		}
2133 	}
2134 	return 0;
2135 }
2136 
2137 static int icm_runtime_resume(struct tb *tb)
2138 {
2139 	/*
2140 	 * We can reuse the same resume functionality than with system
2141 	 * suspend.
2142 	 */
2143 	icm_complete(tb);
2144 	return 0;
2145 }
2146 
2147 static int icm_start(struct tb *tb)
2148 {
2149 	struct icm *icm = tb_priv(tb);
2150 	int ret;
2151 
2152 	if (icm->safe_mode)
2153 		tb->root_switch = tb_switch_alloc_safe_mode(tb, &tb->dev, 0);
2154 	else
2155 		tb->root_switch = tb_switch_alloc(tb, &tb->dev, 0);
2156 	if (IS_ERR(tb->root_switch))
2157 		return PTR_ERR(tb->root_switch);
2158 
2159 	tb->root_switch->no_nvm_upgrade = !icm->can_upgrade_nvm;
2160 	tb->root_switch->rpm = icm->rpm;
2161 
2162 	if (icm->set_uuid)
2163 		icm->set_uuid(tb);
2164 
2165 	ret = tb_switch_add(tb->root_switch);
2166 	if (ret) {
2167 		tb_switch_put(tb->root_switch);
2168 		tb->root_switch = NULL;
2169 	}
2170 
2171 	return ret;
2172 }
2173 
2174 static void icm_stop(struct tb *tb)
2175 {
2176 	struct icm *icm = tb_priv(tb);
2177 
2178 	cancel_delayed_work(&icm->rescan_work);
2179 	tb_switch_remove(tb->root_switch);
2180 	tb->root_switch = NULL;
2181 	nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
2182 	kfree(icm->last_nvm_auth);
2183 	icm->last_nvm_auth = NULL;
2184 }
2185 
2186 static int icm_disconnect_pcie_paths(struct tb *tb)
2187 {
2188 	return nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DISCONNECT_PCIE_PATHS, 0);
2189 }
2190 
2191 static void icm_usb4_switch_nvm_auth_complete(void *data)
2192 {
2193 	struct usb4_switch_nvm_auth *auth = data;
2194 	struct icm *icm = auth->icm;
2195 	struct tb *tb = icm_to_tb(icm);
2196 
2197 	tb_dbg(tb, "NVM_AUTH response for %llx flags %#x status %#x\n",
2198 	       get_route(auth->reply.route_hi, auth->reply.route_lo),
2199 	       auth->reply.hdr.flags, auth->reply.status);
2200 
2201 	mutex_lock(&tb->lock);
2202 	if (WARN_ON(icm->last_nvm_auth))
2203 		kfree(icm->last_nvm_auth);
2204 	icm->last_nvm_auth = auth;
2205 	mutex_unlock(&tb->lock);
2206 }
2207 
2208 static int icm_usb4_switch_nvm_authenticate(struct tb *tb, u64 route)
2209 {
2210 	struct usb4_switch_nvm_auth *auth;
2211 	struct icm *icm = tb_priv(tb);
2212 	struct tb_cfg_request *req;
2213 	int ret;
2214 
2215 	auth = kzalloc(sizeof(*auth), GFP_KERNEL);
2216 	if (!auth)
2217 		return -ENOMEM;
2218 
2219 	auth->icm = icm;
2220 	auth->request.hdr.code = ICM_USB4_SWITCH_OP;
2221 	auth->request.route_hi = upper_32_bits(route);
2222 	auth->request.route_lo = lower_32_bits(route);
2223 	auth->request.opcode = USB4_SWITCH_OP_NVM_AUTH;
2224 
2225 	req = tb_cfg_request_alloc();
2226 	if (!req) {
2227 		ret = -ENOMEM;
2228 		goto err_free_auth;
2229 	}
2230 
2231 	req->match = icm_match;
2232 	req->copy = icm_copy;
2233 	req->request = &auth->request;
2234 	req->request_size = sizeof(auth->request);
2235 	req->request_type = TB_CFG_PKG_ICM_CMD;
2236 	req->response = &auth->reply;
2237 	req->npackets = 1;
2238 	req->response_size = sizeof(auth->reply);
2239 	req->response_type = TB_CFG_PKG_ICM_RESP;
2240 
2241 	tb_dbg(tb, "NVM_AUTH request for %llx\n", route);
2242 
2243 	mutex_lock(&icm->request_lock);
2244 	ret = tb_cfg_request(tb->ctl, req, icm_usb4_switch_nvm_auth_complete,
2245 			     auth);
2246 	mutex_unlock(&icm->request_lock);
2247 
2248 	tb_cfg_request_put(req);
2249 	if (ret)
2250 		goto err_free_auth;
2251 	return 0;
2252 
2253 err_free_auth:
2254 	kfree(auth);
2255 	return ret;
2256 }
2257 
2258 static int icm_usb4_switch_op(struct tb_switch *sw, u16 opcode, u32 *metadata,
2259 			      u8 *status, const void *tx_data, size_t tx_data_len,
2260 			      void *rx_data, size_t rx_data_len)
2261 {
2262 	struct icm_usb4_switch_op_response reply;
2263 	struct icm_usb4_switch_op request;
2264 	struct tb *tb = sw->tb;
2265 	struct icm *icm = tb_priv(tb);
2266 	u64 route = tb_route(sw);
2267 	int ret;
2268 
2269 	/*
2270 	 * USB4 router operation proxy is supported in firmware if the
2271 	 * protocol version is 3 or higher.
2272 	 */
2273 	if (icm->proto_version < 3)
2274 		return -EOPNOTSUPP;
2275 
2276 	/*
2277 	 * NVM_AUTH is a special USB4 proxy operation that does not
2278 	 * return immediately so handle it separately.
2279 	 */
2280 	if (opcode == USB4_SWITCH_OP_NVM_AUTH)
2281 		return icm_usb4_switch_nvm_authenticate(tb, route);
2282 
2283 	memset(&request, 0, sizeof(request));
2284 	request.hdr.code = ICM_USB4_SWITCH_OP;
2285 	request.route_hi = upper_32_bits(route);
2286 	request.route_lo = lower_32_bits(route);
2287 	request.opcode = opcode;
2288 	if (metadata)
2289 		request.metadata = *metadata;
2290 
2291 	if (tx_data_len) {
2292 		request.data_len_valid |= ICM_USB4_SWITCH_DATA_VALID;
2293 		if (tx_data_len < ARRAY_SIZE(request.data))
2294 			request.data_len_valid =
2295 				tx_data_len & ICM_USB4_SWITCH_DATA_LEN_MASK;
2296 		memcpy(request.data, tx_data, tx_data_len * sizeof(u32));
2297 	}
2298 
2299 	memset(&reply, 0, sizeof(reply));
2300 	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
2301 			  1, ICM_TIMEOUT);
2302 	if (ret)
2303 		return ret;
2304 
2305 	if (reply.hdr.flags & ICM_FLAGS_ERROR)
2306 		return -EIO;
2307 
2308 	if (status)
2309 		*status = reply.status;
2310 
2311 	if (metadata)
2312 		*metadata = reply.metadata;
2313 
2314 	if (rx_data_len)
2315 		memcpy(rx_data, reply.data, rx_data_len * sizeof(u32));
2316 
2317 	return 0;
2318 }
2319 
2320 static int icm_usb4_switch_nvm_authenticate_status(struct tb_switch *sw,
2321 						   u32 *status)
2322 {
2323 	struct usb4_switch_nvm_auth *auth;
2324 	struct tb *tb = sw->tb;
2325 	struct icm *icm = tb_priv(tb);
2326 	int ret = 0;
2327 
2328 	if (icm->proto_version < 3)
2329 		return -EOPNOTSUPP;
2330 
2331 	auth = icm->last_nvm_auth;
2332 	icm->last_nvm_auth = NULL;
2333 
2334 	if (auth && auth->reply.route_hi == sw->config.route_hi &&
2335 	    auth->reply.route_lo == sw->config.route_lo) {
2336 		tb_dbg(tb, "NVM_AUTH found for %llx flags %#x status %#x\n",
2337 		       tb_route(sw), auth->reply.hdr.flags, auth->reply.status);
2338 		if (auth->reply.hdr.flags & ICM_FLAGS_ERROR)
2339 			ret = -EIO;
2340 		else
2341 			*status = auth->reply.status;
2342 	} else {
2343 		*status = 0;
2344 	}
2345 
2346 	kfree(auth);
2347 	return ret;
2348 }
2349 
2350 /* Falcon Ridge */
2351 static const struct tb_cm_ops icm_fr_ops = {
2352 	.driver_ready = icm_driver_ready,
2353 	.start = icm_start,
2354 	.stop = icm_stop,
2355 	.suspend = icm_suspend,
2356 	.complete = icm_complete,
2357 	.handle_event = icm_handle_event,
2358 	.approve_switch = icm_fr_approve_switch,
2359 	.add_switch_key = icm_fr_add_switch_key,
2360 	.challenge_switch_key = icm_fr_challenge_switch_key,
2361 	.disconnect_pcie_paths = icm_disconnect_pcie_paths,
2362 	.approve_xdomain_paths = icm_fr_approve_xdomain_paths,
2363 	.disconnect_xdomain_paths = icm_fr_disconnect_xdomain_paths,
2364 };
2365 
2366 /* Alpine Ridge */
2367 static const struct tb_cm_ops icm_ar_ops = {
2368 	.driver_ready = icm_driver_ready,
2369 	.start = icm_start,
2370 	.stop = icm_stop,
2371 	.suspend = icm_suspend,
2372 	.complete = icm_complete,
2373 	.runtime_suspend = icm_runtime_suspend,
2374 	.runtime_resume = icm_runtime_resume,
2375 	.runtime_suspend_switch = icm_runtime_suspend_switch,
2376 	.runtime_resume_switch = icm_runtime_resume_switch,
2377 	.handle_event = icm_handle_event,
2378 	.get_boot_acl = icm_ar_get_boot_acl,
2379 	.set_boot_acl = icm_ar_set_boot_acl,
2380 	.approve_switch = icm_fr_approve_switch,
2381 	.add_switch_key = icm_fr_add_switch_key,
2382 	.challenge_switch_key = icm_fr_challenge_switch_key,
2383 	.disconnect_pcie_paths = icm_disconnect_pcie_paths,
2384 	.approve_xdomain_paths = icm_fr_approve_xdomain_paths,
2385 	.disconnect_xdomain_paths = icm_fr_disconnect_xdomain_paths,
2386 };
2387 
2388 /* Titan Ridge */
2389 static const struct tb_cm_ops icm_tr_ops = {
2390 	.driver_ready = icm_driver_ready,
2391 	.start = icm_start,
2392 	.stop = icm_stop,
2393 	.suspend = icm_suspend,
2394 	.complete = icm_complete,
2395 	.runtime_suspend = icm_runtime_suspend,
2396 	.runtime_resume = icm_runtime_resume,
2397 	.runtime_suspend_switch = icm_runtime_suspend_switch,
2398 	.runtime_resume_switch = icm_runtime_resume_switch,
2399 	.handle_event = icm_handle_event,
2400 	.get_boot_acl = icm_ar_get_boot_acl,
2401 	.set_boot_acl = icm_ar_set_boot_acl,
2402 	.approve_switch = icm_tr_approve_switch,
2403 	.add_switch_key = icm_tr_add_switch_key,
2404 	.challenge_switch_key = icm_tr_challenge_switch_key,
2405 	.disconnect_pcie_paths = icm_disconnect_pcie_paths,
2406 	.approve_xdomain_paths = icm_tr_approve_xdomain_paths,
2407 	.disconnect_xdomain_paths = icm_tr_disconnect_xdomain_paths,
2408 	.usb4_switch_op = icm_usb4_switch_op,
2409 	.usb4_switch_nvm_authenticate_status =
2410 		icm_usb4_switch_nvm_authenticate_status,
2411 };
2412 
2413 /* Ice Lake */
2414 static const struct tb_cm_ops icm_icl_ops = {
2415 	.driver_ready = icm_driver_ready,
2416 	.start = icm_start,
2417 	.stop = icm_stop,
2418 	.complete = icm_complete,
2419 	.runtime_suspend = icm_runtime_suspend,
2420 	.runtime_resume = icm_runtime_resume,
2421 	.handle_event = icm_handle_event,
2422 	.approve_xdomain_paths = icm_tr_approve_xdomain_paths,
2423 	.disconnect_xdomain_paths = icm_tr_disconnect_xdomain_paths,
2424 	.usb4_switch_op = icm_usb4_switch_op,
2425 	.usb4_switch_nvm_authenticate_status =
2426 		icm_usb4_switch_nvm_authenticate_status,
2427 };
2428 
2429 struct tb *icm_probe(struct tb_nhi *nhi)
2430 {
2431 	struct icm *icm;
2432 	struct tb *tb;
2433 
2434 	tb = tb_domain_alloc(nhi, ICM_TIMEOUT, sizeof(struct icm));
2435 	if (!tb)
2436 		return NULL;
2437 
2438 	icm = tb_priv(tb);
2439 	INIT_DELAYED_WORK(&icm->rescan_work, icm_rescan_work);
2440 	mutex_init(&icm->request_lock);
2441 
2442 	switch (nhi->pdev->device) {
2443 	case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI:
2444 	case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI:
2445 		icm->can_upgrade_nvm = true;
2446 		icm->is_supported = icm_fr_is_supported;
2447 		icm->get_route = icm_fr_get_route;
2448 		icm->save_devices = icm_fr_save_devices;
2449 		icm->driver_ready = icm_fr_driver_ready;
2450 		icm->device_connected = icm_fr_device_connected;
2451 		icm->device_disconnected = icm_fr_device_disconnected;
2452 		icm->xdomain_connected = icm_fr_xdomain_connected;
2453 		icm->xdomain_disconnected = icm_fr_xdomain_disconnected;
2454 		tb->cm_ops = &icm_fr_ops;
2455 		break;
2456 
2457 	case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_NHI:
2458 	case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_NHI:
2459 	case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_NHI:
2460 	case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_NHI:
2461 	case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_NHI:
2462 		icm->max_boot_acl = ICM_AR_PREBOOT_ACL_ENTRIES;
2463 		/*
2464 		 * NVM upgrade has not been tested on Apple systems and
2465 		 * they don't provide images publicly either. To be on
2466 		 * the safe side prevent root switch NVM upgrade on Macs
2467 		 * for now.
2468 		 */
2469 		icm->can_upgrade_nvm = !x86_apple_machine;
2470 		icm->is_supported = icm_ar_is_supported;
2471 		icm->cio_reset = icm_ar_cio_reset;
2472 		icm->get_mode = icm_ar_get_mode;
2473 		icm->get_route = icm_ar_get_route;
2474 		icm->save_devices = icm_fr_save_devices;
2475 		icm->driver_ready = icm_ar_driver_ready;
2476 		icm->device_connected = icm_fr_device_connected;
2477 		icm->device_disconnected = icm_fr_device_disconnected;
2478 		icm->xdomain_connected = icm_fr_xdomain_connected;
2479 		icm->xdomain_disconnected = icm_fr_xdomain_disconnected;
2480 		tb->cm_ops = &icm_ar_ops;
2481 		break;
2482 
2483 	case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_NHI:
2484 	case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_NHI:
2485 		icm->max_boot_acl = ICM_AR_PREBOOT_ACL_ENTRIES;
2486 		icm->can_upgrade_nvm = !x86_apple_machine;
2487 		icm->is_supported = icm_ar_is_supported;
2488 		icm->cio_reset = icm_tr_cio_reset;
2489 		icm->get_mode = icm_ar_get_mode;
2490 		icm->driver_ready = icm_tr_driver_ready;
2491 		icm->device_connected = icm_tr_device_connected;
2492 		icm->device_disconnected = icm_tr_device_disconnected;
2493 		icm->xdomain_connected = icm_tr_xdomain_connected;
2494 		icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2495 		tb->cm_ops = &icm_tr_ops;
2496 		break;
2497 
2498 	case PCI_DEVICE_ID_INTEL_ICL_NHI0:
2499 	case PCI_DEVICE_ID_INTEL_ICL_NHI1:
2500 		icm->is_supported = icm_fr_is_supported;
2501 		icm->driver_ready = icm_icl_driver_ready;
2502 		icm->set_uuid = icm_icl_set_uuid;
2503 		icm->device_connected = icm_icl_device_connected;
2504 		icm->device_disconnected = icm_tr_device_disconnected;
2505 		icm->xdomain_connected = icm_tr_xdomain_connected;
2506 		icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2507 		icm->rtd3_veto = icm_icl_rtd3_veto;
2508 		tb->cm_ops = &icm_icl_ops;
2509 		break;
2510 
2511 	case PCI_DEVICE_ID_INTEL_TGL_NHI0:
2512 	case PCI_DEVICE_ID_INTEL_TGL_NHI1:
2513 	case PCI_DEVICE_ID_INTEL_TGL_H_NHI0:
2514 	case PCI_DEVICE_ID_INTEL_TGL_H_NHI1:
2515 	case PCI_DEVICE_ID_INTEL_ADL_NHI0:
2516 	case PCI_DEVICE_ID_INTEL_ADL_NHI1:
2517 	case PCI_DEVICE_ID_INTEL_RPL_NHI0:
2518 	case PCI_DEVICE_ID_INTEL_RPL_NHI1:
2519 	case PCI_DEVICE_ID_INTEL_MTL_M_NHI0:
2520 	case PCI_DEVICE_ID_INTEL_MTL_P_NHI0:
2521 	case PCI_DEVICE_ID_INTEL_MTL_P_NHI1:
2522 		icm->is_supported = icm_tgl_is_supported;
2523 		icm->driver_ready = icm_icl_driver_ready;
2524 		icm->set_uuid = icm_icl_set_uuid;
2525 		icm->device_connected = icm_icl_device_connected;
2526 		icm->device_disconnected = icm_tr_device_disconnected;
2527 		icm->xdomain_connected = icm_tr_xdomain_connected;
2528 		icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2529 		icm->rtd3_veto = icm_icl_rtd3_veto;
2530 		tb->cm_ops = &icm_icl_ops;
2531 		break;
2532 
2533 	case PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_2C_NHI:
2534 	case PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_4C_NHI:
2535 		icm->is_supported = icm_tgl_is_supported;
2536 		icm->get_mode = icm_ar_get_mode;
2537 		icm->driver_ready = icm_tr_driver_ready;
2538 		icm->device_connected = icm_tr_device_connected;
2539 		icm->device_disconnected = icm_tr_device_disconnected;
2540 		icm->xdomain_connected = icm_tr_xdomain_connected;
2541 		icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2542 		tb->cm_ops = &icm_tr_ops;
2543 		break;
2544 	}
2545 
2546 	if (!icm->is_supported || !icm->is_supported(tb)) {
2547 		dev_dbg(&nhi->pdev->dev, "ICM not supported on this controller\n");
2548 		tb_domain_put(tb);
2549 		return NULL;
2550 	}
2551 
2552 	tb_dbg(tb, "using firmware connection manager\n");
2553 
2554 	return tb;
2555 }
2556