1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
4  * Author: Marc Zyngier <marc.zyngier@arm.com>
5  */
6 
7 #include <linux/acpi.h>
8 #include <linux/acpi_iort.h>
9 #include <linux/bitfield.h>
10 #include <linux/bitmap.h>
11 #include <linux/cpu.h>
12 #include <linux/crash_dump.h>
13 #include <linux/delay.h>
14 #include <linux/dma-iommu.h>
15 #include <linux/efi.h>
16 #include <linux/interrupt.h>
17 #include <linux/iopoll.h>
18 #include <linux/irqdomain.h>
19 #include <linux/list.h>
20 #include <linux/log2.h>
21 #include <linux/memblock.h>
22 #include <linux/mm.h>
23 #include <linux/msi.h>
24 #include <linux/of.h>
25 #include <linux/of_address.h>
26 #include <linux/of_irq.h>
27 #include <linux/of_pci.h>
28 #include <linux/of_platform.h>
29 #include <linux/percpu.h>
30 #include <linux/slab.h>
31 #include <linux/syscore_ops.h>
32 
33 #include <linux/irqchip.h>
34 #include <linux/irqchip/arm-gic-v3.h>
35 #include <linux/irqchip/arm-gic-v4.h>
36 
37 #include <asm/cputype.h>
38 #include <asm/exception.h>
39 
40 #include "irq-gic-common.h"
41 
42 #define ITS_FLAGS_CMDQ_NEEDS_FLUSHING		(1ULL << 0)
43 #define ITS_FLAGS_WORKAROUND_CAVIUM_22375	(1ULL << 1)
44 #define ITS_FLAGS_WORKAROUND_CAVIUM_23144	(1ULL << 2)
45 
46 #define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING	(1 << 0)
47 #define RDIST_FLAGS_RD_TABLES_PREALLOCATED	(1 << 1)
48 
49 static u32 lpi_id_bits;
50 
51 /*
52  * We allocate memory for PROPBASE to cover 2 ^ lpi_id_bits LPIs to
53  * deal with (one configuration byte per interrupt). PENDBASE has to
54  * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI).
55  */
56 #define LPI_NRBITS		lpi_id_bits
57 #define LPI_PROPBASE_SZ		ALIGN(BIT(LPI_NRBITS), SZ_64K)
58 #define LPI_PENDBASE_SZ		ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K)
59 
60 #define LPI_PROP_DEFAULT_PRIO	GICD_INT_DEF_PRI
61 
62 /*
63  * Collection structure - just an ID, and a redistributor address to
64  * ping. We use one per CPU as a bag of interrupts assigned to this
65  * CPU.
66  */
67 struct its_collection {
68 	u64			target_address;
69 	u16			col_id;
70 };
71 
72 /*
73  * The ITS_BASER structure - contains memory information, cached
74  * value of BASER register configuration and ITS page size.
75  */
76 struct its_baser {
77 	void		*base;
78 	u64		val;
79 	u32		order;
80 	u32		psz;
81 };
82 
83 struct its_device;
84 
85 /*
86  * The ITS structure - contains most of the infrastructure, with the
87  * top-level MSI domain, the command queue, the collections, and the
88  * list of devices writing to it.
89  *
90  * dev_alloc_lock has to be taken for device allocations, while the
91  * spinlock must be taken to parse data structures such as the device
92  * list.
93  */
94 struct its_node {
95 	raw_spinlock_t		lock;
96 	struct mutex		dev_alloc_lock;
97 	struct list_head	entry;
98 	void __iomem		*base;
99 	void __iomem		*sgir_base;
100 	phys_addr_t		phys_base;
101 	struct its_cmd_block	*cmd_base;
102 	struct its_cmd_block	*cmd_write;
103 	struct its_baser	tables[GITS_BASER_NR_REGS];
104 	struct its_collection	*collections;
105 	struct fwnode_handle	*fwnode_handle;
106 	u64			(*get_msi_base)(struct its_device *its_dev);
107 	u64			typer;
108 	u64			cbaser_save;
109 	u32			ctlr_save;
110 	u32			mpidr;
111 	struct list_head	its_device_list;
112 	u64			flags;
113 	unsigned long		list_nr;
114 	int			numa_node;
115 	unsigned int		msi_domain_flags;
116 	u32			pre_its_base; /* for Socionext Synquacer */
117 	int			vlpi_redist_offset;
118 };
119 
120 #define is_v4(its)		(!!((its)->typer & GITS_TYPER_VLPIS))
121 #define is_v4_1(its)		(!!((its)->typer & GITS_TYPER_VMAPP))
122 #define device_ids(its)		(FIELD_GET(GITS_TYPER_DEVBITS, (its)->typer) + 1)
123 
124 #define ITS_ITT_ALIGN		SZ_256
125 
126 /* The maximum number of VPEID bits supported by VLPI commands */
127 #define ITS_MAX_VPEID_BITS						\
128 	({								\
129 		int nvpeid = 16;					\
130 		if (gic_rdists->has_rvpeid &&				\
131 		    gic_rdists->gicd_typer2 & GICD_TYPER2_VIL)		\
132 			nvpeid = 1 + (gic_rdists->gicd_typer2 &		\
133 				      GICD_TYPER2_VID);			\
134 									\
135 		nvpeid;							\
136 	})
137 #define ITS_MAX_VPEID		(1 << (ITS_MAX_VPEID_BITS))
138 
139 /* Convert page order to size in bytes */
140 #define PAGE_ORDER_TO_SIZE(o)	(PAGE_SIZE << (o))
141 
142 struct event_lpi_map {
143 	unsigned long		*lpi_map;
144 	u16			*col_map;
145 	irq_hw_number_t		lpi_base;
146 	int			nr_lpis;
147 	raw_spinlock_t		vlpi_lock;
148 	struct its_vm		*vm;
149 	struct its_vlpi_map	*vlpi_maps;
150 	int			nr_vlpis;
151 };
152 
153 /*
154  * The ITS view of a device - belongs to an ITS, owns an interrupt
155  * translation table, and a list of interrupts.  If it some of its
156  * LPIs are injected into a guest (GICv4), the event_map.vm field
157  * indicates which one.
158  */
159 struct its_device {
160 	struct list_head	entry;
161 	struct its_node		*its;
162 	struct event_lpi_map	event_map;
163 	void			*itt;
164 	u32			nr_ites;
165 	u32			device_id;
166 	bool			shared;
167 };
168 
169 static struct {
170 	raw_spinlock_t		lock;
171 	struct its_device	*dev;
172 	struct its_vpe		**vpes;
173 	int			next_victim;
174 } vpe_proxy;
175 
176 struct cpu_lpi_count {
177 	atomic_t	managed;
178 	atomic_t	unmanaged;
179 };
180 
181 static DEFINE_PER_CPU(struct cpu_lpi_count, cpu_lpi_count);
182 
183 static LIST_HEAD(its_nodes);
184 static DEFINE_RAW_SPINLOCK(its_lock);
185 static struct rdists *gic_rdists;
186 static struct irq_domain *its_parent;
187 
188 static unsigned long its_list_map;
189 static u16 vmovp_seq_num;
190 static DEFINE_RAW_SPINLOCK(vmovp_lock);
191 
192 static DEFINE_IDA(its_vpeid_ida);
193 
194 #define gic_data_rdist()		(raw_cpu_ptr(gic_rdists->rdist))
195 #define gic_data_rdist_cpu(cpu)		(per_cpu_ptr(gic_rdists->rdist, cpu))
196 #define gic_data_rdist_rd_base()	(gic_data_rdist()->rd_base)
197 #define gic_data_rdist_vlpi_base()	(gic_data_rdist_rd_base() + SZ_128K)
198 
199 /*
200  * Skip ITSs that have no vLPIs mapped, unless we're on GICv4.1, as we
201  * always have vSGIs mapped.
202  */
203 static bool require_its_list_vmovp(struct its_vm *vm, struct its_node *its)
204 {
205 	return (gic_rdists->has_rvpeid || vm->vlpi_count[its->list_nr]);
206 }
207 
208 static u16 get_its_list(struct its_vm *vm)
209 {
210 	struct its_node *its;
211 	unsigned long its_list = 0;
212 
213 	list_for_each_entry(its, &its_nodes, entry) {
214 		if (!is_v4(its))
215 			continue;
216 
217 		if (require_its_list_vmovp(vm, its))
218 			__set_bit(its->list_nr, &its_list);
219 	}
220 
221 	return (u16)its_list;
222 }
223 
224 static inline u32 its_get_event_id(struct irq_data *d)
225 {
226 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
227 	return d->hwirq - its_dev->event_map.lpi_base;
228 }
229 
230 static struct its_collection *dev_event_to_col(struct its_device *its_dev,
231 					       u32 event)
232 {
233 	struct its_node *its = its_dev->its;
234 
235 	return its->collections + its_dev->event_map.col_map[event];
236 }
237 
238 static struct its_vlpi_map *dev_event_to_vlpi_map(struct its_device *its_dev,
239 					       u32 event)
240 {
241 	if (WARN_ON_ONCE(event >= its_dev->event_map.nr_lpis))
242 		return NULL;
243 
244 	return &its_dev->event_map.vlpi_maps[event];
245 }
246 
247 static struct its_vlpi_map *get_vlpi_map(struct irq_data *d)
248 {
249 	if (irqd_is_forwarded_to_vcpu(d)) {
250 		struct its_device *its_dev = irq_data_get_irq_chip_data(d);
251 		u32 event = its_get_event_id(d);
252 
253 		return dev_event_to_vlpi_map(its_dev, event);
254 	}
255 
256 	return NULL;
257 }
258 
259 static int vpe_to_cpuid_lock(struct its_vpe *vpe, unsigned long *flags)
260 {
261 	raw_spin_lock_irqsave(&vpe->vpe_lock, *flags);
262 	return vpe->col_idx;
263 }
264 
265 static void vpe_to_cpuid_unlock(struct its_vpe *vpe, unsigned long flags)
266 {
267 	raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
268 }
269 
270 static int irq_to_cpuid_lock(struct irq_data *d, unsigned long *flags)
271 {
272 	struct its_vlpi_map *map = get_vlpi_map(d);
273 	int cpu;
274 
275 	if (map) {
276 		cpu = vpe_to_cpuid_lock(map->vpe, flags);
277 	} else {
278 		/* Physical LPIs are already locked via the irq_desc lock */
279 		struct its_device *its_dev = irq_data_get_irq_chip_data(d);
280 		cpu = its_dev->event_map.col_map[its_get_event_id(d)];
281 		/* Keep GCC quiet... */
282 		*flags = 0;
283 	}
284 
285 	return cpu;
286 }
287 
288 static void irq_to_cpuid_unlock(struct irq_data *d, unsigned long flags)
289 {
290 	struct its_vlpi_map *map = get_vlpi_map(d);
291 
292 	if (map)
293 		vpe_to_cpuid_unlock(map->vpe, flags);
294 }
295 
296 static struct its_collection *valid_col(struct its_collection *col)
297 {
298 	if (WARN_ON_ONCE(col->target_address & GENMASK_ULL(15, 0)))
299 		return NULL;
300 
301 	return col;
302 }
303 
304 static struct its_vpe *valid_vpe(struct its_node *its, struct its_vpe *vpe)
305 {
306 	if (valid_col(its->collections + vpe->col_idx))
307 		return vpe;
308 
309 	return NULL;
310 }
311 
312 /*
313  * ITS command descriptors - parameters to be encoded in a command
314  * block.
315  */
316 struct its_cmd_desc {
317 	union {
318 		struct {
319 			struct its_device *dev;
320 			u32 event_id;
321 		} its_inv_cmd;
322 
323 		struct {
324 			struct its_device *dev;
325 			u32 event_id;
326 		} its_clear_cmd;
327 
328 		struct {
329 			struct its_device *dev;
330 			u32 event_id;
331 		} its_int_cmd;
332 
333 		struct {
334 			struct its_device *dev;
335 			int valid;
336 		} its_mapd_cmd;
337 
338 		struct {
339 			struct its_collection *col;
340 			int valid;
341 		} its_mapc_cmd;
342 
343 		struct {
344 			struct its_device *dev;
345 			u32 phys_id;
346 			u32 event_id;
347 		} its_mapti_cmd;
348 
349 		struct {
350 			struct its_device *dev;
351 			struct its_collection *col;
352 			u32 event_id;
353 		} its_movi_cmd;
354 
355 		struct {
356 			struct its_device *dev;
357 			u32 event_id;
358 		} its_discard_cmd;
359 
360 		struct {
361 			struct its_collection *col;
362 		} its_invall_cmd;
363 
364 		struct {
365 			struct its_vpe *vpe;
366 		} its_vinvall_cmd;
367 
368 		struct {
369 			struct its_vpe *vpe;
370 			struct its_collection *col;
371 			bool valid;
372 		} its_vmapp_cmd;
373 
374 		struct {
375 			struct its_vpe *vpe;
376 			struct its_device *dev;
377 			u32 virt_id;
378 			u32 event_id;
379 			bool db_enabled;
380 		} its_vmapti_cmd;
381 
382 		struct {
383 			struct its_vpe *vpe;
384 			struct its_device *dev;
385 			u32 event_id;
386 			bool db_enabled;
387 		} its_vmovi_cmd;
388 
389 		struct {
390 			struct its_vpe *vpe;
391 			struct its_collection *col;
392 			u16 seq_num;
393 			u16 its_list;
394 		} its_vmovp_cmd;
395 
396 		struct {
397 			struct its_vpe *vpe;
398 		} its_invdb_cmd;
399 
400 		struct {
401 			struct its_vpe *vpe;
402 			u8 sgi;
403 			u8 priority;
404 			bool enable;
405 			bool group;
406 			bool clear;
407 		} its_vsgi_cmd;
408 	};
409 };
410 
411 /*
412  * The ITS command block, which is what the ITS actually parses.
413  */
414 struct its_cmd_block {
415 	union {
416 		u64	raw_cmd[4];
417 		__le64	raw_cmd_le[4];
418 	};
419 };
420 
421 #define ITS_CMD_QUEUE_SZ		SZ_64K
422 #define ITS_CMD_QUEUE_NR_ENTRIES	(ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block))
423 
424 typedef struct its_collection *(*its_cmd_builder_t)(struct its_node *,
425 						    struct its_cmd_block *,
426 						    struct its_cmd_desc *);
427 
428 typedef struct its_vpe *(*its_cmd_vbuilder_t)(struct its_node *,
429 					      struct its_cmd_block *,
430 					      struct its_cmd_desc *);
431 
432 static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l)
433 {
434 	u64 mask = GENMASK_ULL(h, l);
435 	*raw_cmd &= ~mask;
436 	*raw_cmd |= (val << l) & mask;
437 }
438 
439 static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr)
440 {
441 	its_mask_encode(&cmd->raw_cmd[0], cmd_nr, 7, 0);
442 }
443 
444 static void its_encode_devid(struct its_cmd_block *cmd, u32 devid)
445 {
446 	its_mask_encode(&cmd->raw_cmd[0], devid, 63, 32);
447 }
448 
449 static void its_encode_event_id(struct its_cmd_block *cmd, u32 id)
450 {
451 	its_mask_encode(&cmd->raw_cmd[1], id, 31, 0);
452 }
453 
454 static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id)
455 {
456 	its_mask_encode(&cmd->raw_cmd[1], phys_id, 63, 32);
457 }
458 
459 static void its_encode_size(struct its_cmd_block *cmd, u8 size)
460 {
461 	its_mask_encode(&cmd->raw_cmd[1], size, 4, 0);
462 }
463 
464 static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr)
465 {
466 	its_mask_encode(&cmd->raw_cmd[2], itt_addr >> 8, 51, 8);
467 }
468 
469 static void its_encode_valid(struct its_cmd_block *cmd, int valid)
470 {
471 	its_mask_encode(&cmd->raw_cmd[2], !!valid, 63, 63);
472 }
473 
474 static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr)
475 {
476 	its_mask_encode(&cmd->raw_cmd[2], target_addr >> 16, 51, 16);
477 }
478 
479 static void its_encode_collection(struct its_cmd_block *cmd, u16 col)
480 {
481 	its_mask_encode(&cmd->raw_cmd[2], col, 15, 0);
482 }
483 
484 static void its_encode_vpeid(struct its_cmd_block *cmd, u16 vpeid)
485 {
486 	its_mask_encode(&cmd->raw_cmd[1], vpeid, 47, 32);
487 }
488 
489 static void its_encode_virt_id(struct its_cmd_block *cmd, u32 virt_id)
490 {
491 	its_mask_encode(&cmd->raw_cmd[2], virt_id, 31, 0);
492 }
493 
494 static void its_encode_db_phys_id(struct its_cmd_block *cmd, u32 db_phys_id)
495 {
496 	its_mask_encode(&cmd->raw_cmd[2], db_phys_id, 63, 32);
497 }
498 
499 static void its_encode_db_valid(struct its_cmd_block *cmd, bool db_valid)
500 {
501 	its_mask_encode(&cmd->raw_cmd[2], db_valid, 0, 0);
502 }
503 
504 static void its_encode_seq_num(struct its_cmd_block *cmd, u16 seq_num)
505 {
506 	its_mask_encode(&cmd->raw_cmd[0], seq_num, 47, 32);
507 }
508 
509 static void its_encode_its_list(struct its_cmd_block *cmd, u16 its_list)
510 {
511 	its_mask_encode(&cmd->raw_cmd[1], its_list, 15, 0);
512 }
513 
514 static void its_encode_vpt_addr(struct its_cmd_block *cmd, u64 vpt_pa)
515 {
516 	its_mask_encode(&cmd->raw_cmd[3], vpt_pa >> 16, 51, 16);
517 }
518 
519 static void its_encode_vpt_size(struct its_cmd_block *cmd, u8 vpt_size)
520 {
521 	its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0);
522 }
523 
524 static void its_encode_vconf_addr(struct its_cmd_block *cmd, u64 vconf_pa)
525 {
526 	its_mask_encode(&cmd->raw_cmd[0], vconf_pa >> 16, 51, 16);
527 }
528 
529 static void its_encode_alloc(struct its_cmd_block *cmd, bool alloc)
530 {
531 	its_mask_encode(&cmd->raw_cmd[0], alloc, 8, 8);
532 }
533 
534 static void its_encode_ptz(struct its_cmd_block *cmd, bool ptz)
535 {
536 	its_mask_encode(&cmd->raw_cmd[0], ptz, 9, 9);
537 }
538 
539 static void its_encode_vmapp_default_db(struct its_cmd_block *cmd,
540 					u32 vpe_db_lpi)
541 {
542 	its_mask_encode(&cmd->raw_cmd[1], vpe_db_lpi, 31, 0);
543 }
544 
545 static void its_encode_vmovp_default_db(struct its_cmd_block *cmd,
546 					u32 vpe_db_lpi)
547 {
548 	its_mask_encode(&cmd->raw_cmd[3], vpe_db_lpi, 31, 0);
549 }
550 
551 static void its_encode_db(struct its_cmd_block *cmd, bool db)
552 {
553 	its_mask_encode(&cmd->raw_cmd[2], db, 63, 63);
554 }
555 
556 static void its_encode_sgi_intid(struct its_cmd_block *cmd, u8 sgi)
557 {
558 	its_mask_encode(&cmd->raw_cmd[0], sgi, 35, 32);
559 }
560 
561 static void its_encode_sgi_priority(struct its_cmd_block *cmd, u8 prio)
562 {
563 	its_mask_encode(&cmd->raw_cmd[0], prio >> 4, 23, 20);
564 }
565 
566 static void its_encode_sgi_group(struct its_cmd_block *cmd, bool grp)
567 {
568 	its_mask_encode(&cmd->raw_cmd[0], grp, 10, 10);
569 }
570 
571 static void its_encode_sgi_clear(struct its_cmd_block *cmd, bool clr)
572 {
573 	its_mask_encode(&cmd->raw_cmd[0], clr, 9, 9);
574 }
575 
576 static void its_encode_sgi_enable(struct its_cmd_block *cmd, bool en)
577 {
578 	its_mask_encode(&cmd->raw_cmd[0], en, 8, 8);
579 }
580 
581 static inline void its_fixup_cmd(struct its_cmd_block *cmd)
582 {
583 	/* Let's fixup BE commands */
584 	cmd->raw_cmd_le[0] = cpu_to_le64(cmd->raw_cmd[0]);
585 	cmd->raw_cmd_le[1] = cpu_to_le64(cmd->raw_cmd[1]);
586 	cmd->raw_cmd_le[2] = cpu_to_le64(cmd->raw_cmd[2]);
587 	cmd->raw_cmd_le[3] = cpu_to_le64(cmd->raw_cmd[3]);
588 }
589 
590 static struct its_collection *its_build_mapd_cmd(struct its_node *its,
591 						 struct its_cmd_block *cmd,
592 						 struct its_cmd_desc *desc)
593 {
594 	unsigned long itt_addr;
595 	u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites);
596 
597 	itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt);
598 	itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN);
599 
600 	its_encode_cmd(cmd, GITS_CMD_MAPD);
601 	its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id);
602 	its_encode_size(cmd, size - 1);
603 	its_encode_itt(cmd, itt_addr);
604 	its_encode_valid(cmd, desc->its_mapd_cmd.valid);
605 
606 	its_fixup_cmd(cmd);
607 
608 	return NULL;
609 }
610 
611 static struct its_collection *its_build_mapc_cmd(struct its_node *its,
612 						 struct its_cmd_block *cmd,
613 						 struct its_cmd_desc *desc)
614 {
615 	its_encode_cmd(cmd, GITS_CMD_MAPC);
616 	its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id);
617 	its_encode_target(cmd, desc->its_mapc_cmd.col->target_address);
618 	its_encode_valid(cmd, desc->its_mapc_cmd.valid);
619 
620 	its_fixup_cmd(cmd);
621 
622 	return desc->its_mapc_cmd.col;
623 }
624 
625 static struct its_collection *its_build_mapti_cmd(struct its_node *its,
626 						  struct its_cmd_block *cmd,
627 						  struct its_cmd_desc *desc)
628 {
629 	struct its_collection *col;
630 
631 	col = dev_event_to_col(desc->its_mapti_cmd.dev,
632 			       desc->its_mapti_cmd.event_id);
633 
634 	its_encode_cmd(cmd, GITS_CMD_MAPTI);
635 	its_encode_devid(cmd, desc->its_mapti_cmd.dev->device_id);
636 	its_encode_event_id(cmd, desc->its_mapti_cmd.event_id);
637 	its_encode_phys_id(cmd, desc->its_mapti_cmd.phys_id);
638 	its_encode_collection(cmd, col->col_id);
639 
640 	its_fixup_cmd(cmd);
641 
642 	return valid_col(col);
643 }
644 
645 static struct its_collection *its_build_movi_cmd(struct its_node *its,
646 						 struct its_cmd_block *cmd,
647 						 struct its_cmd_desc *desc)
648 {
649 	struct its_collection *col;
650 
651 	col = dev_event_to_col(desc->its_movi_cmd.dev,
652 			       desc->its_movi_cmd.event_id);
653 
654 	its_encode_cmd(cmd, GITS_CMD_MOVI);
655 	its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id);
656 	its_encode_event_id(cmd, desc->its_movi_cmd.event_id);
657 	its_encode_collection(cmd, desc->its_movi_cmd.col->col_id);
658 
659 	its_fixup_cmd(cmd);
660 
661 	return valid_col(col);
662 }
663 
664 static struct its_collection *its_build_discard_cmd(struct its_node *its,
665 						    struct its_cmd_block *cmd,
666 						    struct its_cmd_desc *desc)
667 {
668 	struct its_collection *col;
669 
670 	col = dev_event_to_col(desc->its_discard_cmd.dev,
671 			       desc->its_discard_cmd.event_id);
672 
673 	its_encode_cmd(cmd, GITS_CMD_DISCARD);
674 	its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id);
675 	its_encode_event_id(cmd, desc->its_discard_cmd.event_id);
676 
677 	its_fixup_cmd(cmd);
678 
679 	return valid_col(col);
680 }
681 
682 static struct its_collection *its_build_inv_cmd(struct its_node *its,
683 						struct its_cmd_block *cmd,
684 						struct its_cmd_desc *desc)
685 {
686 	struct its_collection *col;
687 
688 	col = dev_event_to_col(desc->its_inv_cmd.dev,
689 			       desc->its_inv_cmd.event_id);
690 
691 	its_encode_cmd(cmd, GITS_CMD_INV);
692 	its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
693 	its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
694 
695 	its_fixup_cmd(cmd);
696 
697 	return valid_col(col);
698 }
699 
700 static struct its_collection *its_build_int_cmd(struct its_node *its,
701 						struct its_cmd_block *cmd,
702 						struct its_cmd_desc *desc)
703 {
704 	struct its_collection *col;
705 
706 	col = dev_event_to_col(desc->its_int_cmd.dev,
707 			       desc->its_int_cmd.event_id);
708 
709 	its_encode_cmd(cmd, GITS_CMD_INT);
710 	its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
711 	its_encode_event_id(cmd, desc->its_int_cmd.event_id);
712 
713 	its_fixup_cmd(cmd);
714 
715 	return valid_col(col);
716 }
717 
718 static struct its_collection *its_build_clear_cmd(struct its_node *its,
719 						  struct its_cmd_block *cmd,
720 						  struct its_cmd_desc *desc)
721 {
722 	struct its_collection *col;
723 
724 	col = dev_event_to_col(desc->its_clear_cmd.dev,
725 			       desc->its_clear_cmd.event_id);
726 
727 	its_encode_cmd(cmd, GITS_CMD_CLEAR);
728 	its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
729 	its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
730 
731 	its_fixup_cmd(cmd);
732 
733 	return valid_col(col);
734 }
735 
736 static struct its_collection *its_build_invall_cmd(struct its_node *its,
737 						   struct its_cmd_block *cmd,
738 						   struct its_cmd_desc *desc)
739 {
740 	its_encode_cmd(cmd, GITS_CMD_INVALL);
741 	its_encode_collection(cmd, desc->its_invall_cmd.col->col_id);
742 
743 	its_fixup_cmd(cmd);
744 
745 	return NULL;
746 }
747 
748 static struct its_vpe *its_build_vinvall_cmd(struct its_node *its,
749 					     struct its_cmd_block *cmd,
750 					     struct its_cmd_desc *desc)
751 {
752 	its_encode_cmd(cmd, GITS_CMD_VINVALL);
753 	its_encode_vpeid(cmd, desc->its_vinvall_cmd.vpe->vpe_id);
754 
755 	its_fixup_cmd(cmd);
756 
757 	return valid_vpe(its, desc->its_vinvall_cmd.vpe);
758 }
759 
760 static struct its_vpe *its_build_vmapp_cmd(struct its_node *its,
761 					   struct its_cmd_block *cmd,
762 					   struct its_cmd_desc *desc)
763 {
764 	unsigned long vpt_addr, vconf_addr;
765 	u64 target;
766 	bool alloc;
767 
768 	its_encode_cmd(cmd, GITS_CMD_VMAPP);
769 	its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id);
770 	its_encode_valid(cmd, desc->its_vmapp_cmd.valid);
771 
772 	if (!desc->its_vmapp_cmd.valid) {
773 		if (is_v4_1(its)) {
774 			alloc = !atomic_dec_return(&desc->its_vmapp_cmd.vpe->vmapp_count);
775 			its_encode_alloc(cmd, alloc);
776 		}
777 
778 		goto out;
779 	}
780 
781 	vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page));
782 	target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset;
783 
784 	its_encode_target(cmd, target);
785 	its_encode_vpt_addr(cmd, vpt_addr);
786 	its_encode_vpt_size(cmd, LPI_NRBITS - 1);
787 
788 	if (!is_v4_1(its))
789 		goto out;
790 
791 	vconf_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->its_vm->vprop_page));
792 
793 	alloc = !atomic_fetch_inc(&desc->its_vmapp_cmd.vpe->vmapp_count);
794 
795 	its_encode_alloc(cmd, alloc);
796 
797 	/*
798 	 * GICv4.1 provides a way to get the VLPI state, which needs the vPE
799 	 * to be unmapped first, and in this case, we may remap the vPE
800 	 * back while the VPT is not empty. So we can't assume that the
801 	 * VPT is empty on map. This is why we never advertise PTZ.
802 	 */
803 	its_encode_ptz(cmd, false);
804 	its_encode_vconf_addr(cmd, vconf_addr);
805 	its_encode_vmapp_default_db(cmd, desc->its_vmapp_cmd.vpe->vpe_db_lpi);
806 
807 out:
808 	its_fixup_cmd(cmd);
809 
810 	return valid_vpe(its, desc->its_vmapp_cmd.vpe);
811 }
812 
813 static struct its_vpe *its_build_vmapti_cmd(struct its_node *its,
814 					    struct its_cmd_block *cmd,
815 					    struct its_cmd_desc *desc)
816 {
817 	u32 db;
818 
819 	if (!is_v4_1(its) && desc->its_vmapti_cmd.db_enabled)
820 		db = desc->its_vmapti_cmd.vpe->vpe_db_lpi;
821 	else
822 		db = 1023;
823 
824 	its_encode_cmd(cmd, GITS_CMD_VMAPTI);
825 	its_encode_devid(cmd, desc->its_vmapti_cmd.dev->device_id);
826 	its_encode_vpeid(cmd, desc->its_vmapti_cmd.vpe->vpe_id);
827 	its_encode_event_id(cmd, desc->its_vmapti_cmd.event_id);
828 	its_encode_db_phys_id(cmd, db);
829 	its_encode_virt_id(cmd, desc->its_vmapti_cmd.virt_id);
830 
831 	its_fixup_cmd(cmd);
832 
833 	return valid_vpe(its, desc->its_vmapti_cmd.vpe);
834 }
835 
836 static struct its_vpe *its_build_vmovi_cmd(struct its_node *its,
837 					   struct its_cmd_block *cmd,
838 					   struct its_cmd_desc *desc)
839 {
840 	u32 db;
841 
842 	if (!is_v4_1(its) && desc->its_vmovi_cmd.db_enabled)
843 		db = desc->its_vmovi_cmd.vpe->vpe_db_lpi;
844 	else
845 		db = 1023;
846 
847 	its_encode_cmd(cmd, GITS_CMD_VMOVI);
848 	its_encode_devid(cmd, desc->its_vmovi_cmd.dev->device_id);
849 	its_encode_vpeid(cmd, desc->its_vmovi_cmd.vpe->vpe_id);
850 	its_encode_event_id(cmd, desc->its_vmovi_cmd.event_id);
851 	its_encode_db_phys_id(cmd, db);
852 	its_encode_db_valid(cmd, true);
853 
854 	its_fixup_cmd(cmd);
855 
856 	return valid_vpe(its, desc->its_vmovi_cmd.vpe);
857 }
858 
859 static struct its_vpe *its_build_vmovp_cmd(struct its_node *its,
860 					   struct its_cmd_block *cmd,
861 					   struct its_cmd_desc *desc)
862 {
863 	u64 target;
864 
865 	target = desc->its_vmovp_cmd.col->target_address + its->vlpi_redist_offset;
866 	its_encode_cmd(cmd, GITS_CMD_VMOVP);
867 	its_encode_seq_num(cmd, desc->its_vmovp_cmd.seq_num);
868 	its_encode_its_list(cmd, desc->its_vmovp_cmd.its_list);
869 	its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id);
870 	its_encode_target(cmd, target);
871 
872 	if (is_v4_1(its)) {
873 		its_encode_db(cmd, true);
874 		its_encode_vmovp_default_db(cmd, desc->its_vmovp_cmd.vpe->vpe_db_lpi);
875 	}
876 
877 	its_fixup_cmd(cmd);
878 
879 	return valid_vpe(its, desc->its_vmovp_cmd.vpe);
880 }
881 
882 static struct its_vpe *its_build_vinv_cmd(struct its_node *its,
883 					  struct its_cmd_block *cmd,
884 					  struct its_cmd_desc *desc)
885 {
886 	struct its_vlpi_map *map;
887 
888 	map = dev_event_to_vlpi_map(desc->its_inv_cmd.dev,
889 				    desc->its_inv_cmd.event_id);
890 
891 	its_encode_cmd(cmd, GITS_CMD_INV);
892 	its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
893 	its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
894 
895 	its_fixup_cmd(cmd);
896 
897 	return valid_vpe(its, map->vpe);
898 }
899 
900 static struct its_vpe *its_build_vint_cmd(struct its_node *its,
901 					  struct its_cmd_block *cmd,
902 					  struct its_cmd_desc *desc)
903 {
904 	struct its_vlpi_map *map;
905 
906 	map = dev_event_to_vlpi_map(desc->its_int_cmd.dev,
907 				    desc->its_int_cmd.event_id);
908 
909 	its_encode_cmd(cmd, GITS_CMD_INT);
910 	its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
911 	its_encode_event_id(cmd, desc->its_int_cmd.event_id);
912 
913 	its_fixup_cmd(cmd);
914 
915 	return valid_vpe(its, map->vpe);
916 }
917 
918 static struct its_vpe *its_build_vclear_cmd(struct its_node *its,
919 					    struct its_cmd_block *cmd,
920 					    struct its_cmd_desc *desc)
921 {
922 	struct its_vlpi_map *map;
923 
924 	map = dev_event_to_vlpi_map(desc->its_clear_cmd.dev,
925 				    desc->its_clear_cmd.event_id);
926 
927 	its_encode_cmd(cmd, GITS_CMD_CLEAR);
928 	its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
929 	its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
930 
931 	its_fixup_cmd(cmd);
932 
933 	return valid_vpe(its, map->vpe);
934 }
935 
936 static struct its_vpe *its_build_invdb_cmd(struct its_node *its,
937 					   struct its_cmd_block *cmd,
938 					   struct its_cmd_desc *desc)
939 {
940 	if (WARN_ON(!is_v4_1(its)))
941 		return NULL;
942 
943 	its_encode_cmd(cmd, GITS_CMD_INVDB);
944 	its_encode_vpeid(cmd, desc->its_invdb_cmd.vpe->vpe_id);
945 
946 	its_fixup_cmd(cmd);
947 
948 	return valid_vpe(its, desc->its_invdb_cmd.vpe);
949 }
950 
951 static struct its_vpe *its_build_vsgi_cmd(struct its_node *its,
952 					  struct its_cmd_block *cmd,
953 					  struct its_cmd_desc *desc)
954 {
955 	if (WARN_ON(!is_v4_1(its)))
956 		return NULL;
957 
958 	its_encode_cmd(cmd, GITS_CMD_VSGI);
959 	its_encode_vpeid(cmd, desc->its_vsgi_cmd.vpe->vpe_id);
960 	its_encode_sgi_intid(cmd, desc->its_vsgi_cmd.sgi);
961 	its_encode_sgi_priority(cmd, desc->its_vsgi_cmd.priority);
962 	its_encode_sgi_group(cmd, desc->its_vsgi_cmd.group);
963 	its_encode_sgi_clear(cmd, desc->its_vsgi_cmd.clear);
964 	its_encode_sgi_enable(cmd, desc->its_vsgi_cmd.enable);
965 
966 	its_fixup_cmd(cmd);
967 
968 	return valid_vpe(its, desc->its_vsgi_cmd.vpe);
969 }
970 
971 static u64 its_cmd_ptr_to_offset(struct its_node *its,
972 				 struct its_cmd_block *ptr)
973 {
974 	return (ptr - its->cmd_base) * sizeof(*ptr);
975 }
976 
977 static int its_queue_full(struct its_node *its)
978 {
979 	int widx;
980 	int ridx;
981 
982 	widx = its->cmd_write - its->cmd_base;
983 	ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block);
984 
985 	/* This is incredibly unlikely to happen, unless the ITS locks up. */
986 	if (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx)
987 		return 1;
988 
989 	return 0;
990 }
991 
992 static struct its_cmd_block *its_allocate_entry(struct its_node *its)
993 {
994 	struct its_cmd_block *cmd;
995 	u32 count = 1000000;	/* 1s! */
996 
997 	while (its_queue_full(its)) {
998 		count--;
999 		if (!count) {
1000 			pr_err_ratelimited("ITS queue not draining\n");
1001 			return NULL;
1002 		}
1003 		cpu_relax();
1004 		udelay(1);
1005 	}
1006 
1007 	cmd = its->cmd_write++;
1008 
1009 	/* Handle queue wrapping */
1010 	if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES))
1011 		its->cmd_write = its->cmd_base;
1012 
1013 	/* Clear command  */
1014 	cmd->raw_cmd[0] = 0;
1015 	cmd->raw_cmd[1] = 0;
1016 	cmd->raw_cmd[2] = 0;
1017 	cmd->raw_cmd[3] = 0;
1018 
1019 	return cmd;
1020 }
1021 
1022 static struct its_cmd_block *its_post_commands(struct its_node *its)
1023 {
1024 	u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write);
1025 
1026 	writel_relaxed(wr, its->base + GITS_CWRITER);
1027 
1028 	return its->cmd_write;
1029 }
1030 
1031 static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd)
1032 {
1033 	/*
1034 	 * Make sure the commands written to memory are observable by
1035 	 * the ITS.
1036 	 */
1037 	if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING)
1038 		gic_flush_dcache_to_poc(cmd, sizeof(*cmd));
1039 	else
1040 		dsb(ishst);
1041 }
1042 
1043 static int its_wait_for_range_completion(struct its_node *its,
1044 					 u64	prev_idx,
1045 					 struct its_cmd_block *to)
1046 {
1047 	u64 rd_idx, to_idx, linear_idx;
1048 	u32 count = 1000000;	/* 1s! */
1049 
1050 	/* Linearize to_idx if the command set has wrapped around */
1051 	to_idx = its_cmd_ptr_to_offset(its, to);
1052 	if (to_idx < prev_idx)
1053 		to_idx += ITS_CMD_QUEUE_SZ;
1054 
1055 	linear_idx = prev_idx;
1056 
1057 	while (1) {
1058 		s64 delta;
1059 
1060 		rd_idx = readl_relaxed(its->base + GITS_CREADR);
1061 
1062 		/*
1063 		 * Compute the read pointer progress, taking the
1064 		 * potential wrap-around into account.
1065 		 */
1066 		delta = rd_idx - prev_idx;
1067 		if (rd_idx < prev_idx)
1068 			delta += ITS_CMD_QUEUE_SZ;
1069 
1070 		linear_idx += delta;
1071 		if (linear_idx >= to_idx)
1072 			break;
1073 
1074 		count--;
1075 		if (!count) {
1076 			pr_err_ratelimited("ITS queue timeout (%llu %llu)\n",
1077 					   to_idx, linear_idx);
1078 			return -1;
1079 		}
1080 		prev_idx = rd_idx;
1081 		cpu_relax();
1082 		udelay(1);
1083 	}
1084 
1085 	return 0;
1086 }
1087 
1088 /* Warning, macro hell follows */
1089 #define BUILD_SINGLE_CMD_FUNC(name, buildtype, synctype, buildfn)	\
1090 void name(struct its_node *its,						\
1091 	  buildtype builder,						\
1092 	  struct its_cmd_desc *desc)					\
1093 {									\
1094 	struct its_cmd_block *cmd, *sync_cmd, *next_cmd;		\
1095 	synctype *sync_obj;						\
1096 	unsigned long flags;						\
1097 	u64 rd_idx;							\
1098 									\
1099 	raw_spin_lock_irqsave(&its->lock, flags);			\
1100 									\
1101 	cmd = its_allocate_entry(its);					\
1102 	if (!cmd) {		/* We're soooooo screewed... */		\
1103 		raw_spin_unlock_irqrestore(&its->lock, flags);		\
1104 		return;							\
1105 	}								\
1106 	sync_obj = builder(its, cmd, desc);				\
1107 	its_flush_cmd(its, cmd);					\
1108 									\
1109 	if (sync_obj) {							\
1110 		sync_cmd = its_allocate_entry(its);			\
1111 		if (!sync_cmd)						\
1112 			goto post;					\
1113 									\
1114 		buildfn(its, sync_cmd, sync_obj);			\
1115 		its_flush_cmd(its, sync_cmd);				\
1116 	}								\
1117 									\
1118 post:									\
1119 	rd_idx = readl_relaxed(its->base + GITS_CREADR);		\
1120 	next_cmd = its_post_commands(its);				\
1121 	raw_spin_unlock_irqrestore(&its->lock, flags);			\
1122 									\
1123 	if (its_wait_for_range_completion(its, rd_idx, next_cmd))	\
1124 		pr_err_ratelimited("ITS cmd %ps failed\n", builder);	\
1125 }
1126 
1127 static void its_build_sync_cmd(struct its_node *its,
1128 			       struct its_cmd_block *sync_cmd,
1129 			       struct its_collection *sync_col)
1130 {
1131 	its_encode_cmd(sync_cmd, GITS_CMD_SYNC);
1132 	its_encode_target(sync_cmd, sync_col->target_address);
1133 
1134 	its_fixup_cmd(sync_cmd);
1135 }
1136 
1137 static BUILD_SINGLE_CMD_FUNC(its_send_single_command, its_cmd_builder_t,
1138 			     struct its_collection, its_build_sync_cmd)
1139 
1140 static void its_build_vsync_cmd(struct its_node *its,
1141 				struct its_cmd_block *sync_cmd,
1142 				struct its_vpe *sync_vpe)
1143 {
1144 	its_encode_cmd(sync_cmd, GITS_CMD_VSYNC);
1145 	its_encode_vpeid(sync_cmd, sync_vpe->vpe_id);
1146 
1147 	its_fixup_cmd(sync_cmd);
1148 }
1149 
1150 static BUILD_SINGLE_CMD_FUNC(its_send_single_vcommand, its_cmd_vbuilder_t,
1151 			     struct its_vpe, its_build_vsync_cmd)
1152 
1153 static void its_send_int(struct its_device *dev, u32 event_id)
1154 {
1155 	struct its_cmd_desc desc;
1156 
1157 	desc.its_int_cmd.dev = dev;
1158 	desc.its_int_cmd.event_id = event_id;
1159 
1160 	its_send_single_command(dev->its, its_build_int_cmd, &desc);
1161 }
1162 
1163 static void its_send_clear(struct its_device *dev, u32 event_id)
1164 {
1165 	struct its_cmd_desc desc;
1166 
1167 	desc.its_clear_cmd.dev = dev;
1168 	desc.its_clear_cmd.event_id = event_id;
1169 
1170 	its_send_single_command(dev->its, its_build_clear_cmd, &desc);
1171 }
1172 
1173 static void its_send_inv(struct its_device *dev, u32 event_id)
1174 {
1175 	struct its_cmd_desc desc;
1176 
1177 	desc.its_inv_cmd.dev = dev;
1178 	desc.its_inv_cmd.event_id = event_id;
1179 
1180 	its_send_single_command(dev->its, its_build_inv_cmd, &desc);
1181 }
1182 
1183 static void its_send_mapd(struct its_device *dev, int valid)
1184 {
1185 	struct its_cmd_desc desc;
1186 
1187 	desc.its_mapd_cmd.dev = dev;
1188 	desc.its_mapd_cmd.valid = !!valid;
1189 
1190 	its_send_single_command(dev->its, its_build_mapd_cmd, &desc);
1191 }
1192 
1193 static void its_send_mapc(struct its_node *its, struct its_collection *col,
1194 			  int valid)
1195 {
1196 	struct its_cmd_desc desc;
1197 
1198 	desc.its_mapc_cmd.col = col;
1199 	desc.its_mapc_cmd.valid = !!valid;
1200 
1201 	its_send_single_command(its, its_build_mapc_cmd, &desc);
1202 }
1203 
1204 static void its_send_mapti(struct its_device *dev, u32 irq_id, u32 id)
1205 {
1206 	struct its_cmd_desc desc;
1207 
1208 	desc.its_mapti_cmd.dev = dev;
1209 	desc.its_mapti_cmd.phys_id = irq_id;
1210 	desc.its_mapti_cmd.event_id = id;
1211 
1212 	its_send_single_command(dev->its, its_build_mapti_cmd, &desc);
1213 }
1214 
1215 static void its_send_movi(struct its_device *dev,
1216 			  struct its_collection *col, u32 id)
1217 {
1218 	struct its_cmd_desc desc;
1219 
1220 	desc.its_movi_cmd.dev = dev;
1221 	desc.its_movi_cmd.col = col;
1222 	desc.its_movi_cmd.event_id = id;
1223 
1224 	its_send_single_command(dev->its, its_build_movi_cmd, &desc);
1225 }
1226 
1227 static void its_send_discard(struct its_device *dev, u32 id)
1228 {
1229 	struct its_cmd_desc desc;
1230 
1231 	desc.its_discard_cmd.dev = dev;
1232 	desc.its_discard_cmd.event_id = id;
1233 
1234 	its_send_single_command(dev->its, its_build_discard_cmd, &desc);
1235 }
1236 
1237 static void its_send_invall(struct its_node *its, struct its_collection *col)
1238 {
1239 	struct its_cmd_desc desc;
1240 
1241 	desc.its_invall_cmd.col = col;
1242 
1243 	its_send_single_command(its, its_build_invall_cmd, &desc);
1244 }
1245 
1246 static void its_send_vmapti(struct its_device *dev, u32 id)
1247 {
1248 	struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
1249 	struct its_cmd_desc desc;
1250 
1251 	desc.its_vmapti_cmd.vpe = map->vpe;
1252 	desc.its_vmapti_cmd.dev = dev;
1253 	desc.its_vmapti_cmd.virt_id = map->vintid;
1254 	desc.its_vmapti_cmd.event_id = id;
1255 	desc.its_vmapti_cmd.db_enabled = map->db_enabled;
1256 
1257 	its_send_single_vcommand(dev->its, its_build_vmapti_cmd, &desc);
1258 }
1259 
1260 static void its_send_vmovi(struct its_device *dev, u32 id)
1261 {
1262 	struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
1263 	struct its_cmd_desc desc;
1264 
1265 	desc.its_vmovi_cmd.vpe = map->vpe;
1266 	desc.its_vmovi_cmd.dev = dev;
1267 	desc.its_vmovi_cmd.event_id = id;
1268 	desc.its_vmovi_cmd.db_enabled = map->db_enabled;
1269 
1270 	its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc);
1271 }
1272 
1273 static void its_send_vmapp(struct its_node *its,
1274 			   struct its_vpe *vpe, bool valid)
1275 {
1276 	struct its_cmd_desc desc;
1277 
1278 	desc.its_vmapp_cmd.vpe = vpe;
1279 	desc.its_vmapp_cmd.valid = valid;
1280 	desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx];
1281 
1282 	its_send_single_vcommand(its, its_build_vmapp_cmd, &desc);
1283 }
1284 
1285 static void its_send_vmovp(struct its_vpe *vpe)
1286 {
1287 	struct its_cmd_desc desc = {};
1288 	struct its_node *its;
1289 	unsigned long flags;
1290 	int col_id = vpe->col_idx;
1291 
1292 	desc.its_vmovp_cmd.vpe = vpe;
1293 
1294 	if (!its_list_map) {
1295 		its = list_first_entry(&its_nodes, struct its_node, entry);
1296 		desc.its_vmovp_cmd.col = &its->collections[col_id];
1297 		its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
1298 		return;
1299 	}
1300 
1301 	/*
1302 	 * Yet another marvel of the architecture. If using the
1303 	 * its_list "feature", we need to make sure that all ITSs
1304 	 * receive all VMOVP commands in the same order. The only way
1305 	 * to guarantee this is to make vmovp a serialization point.
1306 	 *
1307 	 * Wall <-- Head.
1308 	 */
1309 	raw_spin_lock_irqsave(&vmovp_lock, flags);
1310 
1311 	desc.its_vmovp_cmd.seq_num = vmovp_seq_num++;
1312 	desc.its_vmovp_cmd.its_list = get_its_list(vpe->its_vm);
1313 
1314 	/* Emit VMOVPs */
1315 	list_for_each_entry(its, &its_nodes, entry) {
1316 		if (!is_v4(its))
1317 			continue;
1318 
1319 		if (!require_its_list_vmovp(vpe->its_vm, its))
1320 			continue;
1321 
1322 		desc.its_vmovp_cmd.col = &its->collections[col_id];
1323 		its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
1324 	}
1325 
1326 	raw_spin_unlock_irqrestore(&vmovp_lock, flags);
1327 }
1328 
1329 static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe)
1330 {
1331 	struct its_cmd_desc desc;
1332 
1333 	desc.its_vinvall_cmd.vpe = vpe;
1334 	its_send_single_vcommand(its, its_build_vinvall_cmd, &desc);
1335 }
1336 
1337 static void its_send_vinv(struct its_device *dev, u32 event_id)
1338 {
1339 	struct its_cmd_desc desc;
1340 
1341 	/*
1342 	 * There is no real VINV command. This is just a normal INV,
1343 	 * with a VSYNC instead of a SYNC.
1344 	 */
1345 	desc.its_inv_cmd.dev = dev;
1346 	desc.its_inv_cmd.event_id = event_id;
1347 
1348 	its_send_single_vcommand(dev->its, its_build_vinv_cmd, &desc);
1349 }
1350 
1351 static void its_send_vint(struct its_device *dev, u32 event_id)
1352 {
1353 	struct its_cmd_desc desc;
1354 
1355 	/*
1356 	 * There is no real VINT command. This is just a normal INT,
1357 	 * with a VSYNC instead of a SYNC.
1358 	 */
1359 	desc.its_int_cmd.dev = dev;
1360 	desc.its_int_cmd.event_id = event_id;
1361 
1362 	its_send_single_vcommand(dev->its, its_build_vint_cmd, &desc);
1363 }
1364 
1365 static void its_send_vclear(struct its_device *dev, u32 event_id)
1366 {
1367 	struct its_cmd_desc desc;
1368 
1369 	/*
1370 	 * There is no real VCLEAR command. This is just a normal CLEAR,
1371 	 * with a VSYNC instead of a SYNC.
1372 	 */
1373 	desc.its_clear_cmd.dev = dev;
1374 	desc.its_clear_cmd.event_id = event_id;
1375 
1376 	its_send_single_vcommand(dev->its, its_build_vclear_cmd, &desc);
1377 }
1378 
1379 static void its_send_invdb(struct its_node *its, struct its_vpe *vpe)
1380 {
1381 	struct its_cmd_desc desc;
1382 
1383 	desc.its_invdb_cmd.vpe = vpe;
1384 	its_send_single_vcommand(its, its_build_invdb_cmd, &desc);
1385 }
1386 
1387 /*
1388  * irqchip functions - assumes MSI, mostly.
1389  */
1390 static void lpi_write_config(struct irq_data *d, u8 clr, u8 set)
1391 {
1392 	struct its_vlpi_map *map = get_vlpi_map(d);
1393 	irq_hw_number_t hwirq;
1394 	void *va;
1395 	u8 *cfg;
1396 
1397 	if (map) {
1398 		va = page_address(map->vm->vprop_page);
1399 		hwirq = map->vintid;
1400 
1401 		/* Remember the updated property */
1402 		map->properties &= ~clr;
1403 		map->properties |= set | LPI_PROP_GROUP1;
1404 	} else {
1405 		va = gic_rdists->prop_table_va;
1406 		hwirq = d->hwirq;
1407 	}
1408 
1409 	cfg = va + hwirq - 8192;
1410 	*cfg &= ~clr;
1411 	*cfg |= set | LPI_PROP_GROUP1;
1412 
1413 	/*
1414 	 * Make the above write visible to the redistributors.
1415 	 * And yes, we're flushing exactly: One. Single. Byte.
1416 	 * Humpf...
1417 	 */
1418 	if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING)
1419 		gic_flush_dcache_to_poc(cfg, sizeof(*cfg));
1420 	else
1421 		dsb(ishst);
1422 }
1423 
1424 static void wait_for_syncr(void __iomem *rdbase)
1425 {
1426 	while (readl_relaxed(rdbase + GICR_SYNCR) & 1)
1427 		cpu_relax();
1428 }
1429 
1430 static void direct_lpi_inv(struct irq_data *d)
1431 {
1432 	struct its_vlpi_map *map = get_vlpi_map(d);
1433 	void __iomem *rdbase;
1434 	unsigned long flags;
1435 	u64 val;
1436 	int cpu;
1437 
1438 	if (map) {
1439 		struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1440 
1441 		WARN_ON(!is_v4_1(its_dev->its));
1442 
1443 		val  = GICR_INVLPIR_V;
1444 		val |= FIELD_PREP(GICR_INVLPIR_VPEID, map->vpe->vpe_id);
1445 		val |= FIELD_PREP(GICR_INVLPIR_INTID, map->vintid);
1446 	} else {
1447 		val = d->hwirq;
1448 	}
1449 
1450 	/* Target the redistributor this LPI is currently routed to */
1451 	cpu = irq_to_cpuid_lock(d, &flags);
1452 	raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
1453 	rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
1454 	gic_write_lpir(val, rdbase + GICR_INVLPIR);
1455 
1456 	wait_for_syncr(rdbase);
1457 	raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
1458 	irq_to_cpuid_unlock(d, flags);
1459 }
1460 
1461 static void lpi_update_config(struct irq_data *d, u8 clr, u8 set)
1462 {
1463 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1464 
1465 	lpi_write_config(d, clr, set);
1466 	if (gic_rdists->has_direct_lpi &&
1467 	    (is_v4_1(its_dev->its) || !irqd_is_forwarded_to_vcpu(d)))
1468 		direct_lpi_inv(d);
1469 	else if (!irqd_is_forwarded_to_vcpu(d))
1470 		its_send_inv(its_dev, its_get_event_id(d));
1471 	else
1472 		its_send_vinv(its_dev, its_get_event_id(d));
1473 }
1474 
1475 static void its_vlpi_set_doorbell(struct irq_data *d, bool enable)
1476 {
1477 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1478 	u32 event = its_get_event_id(d);
1479 	struct its_vlpi_map *map;
1480 
1481 	/*
1482 	 * GICv4.1 does away with the per-LPI nonsense, nothing to do
1483 	 * here.
1484 	 */
1485 	if (is_v4_1(its_dev->its))
1486 		return;
1487 
1488 	map = dev_event_to_vlpi_map(its_dev, event);
1489 
1490 	if (map->db_enabled == enable)
1491 		return;
1492 
1493 	map->db_enabled = enable;
1494 
1495 	/*
1496 	 * More fun with the architecture:
1497 	 *
1498 	 * Ideally, we'd issue a VMAPTI to set the doorbell to its LPI
1499 	 * value or to 1023, depending on the enable bit. But that
1500 	 * would be issuing a mapping for an /existing/ DevID+EventID
1501 	 * pair, which is UNPREDICTABLE. Instead, let's issue a VMOVI
1502 	 * to the /same/ vPE, using this opportunity to adjust the
1503 	 * doorbell. Mouahahahaha. We loves it, Precious.
1504 	 */
1505 	its_send_vmovi(its_dev, event);
1506 }
1507 
1508 static void its_mask_irq(struct irq_data *d)
1509 {
1510 	if (irqd_is_forwarded_to_vcpu(d))
1511 		its_vlpi_set_doorbell(d, false);
1512 
1513 	lpi_update_config(d, LPI_PROP_ENABLED, 0);
1514 }
1515 
1516 static void its_unmask_irq(struct irq_data *d)
1517 {
1518 	if (irqd_is_forwarded_to_vcpu(d))
1519 		its_vlpi_set_doorbell(d, true);
1520 
1521 	lpi_update_config(d, 0, LPI_PROP_ENABLED);
1522 }
1523 
1524 static __maybe_unused u32 its_read_lpi_count(struct irq_data *d, int cpu)
1525 {
1526 	if (irqd_affinity_is_managed(d))
1527 		return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1528 
1529 	return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1530 }
1531 
1532 static void its_inc_lpi_count(struct irq_data *d, int cpu)
1533 {
1534 	if (irqd_affinity_is_managed(d))
1535 		atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1536 	else
1537 		atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1538 }
1539 
1540 static void its_dec_lpi_count(struct irq_data *d, int cpu)
1541 {
1542 	if (irqd_affinity_is_managed(d))
1543 		atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1544 	else
1545 		atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1546 }
1547 
1548 static unsigned int cpumask_pick_least_loaded(struct irq_data *d,
1549 					      const struct cpumask *cpu_mask)
1550 {
1551 	unsigned int cpu = nr_cpu_ids, tmp;
1552 	int count = S32_MAX;
1553 
1554 	for_each_cpu(tmp, cpu_mask) {
1555 		int this_count = its_read_lpi_count(d, tmp);
1556 		if (this_count < count) {
1557 			cpu = tmp;
1558 		        count = this_count;
1559 		}
1560 	}
1561 
1562 	return cpu;
1563 }
1564 
1565 /*
1566  * As suggested by Thomas Gleixner in:
1567  * https://lore.kernel.org/r/87h80q2aoc.fsf@nanos.tec.linutronix.de
1568  */
1569 static int its_select_cpu(struct irq_data *d,
1570 			  const struct cpumask *aff_mask)
1571 {
1572 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1573 	cpumask_var_t tmpmask;
1574 	int cpu, node;
1575 
1576 	if (!alloc_cpumask_var(&tmpmask, GFP_ATOMIC))
1577 		return -ENOMEM;
1578 
1579 	node = its_dev->its->numa_node;
1580 
1581 	if (!irqd_affinity_is_managed(d)) {
1582 		/* First try the NUMA node */
1583 		if (node != NUMA_NO_NODE) {
1584 			/*
1585 			 * Try the intersection of the affinity mask and the
1586 			 * node mask (and the online mask, just to be safe).
1587 			 */
1588 			cpumask_and(tmpmask, cpumask_of_node(node), aff_mask);
1589 			cpumask_and(tmpmask, tmpmask, cpu_online_mask);
1590 
1591 			/*
1592 			 * Ideally, we would check if the mask is empty, and
1593 			 * try again on the full node here.
1594 			 *
1595 			 * But it turns out that the way ACPI describes the
1596 			 * affinity for ITSs only deals about memory, and
1597 			 * not target CPUs, so it cannot describe a single
1598 			 * ITS placed next to two NUMA nodes.
1599 			 *
1600 			 * Instead, just fallback on the online mask. This
1601 			 * diverges from Thomas' suggestion above.
1602 			 */
1603 			cpu = cpumask_pick_least_loaded(d, tmpmask);
1604 			if (cpu < nr_cpu_ids)
1605 				goto out;
1606 
1607 			/* If we can't cross sockets, give up */
1608 			if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144))
1609 				goto out;
1610 
1611 			/* If the above failed, expand the search */
1612 		}
1613 
1614 		/* Try the intersection of the affinity and online masks */
1615 		cpumask_and(tmpmask, aff_mask, cpu_online_mask);
1616 
1617 		/* If that doesn't fly, the online mask is the last resort */
1618 		if (cpumask_empty(tmpmask))
1619 			cpumask_copy(tmpmask, cpu_online_mask);
1620 
1621 		cpu = cpumask_pick_least_loaded(d, tmpmask);
1622 	} else {
1623 		cpumask_and(tmpmask, irq_data_get_affinity_mask(d), cpu_online_mask);
1624 
1625 		/* If we cannot cross sockets, limit the search to that node */
1626 		if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) &&
1627 		    node != NUMA_NO_NODE)
1628 			cpumask_and(tmpmask, tmpmask, cpumask_of_node(node));
1629 
1630 		cpu = cpumask_pick_least_loaded(d, tmpmask);
1631 	}
1632 out:
1633 	free_cpumask_var(tmpmask);
1634 
1635 	pr_debug("IRQ%d -> %*pbl CPU%d\n", d->irq, cpumask_pr_args(aff_mask), cpu);
1636 	return cpu;
1637 }
1638 
1639 static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
1640 			    bool force)
1641 {
1642 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1643 	struct its_collection *target_col;
1644 	u32 id = its_get_event_id(d);
1645 	int cpu, prev_cpu;
1646 
1647 	/* A forwarded interrupt should use irq_set_vcpu_affinity */
1648 	if (irqd_is_forwarded_to_vcpu(d))
1649 		return -EINVAL;
1650 
1651 	prev_cpu = its_dev->event_map.col_map[id];
1652 	its_dec_lpi_count(d, prev_cpu);
1653 
1654 	if (!force)
1655 		cpu = its_select_cpu(d, mask_val);
1656 	else
1657 		cpu = cpumask_pick_least_loaded(d, mask_val);
1658 
1659 	if (cpu < 0 || cpu >= nr_cpu_ids)
1660 		goto err;
1661 
1662 	/* don't set the affinity when the target cpu is same as current one */
1663 	if (cpu != prev_cpu) {
1664 		target_col = &its_dev->its->collections[cpu];
1665 		its_send_movi(its_dev, target_col, id);
1666 		its_dev->event_map.col_map[id] = cpu;
1667 		irq_data_update_effective_affinity(d, cpumask_of(cpu));
1668 	}
1669 
1670 	its_inc_lpi_count(d, cpu);
1671 
1672 	return IRQ_SET_MASK_OK_DONE;
1673 
1674 err:
1675 	its_inc_lpi_count(d, prev_cpu);
1676 	return -EINVAL;
1677 }
1678 
1679 static u64 its_irq_get_msi_base(struct its_device *its_dev)
1680 {
1681 	struct its_node *its = its_dev->its;
1682 
1683 	return its->phys_base + GITS_TRANSLATER;
1684 }
1685 
1686 static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
1687 {
1688 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1689 	struct its_node *its;
1690 	u64 addr;
1691 
1692 	its = its_dev->its;
1693 	addr = its->get_msi_base(its_dev);
1694 
1695 	msg->address_lo		= lower_32_bits(addr);
1696 	msg->address_hi		= upper_32_bits(addr);
1697 	msg->data		= its_get_event_id(d);
1698 
1699 	iommu_dma_compose_msi_msg(irq_data_get_msi_desc(d), msg);
1700 }
1701 
1702 static int its_irq_set_irqchip_state(struct irq_data *d,
1703 				     enum irqchip_irq_state which,
1704 				     bool state)
1705 {
1706 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1707 	u32 event = its_get_event_id(d);
1708 
1709 	if (which != IRQCHIP_STATE_PENDING)
1710 		return -EINVAL;
1711 
1712 	if (irqd_is_forwarded_to_vcpu(d)) {
1713 		if (state)
1714 			its_send_vint(its_dev, event);
1715 		else
1716 			its_send_vclear(its_dev, event);
1717 	} else {
1718 		if (state)
1719 			its_send_int(its_dev, event);
1720 		else
1721 			its_send_clear(its_dev, event);
1722 	}
1723 
1724 	return 0;
1725 }
1726 
1727 static int its_irq_retrigger(struct irq_data *d)
1728 {
1729 	return !its_irq_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true);
1730 }
1731 
1732 /*
1733  * Two favourable cases:
1734  *
1735  * (a) Either we have a GICv4.1, and all vPEs have to be mapped at all times
1736  *     for vSGI delivery
1737  *
1738  * (b) Or the ITSs do not use a list map, meaning that VMOVP is cheap enough
1739  *     and we're better off mapping all VPEs always
1740  *
1741  * If neither (a) nor (b) is true, then we map vPEs on demand.
1742  *
1743  */
1744 static bool gic_requires_eager_mapping(void)
1745 {
1746 	if (!its_list_map || gic_rdists->has_rvpeid)
1747 		return true;
1748 
1749 	return false;
1750 }
1751 
1752 static void its_map_vm(struct its_node *its, struct its_vm *vm)
1753 {
1754 	unsigned long flags;
1755 
1756 	if (gic_requires_eager_mapping())
1757 		return;
1758 
1759 	raw_spin_lock_irqsave(&vmovp_lock, flags);
1760 
1761 	/*
1762 	 * If the VM wasn't mapped yet, iterate over the vpes and get
1763 	 * them mapped now.
1764 	 */
1765 	vm->vlpi_count[its->list_nr]++;
1766 
1767 	if (vm->vlpi_count[its->list_nr] == 1) {
1768 		int i;
1769 
1770 		for (i = 0; i < vm->nr_vpes; i++) {
1771 			struct its_vpe *vpe = vm->vpes[i];
1772 			struct irq_data *d = irq_get_irq_data(vpe->irq);
1773 
1774 			/* Map the VPE to the first possible CPU */
1775 			vpe->col_idx = cpumask_first(cpu_online_mask);
1776 			its_send_vmapp(its, vpe, true);
1777 			its_send_vinvall(its, vpe);
1778 			irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
1779 		}
1780 	}
1781 
1782 	raw_spin_unlock_irqrestore(&vmovp_lock, flags);
1783 }
1784 
1785 static void its_unmap_vm(struct its_node *its, struct its_vm *vm)
1786 {
1787 	unsigned long flags;
1788 
1789 	/* Not using the ITS list? Everything is always mapped. */
1790 	if (gic_requires_eager_mapping())
1791 		return;
1792 
1793 	raw_spin_lock_irqsave(&vmovp_lock, flags);
1794 
1795 	if (!--vm->vlpi_count[its->list_nr]) {
1796 		int i;
1797 
1798 		for (i = 0; i < vm->nr_vpes; i++)
1799 			its_send_vmapp(its, vm->vpes[i], false);
1800 	}
1801 
1802 	raw_spin_unlock_irqrestore(&vmovp_lock, flags);
1803 }
1804 
1805 static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info)
1806 {
1807 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1808 	u32 event = its_get_event_id(d);
1809 	int ret = 0;
1810 
1811 	if (!info->map)
1812 		return -EINVAL;
1813 
1814 	raw_spin_lock(&its_dev->event_map.vlpi_lock);
1815 
1816 	if (!its_dev->event_map.vm) {
1817 		struct its_vlpi_map *maps;
1818 
1819 		maps = kcalloc(its_dev->event_map.nr_lpis, sizeof(*maps),
1820 			       GFP_ATOMIC);
1821 		if (!maps) {
1822 			ret = -ENOMEM;
1823 			goto out;
1824 		}
1825 
1826 		its_dev->event_map.vm = info->map->vm;
1827 		its_dev->event_map.vlpi_maps = maps;
1828 	} else if (its_dev->event_map.vm != info->map->vm) {
1829 		ret = -EINVAL;
1830 		goto out;
1831 	}
1832 
1833 	/* Get our private copy of the mapping information */
1834 	its_dev->event_map.vlpi_maps[event] = *info->map;
1835 
1836 	if (irqd_is_forwarded_to_vcpu(d)) {
1837 		/* Already mapped, move it around */
1838 		its_send_vmovi(its_dev, event);
1839 	} else {
1840 		/* Ensure all the VPEs are mapped on this ITS */
1841 		its_map_vm(its_dev->its, info->map->vm);
1842 
1843 		/*
1844 		 * Flag the interrupt as forwarded so that we can
1845 		 * start poking the virtual property table.
1846 		 */
1847 		irqd_set_forwarded_to_vcpu(d);
1848 
1849 		/* Write out the property to the prop table */
1850 		lpi_write_config(d, 0xff, info->map->properties);
1851 
1852 		/* Drop the physical mapping */
1853 		its_send_discard(its_dev, event);
1854 
1855 		/* and install the virtual one */
1856 		its_send_vmapti(its_dev, event);
1857 
1858 		/* Increment the number of VLPIs */
1859 		its_dev->event_map.nr_vlpis++;
1860 	}
1861 
1862 out:
1863 	raw_spin_unlock(&its_dev->event_map.vlpi_lock);
1864 	return ret;
1865 }
1866 
1867 static int its_vlpi_get(struct irq_data *d, struct its_cmd_info *info)
1868 {
1869 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1870 	struct its_vlpi_map *map;
1871 	int ret = 0;
1872 
1873 	raw_spin_lock(&its_dev->event_map.vlpi_lock);
1874 
1875 	map = get_vlpi_map(d);
1876 
1877 	if (!its_dev->event_map.vm || !map) {
1878 		ret = -EINVAL;
1879 		goto out;
1880 	}
1881 
1882 	/* Copy our mapping information to the incoming request */
1883 	*info->map = *map;
1884 
1885 out:
1886 	raw_spin_unlock(&its_dev->event_map.vlpi_lock);
1887 	return ret;
1888 }
1889 
1890 static int its_vlpi_unmap(struct irq_data *d)
1891 {
1892 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1893 	u32 event = its_get_event_id(d);
1894 	int ret = 0;
1895 
1896 	raw_spin_lock(&its_dev->event_map.vlpi_lock);
1897 
1898 	if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) {
1899 		ret = -EINVAL;
1900 		goto out;
1901 	}
1902 
1903 	/* Drop the virtual mapping */
1904 	its_send_discard(its_dev, event);
1905 
1906 	/* and restore the physical one */
1907 	irqd_clr_forwarded_to_vcpu(d);
1908 	its_send_mapti(its_dev, d->hwirq, event);
1909 	lpi_update_config(d, 0xff, (LPI_PROP_DEFAULT_PRIO |
1910 				    LPI_PROP_ENABLED |
1911 				    LPI_PROP_GROUP1));
1912 
1913 	/* Potentially unmap the VM from this ITS */
1914 	its_unmap_vm(its_dev->its, its_dev->event_map.vm);
1915 
1916 	/*
1917 	 * Drop the refcount and make the device available again if
1918 	 * this was the last VLPI.
1919 	 */
1920 	if (!--its_dev->event_map.nr_vlpis) {
1921 		its_dev->event_map.vm = NULL;
1922 		kfree(its_dev->event_map.vlpi_maps);
1923 	}
1924 
1925 out:
1926 	raw_spin_unlock(&its_dev->event_map.vlpi_lock);
1927 	return ret;
1928 }
1929 
1930 static int its_vlpi_prop_update(struct irq_data *d, struct its_cmd_info *info)
1931 {
1932 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1933 
1934 	if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d))
1935 		return -EINVAL;
1936 
1937 	if (info->cmd_type == PROP_UPDATE_AND_INV_VLPI)
1938 		lpi_update_config(d, 0xff, info->config);
1939 	else
1940 		lpi_write_config(d, 0xff, info->config);
1941 	its_vlpi_set_doorbell(d, !!(info->config & LPI_PROP_ENABLED));
1942 
1943 	return 0;
1944 }
1945 
1946 static int its_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
1947 {
1948 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1949 	struct its_cmd_info *info = vcpu_info;
1950 
1951 	/* Need a v4 ITS */
1952 	if (!is_v4(its_dev->its))
1953 		return -EINVAL;
1954 
1955 	/* Unmap request? */
1956 	if (!info)
1957 		return its_vlpi_unmap(d);
1958 
1959 	switch (info->cmd_type) {
1960 	case MAP_VLPI:
1961 		return its_vlpi_map(d, info);
1962 
1963 	case GET_VLPI:
1964 		return its_vlpi_get(d, info);
1965 
1966 	case PROP_UPDATE_VLPI:
1967 	case PROP_UPDATE_AND_INV_VLPI:
1968 		return its_vlpi_prop_update(d, info);
1969 
1970 	default:
1971 		return -EINVAL;
1972 	}
1973 }
1974 
1975 static struct irq_chip its_irq_chip = {
1976 	.name			= "ITS",
1977 	.irq_mask		= its_mask_irq,
1978 	.irq_unmask		= its_unmask_irq,
1979 	.irq_eoi		= irq_chip_eoi_parent,
1980 	.irq_set_affinity	= its_set_affinity,
1981 	.irq_compose_msi_msg	= its_irq_compose_msi_msg,
1982 	.irq_set_irqchip_state	= its_irq_set_irqchip_state,
1983 	.irq_retrigger		= its_irq_retrigger,
1984 	.irq_set_vcpu_affinity	= its_irq_set_vcpu_affinity,
1985 };
1986 
1987 
1988 /*
1989  * How we allocate LPIs:
1990  *
1991  * lpi_range_list contains ranges of LPIs that are to available to
1992  * allocate from. To allocate LPIs, just pick the first range that
1993  * fits the required allocation, and reduce it by the required
1994  * amount. Once empty, remove the range from the list.
1995  *
1996  * To free a range of LPIs, add a free range to the list, sort it and
1997  * merge the result if the new range happens to be adjacent to an
1998  * already free block.
1999  *
2000  * The consequence of the above is that allocation is cost is low, but
2001  * freeing is expensive. We assumes that freeing rarely occurs.
2002  */
2003 #define ITS_MAX_LPI_NRBITS	16 /* 64K LPIs */
2004 
2005 static DEFINE_MUTEX(lpi_range_lock);
2006 static LIST_HEAD(lpi_range_list);
2007 
2008 struct lpi_range {
2009 	struct list_head	entry;
2010 	u32			base_id;
2011 	u32			span;
2012 };
2013 
2014 static struct lpi_range *mk_lpi_range(u32 base, u32 span)
2015 {
2016 	struct lpi_range *range;
2017 
2018 	range = kmalloc(sizeof(*range), GFP_KERNEL);
2019 	if (range) {
2020 		range->base_id = base;
2021 		range->span = span;
2022 	}
2023 
2024 	return range;
2025 }
2026 
2027 static int alloc_lpi_range(u32 nr_lpis, u32 *base)
2028 {
2029 	struct lpi_range *range, *tmp;
2030 	int err = -ENOSPC;
2031 
2032 	mutex_lock(&lpi_range_lock);
2033 
2034 	list_for_each_entry_safe(range, tmp, &lpi_range_list, entry) {
2035 		if (range->span >= nr_lpis) {
2036 			*base = range->base_id;
2037 			range->base_id += nr_lpis;
2038 			range->span -= nr_lpis;
2039 
2040 			if (range->span == 0) {
2041 				list_del(&range->entry);
2042 				kfree(range);
2043 			}
2044 
2045 			err = 0;
2046 			break;
2047 		}
2048 	}
2049 
2050 	mutex_unlock(&lpi_range_lock);
2051 
2052 	pr_debug("ITS: alloc %u:%u\n", *base, nr_lpis);
2053 	return err;
2054 }
2055 
2056 static void merge_lpi_ranges(struct lpi_range *a, struct lpi_range *b)
2057 {
2058 	if (&a->entry == &lpi_range_list || &b->entry == &lpi_range_list)
2059 		return;
2060 	if (a->base_id + a->span != b->base_id)
2061 		return;
2062 	b->base_id = a->base_id;
2063 	b->span += a->span;
2064 	list_del(&a->entry);
2065 	kfree(a);
2066 }
2067 
2068 static int free_lpi_range(u32 base, u32 nr_lpis)
2069 {
2070 	struct lpi_range *new, *old;
2071 
2072 	new = mk_lpi_range(base, nr_lpis);
2073 	if (!new)
2074 		return -ENOMEM;
2075 
2076 	mutex_lock(&lpi_range_lock);
2077 
2078 	list_for_each_entry_reverse(old, &lpi_range_list, entry) {
2079 		if (old->base_id < base)
2080 			break;
2081 	}
2082 	/*
2083 	 * old is the last element with ->base_id smaller than base,
2084 	 * so new goes right after it. If there are no elements with
2085 	 * ->base_id smaller than base, &old->entry ends up pointing
2086 	 * at the head of the list, and inserting new it the start of
2087 	 * the list is the right thing to do in that case as well.
2088 	 */
2089 	list_add(&new->entry, &old->entry);
2090 	/*
2091 	 * Now check if we can merge with the preceding and/or
2092 	 * following ranges.
2093 	 */
2094 	merge_lpi_ranges(old, new);
2095 	merge_lpi_ranges(new, list_next_entry(new, entry));
2096 
2097 	mutex_unlock(&lpi_range_lock);
2098 	return 0;
2099 }
2100 
2101 static int __init its_lpi_init(u32 id_bits)
2102 {
2103 	u32 lpis = (1UL << id_bits) - 8192;
2104 	u32 numlpis;
2105 	int err;
2106 
2107 	numlpis = 1UL << GICD_TYPER_NUM_LPIS(gic_rdists->gicd_typer);
2108 
2109 	if (numlpis > 2 && !WARN_ON(numlpis > lpis)) {
2110 		lpis = numlpis;
2111 		pr_info("ITS: Using hypervisor restricted LPI range [%u]\n",
2112 			lpis);
2113 	}
2114 
2115 	/*
2116 	 * Initializing the allocator is just the same as freeing the
2117 	 * full range of LPIs.
2118 	 */
2119 	err = free_lpi_range(8192, lpis);
2120 	pr_debug("ITS: Allocator initialized for %u LPIs\n", lpis);
2121 	return err;
2122 }
2123 
2124 static unsigned long *its_lpi_alloc(int nr_irqs, u32 *base, int *nr_ids)
2125 {
2126 	unsigned long *bitmap = NULL;
2127 	int err = 0;
2128 
2129 	do {
2130 		err = alloc_lpi_range(nr_irqs, base);
2131 		if (!err)
2132 			break;
2133 
2134 		nr_irqs /= 2;
2135 	} while (nr_irqs > 0);
2136 
2137 	if (!nr_irqs)
2138 		err = -ENOSPC;
2139 
2140 	if (err)
2141 		goto out;
2142 
2143 	bitmap = bitmap_zalloc(nr_irqs, GFP_ATOMIC);
2144 	if (!bitmap)
2145 		goto out;
2146 
2147 	*nr_ids = nr_irqs;
2148 
2149 out:
2150 	if (!bitmap)
2151 		*base = *nr_ids = 0;
2152 
2153 	return bitmap;
2154 }
2155 
2156 static void its_lpi_free(unsigned long *bitmap, u32 base, u32 nr_ids)
2157 {
2158 	WARN_ON(free_lpi_range(base, nr_ids));
2159 	bitmap_free(bitmap);
2160 }
2161 
2162 static void gic_reset_prop_table(void *va)
2163 {
2164 	/* Priority 0xa0, Group-1, disabled */
2165 	memset(va, LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1, LPI_PROPBASE_SZ);
2166 
2167 	/* Make sure the GIC will observe the written configuration */
2168 	gic_flush_dcache_to_poc(va, LPI_PROPBASE_SZ);
2169 }
2170 
2171 static struct page *its_allocate_prop_table(gfp_t gfp_flags)
2172 {
2173 	struct page *prop_page;
2174 
2175 	prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ));
2176 	if (!prop_page)
2177 		return NULL;
2178 
2179 	gic_reset_prop_table(page_address(prop_page));
2180 
2181 	return prop_page;
2182 }
2183 
2184 static void its_free_prop_table(struct page *prop_page)
2185 {
2186 	free_pages((unsigned long)page_address(prop_page),
2187 		   get_order(LPI_PROPBASE_SZ));
2188 }
2189 
2190 static bool gic_check_reserved_range(phys_addr_t addr, unsigned long size)
2191 {
2192 	phys_addr_t start, end, addr_end;
2193 	u64 i;
2194 
2195 	/*
2196 	 * We don't bother checking for a kdump kernel as by
2197 	 * construction, the LPI tables are out of this kernel's
2198 	 * memory map.
2199 	 */
2200 	if (is_kdump_kernel())
2201 		return true;
2202 
2203 	addr_end = addr + size - 1;
2204 
2205 	for_each_reserved_mem_range(i, &start, &end) {
2206 		if (addr >= start && addr_end <= end)
2207 			return true;
2208 	}
2209 
2210 	/* Not found, not a good sign... */
2211 	pr_warn("GICv3: Expected reserved range [%pa:%pa], not found\n",
2212 		&addr, &addr_end);
2213 	add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
2214 	return false;
2215 }
2216 
2217 static int gic_reserve_range(phys_addr_t addr, unsigned long size)
2218 {
2219 	if (efi_enabled(EFI_CONFIG_TABLES))
2220 		return efi_mem_reserve_persistent(addr, size);
2221 
2222 	return 0;
2223 }
2224 
2225 static int __init its_setup_lpi_prop_table(void)
2226 {
2227 	if (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) {
2228 		u64 val;
2229 
2230 		val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
2231 		lpi_id_bits = (val & GICR_PROPBASER_IDBITS_MASK) + 1;
2232 
2233 		gic_rdists->prop_table_pa = val & GENMASK_ULL(51, 12);
2234 		gic_rdists->prop_table_va = memremap(gic_rdists->prop_table_pa,
2235 						     LPI_PROPBASE_SZ,
2236 						     MEMREMAP_WB);
2237 		gic_reset_prop_table(gic_rdists->prop_table_va);
2238 	} else {
2239 		struct page *page;
2240 
2241 		lpi_id_bits = min_t(u32,
2242 				    GICD_TYPER_ID_BITS(gic_rdists->gicd_typer),
2243 				    ITS_MAX_LPI_NRBITS);
2244 		page = its_allocate_prop_table(GFP_NOWAIT);
2245 		if (!page) {
2246 			pr_err("Failed to allocate PROPBASE\n");
2247 			return -ENOMEM;
2248 		}
2249 
2250 		gic_rdists->prop_table_pa = page_to_phys(page);
2251 		gic_rdists->prop_table_va = page_address(page);
2252 		WARN_ON(gic_reserve_range(gic_rdists->prop_table_pa,
2253 					  LPI_PROPBASE_SZ));
2254 	}
2255 
2256 	pr_info("GICv3: using LPI property table @%pa\n",
2257 		&gic_rdists->prop_table_pa);
2258 
2259 	return its_lpi_init(lpi_id_bits);
2260 }
2261 
2262 static const char *its_base_type_string[] = {
2263 	[GITS_BASER_TYPE_DEVICE]	= "Devices",
2264 	[GITS_BASER_TYPE_VCPU]		= "Virtual CPUs",
2265 	[GITS_BASER_TYPE_RESERVED3]	= "Reserved (3)",
2266 	[GITS_BASER_TYPE_COLLECTION]	= "Interrupt Collections",
2267 	[GITS_BASER_TYPE_RESERVED5] 	= "Reserved (5)",
2268 	[GITS_BASER_TYPE_RESERVED6] 	= "Reserved (6)",
2269 	[GITS_BASER_TYPE_RESERVED7] 	= "Reserved (7)",
2270 };
2271 
2272 static u64 its_read_baser(struct its_node *its, struct its_baser *baser)
2273 {
2274 	u32 idx = baser - its->tables;
2275 
2276 	return gits_read_baser(its->base + GITS_BASER + (idx << 3));
2277 }
2278 
2279 static void its_write_baser(struct its_node *its, struct its_baser *baser,
2280 			    u64 val)
2281 {
2282 	u32 idx = baser - its->tables;
2283 
2284 	gits_write_baser(val, its->base + GITS_BASER + (idx << 3));
2285 	baser->val = its_read_baser(its, baser);
2286 }
2287 
2288 static int its_setup_baser(struct its_node *its, struct its_baser *baser,
2289 			   u64 cache, u64 shr, u32 order, bool indirect)
2290 {
2291 	u64 val = its_read_baser(its, baser);
2292 	u64 esz = GITS_BASER_ENTRY_SIZE(val);
2293 	u64 type = GITS_BASER_TYPE(val);
2294 	u64 baser_phys, tmp;
2295 	u32 alloc_pages, psz;
2296 	struct page *page;
2297 	void *base;
2298 
2299 	psz = baser->psz;
2300 	alloc_pages = (PAGE_ORDER_TO_SIZE(order) / psz);
2301 	if (alloc_pages > GITS_BASER_PAGES_MAX) {
2302 		pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n",
2303 			&its->phys_base, its_base_type_string[type],
2304 			alloc_pages, GITS_BASER_PAGES_MAX);
2305 		alloc_pages = GITS_BASER_PAGES_MAX;
2306 		order = get_order(GITS_BASER_PAGES_MAX * psz);
2307 	}
2308 
2309 	page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, order);
2310 	if (!page)
2311 		return -ENOMEM;
2312 
2313 	base = (void *)page_address(page);
2314 	baser_phys = virt_to_phys(base);
2315 
2316 	/* Check if the physical address of the memory is above 48bits */
2317 	if (IS_ENABLED(CONFIG_ARM64_64K_PAGES) && (baser_phys >> 48)) {
2318 
2319 		/* 52bit PA is supported only when PageSize=64K */
2320 		if (psz != SZ_64K) {
2321 			pr_err("ITS: no 52bit PA support when psz=%d\n", psz);
2322 			free_pages((unsigned long)base, order);
2323 			return -ENXIO;
2324 		}
2325 
2326 		/* Convert 52bit PA to 48bit field */
2327 		baser_phys = GITS_BASER_PHYS_52_to_48(baser_phys);
2328 	}
2329 
2330 retry_baser:
2331 	val = (baser_phys					 |
2332 		(type << GITS_BASER_TYPE_SHIFT)			 |
2333 		((esz - 1) << GITS_BASER_ENTRY_SIZE_SHIFT)	 |
2334 		((alloc_pages - 1) << GITS_BASER_PAGES_SHIFT)	 |
2335 		cache						 |
2336 		shr						 |
2337 		GITS_BASER_VALID);
2338 
2339 	val |=	indirect ? GITS_BASER_INDIRECT : 0x0;
2340 
2341 	switch (psz) {
2342 	case SZ_4K:
2343 		val |= GITS_BASER_PAGE_SIZE_4K;
2344 		break;
2345 	case SZ_16K:
2346 		val |= GITS_BASER_PAGE_SIZE_16K;
2347 		break;
2348 	case SZ_64K:
2349 		val |= GITS_BASER_PAGE_SIZE_64K;
2350 		break;
2351 	}
2352 
2353 	its_write_baser(its, baser, val);
2354 	tmp = baser->val;
2355 
2356 	if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) {
2357 		/*
2358 		 * Shareability didn't stick. Just use
2359 		 * whatever the read reported, which is likely
2360 		 * to be the only thing this redistributor
2361 		 * supports. If that's zero, make it
2362 		 * non-cacheable as well.
2363 		 */
2364 		shr = tmp & GITS_BASER_SHAREABILITY_MASK;
2365 		if (!shr) {
2366 			cache = GITS_BASER_nC;
2367 			gic_flush_dcache_to_poc(base, PAGE_ORDER_TO_SIZE(order));
2368 		}
2369 		goto retry_baser;
2370 	}
2371 
2372 	if (val != tmp) {
2373 		pr_err("ITS@%pa: %s doesn't stick: %llx %llx\n",
2374 		       &its->phys_base, its_base_type_string[type],
2375 		       val, tmp);
2376 		free_pages((unsigned long)base, order);
2377 		return -ENXIO;
2378 	}
2379 
2380 	baser->order = order;
2381 	baser->base = base;
2382 	baser->psz = psz;
2383 	tmp = indirect ? GITS_LVL1_ENTRY_SIZE : esz;
2384 
2385 	pr_info("ITS@%pa: allocated %d %s @%lx (%s, esz %d, psz %dK, shr %d)\n",
2386 		&its->phys_base, (int)(PAGE_ORDER_TO_SIZE(order) / (int)tmp),
2387 		its_base_type_string[type],
2388 		(unsigned long)virt_to_phys(base),
2389 		indirect ? "indirect" : "flat", (int)esz,
2390 		psz / SZ_1K, (int)shr >> GITS_BASER_SHAREABILITY_SHIFT);
2391 
2392 	return 0;
2393 }
2394 
2395 static bool its_parse_indirect_baser(struct its_node *its,
2396 				     struct its_baser *baser,
2397 				     u32 *order, u32 ids)
2398 {
2399 	u64 tmp = its_read_baser(its, baser);
2400 	u64 type = GITS_BASER_TYPE(tmp);
2401 	u64 esz = GITS_BASER_ENTRY_SIZE(tmp);
2402 	u64 val = GITS_BASER_InnerShareable | GITS_BASER_RaWaWb;
2403 	u32 new_order = *order;
2404 	u32 psz = baser->psz;
2405 	bool indirect = false;
2406 
2407 	/* No need to enable Indirection if memory requirement < (psz*2)bytes */
2408 	if ((esz << ids) > (psz * 2)) {
2409 		/*
2410 		 * Find out whether hw supports a single or two-level table by
2411 		 * table by reading bit at offset '62' after writing '1' to it.
2412 		 */
2413 		its_write_baser(its, baser, val | GITS_BASER_INDIRECT);
2414 		indirect = !!(baser->val & GITS_BASER_INDIRECT);
2415 
2416 		if (indirect) {
2417 			/*
2418 			 * The size of the lvl2 table is equal to ITS page size
2419 			 * which is 'psz'. For computing lvl1 table size,
2420 			 * subtract ID bits that sparse lvl2 table from 'ids'
2421 			 * which is reported by ITS hardware times lvl1 table
2422 			 * entry size.
2423 			 */
2424 			ids -= ilog2(psz / (int)esz);
2425 			esz = GITS_LVL1_ENTRY_SIZE;
2426 		}
2427 	}
2428 
2429 	/*
2430 	 * Allocate as many entries as required to fit the
2431 	 * range of device IDs that the ITS can grok... The ID
2432 	 * space being incredibly sparse, this results in a
2433 	 * massive waste of memory if two-level device table
2434 	 * feature is not supported by hardware.
2435 	 */
2436 	new_order = max_t(u32, get_order(esz << ids), new_order);
2437 	if (new_order >= MAX_ORDER) {
2438 		new_order = MAX_ORDER - 1;
2439 		ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / (int)esz);
2440 		pr_warn("ITS@%pa: %s Table too large, reduce ids %llu->%u\n",
2441 			&its->phys_base, its_base_type_string[type],
2442 			device_ids(its), ids);
2443 	}
2444 
2445 	*order = new_order;
2446 
2447 	return indirect;
2448 }
2449 
2450 static u32 compute_common_aff(u64 val)
2451 {
2452 	u32 aff, clpiaff;
2453 
2454 	aff = FIELD_GET(GICR_TYPER_AFFINITY, val);
2455 	clpiaff = FIELD_GET(GICR_TYPER_COMMON_LPI_AFF, val);
2456 
2457 	return aff & ~(GENMASK(31, 0) >> (clpiaff * 8));
2458 }
2459 
2460 static u32 compute_its_aff(struct its_node *its)
2461 {
2462 	u64 val;
2463 	u32 svpet;
2464 
2465 	/*
2466 	 * Reencode the ITS SVPET and MPIDR as a GICR_TYPER, and compute
2467 	 * the resulting affinity. We then use that to see if this match
2468 	 * our own affinity.
2469 	 */
2470 	svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer);
2471 	val  = FIELD_PREP(GICR_TYPER_COMMON_LPI_AFF, svpet);
2472 	val |= FIELD_PREP(GICR_TYPER_AFFINITY, its->mpidr);
2473 	return compute_common_aff(val);
2474 }
2475 
2476 static struct its_node *find_sibling_its(struct its_node *cur_its)
2477 {
2478 	struct its_node *its;
2479 	u32 aff;
2480 
2481 	if (!FIELD_GET(GITS_TYPER_SVPET, cur_its->typer))
2482 		return NULL;
2483 
2484 	aff = compute_its_aff(cur_its);
2485 
2486 	list_for_each_entry(its, &its_nodes, entry) {
2487 		u64 baser;
2488 
2489 		if (!is_v4_1(its) || its == cur_its)
2490 			continue;
2491 
2492 		if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
2493 			continue;
2494 
2495 		if (aff != compute_its_aff(its))
2496 			continue;
2497 
2498 		/* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
2499 		baser = its->tables[2].val;
2500 		if (!(baser & GITS_BASER_VALID))
2501 			continue;
2502 
2503 		return its;
2504 	}
2505 
2506 	return NULL;
2507 }
2508 
2509 static void its_free_tables(struct its_node *its)
2510 {
2511 	int i;
2512 
2513 	for (i = 0; i < GITS_BASER_NR_REGS; i++) {
2514 		if (its->tables[i].base) {
2515 			free_pages((unsigned long)its->tables[i].base,
2516 				   its->tables[i].order);
2517 			its->tables[i].base = NULL;
2518 		}
2519 	}
2520 }
2521 
2522 static int its_probe_baser_psz(struct its_node *its, struct its_baser *baser)
2523 {
2524 	u64 psz = SZ_64K;
2525 
2526 	while (psz) {
2527 		u64 val, gpsz;
2528 
2529 		val = its_read_baser(its, baser);
2530 		val &= ~GITS_BASER_PAGE_SIZE_MASK;
2531 
2532 		switch (psz) {
2533 		case SZ_64K:
2534 			gpsz = GITS_BASER_PAGE_SIZE_64K;
2535 			break;
2536 		case SZ_16K:
2537 			gpsz = GITS_BASER_PAGE_SIZE_16K;
2538 			break;
2539 		case SZ_4K:
2540 		default:
2541 			gpsz = GITS_BASER_PAGE_SIZE_4K;
2542 			break;
2543 		}
2544 
2545 		gpsz >>= GITS_BASER_PAGE_SIZE_SHIFT;
2546 
2547 		val |= FIELD_PREP(GITS_BASER_PAGE_SIZE_MASK, gpsz);
2548 		its_write_baser(its, baser, val);
2549 
2550 		if (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser->val) == gpsz)
2551 			break;
2552 
2553 		switch (psz) {
2554 		case SZ_64K:
2555 			psz = SZ_16K;
2556 			break;
2557 		case SZ_16K:
2558 			psz = SZ_4K;
2559 			break;
2560 		case SZ_4K:
2561 		default:
2562 			return -1;
2563 		}
2564 	}
2565 
2566 	baser->psz = psz;
2567 	return 0;
2568 }
2569 
2570 static int its_alloc_tables(struct its_node *its)
2571 {
2572 	u64 shr = GITS_BASER_InnerShareable;
2573 	u64 cache = GITS_BASER_RaWaWb;
2574 	int err, i;
2575 
2576 	if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375)
2577 		/* erratum 24313: ignore memory access type */
2578 		cache = GITS_BASER_nCnB;
2579 
2580 	for (i = 0; i < GITS_BASER_NR_REGS; i++) {
2581 		struct its_baser *baser = its->tables + i;
2582 		u64 val = its_read_baser(its, baser);
2583 		u64 type = GITS_BASER_TYPE(val);
2584 		bool indirect = false;
2585 		u32 order;
2586 
2587 		if (type == GITS_BASER_TYPE_NONE)
2588 			continue;
2589 
2590 		if (its_probe_baser_psz(its, baser)) {
2591 			its_free_tables(its);
2592 			return -ENXIO;
2593 		}
2594 
2595 		order = get_order(baser->psz);
2596 
2597 		switch (type) {
2598 		case GITS_BASER_TYPE_DEVICE:
2599 			indirect = its_parse_indirect_baser(its, baser, &order,
2600 							    device_ids(its));
2601 			break;
2602 
2603 		case GITS_BASER_TYPE_VCPU:
2604 			if (is_v4_1(its)) {
2605 				struct its_node *sibling;
2606 
2607 				WARN_ON(i != 2);
2608 				if ((sibling = find_sibling_its(its))) {
2609 					*baser = sibling->tables[2];
2610 					its_write_baser(its, baser, baser->val);
2611 					continue;
2612 				}
2613 			}
2614 
2615 			indirect = its_parse_indirect_baser(its, baser, &order,
2616 							    ITS_MAX_VPEID_BITS);
2617 			break;
2618 		}
2619 
2620 		err = its_setup_baser(its, baser, cache, shr, order, indirect);
2621 		if (err < 0) {
2622 			its_free_tables(its);
2623 			return err;
2624 		}
2625 
2626 		/* Update settings which will be used for next BASERn */
2627 		cache = baser->val & GITS_BASER_CACHEABILITY_MASK;
2628 		shr = baser->val & GITS_BASER_SHAREABILITY_MASK;
2629 	}
2630 
2631 	return 0;
2632 }
2633 
2634 static u64 inherit_vpe_l1_table_from_its(void)
2635 {
2636 	struct its_node *its;
2637 	u64 val;
2638 	u32 aff;
2639 
2640 	val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
2641 	aff = compute_common_aff(val);
2642 
2643 	list_for_each_entry(its, &its_nodes, entry) {
2644 		u64 baser, addr;
2645 
2646 		if (!is_v4_1(its))
2647 			continue;
2648 
2649 		if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
2650 			continue;
2651 
2652 		if (aff != compute_its_aff(its))
2653 			continue;
2654 
2655 		/* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
2656 		baser = its->tables[2].val;
2657 		if (!(baser & GITS_BASER_VALID))
2658 			continue;
2659 
2660 		/* We have a winner! */
2661 		gic_data_rdist()->vpe_l1_base = its->tables[2].base;
2662 
2663 		val  = GICR_VPROPBASER_4_1_VALID;
2664 		if (baser & GITS_BASER_INDIRECT)
2665 			val |= GICR_VPROPBASER_4_1_INDIRECT;
2666 		val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE,
2667 				  FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser));
2668 		switch (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser)) {
2669 		case GIC_PAGE_SIZE_64K:
2670 			addr = GITS_BASER_ADDR_48_to_52(baser);
2671 			break;
2672 		default:
2673 			addr = baser & GENMASK_ULL(47, 12);
2674 			break;
2675 		}
2676 		val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, addr >> 12);
2677 		val |= FIELD_PREP(GICR_VPROPBASER_SHAREABILITY_MASK,
2678 				  FIELD_GET(GITS_BASER_SHAREABILITY_MASK, baser));
2679 		val |= FIELD_PREP(GICR_VPROPBASER_INNER_CACHEABILITY_MASK,
2680 				  FIELD_GET(GITS_BASER_INNER_CACHEABILITY_MASK, baser));
2681 		val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, GITS_BASER_NR_PAGES(baser) - 1);
2682 
2683 		return val;
2684 	}
2685 
2686 	return 0;
2687 }
2688 
2689 static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask)
2690 {
2691 	u32 aff;
2692 	u64 val;
2693 	int cpu;
2694 
2695 	val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
2696 	aff = compute_common_aff(val);
2697 
2698 	for_each_possible_cpu(cpu) {
2699 		void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
2700 
2701 		if (!base || cpu == smp_processor_id())
2702 			continue;
2703 
2704 		val = gic_read_typer(base + GICR_TYPER);
2705 		if (aff != compute_common_aff(val))
2706 			continue;
2707 
2708 		/*
2709 		 * At this point, we have a victim. This particular CPU
2710 		 * has already booted, and has an affinity that matches
2711 		 * ours wrt CommonLPIAff. Let's use its own VPROPBASER.
2712 		 * Make sure we don't write the Z bit in that case.
2713 		 */
2714 		val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
2715 		val &= ~GICR_VPROPBASER_4_1_Z;
2716 
2717 		gic_data_rdist()->vpe_l1_base = gic_data_rdist_cpu(cpu)->vpe_l1_base;
2718 		*mask = gic_data_rdist_cpu(cpu)->vpe_table_mask;
2719 
2720 		return val;
2721 	}
2722 
2723 	return 0;
2724 }
2725 
2726 static bool allocate_vpe_l2_table(int cpu, u32 id)
2727 {
2728 	void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
2729 	unsigned int psz, esz, idx, npg, gpsz;
2730 	u64 val;
2731 	struct page *page;
2732 	__le64 *table;
2733 
2734 	if (!gic_rdists->has_rvpeid)
2735 		return true;
2736 
2737 	/* Skip non-present CPUs */
2738 	if (!base)
2739 		return true;
2740 
2741 	val  = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
2742 
2743 	esz  = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val) + 1;
2744 	gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
2745 	npg  = FIELD_GET(GICR_VPROPBASER_4_1_SIZE, val) + 1;
2746 
2747 	switch (gpsz) {
2748 	default:
2749 		WARN_ON(1);
2750 		fallthrough;
2751 	case GIC_PAGE_SIZE_4K:
2752 		psz = SZ_4K;
2753 		break;
2754 	case GIC_PAGE_SIZE_16K:
2755 		psz = SZ_16K;
2756 		break;
2757 	case GIC_PAGE_SIZE_64K:
2758 		psz = SZ_64K;
2759 		break;
2760 	}
2761 
2762 	/* Don't allow vpe_id that exceeds single, flat table limit */
2763 	if (!(val & GICR_VPROPBASER_4_1_INDIRECT))
2764 		return (id < (npg * psz / (esz * SZ_8)));
2765 
2766 	/* Compute 1st level table index & check if that exceeds table limit */
2767 	idx = id >> ilog2(psz / (esz * SZ_8));
2768 	if (idx >= (npg * psz / GITS_LVL1_ENTRY_SIZE))
2769 		return false;
2770 
2771 	table = gic_data_rdist_cpu(cpu)->vpe_l1_base;
2772 
2773 	/* Allocate memory for 2nd level table */
2774 	if (!table[idx]) {
2775 		page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(psz));
2776 		if (!page)
2777 			return false;
2778 
2779 		/* Flush Lvl2 table to PoC if hw doesn't support coherency */
2780 		if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
2781 			gic_flush_dcache_to_poc(page_address(page), psz);
2782 
2783 		table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID);
2784 
2785 		/* Flush Lvl1 entry to PoC if hw doesn't support coherency */
2786 		if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
2787 			gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
2788 
2789 		/* Ensure updated table contents are visible to RD hardware */
2790 		dsb(sy);
2791 	}
2792 
2793 	return true;
2794 }
2795 
2796 static int allocate_vpe_l1_table(void)
2797 {
2798 	void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
2799 	u64 val, gpsz, npg, pa;
2800 	unsigned int psz = SZ_64K;
2801 	unsigned int np, epp, esz;
2802 	struct page *page;
2803 
2804 	if (!gic_rdists->has_rvpeid)
2805 		return 0;
2806 
2807 	/*
2808 	 * if VPENDBASER.Valid is set, disable any previously programmed
2809 	 * VPE by setting PendingLast while clearing Valid. This has the
2810 	 * effect of making sure no doorbell will be generated and we can
2811 	 * then safely clear VPROPBASER.Valid.
2812 	 */
2813 	if (gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER) & GICR_VPENDBASER_Valid)
2814 		gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast,
2815 				      vlpi_base + GICR_VPENDBASER);
2816 
2817 	/*
2818 	 * If we can inherit the configuration from another RD, let's do
2819 	 * so. Otherwise, we have to go through the allocation process. We
2820 	 * assume that all RDs have the exact same requirements, as
2821 	 * nothing will work otherwise.
2822 	 */
2823 	val = inherit_vpe_l1_table_from_rd(&gic_data_rdist()->vpe_table_mask);
2824 	if (val & GICR_VPROPBASER_4_1_VALID)
2825 		goto out;
2826 
2827 	gic_data_rdist()->vpe_table_mask = kzalloc(sizeof(cpumask_t), GFP_ATOMIC);
2828 	if (!gic_data_rdist()->vpe_table_mask)
2829 		return -ENOMEM;
2830 
2831 	val = inherit_vpe_l1_table_from_its();
2832 	if (val & GICR_VPROPBASER_4_1_VALID)
2833 		goto out;
2834 
2835 	/* First probe the page size */
2836 	val = FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, GIC_PAGE_SIZE_64K);
2837 	gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
2838 	val = gicr_read_vpropbaser(vlpi_base + GICR_VPROPBASER);
2839 	gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
2840 	esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val);
2841 
2842 	switch (gpsz) {
2843 	default:
2844 		gpsz = GIC_PAGE_SIZE_4K;
2845 		fallthrough;
2846 	case GIC_PAGE_SIZE_4K:
2847 		psz = SZ_4K;
2848 		break;
2849 	case GIC_PAGE_SIZE_16K:
2850 		psz = SZ_16K;
2851 		break;
2852 	case GIC_PAGE_SIZE_64K:
2853 		psz = SZ_64K;
2854 		break;
2855 	}
2856 
2857 	/*
2858 	 * Start populating the register from scratch, including RO fields
2859 	 * (which we want to print in debug cases...)
2860 	 */
2861 	val = 0;
2862 	val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, gpsz);
2863 	val |= FIELD_PREP(GICR_VPROPBASER_4_1_ENTRY_SIZE, esz);
2864 
2865 	/* How many entries per GIC page? */
2866 	esz++;
2867 	epp = psz / (esz * SZ_8);
2868 
2869 	/*
2870 	 * If we need more than just a single L1 page, flag the table
2871 	 * as indirect and compute the number of required L1 pages.
2872 	 */
2873 	if (epp < ITS_MAX_VPEID) {
2874 		int nl2;
2875 
2876 		val |= GICR_VPROPBASER_4_1_INDIRECT;
2877 
2878 		/* Number of L2 pages required to cover the VPEID space */
2879 		nl2 = DIV_ROUND_UP(ITS_MAX_VPEID, epp);
2880 
2881 		/* Number of L1 pages to point to the L2 pages */
2882 		npg = DIV_ROUND_UP(nl2 * SZ_8, psz);
2883 	} else {
2884 		npg = 1;
2885 	}
2886 
2887 	val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, npg - 1);
2888 
2889 	/* Right, that's the number of CPU pages we need for L1 */
2890 	np = DIV_ROUND_UP(npg * psz, PAGE_SIZE);
2891 
2892 	pr_debug("np = %d, npg = %lld, psz = %d, epp = %d, esz = %d\n",
2893 		 np, npg, psz, epp, esz);
2894 	page = alloc_pages(GFP_ATOMIC | __GFP_ZERO, get_order(np * PAGE_SIZE));
2895 	if (!page)
2896 		return -ENOMEM;
2897 
2898 	gic_data_rdist()->vpe_l1_base = page_address(page);
2899 	pa = virt_to_phys(page_address(page));
2900 	WARN_ON(!IS_ALIGNED(pa, psz));
2901 
2902 	val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, pa >> 12);
2903 	val |= GICR_VPROPBASER_RaWb;
2904 	val |= GICR_VPROPBASER_InnerShareable;
2905 	val |= GICR_VPROPBASER_4_1_Z;
2906 	val |= GICR_VPROPBASER_4_1_VALID;
2907 
2908 out:
2909 	gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
2910 	cpumask_set_cpu(smp_processor_id(), gic_data_rdist()->vpe_table_mask);
2911 
2912 	pr_debug("CPU%d: VPROPBASER = %llx %*pbl\n",
2913 		 smp_processor_id(), val,
2914 		 cpumask_pr_args(gic_data_rdist()->vpe_table_mask));
2915 
2916 	return 0;
2917 }
2918 
2919 static int its_alloc_collections(struct its_node *its)
2920 {
2921 	int i;
2922 
2923 	its->collections = kcalloc(nr_cpu_ids, sizeof(*its->collections),
2924 				   GFP_KERNEL);
2925 	if (!its->collections)
2926 		return -ENOMEM;
2927 
2928 	for (i = 0; i < nr_cpu_ids; i++)
2929 		its->collections[i].target_address = ~0ULL;
2930 
2931 	return 0;
2932 }
2933 
2934 static struct page *its_allocate_pending_table(gfp_t gfp_flags)
2935 {
2936 	struct page *pend_page;
2937 
2938 	pend_page = alloc_pages(gfp_flags | __GFP_ZERO,
2939 				get_order(LPI_PENDBASE_SZ));
2940 	if (!pend_page)
2941 		return NULL;
2942 
2943 	/* Make sure the GIC will observe the zero-ed page */
2944 	gic_flush_dcache_to_poc(page_address(pend_page), LPI_PENDBASE_SZ);
2945 
2946 	return pend_page;
2947 }
2948 
2949 static void its_free_pending_table(struct page *pt)
2950 {
2951 	free_pages((unsigned long)page_address(pt), get_order(LPI_PENDBASE_SZ));
2952 }
2953 
2954 /*
2955  * Booting with kdump and LPIs enabled is generally fine. Any other
2956  * case is wrong in the absence of firmware/EFI support.
2957  */
2958 static bool enabled_lpis_allowed(void)
2959 {
2960 	phys_addr_t addr;
2961 	u64 val;
2962 
2963 	/* Check whether the property table is in a reserved region */
2964 	val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
2965 	addr = val & GENMASK_ULL(51, 12);
2966 
2967 	return gic_check_reserved_range(addr, LPI_PROPBASE_SZ);
2968 }
2969 
2970 static int __init allocate_lpi_tables(void)
2971 {
2972 	u64 val;
2973 	int err, cpu;
2974 
2975 	/*
2976 	 * If LPIs are enabled while we run this from the boot CPU,
2977 	 * flag the RD tables as pre-allocated if the stars do align.
2978 	 */
2979 	val = readl_relaxed(gic_data_rdist_rd_base() + GICR_CTLR);
2980 	if ((val & GICR_CTLR_ENABLE_LPIS) && enabled_lpis_allowed()) {
2981 		gic_rdists->flags |= (RDIST_FLAGS_RD_TABLES_PREALLOCATED |
2982 				      RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING);
2983 		pr_info("GICv3: Using preallocated redistributor tables\n");
2984 	}
2985 
2986 	err = its_setup_lpi_prop_table();
2987 	if (err)
2988 		return err;
2989 
2990 	/*
2991 	 * We allocate all the pending tables anyway, as we may have a
2992 	 * mix of RDs that have had LPIs enabled, and some that
2993 	 * don't. We'll free the unused ones as each CPU comes online.
2994 	 */
2995 	for_each_possible_cpu(cpu) {
2996 		struct page *pend_page;
2997 
2998 		pend_page = its_allocate_pending_table(GFP_NOWAIT);
2999 		if (!pend_page) {
3000 			pr_err("Failed to allocate PENDBASE for CPU%d\n", cpu);
3001 			return -ENOMEM;
3002 		}
3003 
3004 		gic_data_rdist_cpu(cpu)->pend_page = pend_page;
3005 	}
3006 
3007 	return 0;
3008 }
3009 
3010 static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set)
3011 {
3012 	u32 count = 1000000;	/* 1s! */
3013 	bool clean;
3014 	u64 val;
3015 
3016 	val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
3017 	val &= ~GICR_VPENDBASER_Valid;
3018 	val &= ~clr;
3019 	val |= set;
3020 	gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
3021 
3022 	do {
3023 		val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
3024 		clean = !(val & GICR_VPENDBASER_Dirty);
3025 		if (!clean) {
3026 			count--;
3027 			cpu_relax();
3028 			udelay(1);
3029 		}
3030 	} while (!clean && count);
3031 
3032 	if (unlikely(val & GICR_VPENDBASER_Dirty)) {
3033 		pr_err_ratelimited("ITS virtual pending table not cleaning\n");
3034 		val |= GICR_VPENDBASER_PendingLast;
3035 	}
3036 
3037 	return val;
3038 }
3039 
3040 static void its_cpu_init_lpis(void)
3041 {
3042 	void __iomem *rbase = gic_data_rdist_rd_base();
3043 	struct page *pend_page;
3044 	phys_addr_t paddr;
3045 	u64 val, tmp;
3046 
3047 	if (gic_data_rdist()->lpi_enabled)
3048 		return;
3049 
3050 	val = readl_relaxed(rbase + GICR_CTLR);
3051 	if ((gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) &&
3052 	    (val & GICR_CTLR_ENABLE_LPIS)) {
3053 		/*
3054 		 * Check that we get the same property table on all
3055 		 * RDs. If we don't, this is hopeless.
3056 		 */
3057 		paddr = gicr_read_propbaser(rbase + GICR_PROPBASER);
3058 		paddr &= GENMASK_ULL(51, 12);
3059 		if (WARN_ON(gic_rdists->prop_table_pa != paddr))
3060 			add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
3061 
3062 		paddr = gicr_read_pendbaser(rbase + GICR_PENDBASER);
3063 		paddr &= GENMASK_ULL(51, 16);
3064 
3065 		WARN_ON(!gic_check_reserved_range(paddr, LPI_PENDBASE_SZ));
3066 		its_free_pending_table(gic_data_rdist()->pend_page);
3067 		gic_data_rdist()->pend_page = NULL;
3068 
3069 		goto out;
3070 	}
3071 
3072 	pend_page = gic_data_rdist()->pend_page;
3073 	paddr = page_to_phys(pend_page);
3074 	WARN_ON(gic_reserve_range(paddr, LPI_PENDBASE_SZ));
3075 
3076 	/* set PROPBASE */
3077 	val = (gic_rdists->prop_table_pa |
3078 	       GICR_PROPBASER_InnerShareable |
3079 	       GICR_PROPBASER_RaWaWb |
3080 	       ((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK));
3081 
3082 	gicr_write_propbaser(val, rbase + GICR_PROPBASER);
3083 	tmp = gicr_read_propbaser(rbase + GICR_PROPBASER);
3084 
3085 	if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) {
3086 		if (!(tmp & GICR_PROPBASER_SHAREABILITY_MASK)) {
3087 			/*
3088 			 * The HW reports non-shareable, we must
3089 			 * remove the cacheability attributes as
3090 			 * well.
3091 			 */
3092 			val &= ~(GICR_PROPBASER_SHAREABILITY_MASK |
3093 				 GICR_PROPBASER_CACHEABILITY_MASK);
3094 			val |= GICR_PROPBASER_nC;
3095 			gicr_write_propbaser(val, rbase + GICR_PROPBASER);
3096 		}
3097 		pr_info_once("GIC: using cache flushing for LPI property table\n");
3098 		gic_rdists->flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING;
3099 	}
3100 
3101 	/* set PENDBASE */
3102 	val = (page_to_phys(pend_page) |
3103 	       GICR_PENDBASER_InnerShareable |
3104 	       GICR_PENDBASER_RaWaWb);
3105 
3106 	gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
3107 	tmp = gicr_read_pendbaser(rbase + GICR_PENDBASER);
3108 
3109 	if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) {
3110 		/*
3111 		 * The HW reports non-shareable, we must remove the
3112 		 * cacheability attributes as well.
3113 		 */
3114 		val &= ~(GICR_PENDBASER_SHAREABILITY_MASK |
3115 			 GICR_PENDBASER_CACHEABILITY_MASK);
3116 		val |= GICR_PENDBASER_nC;
3117 		gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
3118 	}
3119 
3120 	/* Enable LPIs */
3121 	val = readl_relaxed(rbase + GICR_CTLR);
3122 	val |= GICR_CTLR_ENABLE_LPIS;
3123 	writel_relaxed(val, rbase + GICR_CTLR);
3124 
3125 	if (gic_rdists->has_vlpis && !gic_rdists->has_rvpeid) {
3126 		void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3127 
3128 		/*
3129 		 * It's possible for CPU to receive VLPIs before it is
3130 		 * scheduled as a vPE, especially for the first CPU, and the
3131 		 * VLPI with INTID larger than 2^(IDbits+1) will be considered
3132 		 * as out of range and dropped by GIC.
3133 		 * So we initialize IDbits to known value to avoid VLPI drop.
3134 		 */
3135 		val = (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
3136 		pr_debug("GICv4: CPU%d: Init IDbits to 0x%llx for GICR_VPROPBASER\n",
3137 			smp_processor_id(), val);
3138 		gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
3139 
3140 		/*
3141 		 * Also clear Valid bit of GICR_VPENDBASER, in case some
3142 		 * ancient programming gets left in and has possibility of
3143 		 * corrupting memory.
3144 		 */
3145 		val = its_clear_vpend_valid(vlpi_base, 0, 0);
3146 	}
3147 
3148 	if (allocate_vpe_l1_table()) {
3149 		/*
3150 		 * If the allocation has failed, we're in massive trouble.
3151 		 * Disable direct injection, and pray that no VM was
3152 		 * already running...
3153 		 */
3154 		gic_rdists->has_rvpeid = false;
3155 		gic_rdists->has_vlpis = false;
3156 	}
3157 
3158 	/* Make sure the GIC has seen the above */
3159 	dsb(sy);
3160 out:
3161 	gic_data_rdist()->lpi_enabled = true;
3162 	pr_info("GICv3: CPU%d: using %s LPI pending table @%pa\n",
3163 		smp_processor_id(),
3164 		gic_data_rdist()->pend_page ? "allocated" : "reserved",
3165 		&paddr);
3166 }
3167 
3168 static void its_cpu_init_collection(struct its_node *its)
3169 {
3170 	int cpu = smp_processor_id();
3171 	u64 target;
3172 
3173 	/* avoid cross node collections and its mapping */
3174 	if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) {
3175 		struct device_node *cpu_node;
3176 
3177 		cpu_node = of_get_cpu_node(cpu, NULL);
3178 		if (its->numa_node != NUMA_NO_NODE &&
3179 			its->numa_node != of_node_to_nid(cpu_node))
3180 			return;
3181 	}
3182 
3183 	/*
3184 	 * We now have to bind each collection to its target
3185 	 * redistributor.
3186 	 */
3187 	if (gic_read_typer(its->base + GITS_TYPER) & GITS_TYPER_PTA) {
3188 		/*
3189 		 * This ITS wants the physical address of the
3190 		 * redistributor.
3191 		 */
3192 		target = gic_data_rdist()->phys_base;
3193 	} else {
3194 		/* This ITS wants a linear CPU number. */
3195 		target = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
3196 		target = GICR_TYPER_CPU_NUMBER(target) << 16;
3197 	}
3198 
3199 	/* Perform collection mapping */
3200 	its->collections[cpu].target_address = target;
3201 	its->collections[cpu].col_id = cpu;
3202 
3203 	its_send_mapc(its, &its->collections[cpu], 1);
3204 	its_send_invall(its, &its->collections[cpu]);
3205 }
3206 
3207 static void its_cpu_init_collections(void)
3208 {
3209 	struct its_node *its;
3210 
3211 	raw_spin_lock(&its_lock);
3212 
3213 	list_for_each_entry(its, &its_nodes, entry)
3214 		its_cpu_init_collection(its);
3215 
3216 	raw_spin_unlock(&its_lock);
3217 }
3218 
3219 static struct its_device *its_find_device(struct its_node *its, u32 dev_id)
3220 {
3221 	struct its_device *its_dev = NULL, *tmp;
3222 	unsigned long flags;
3223 
3224 	raw_spin_lock_irqsave(&its->lock, flags);
3225 
3226 	list_for_each_entry(tmp, &its->its_device_list, entry) {
3227 		if (tmp->device_id == dev_id) {
3228 			its_dev = tmp;
3229 			break;
3230 		}
3231 	}
3232 
3233 	raw_spin_unlock_irqrestore(&its->lock, flags);
3234 
3235 	return its_dev;
3236 }
3237 
3238 static struct its_baser *its_get_baser(struct its_node *its, u32 type)
3239 {
3240 	int i;
3241 
3242 	for (i = 0; i < GITS_BASER_NR_REGS; i++) {
3243 		if (GITS_BASER_TYPE(its->tables[i].val) == type)
3244 			return &its->tables[i];
3245 	}
3246 
3247 	return NULL;
3248 }
3249 
3250 static bool its_alloc_table_entry(struct its_node *its,
3251 				  struct its_baser *baser, u32 id)
3252 {
3253 	struct page *page;
3254 	u32 esz, idx;
3255 	__le64 *table;
3256 
3257 	/* Don't allow device id that exceeds single, flat table limit */
3258 	esz = GITS_BASER_ENTRY_SIZE(baser->val);
3259 	if (!(baser->val & GITS_BASER_INDIRECT))
3260 		return (id < (PAGE_ORDER_TO_SIZE(baser->order) / esz));
3261 
3262 	/* Compute 1st level table index & check if that exceeds table limit */
3263 	idx = id >> ilog2(baser->psz / esz);
3264 	if (idx >= (PAGE_ORDER_TO_SIZE(baser->order) / GITS_LVL1_ENTRY_SIZE))
3265 		return false;
3266 
3267 	table = baser->base;
3268 
3269 	/* Allocate memory for 2nd level table */
3270 	if (!table[idx]) {
3271 		page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO,
3272 					get_order(baser->psz));
3273 		if (!page)
3274 			return false;
3275 
3276 		/* Flush Lvl2 table to PoC if hw doesn't support coherency */
3277 		if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
3278 			gic_flush_dcache_to_poc(page_address(page), baser->psz);
3279 
3280 		table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID);
3281 
3282 		/* Flush Lvl1 entry to PoC if hw doesn't support coherency */
3283 		if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
3284 			gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
3285 
3286 		/* Ensure updated table contents are visible to ITS hardware */
3287 		dsb(sy);
3288 	}
3289 
3290 	return true;
3291 }
3292 
3293 static bool its_alloc_device_table(struct its_node *its, u32 dev_id)
3294 {
3295 	struct its_baser *baser;
3296 
3297 	baser = its_get_baser(its, GITS_BASER_TYPE_DEVICE);
3298 
3299 	/* Don't allow device id that exceeds ITS hardware limit */
3300 	if (!baser)
3301 		return (ilog2(dev_id) < device_ids(its));
3302 
3303 	return its_alloc_table_entry(its, baser, dev_id);
3304 }
3305 
3306 static bool its_alloc_vpe_table(u32 vpe_id)
3307 {
3308 	struct its_node *its;
3309 	int cpu;
3310 
3311 	/*
3312 	 * Make sure the L2 tables are allocated on *all* v4 ITSs. We
3313 	 * could try and only do it on ITSs corresponding to devices
3314 	 * that have interrupts targeted at this VPE, but the
3315 	 * complexity becomes crazy (and you have tons of memory
3316 	 * anyway, right?).
3317 	 */
3318 	list_for_each_entry(its, &its_nodes, entry) {
3319 		struct its_baser *baser;
3320 
3321 		if (!is_v4(its))
3322 			continue;
3323 
3324 		baser = its_get_baser(its, GITS_BASER_TYPE_VCPU);
3325 		if (!baser)
3326 			return false;
3327 
3328 		if (!its_alloc_table_entry(its, baser, vpe_id))
3329 			return false;
3330 	}
3331 
3332 	/* Non v4.1? No need to iterate RDs and go back early. */
3333 	if (!gic_rdists->has_rvpeid)
3334 		return true;
3335 
3336 	/*
3337 	 * Make sure the L2 tables are allocated for all copies of
3338 	 * the L1 table on *all* v4.1 RDs.
3339 	 */
3340 	for_each_possible_cpu(cpu) {
3341 		if (!allocate_vpe_l2_table(cpu, vpe_id))
3342 			return false;
3343 	}
3344 
3345 	return true;
3346 }
3347 
3348 static struct its_device *its_create_device(struct its_node *its, u32 dev_id,
3349 					    int nvecs, bool alloc_lpis)
3350 {
3351 	struct its_device *dev;
3352 	unsigned long *lpi_map = NULL;
3353 	unsigned long flags;
3354 	u16 *col_map = NULL;
3355 	void *itt;
3356 	int lpi_base;
3357 	int nr_lpis;
3358 	int nr_ites;
3359 	int sz;
3360 
3361 	if (!its_alloc_device_table(its, dev_id))
3362 		return NULL;
3363 
3364 	if (WARN_ON(!is_power_of_2(nvecs)))
3365 		nvecs = roundup_pow_of_two(nvecs);
3366 
3367 	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
3368 	/*
3369 	 * Even if the device wants a single LPI, the ITT must be
3370 	 * sized as a power of two (and you need at least one bit...).
3371 	 */
3372 	nr_ites = max(2, nvecs);
3373 	sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1);
3374 	sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1;
3375 	itt = kzalloc_node(sz, GFP_KERNEL, its->numa_node);
3376 	if (alloc_lpis) {
3377 		lpi_map = its_lpi_alloc(nvecs, &lpi_base, &nr_lpis);
3378 		if (lpi_map)
3379 			col_map = kcalloc(nr_lpis, sizeof(*col_map),
3380 					  GFP_KERNEL);
3381 	} else {
3382 		col_map = kcalloc(nr_ites, sizeof(*col_map), GFP_KERNEL);
3383 		nr_lpis = 0;
3384 		lpi_base = 0;
3385 	}
3386 
3387 	if (!dev || !itt ||  !col_map || (!lpi_map && alloc_lpis)) {
3388 		kfree(dev);
3389 		kfree(itt);
3390 		bitmap_free(lpi_map);
3391 		kfree(col_map);
3392 		return NULL;
3393 	}
3394 
3395 	gic_flush_dcache_to_poc(itt, sz);
3396 
3397 	dev->its = its;
3398 	dev->itt = itt;
3399 	dev->nr_ites = nr_ites;
3400 	dev->event_map.lpi_map = lpi_map;
3401 	dev->event_map.col_map = col_map;
3402 	dev->event_map.lpi_base = lpi_base;
3403 	dev->event_map.nr_lpis = nr_lpis;
3404 	raw_spin_lock_init(&dev->event_map.vlpi_lock);
3405 	dev->device_id = dev_id;
3406 	INIT_LIST_HEAD(&dev->entry);
3407 
3408 	raw_spin_lock_irqsave(&its->lock, flags);
3409 	list_add(&dev->entry, &its->its_device_list);
3410 	raw_spin_unlock_irqrestore(&its->lock, flags);
3411 
3412 	/* Map device to its ITT */
3413 	its_send_mapd(dev, 1);
3414 
3415 	return dev;
3416 }
3417 
3418 static void its_free_device(struct its_device *its_dev)
3419 {
3420 	unsigned long flags;
3421 
3422 	raw_spin_lock_irqsave(&its_dev->its->lock, flags);
3423 	list_del(&its_dev->entry);
3424 	raw_spin_unlock_irqrestore(&its_dev->its->lock, flags);
3425 	kfree(its_dev->event_map.col_map);
3426 	kfree(its_dev->itt);
3427 	kfree(its_dev);
3428 }
3429 
3430 static int its_alloc_device_irq(struct its_device *dev, int nvecs, irq_hw_number_t *hwirq)
3431 {
3432 	int idx;
3433 
3434 	/* Find a free LPI region in lpi_map and allocate them. */
3435 	idx = bitmap_find_free_region(dev->event_map.lpi_map,
3436 				      dev->event_map.nr_lpis,
3437 				      get_count_order(nvecs));
3438 	if (idx < 0)
3439 		return -ENOSPC;
3440 
3441 	*hwirq = dev->event_map.lpi_base + idx;
3442 
3443 	return 0;
3444 }
3445 
3446 static int its_msi_prepare(struct irq_domain *domain, struct device *dev,
3447 			   int nvec, msi_alloc_info_t *info)
3448 {
3449 	struct its_node *its;
3450 	struct its_device *its_dev;
3451 	struct msi_domain_info *msi_info;
3452 	u32 dev_id;
3453 	int err = 0;
3454 
3455 	/*
3456 	 * We ignore "dev" entirely, and rely on the dev_id that has
3457 	 * been passed via the scratchpad. This limits this domain's
3458 	 * usefulness to upper layers that definitely know that they
3459 	 * are built on top of the ITS.
3460 	 */
3461 	dev_id = info->scratchpad[0].ul;
3462 
3463 	msi_info = msi_get_domain_info(domain);
3464 	its = msi_info->data;
3465 
3466 	if (!gic_rdists->has_direct_lpi &&
3467 	    vpe_proxy.dev &&
3468 	    vpe_proxy.dev->its == its &&
3469 	    dev_id == vpe_proxy.dev->device_id) {
3470 		/* Bad luck. Get yourself a better implementation */
3471 		WARN_ONCE(1, "DevId %x clashes with GICv4 VPE proxy device\n",
3472 			  dev_id);
3473 		return -EINVAL;
3474 	}
3475 
3476 	mutex_lock(&its->dev_alloc_lock);
3477 	its_dev = its_find_device(its, dev_id);
3478 	if (its_dev) {
3479 		/*
3480 		 * We already have seen this ID, probably through
3481 		 * another alias (PCI bridge of some sort). No need to
3482 		 * create the device.
3483 		 */
3484 		its_dev->shared = true;
3485 		pr_debug("Reusing ITT for devID %x\n", dev_id);
3486 		goto out;
3487 	}
3488 
3489 	its_dev = its_create_device(its, dev_id, nvec, true);
3490 	if (!its_dev) {
3491 		err = -ENOMEM;
3492 		goto out;
3493 	}
3494 
3495 	if (info->flags & MSI_ALLOC_FLAGS_PROXY_DEVICE)
3496 		its_dev->shared = true;
3497 
3498 	pr_debug("ITT %d entries, %d bits\n", nvec, ilog2(nvec));
3499 out:
3500 	mutex_unlock(&its->dev_alloc_lock);
3501 	info->scratchpad[0].ptr = its_dev;
3502 	return err;
3503 }
3504 
3505 static struct msi_domain_ops its_msi_domain_ops = {
3506 	.msi_prepare	= its_msi_prepare,
3507 };
3508 
3509 static int its_irq_gic_domain_alloc(struct irq_domain *domain,
3510 				    unsigned int virq,
3511 				    irq_hw_number_t hwirq)
3512 {
3513 	struct irq_fwspec fwspec;
3514 
3515 	if (irq_domain_get_of_node(domain->parent)) {
3516 		fwspec.fwnode = domain->parent->fwnode;
3517 		fwspec.param_count = 3;
3518 		fwspec.param[0] = GIC_IRQ_TYPE_LPI;
3519 		fwspec.param[1] = hwirq;
3520 		fwspec.param[2] = IRQ_TYPE_EDGE_RISING;
3521 	} else if (is_fwnode_irqchip(domain->parent->fwnode)) {
3522 		fwspec.fwnode = domain->parent->fwnode;
3523 		fwspec.param_count = 2;
3524 		fwspec.param[0] = hwirq;
3525 		fwspec.param[1] = IRQ_TYPE_EDGE_RISING;
3526 	} else {
3527 		return -EINVAL;
3528 	}
3529 
3530 	return irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec);
3531 }
3532 
3533 static int its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
3534 				unsigned int nr_irqs, void *args)
3535 {
3536 	msi_alloc_info_t *info = args;
3537 	struct its_device *its_dev = info->scratchpad[0].ptr;
3538 	struct its_node *its = its_dev->its;
3539 	struct irq_data *irqd;
3540 	irq_hw_number_t hwirq;
3541 	int err;
3542 	int i;
3543 
3544 	err = its_alloc_device_irq(its_dev, nr_irqs, &hwirq);
3545 	if (err)
3546 		return err;
3547 
3548 	err = iommu_dma_prepare_msi(info->desc, its->get_msi_base(its_dev));
3549 	if (err)
3550 		return err;
3551 
3552 	for (i = 0; i < nr_irqs; i++) {
3553 		err = its_irq_gic_domain_alloc(domain, virq + i, hwirq + i);
3554 		if (err)
3555 			return err;
3556 
3557 		irq_domain_set_hwirq_and_chip(domain, virq + i,
3558 					      hwirq + i, &its_irq_chip, its_dev);
3559 		irqd = irq_get_irq_data(virq + i);
3560 		irqd_set_single_target(irqd);
3561 		irqd_set_affinity_on_activate(irqd);
3562 		pr_debug("ID:%d pID:%d vID:%d\n",
3563 			 (int)(hwirq + i - its_dev->event_map.lpi_base),
3564 			 (int)(hwirq + i), virq + i);
3565 	}
3566 
3567 	return 0;
3568 }
3569 
3570 static int its_irq_domain_activate(struct irq_domain *domain,
3571 				   struct irq_data *d, bool reserve)
3572 {
3573 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3574 	u32 event = its_get_event_id(d);
3575 	int cpu;
3576 
3577 	cpu = its_select_cpu(d, cpu_online_mask);
3578 	if (cpu < 0 || cpu >= nr_cpu_ids)
3579 		return -EINVAL;
3580 
3581 	its_inc_lpi_count(d, cpu);
3582 	its_dev->event_map.col_map[event] = cpu;
3583 	irq_data_update_effective_affinity(d, cpumask_of(cpu));
3584 
3585 	/* Map the GIC IRQ and event to the device */
3586 	its_send_mapti(its_dev, d->hwirq, event);
3587 	return 0;
3588 }
3589 
3590 static void its_irq_domain_deactivate(struct irq_domain *domain,
3591 				      struct irq_data *d)
3592 {
3593 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3594 	u32 event = its_get_event_id(d);
3595 
3596 	its_dec_lpi_count(d, its_dev->event_map.col_map[event]);
3597 	/* Stop the delivery of interrupts */
3598 	its_send_discard(its_dev, event);
3599 }
3600 
3601 static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq,
3602 				unsigned int nr_irqs)
3603 {
3604 	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
3605 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3606 	struct its_node *its = its_dev->its;
3607 	int i;
3608 
3609 	bitmap_release_region(its_dev->event_map.lpi_map,
3610 			      its_get_event_id(irq_domain_get_irq_data(domain, virq)),
3611 			      get_count_order(nr_irqs));
3612 
3613 	for (i = 0; i < nr_irqs; i++) {
3614 		struct irq_data *data = irq_domain_get_irq_data(domain,
3615 								virq + i);
3616 		/* Nuke the entry in the domain */
3617 		irq_domain_reset_irq_data(data);
3618 	}
3619 
3620 	mutex_lock(&its->dev_alloc_lock);
3621 
3622 	/*
3623 	 * If all interrupts have been freed, start mopping the
3624 	 * floor. This is conditioned on the device not being shared.
3625 	 */
3626 	if (!its_dev->shared &&
3627 	    bitmap_empty(its_dev->event_map.lpi_map,
3628 			 its_dev->event_map.nr_lpis)) {
3629 		its_lpi_free(its_dev->event_map.lpi_map,
3630 			     its_dev->event_map.lpi_base,
3631 			     its_dev->event_map.nr_lpis);
3632 
3633 		/* Unmap device/itt */
3634 		its_send_mapd(its_dev, 0);
3635 		its_free_device(its_dev);
3636 	}
3637 
3638 	mutex_unlock(&its->dev_alloc_lock);
3639 
3640 	irq_domain_free_irqs_parent(domain, virq, nr_irqs);
3641 }
3642 
3643 static const struct irq_domain_ops its_domain_ops = {
3644 	.alloc			= its_irq_domain_alloc,
3645 	.free			= its_irq_domain_free,
3646 	.activate		= its_irq_domain_activate,
3647 	.deactivate		= its_irq_domain_deactivate,
3648 };
3649 
3650 /*
3651  * This is insane.
3652  *
3653  * If a GICv4.0 doesn't implement Direct LPIs (which is extremely
3654  * likely), the only way to perform an invalidate is to use a fake
3655  * device to issue an INV command, implying that the LPI has first
3656  * been mapped to some event on that device. Since this is not exactly
3657  * cheap, we try to keep that mapping around as long as possible, and
3658  * only issue an UNMAP if we're short on available slots.
3659  *
3660  * Broken by design(tm).
3661  *
3662  * GICv4.1, on the other hand, mandates that we're able to invalidate
3663  * by writing to a MMIO register. It doesn't implement the whole of
3664  * DirectLPI, but that's good enough. And most of the time, we don't
3665  * even have to invalidate anything, as the redistributor can be told
3666  * whether to generate a doorbell or not (we thus leave it enabled,
3667  * always).
3668  */
3669 static void its_vpe_db_proxy_unmap_locked(struct its_vpe *vpe)
3670 {
3671 	/* GICv4.1 doesn't use a proxy, so nothing to do here */
3672 	if (gic_rdists->has_rvpeid)
3673 		return;
3674 
3675 	/* Already unmapped? */
3676 	if (vpe->vpe_proxy_event == -1)
3677 		return;
3678 
3679 	its_send_discard(vpe_proxy.dev, vpe->vpe_proxy_event);
3680 	vpe_proxy.vpes[vpe->vpe_proxy_event] = NULL;
3681 
3682 	/*
3683 	 * We don't track empty slots at all, so let's move the
3684 	 * next_victim pointer if we can quickly reuse that slot
3685 	 * instead of nuking an existing entry. Not clear that this is
3686 	 * always a win though, and this might just generate a ripple
3687 	 * effect... Let's just hope VPEs don't migrate too often.
3688 	 */
3689 	if (vpe_proxy.vpes[vpe_proxy.next_victim])
3690 		vpe_proxy.next_victim = vpe->vpe_proxy_event;
3691 
3692 	vpe->vpe_proxy_event = -1;
3693 }
3694 
3695 static void its_vpe_db_proxy_unmap(struct its_vpe *vpe)
3696 {
3697 	/* GICv4.1 doesn't use a proxy, so nothing to do here */
3698 	if (gic_rdists->has_rvpeid)
3699 		return;
3700 
3701 	if (!gic_rdists->has_direct_lpi) {
3702 		unsigned long flags;
3703 
3704 		raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3705 		its_vpe_db_proxy_unmap_locked(vpe);
3706 		raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3707 	}
3708 }
3709 
3710 static void its_vpe_db_proxy_map_locked(struct its_vpe *vpe)
3711 {
3712 	/* GICv4.1 doesn't use a proxy, so nothing to do here */
3713 	if (gic_rdists->has_rvpeid)
3714 		return;
3715 
3716 	/* Already mapped? */
3717 	if (vpe->vpe_proxy_event != -1)
3718 		return;
3719 
3720 	/* This slot was already allocated. Kick the other VPE out. */
3721 	if (vpe_proxy.vpes[vpe_proxy.next_victim])
3722 		its_vpe_db_proxy_unmap_locked(vpe_proxy.vpes[vpe_proxy.next_victim]);
3723 
3724 	/* Map the new VPE instead */
3725 	vpe_proxy.vpes[vpe_proxy.next_victim] = vpe;
3726 	vpe->vpe_proxy_event = vpe_proxy.next_victim;
3727 	vpe_proxy.next_victim = (vpe_proxy.next_victim + 1) % vpe_proxy.dev->nr_ites;
3728 
3729 	vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = vpe->col_idx;
3730 	its_send_mapti(vpe_proxy.dev, vpe->vpe_db_lpi, vpe->vpe_proxy_event);
3731 }
3732 
3733 static void its_vpe_db_proxy_move(struct its_vpe *vpe, int from, int to)
3734 {
3735 	unsigned long flags;
3736 	struct its_collection *target_col;
3737 
3738 	/* GICv4.1 doesn't use a proxy, so nothing to do here */
3739 	if (gic_rdists->has_rvpeid)
3740 		return;
3741 
3742 	if (gic_rdists->has_direct_lpi) {
3743 		void __iomem *rdbase;
3744 
3745 		rdbase = per_cpu_ptr(gic_rdists->rdist, from)->rd_base;
3746 		gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
3747 		wait_for_syncr(rdbase);
3748 
3749 		return;
3750 	}
3751 
3752 	raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3753 
3754 	its_vpe_db_proxy_map_locked(vpe);
3755 
3756 	target_col = &vpe_proxy.dev->its->collections[to];
3757 	its_send_movi(vpe_proxy.dev, target_col, vpe->vpe_proxy_event);
3758 	vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = to;
3759 
3760 	raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3761 }
3762 
3763 static int its_vpe_set_affinity(struct irq_data *d,
3764 				const struct cpumask *mask_val,
3765 				bool force)
3766 {
3767 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3768 	int from, cpu = cpumask_first(mask_val);
3769 	unsigned long flags;
3770 
3771 	/*
3772 	 * Changing affinity is mega expensive, so let's be as lazy as
3773 	 * we can and only do it if we really have to. Also, if mapped
3774 	 * into the proxy device, we need to move the doorbell
3775 	 * interrupt to its new location.
3776 	 *
3777 	 * Another thing is that changing the affinity of a vPE affects
3778 	 * *other interrupts* such as all the vLPIs that are routed to
3779 	 * this vPE. This means that the irq_desc lock is not enough to
3780 	 * protect us, and that we must ensure nobody samples vpe->col_idx
3781 	 * during the update, hence the lock below which must also be
3782 	 * taken on any vLPI handling path that evaluates vpe->col_idx.
3783 	 */
3784 	from = vpe_to_cpuid_lock(vpe, &flags);
3785 	if (from == cpu)
3786 		goto out;
3787 
3788 	vpe->col_idx = cpu;
3789 
3790 	/*
3791 	 * GICv4.1 allows us to skip VMOVP if moving to a cpu whose RD
3792 	 * is sharing its VPE table with the current one.
3793 	 */
3794 	if (gic_data_rdist_cpu(cpu)->vpe_table_mask &&
3795 	    cpumask_test_cpu(from, gic_data_rdist_cpu(cpu)->vpe_table_mask))
3796 		goto out;
3797 
3798 	its_send_vmovp(vpe);
3799 	its_vpe_db_proxy_move(vpe, from, cpu);
3800 
3801 out:
3802 	irq_data_update_effective_affinity(d, cpumask_of(cpu));
3803 	vpe_to_cpuid_unlock(vpe, flags);
3804 
3805 	return IRQ_SET_MASK_OK_DONE;
3806 }
3807 
3808 static void its_wait_vpt_parse_complete(void)
3809 {
3810 	void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3811 	u64 val;
3812 
3813 	if (!gic_rdists->has_vpend_valid_dirty)
3814 		return;
3815 
3816 	WARN_ON_ONCE(readq_relaxed_poll_timeout_atomic(vlpi_base + GICR_VPENDBASER,
3817 						       val,
3818 						       !(val & GICR_VPENDBASER_Dirty),
3819 						       1, 500));
3820 }
3821 
3822 static void its_vpe_schedule(struct its_vpe *vpe)
3823 {
3824 	void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3825 	u64 val;
3826 
3827 	/* Schedule the VPE */
3828 	val  = virt_to_phys(page_address(vpe->its_vm->vprop_page)) &
3829 		GENMASK_ULL(51, 12);
3830 	val |= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
3831 	val |= GICR_VPROPBASER_RaWb;
3832 	val |= GICR_VPROPBASER_InnerShareable;
3833 	gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
3834 
3835 	val  = virt_to_phys(page_address(vpe->vpt_page)) &
3836 		GENMASK_ULL(51, 16);
3837 	val |= GICR_VPENDBASER_RaWaWb;
3838 	val |= GICR_VPENDBASER_InnerShareable;
3839 	/*
3840 	 * There is no good way of finding out if the pending table is
3841 	 * empty as we can race against the doorbell interrupt very
3842 	 * easily. So in the end, vpe->pending_last is only an
3843 	 * indication that the vcpu has something pending, not one
3844 	 * that the pending table is empty. A good implementation
3845 	 * would be able to read its coarse map pretty quickly anyway,
3846 	 * making this a tolerable issue.
3847 	 */
3848 	val |= GICR_VPENDBASER_PendingLast;
3849 	val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0;
3850 	val |= GICR_VPENDBASER_Valid;
3851 	gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
3852 }
3853 
3854 static void its_vpe_deschedule(struct its_vpe *vpe)
3855 {
3856 	void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3857 	u64 val;
3858 
3859 	val = its_clear_vpend_valid(vlpi_base, 0, 0);
3860 
3861 	vpe->idai = !!(val & GICR_VPENDBASER_IDAI);
3862 	vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
3863 }
3864 
3865 static void its_vpe_invall(struct its_vpe *vpe)
3866 {
3867 	struct its_node *its;
3868 
3869 	list_for_each_entry(its, &its_nodes, entry) {
3870 		if (!is_v4(its))
3871 			continue;
3872 
3873 		if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr])
3874 			continue;
3875 
3876 		/*
3877 		 * Sending a VINVALL to a single ITS is enough, as all
3878 		 * we need is to reach the redistributors.
3879 		 */
3880 		its_send_vinvall(its, vpe);
3881 		return;
3882 	}
3883 }
3884 
3885 static int its_vpe_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
3886 {
3887 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3888 	struct its_cmd_info *info = vcpu_info;
3889 
3890 	switch (info->cmd_type) {
3891 	case SCHEDULE_VPE:
3892 		its_vpe_schedule(vpe);
3893 		return 0;
3894 
3895 	case DESCHEDULE_VPE:
3896 		its_vpe_deschedule(vpe);
3897 		return 0;
3898 
3899 	case COMMIT_VPE:
3900 		its_wait_vpt_parse_complete();
3901 		return 0;
3902 
3903 	case INVALL_VPE:
3904 		its_vpe_invall(vpe);
3905 		return 0;
3906 
3907 	default:
3908 		return -EINVAL;
3909 	}
3910 }
3911 
3912 static void its_vpe_send_cmd(struct its_vpe *vpe,
3913 			     void (*cmd)(struct its_device *, u32))
3914 {
3915 	unsigned long flags;
3916 
3917 	raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3918 
3919 	its_vpe_db_proxy_map_locked(vpe);
3920 	cmd(vpe_proxy.dev, vpe->vpe_proxy_event);
3921 
3922 	raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3923 }
3924 
3925 static void its_vpe_send_inv(struct irq_data *d)
3926 {
3927 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3928 
3929 	if (gic_rdists->has_direct_lpi) {
3930 		void __iomem *rdbase;
3931 
3932 		/* Target the redistributor this VPE is currently known on */
3933 		raw_spin_lock(&gic_data_rdist_cpu(vpe->col_idx)->rd_lock);
3934 		rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;
3935 		gic_write_lpir(d->parent_data->hwirq, rdbase + GICR_INVLPIR);
3936 		wait_for_syncr(rdbase);
3937 		raw_spin_unlock(&gic_data_rdist_cpu(vpe->col_idx)->rd_lock);
3938 	} else {
3939 		its_vpe_send_cmd(vpe, its_send_inv);
3940 	}
3941 }
3942 
3943 static void its_vpe_mask_irq(struct irq_data *d)
3944 {
3945 	/*
3946 	 * We need to unmask the LPI, which is described by the parent
3947 	 * irq_data. Instead of calling into the parent (which won't
3948 	 * exactly do the right thing, let's simply use the
3949 	 * parent_data pointer. Yes, I'm naughty.
3950 	 */
3951 	lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
3952 	its_vpe_send_inv(d);
3953 }
3954 
3955 static void its_vpe_unmask_irq(struct irq_data *d)
3956 {
3957 	/* Same hack as above... */
3958 	lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
3959 	its_vpe_send_inv(d);
3960 }
3961 
3962 static int its_vpe_set_irqchip_state(struct irq_data *d,
3963 				     enum irqchip_irq_state which,
3964 				     bool state)
3965 {
3966 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3967 
3968 	if (which != IRQCHIP_STATE_PENDING)
3969 		return -EINVAL;
3970 
3971 	if (gic_rdists->has_direct_lpi) {
3972 		void __iomem *rdbase;
3973 
3974 		rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;
3975 		if (state) {
3976 			gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR);
3977 		} else {
3978 			gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
3979 			wait_for_syncr(rdbase);
3980 		}
3981 	} else {
3982 		if (state)
3983 			its_vpe_send_cmd(vpe, its_send_int);
3984 		else
3985 			its_vpe_send_cmd(vpe, its_send_clear);
3986 	}
3987 
3988 	return 0;
3989 }
3990 
3991 static int its_vpe_retrigger(struct irq_data *d)
3992 {
3993 	return !its_vpe_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true);
3994 }
3995 
3996 static struct irq_chip its_vpe_irq_chip = {
3997 	.name			= "GICv4-vpe",
3998 	.irq_mask		= its_vpe_mask_irq,
3999 	.irq_unmask		= its_vpe_unmask_irq,
4000 	.irq_eoi		= irq_chip_eoi_parent,
4001 	.irq_set_affinity	= its_vpe_set_affinity,
4002 	.irq_retrigger		= its_vpe_retrigger,
4003 	.irq_set_irqchip_state	= its_vpe_set_irqchip_state,
4004 	.irq_set_vcpu_affinity	= its_vpe_set_vcpu_affinity,
4005 };
4006 
4007 static struct its_node *find_4_1_its(void)
4008 {
4009 	static struct its_node *its = NULL;
4010 
4011 	if (!its) {
4012 		list_for_each_entry(its, &its_nodes, entry) {
4013 			if (is_v4_1(its))
4014 				return its;
4015 		}
4016 
4017 		/* Oops? */
4018 		its = NULL;
4019 	}
4020 
4021 	return its;
4022 }
4023 
4024 static void its_vpe_4_1_send_inv(struct irq_data *d)
4025 {
4026 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4027 	struct its_node *its;
4028 
4029 	/*
4030 	 * GICv4.1 wants doorbells to be invalidated using the
4031 	 * INVDB command in order to be broadcast to all RDs. Send
4032 	 * it to the first valid ITS, and let the HW do its magic.
4033 	 */
4034 	its = find_4_1_its();
4035 	if (its)
4036 		its_send_invdb(its, vpe);
4037 }
4038 
4039 static void its_vpe_4_1_mask_irq(struct irq_data *d)
4040 {
4041 	lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
4042 	its_vpe_4_1_send_inv(d);
4043 }
4044 
4045 static void its_vpe_4_1_unmask_irq(struct irq_data *d)
4046 {
4047 	lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
4048 	its_vpe_4_1_send_inv(d);
4049 }
4050 
4051 static void its_vpe_4_1_schedule(struct its_vpe *vpe,
4052 				 struct its_cmd_info *info)
4053 {
4054 	void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
4055 	u64 val = 0;
4056 
4057 	/* Schedule the VPE */
4058 	val |= GICR_VPENDBASER_Valid;
4059 	val |= info->g0en ? GICR_VPENDBASER_4_1_VGRP0EN : 0;
4060 	val |= info->g1en ? GICR_VPENDBASER_4_1_VGRP1EN : 0;
4061 	val |= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id);
4062 
4063 	gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
4064 }
4065 
4066 static void its_vpe_4_1_deschedule(struct its_vpe *vpe,
4067 				   struct its_cmd_info *info)
4068 {
4069 	void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
4070 	u64 val;
4071 
4072 	if (info->req_db) {
4073 		unsigned long flags;
4074 
4075 		/*
4076 		 * vPE is going to block: make the vPE non-resident with
4077 		 * PendingLast clear and DB set. The GIC guarantees that if
4078 		 * we read-back PendingLast clear, then a doorbell will be
4079 		 * delivered when an interrupt comes.
4080 		 *
4081 		 * Note the locking to deal with the concurrent update of
4082 		 * pending_last from the doorbell interrupt handler that can
4083 		 * run concurrently.
4084 		 */
4085 		raw_spin_lock_irqsave(&vpe->vpe_lock, flags);
4086 		val = its_clear_vpend_valid(vlpi_base,
4087 					    GICR_VPENDBASER_PendingLast,
4088 					    GICR_VPENDBASER_4_1_DB);
4089 		vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
4090 		raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
4091 	} else {
4092 		/*
4093 		 * We're not blocking, so just make the vPE non-resident
4094 		 * with PendingLast set, indicating that we'll be back.
4095 		 */
4096 		val = its_clear_vpend_valid(vlpi_base,
4097 					    0,
4098 					    GICR_VPENDBASER_PendingLast);
4099 		vpe->pending_last = true;
4100 	}
4101 }
4102 
4103 static void its_vpe_4_1_invall(struct its_vpe *vpe)
4104 {
4105 	void __iomem *rdbase;
4106 	unsigned long flags;
4107 	u64 val;
4108 	int cpu;
4109 
4110 	val  = GICR_INVALLR_V;
4111 	val |= FIELD_PREP(GICR_INVALLR_VPEID, vpe->vpe_id);
4112 
4113 	/* Target the redistributor this vPE is currently known on */
4114 	cpu = vpe_to_cpuid_lock(vpe, &flags);
4115 	raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
4116 	rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
4117 	gic_write_lpir(val, rdbase + GICR_INVALLR);
4118 
4119 	wait_for_syncr(rdbase);
4120 	raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
4121 	vpe_to_cpuid_unlock(vpe, flags);
4122 }
4123 
4124 static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
4125 {
4126 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4127 	struct its_cmd_info *info = vcpu_info;
4128 
4129 	switch (info->cmd_type) {
4130 	case SCHEDULE_VPE:
4131 		its_vpe_4_1_schedule(vpe, info);
4132 		return 0;
4133 
4134 	case DESCHEDULE_VPE:
4135 		its_vpe_4_1_deschedule(vpe, info);
4136 		return 0;
4137 
4138 	case COMMIT_VPE:
4139 		its_wait_vpt_parse_complete();
4140 		return 0;
4141 
4142 	case INVALL_VPE:
4143 		its_vpe_4_1_invall(vpe);
4144 		return 0;
4145 
4146 	default:
4147 		return -EINVAL;
4148 	}
4149 }
4150 
4151 static struct irq_chip its_vpe_4_1_irq_chip = {
4152 	.name			= "GICv4.1-vpe",
4153 	.irq_mask		= its_vpe_4_1_mask_irq,
4154 	.irq_unmask		= its_vpe_4_1_unmask_irq,
4155 	.irq_eoi		= irq_chip_eoi_parent,
4156 	.irq_set_affinity	= its_vpe_set_affinity,
4157 	.irq_set_vcpu_affinity	= its_vpe_4_1_set_vcpu_affinity,
4158 };
4159 
4160 static void its_configure_sgi(struct irq_data *d, bool clear)
4161 {
4162 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4163 	struct its_cmd_desc desc;
4164 
4165 	desc.its_vsgi_cmd.vpe = vpe;
4166 	desc.its_vsgi_cmd.sgi = d->hwirq;
4167 	desc.its_vsgi_cmd.priority = vpe->sgi_config[d->hwirq].priority;
4168 	desc.its_vsgi_cmd.enable = vpe->sgi_config[d->hwirq].enabled;
4169 	desc.its_vsgi_cmd.group = vpe->sgi_config[d->hwirq].group;
4170 	desc.its_vsgi_cmd.clear = clear;
4171 
4172 	/*
4173 	 * GICv4.1 allows us to send VSGI commands to any ITS as long as the
4174 	 * destination VPE is mapped there. Since we map them eagerly at
4175 	 * activation time, we're pretty sure the first GICv4.1 ITS will do.
4176 	 */
4177 	its_send_single_vcommand(find_4_1_its(), its_build_vsgi_cmd, &desc);
4178 }
4179 
4180 static void its_sgi_mask_irq(struct irq_data *d)
4181 {
4182 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4183 
4184 	vpe->sgi_config[d->hwirq].enabled = false;
4185 	its_configure_sgi(d, false);
4186 }
4187 
4188 static void its_sgi_unmask_irq(struct irq_data *d)
4189 {
4190 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4191 
4192 	vpe->sgi_config[d->hwirq].enabled = true;
4193 	its_configure_sgi(d, false);
4194 }
4195 
4196 static int its_sgi_set_affinity(struct irq_data *d,
4197 				const struct cpumask *mask_val,
4198 				bool force)
4199 {
4200 	/*
4201 	 * There is no notion of affinity for virtual SGIs, at least
4202 	 * not on the host (since they can only be targeting a vPE).
4203 	 * Tell the kernel we've done whatever it asked for.
4204 	 */
4205 	irq_data_update_effective_affinity(d, mask_val);
4206 	return IRQ_SET_MASK_OK;
4207 }
4208 
4209 static int its_sgi_set_irqchip_state(struct irq_data *d,
4210 				     enum irqchip_irq_state which,
4211 				     bool state)
4212 {
4213 	if (which != IRQCHIP_STATE_PENDING)
4214 		return -EINVAL;
4215 
4216 	if (state) {
4217 		struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4218 		struct its_node *its = find_4_1_its();
4219 		u64 val;
4220 
4221 		val  = FIELD_PREP(GITS_SGIR_VPEID, vpe->vpe_id);
4222 		val |= FIELD_PREP(GITS_SGIR_VINTID, d->hwirq);
4223 		writeq_relaxed(val, its->sgir_base + GITS_SGIR - SZ_128K);
4224 	} else {
4225 		its_configure_sgi(d, true);
4226 	}
4227 
4228 	return 0;
4229 }
4230 
4231 static int its_sgi_get_irqchip_state(struct irq_data *d,
4232 				     enum irqchip_irq_state which, bool *val)
4233 {
4234 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4235 	void __iomem *base;
4236 	unsigned long flags;
4237 	u32 count = 1000000;	/* 1s! */
4238 	u32 status;
4239 	int cpu;
4240 
4241 	if (which != IRQCHIP_STATE_PENDING)
4242 		return -EINVAL;
4243 
4244 	/*
4245 	 * Locking galore! We can race against two different events:
4246 	 *
4247 	 * - Concurrent vPE affinity change: we must make sure it cannot
4248 	 *   happen, or we'll talk to the wrong redistributor. This is
4249 	 *   identical to what happens with vLPIs.
4250 	 *
4251 	 * - Concurrent VSGIPENDR access: As it involves accessing two
4252 	 *   MMIO registers, this must be made atomic one way or another.
4253 	 */
4254 	cpu = vpe_to_cpuid_lock(vpe, &flags);
4255 	raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
4256 	base = gic_data_rdist_cpu(cpu)->rd_base + SZ_128K;
4257 	writel_relaxed(vpe->vpe_id, base + GICR_VSGIR);
4258 	do {
4259 		status = readl_relaxed(base + GICR_VSGIPENDR);
4260 		if (!(status & GICR_VSGIPENDR_BUSY))
4261 			goto out;
4262 
4263 		count--;
4264 		if (!count) {
4265 			pr_err_ratelimited("Unable to get SGI status\n");
4266 			goto out;
4267 		}
4268 		cpu_relax();
4269 		udelay(1);
4270 	} while (count);
4271 
4272 out:
4273 	raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
4274 	vpe_to_cpuid_unlock(vpe, flags);
4275 
4276 	if (!count)
4277 		return -ENXIO;
4278 
4279 	*val = !!(status & (1 << d->hwirq));
4280 
4281 	return 0;
4282 }
4283 
4284 static int its_sgi_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
4285 {
4286 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4287 	struct its_cmd_info *info = vcpu_info;
4288 
4289 	switch (info->cmd_type) {
4290 	case PROP_UPDATE_VSGI:
4291 		vpe->sgi_config[d->hwirq].priority = info->priority;
4292 		vpe->sgi_config[d->hwirq].group = info->group;
4293 		its_configure_sgi(d, false);
4294 		return 0;
4295 
4296 	default:
4297 		return -EINVAL;
4298 	}
4299 }
4300 
4301 static struct irq_chip its_sgi_irq_chip = {
4302 	.name			= "GICv4.1-sgi",
4303 	.irq_mask		= its_sgi_mask_irq,
4304 	.irq_unmask		= its_sgi_unmask_irq,
4305 	.irq_set_affinity	= its_sgi_set_affinity,
4306 	.irq_set_irqchip_state	= its_sgi_set_irqchip_state,
4307 	.irq_get_irqchip_state	= its_sgi_get_irqchip_state,
4308 	.irq_set_vcpu_affinity	= its_sgi_set_vcpu_affinity,
4309 };
4310 
4311 static int its_sgi_irq_domain_alloc(struct irq_domain *domain,
4312 				    unsigned int virq, unsigned int nr_irqs,
4313 				    void *args)
4314 {
4315 	struct its_vpe *vpe = args;
4316 	int i;
4317 
4318 	/* Yes, we do want 16 SGIs */
4319 	WARN_ON(nr_irqs != 16);
4320 
4321 	for (i = 0; i < 16; i++) {
4322 		vpe->sgi_config[i].priority = 0;
4323 		vpe->sgi_config[i].enabled = false;
4324 		vpe->sgi_config[i].group = false;
4325 
4326 		irq_domain_set_hwirq_and_chip(domain, virq + i, i,
4327 					      &its_sgi_irq_chip, vpe);
4328 		irq_set_status_flags(virq + i, IRQ_DISABLE_UNLAZY);
4329 	}
4330 
4331 	return 0;
4332 }
4333 
4334 static void its_sgi_irq_domain_free(struct irq_domain *domain,
4335 				    unsigned int virq,
4336 				    unsigned int nr_irqs)
4337 {
4338 	/* Nothing to do */
4339 }
4340 
4341 static int its_sgi_irq_domain_activate(struct irq_domain *domain,
4342 				       struct irq_data *d, bool reserve)
4343 {
4344 	/* Write out the initial SGI configuration */
4345 	its_configure_sgi(d, false);
4346 	return 0;
4347 }
4348 
4349 static void its_sgi_irq_domain_deactivate(struct irq_domain *domain,
4350 					  struct irq_data *d)
4351 {
4352 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4353 
4354 	/*
4355 	 * The VSGI command is awkward:
4356 	 *
4357 	 * - To change the configuration, CLEAR must be set to false,
4358 	 *   leaving the pending bit unchanged.
4359 	 * - To clear the pending bit, CLEAR must be set to true, leaving
4360 	 *   the configuration unchanged.
4361 	 *
4362 	 * You just can't do both at once, hence the two commands below.
4363 	 */
4364 	vpe->sgi_config[d->hwirq].enabled = false;
4365 	its_configure_sgi(d, false);
4366 	its_configure_sgi(d, true);
4367 }
4368 
4369 static const struct irq_domain_ops its_sgi_domain_ops = {
4370 	.alloc		= its_sgi_irq_domain_alloc,
4371 	.free		= its_sgi_irq_domain_free,
4372 	.activate	= its_sgi_irq_domain_activate,
4373 	.deactivate	= its_sgi_irq_domain_deactivate,
4374 };
4375 
4376 static int its_vpe_id_alloc(void)
4377 {
4378 	return ida_simple_get(&its_vpeid_ida, 0, ITS_MAX_VPEID, GFP_KERNEL);
4379 }
4380 
4381 static void its_vpe_id_free(u16 id)
4382 {
4383 	ida_simple_remove(&its_vpeid_ida, id);
4384 }
4385 
4386 static int its_vpe_init(struct its_vpe *vpe)
4387 {
4388 	struct page *vpt_page;
4389 	int vpe_id;
4390 
4391 	/* Allocate vpe_id */
4392 	vpe_id = its_vpe_id_alloc();
4393 	if (vpe_id < 0)
4394 		return vpe_id;
4395 
4396 	/* Allocate VPT */
4397 	vpt_page = its_allocate_pending_table(GFP_KERNEL);
4398 	if (!vpt_page) {
4399 		its_vpe_id_free(vpe_id);
4400 		return -ENOMEM;
4401 	}
4402 
4403 	if (!its_alloc_vpe_table(vpe_id)) {
4404 		its_vpe_id_free(vpe_id);
4405 		its_free_pending_table(vpt_page);
4406 		return -ENOMEM;
4407 	}
4408 
4409 	raw_spin_lock_init(&vpe->vpe_lock);
4410 	vpe->vpe_id = vpe_id;
4411 	vpe->vpt_page = vpt_page;
4412 	if (gic_rdists->has_rvpeid)
4413 		atomic_set(&vpe->vmapp_count, 0);
4414 	else
4415 		vpe->vpe_proxy_event = -1;
4416 
4417 	return 0;
4418 }
4419 
4420 static void its_vpe_teardown(struct its_vpe *vpe)
4421 {
4422 	its_vpe_db_proxy_unmap(vpe);
4423 	its_vpe_id_free(vpe->vpe_id);
4424 	its_free_pending_table(vpe->vpt_page);
4425 }
4426 
4427 static void its_vpe_irq_domain_free(struct irq_domain *domain,
4428 				    unsigned int virq,
4429 				    unsigned int nr_irqs)
4430 {
4431 	struct its_vm *vm = domain->host_data;
4432 	int i;
4433 
4434 	irq_domain_free_irqs_parent(domain, virq, nr_irqs);
4435 
4436 	for (i = 0; i < nr_irqs; i++) {
4437 		struct irq_data *data = irq_domain_get_irq_data(domain,
4438 								virq + i);
4439 		struct its_vpe *vpe = irq_data_get_irq_chip_data(data);
4440 
4441 		BUG_ON(vm != vpe->its_vm);
4442 
4443 		clear_bit(data->hwirq, vm->db_bitmap);
4444 		its_vpe_teardown(vpe);
4445 		irq_domain_reset_irq_data(data);
4446 	}
4447 
4448 	if (bitmap_empty(vm->db_bitmap, vm->nr_db_lpis)) {
4449 		its_lpi_free(vm->db_bitmap, vm->db_lpi_base, vm->nr_db_lpis);
4450 		its_free_prop_table(vm->vprop_page);
4451 	}
4452 }
4453 
4454 static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
4455 				    unsigned int nr_irqs, void *args)
4456 {
4457 	struct irq_chip *irqchip = &its_vpe_irq_chip;
4458 	struct its_vm *vm = args;
4459 	unsigned long *bitmap;
4460 	struct page *vprop_page;
4461 	int base, nr_ids, i, err = 0;
4462 
4463 	BUG_ON(!vm);
4464 
4465 	bitmap = its_lpi_alloc(roundup_pow_of_two(nr_irqs), &base, &nr_ids);
4466 	if (!bitmap)
4467 		return -ENOMEM;
4468 
4469 	if (nr_ids < nr_irqs) {
4470 		its_lpi_free(bitmap, base, nr_ids);
4471 		return -ENOMEM;
4472 	}
4473 
4474 	vprop_page = its_allocate_prop_table(GFP_KERNEL);
4475 	if (!vprop_page) {
4476 		its_lpi_free(bitmap, base, nr_ids);
4477 		return -ENOMEM;
4478 	}
4479 
4480 	vm->db_bitmap = bitmap;
4481 	vm->db_lpi_base = base;
4482 	vm->nr_db_lpis = nr_ids;
4483 	vm->vprop_page = vprop_page;
4484 
4485 	if (gic_rdists->has_rvpeid)
4486 		irqchip = &its_vpe_4_1_irq_chip;
4487 
4488 	for (i = 0; i < nr_irqs; i++) {
4489 		vm->vpes[i]->vpe_db_lpi = base + i;
4490 		err = its_vpe_init(vm->vpes[i]);
4491 		if (err)
4492 			break;
4493 		err = its_irq_gic_domain_alloc(domain, virq + i,
4494 					       vm->vpes[i]->vpe_db_lpi);
4495 		if (err)
4496 			break;
4497 		irq_domain_set_hwirq_and_chip(domain, virq + i, i,
4498 					      irqchip, vm->vpes[i]);
4499 		set_bit(i, bitmap);
4500 	}
4501 
4502 	if (err) {
4503 		if (i > 0)
4504 			its_vpe_irq_domain_free(domain, virq, i);
4505 
4506 		its_lpi_free(bitmap, base, nr_ids);
4507 		its_free_prop_table(vprop_page);
4508 	}
4509 
4510 	return err;
4511 }
4512 
4513 static int its_vpe_irq_domain_activate(struct irq_domain *domain,
4514 				       struct irq_data *d, bool reserve)
4515 {
4516 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4517 	struct its_node *its;
4518 
4519 	/*
4520 	 * If we use the list map, we issue VMAPP on demand... Unless
4521 	 * we're on a GICv4.1 and we eagerly map the VPE on all ITSs
4522 	 * so that VSGIs can work.
4523 	 */
4524 	if (!gic_requires_eager_mapping())
4525 		return 0;
4526 
4527 	/* Map the VPE to the first possible CPU */
4528 	vpe->col_idx = cpumask_first(cpu_online_mask);
4529 
4530 	list_for_each_entry(its, &its_nodes, entry) {
4531 		if (!is_v4(its))
4532 			continue;
4533 
4534 		its_send_vmapp(its, vpe, true);
4535 		its_send_vinvall(its, vpe);
4536 	}
4537 
4538 	irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
4539 
4540 	return 0;
4541 }
4542 
4543 static void its_vpe_irq_domain_deactivate(struct irq_domain *domain,
4544 					  struct irq_data *d)
4545 {
4546 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4547 	struct its_node *its;
4548 
4549 	/*
4550 	 * If we use the list map on GICv4.0, we unmap the VPE once no
4551 	 * VLPIs are associated with the VM.
4552 	 */
4553 	if (!gic_requires_eager_mapping())
4554 		return;
4555 
4556 	list_for_each_entry(its, &its_nodes, entry) {
4557 		if (!is_v4(its))
4558 			continue;
4559 
4560 		its_send_vmapp(its, vpe, false);
4561 	}
4562 
4563 	/*
4564 	 * There may be a direct read to the VPT after unmapping the
4565 	 * vPE, to guarantee the validity of this, we make the VPT
4566 	 * memory coherent with the CPU caches here.
4567 	 */
4568 	if (find_4_1_its() && !atomic_read(&vpe->vmapp_count))
4569 		gic_flush_dcache_to_poc(page_address(vpe->vpt_page),
4570 					LPI_PENDBASE_SZ);
4571 }
4572 
4573 static const struct irq_domain_ops its_vpe_domain_ops = {
4574 	.alloc			= its_vpe_irq_domain_alloc,
4575 	.free			= its_vpe_irq_domain_free,
4576 	.activate		= its_vpe_irq_domain_activate,
4577 	.deactivate		= its_vpe_irq_domain_deactivate,
4578 };
4579 
4580 static int its_force_quiescent(void __iomem *base)
4581 {
4582 	u32 count = 1000000;	/* 1s */
4583 	u32 val;
4584 
4585 	val = readl_relaxed(base + GITS_CTLR);
4586 	/*
4587 	 * GIC architecture specification requires the ITS to be both
4588 	 * disabled and quiescent for writes to GITS_BASER<n> or
4589 	 * GITS_CBASER to not have UNPREDICTABLE results.
4590 	 */
4591 	if ((val & GITS_CTLR_QUIESCENT) && !(val & GITS_CTLR_ENABLE))
4592 		return 0;
4593 
4594 	/* Disable the generation of all interrupts to this ITS */
4595 	val &= ~(GITS_CTLR_ENABLE | GITS_CTLR_ImDe);
4596 	writel_relaxed(val, base + GITS_CTLR);
4597 
4598 	/* Poll GITS_CTLR and wait until ITS becomes quiescent */
4599 	while (1) {
4600 		val = readl_relaxed(base + GITS_CTLR);
4601 		if (val & GITS_CTLR_QUIESCENT)
4602 			return 0;
4603 
4604 		count--;
4605 		if (!count)
4606 			return -EBUSY;
4607 
4608 		cpu_relax();
4609 		udelay(1);
4610 	}
4611 }
4612 
4613 static bool __maybe_unused its_enable_quirk_cavium_22375(void *data)
4614 {
4615 	struct its_node *its = data;
4616 
4617 	/* erratum 22375: only alloc 8MB table size (20 bits) */
4618 	its->typer &= ~GITS_TYPER_DEVBITS;
4619 	its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, 20 - 1);
4620 	its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375;
4621 
4622 	return true;
4623 }
4624 
4625 static bool __maybe_unused its_enable_quirk_cavium_23144(void *data)
4626 {
4627 	struct its_node *its = data;
4628 
4629 	its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_23144;
4630 
4631 	return true;
4632 }
4633 
4634 static bool __maybe_unused its_enable_quirk_qdf2400_e0065(void *data)
4635 {
4636 	struct its_node *its = data;
4637 
4638 	/* On QDF2400, the size of the ITE is 16Bytes */
4639 	its->typer &= ~GITS_TYPER_ITT_ENTRY_SIZE;
4640 	its->typer |= FIELD_PREP(GITS_TYPER_ITT_ENTRY_SIZE, 16 - 1);
4641 
4642 	return true;
4643 }
4644 
4645 static u64 its_irq_get_msi_base_pre_its(struct its_device *its_dev)
4646 {
4647 	struct its_node *its = its_dev->its;
4648 
4649 	/*
4650 	 * The Socionext Synquacer SoC has a so-called 'pre-ITS',
4651 	 * which maps 32-bit writes targeted at a separate window of
4652 	 * size '4 << device_id_bits' onto writes to GITS_TRANSLATER
4653 	 * with device ID taken from bits [device_id_bits + 1:2] of
4654 	 * the window offset.
4655 	 */
4656 	return its->pre_its_base + (its_dev->device_id << 2);
4657 }
4658 
4659 static bool __maybe_unused its_enable_quirk_socionext_synquacer(void *data)
4660 {
4661 	struct its_node *its = data;
4662 	u32 pre_its_window[2];
4663 	u32 ids;
4664 
4665 	if (!fwnode_property_read_u32_array(its->fwnode_handle,
4666 					   "socionext,synquacer-pre-its",
4667 					   pre_its_window,
4668 					   ARRAY_SIZE(pre_its_window))) {
4669 
4670 		its->pre_its_base = pre_its_window[0];
4671 		its->get_msi_base = its_irq_get_msi_base_pre_its;
4672 
4673 		ids = ilog2(pre_its_window[1]) - 2;
4674 		if (device_ids(its) > ids) {
4675 			its->typer &= ~GITS_TYPER_DEVBITS;
4676 			its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, ids - 1);
4677 		}
4678 
4679 		/* the pre-ITS breaks isolation, so disable MSI remapping */
4680 		its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_MSI_REMAP;
4681 		return true;
4682 	}
4683 	return false;
4684 }
4685 
4686 static bool __maybe_unused its_enable_quirk_hip07_161600802(void *data)
4687 {
4688 	struct its_node *its = data;
4689 
4690 	/*
4691 	 * Hip07 insists on using the wrong address for the VLPI
4692 	 * page. Trick it into doing the right thing...
4693 	 */
4694 	its->vlpi_redist_offset = SZ_128K;
4695 	return true;
4696 }
4697 
4698 static const struct gic_quirk its_quirks[] = {
4699 #ifdef CONFIG_CAVIUM_ERRATUM_22375
4700 	{
4701 		.desc	= "ITS: Cavium errata 22375, 24313",
4702 		.iidr	= 0xa100034c,	/* ThunderX pass 1.x */
4703 		.mask	= 0xffff0fff,
4704 		.init	= its_enable_quirk_cavium_22375,
4705 	},
4706 #endif
4707 #ifdef CONFIG_CAVIUM_ERRATUM_23144
4708 	{
4709 		.desc	= "ITS: Cavium erratum 23144",
4710 		.iidr	= 0xa100034c,	/* ThunderX pass 1.x */
4711 		.mask	= 0xffff0fff,
4712 		.init	= its_enable_quirk_cavium_23144,
4713 	},
4714 #endif
4715 #ifdef CONFIG_QCOM_QDF2400_ERRATUM_0065
4716 	{
4717 		.desc	= "ITS: QDF2400 erratum 0065",
4718 		.iidr	= 0x00001070, /* QDF2400 ITS rev 1.x */
4719 		.mask	= 0xffffffff,
4720 		.init	= its_enable_quirk_qdf2400_e0065,
4721 	},
4722 #endif
4723 #ifdef CONFIG_SOCIONEXT_SYNQUACER_PREITS
4724 	{
4725 		/*
4726 		 * The Socionext Synquacer SoC incorporates ARM's own GIC-500
4727 		 * implementation, but with a 'pre-ITS' added that requires
4728 		 * special handling in software.
4729 		 */
4730 		.desc	= "ITS: Socionext Synquacer pre-ITS",
4731 		.iidr	= 0x0001143b,
4732 		.mask	= 0xffffffff,
4733 		.init	= its_enable_quirk_socionext_synquacer,
4734 	},
4735 #endif
4736 #ifdef CONFIG_HISILICON_ERRATUM_161600802
4737 	{
4738 		.desc	= "ITS: Hip07 erratum 161600802",
4739 		.iidr	= 0x00000004,
4740 		.mask	= 0xffffffff,
4741 		.init	= its_enable_quirk_hip07_161600802,
4742 	},
4743 #endif
4744 	{
4745 	}
4746 };
4747 
4748 static void its_enable_quirks(struct its_node *its)
4749 {
4750 	u32 iidr = readl_relaxed(its->base + GITS_IIDR);
4751 
4752 	gic_enable_quirks(iidr, its_quirks, its);
4753 }
4754 
4755 static int its_save_disable(void)
4756 {
4757 	struct its_node *its;
4758 	int err = 0;
4759 
4760 	raw_spin_lock(&its_lock);
4761 	list_for_each_entry(its, &its_nodes, entry) {
4762 		void __iomem *base;
4763 
4764 		base = its->base;
4765 		its->ctlr_save = readl_relaxed(base + GITS_CTLR);
4766 		err = its_force_quiescent(base);
4767 		if (err) {
4768 			pr_err("ITS@%pa: failed to quiesce: %d\n",
4769 			       &its->phys_base, err);
4770 			writel_relaxed(its->ctlr_save, base + GITS_CTLR);
4771 			goto err;
4772 		}
4773 
4774 		its->cbaser_save = gits_read_cbaser(base + GITS_CBASER);
4775 	}
4776 
4777 err:
4778 	if (err) {
4779 		list_for_each_entry_continue_reverse(its, &its_nodes, entry) {
4780 			void __iomem *base;
4781 
4782 			base = its->base;
4783 			writel_relaxed(its->ctlr_save, base + GITS_CTLR);
4784 		}
4785 	}
4786 	raw_spin_unlock(&its_lock);
4787 
4788 	return err;
4789 }
4790 
4791 static void its_restore_enable(void)
4792 {
4793 	struct its_node *its;
4794 	int ret;
4795 
4796 	raw_spin_lock(&its_lock);
4797 	list_for_each_entry(its, &its_nodes, entry) {
4798 		void __iomem *base;
4799 		int i;
4800 
4801 		base = its->base;
4802 
4803 		/*
4804 		 * Make sure that the ITS is disabled. If it fails to quiesce,
4805 		 * don't restore it since writing to CBASER or BASER<n>
4806 		 * registers is undefined according to the GIC v3 ITS
4807 		 * Specification.
4808 		 *
4809 		 * Firmware resuming with the ITS enabled is terminally broken.
4810 		 */
4811 		WARN_ON(readl_relaxed(base + GITS_CTLR) & GITS_CTLR_ENABLE);
4812 		ret = its_force_quiescent(base);
4813 		if (ret) {
4814 			pr_err("ITS@%pa: failed to quiesce on resume: %d\n",
4815 			       &its->phys_base, ret);
4816 			continue;
4817 		}
4818 
4819 		gits_write_cbaser(its->cbaser_save, base + GITS_CBASER);
4820 
4821 		/*
4822 		 * Writing CBASER resets CREADR to 0, so make CWRITER and
4823 		 * cmd_write line up with it.
4824 		 */
4825 		its->cmd_write = its->cmd_base;
4826 		gits_write_cwriter(0, base + GITS_CWRITER);
4827 
4828 		/* Restore GITS_BASER from the value cache. */
4829 		for (i = 0; i < GITS_BASER_NR_REGS; i++) {
4830 			struct its_baser *baser = &its->tables[i];
4831 
4832 			if (!(baser->val & GITS_BASER_VALID))
4833 				continue;
4834 
4835 			its_write_baser(its, baser, baser->val);
4836 		}
4837 		writel_relaxed(its->ctlr_save, base + GITS_CTLR);
4838 
4839 		/*
4840 		 * Reinit the collection if it's stored in the ITS. This is
4841 		 * indicated by the col_id being less than the HCC field.
4842 		 * CID < HCC as specified in the GIC v3 Documentation.
4843 		 */
4844 		if (its->collections[smp_processor_id()].col_id <
4845 		    GITS_TYPER_HCC(gic_read_typer(base + GITS_TYPER)))
4846 			its_cpu_init_collection(its);
4847 	}
4848 	raw_spin_unlock(&its_lock);
4849 }
4850 
4851 static struct syscore_ops its_syscore_ops = {
4852 	.suspend = its_save_disable,
4853 	.resume = its_restore_enable,
4854 };
4855 
4856 static int its_init_domain(struct fwnode_handle *handle, struct its_node *its)
4857 {
4858 	struct irq_domain *inner_domain;
4859 	struct msi_domain_info *info;
4860 
4861 	info = kzalloc(sizeof(*info), GFP_KERNEL);
4862 	if (!info)
4863 		return -ENOMEM;
4864 
4865 	inner_domain = irq_domain_create_tree(handle, &its_domain_ops, its);
4866 	if (!inner_domain) {
4867 		kfree(info);
4868 		return -ENOMEM;
4869 	}
4870 
4871 	inner_domain->parent = its_parent;
4872 	irq_domain_update_bus_token(inner_domain, DOMAIN_BUS_NEXUS);
4873 	inner_domain->flags |= its->msi_domain_flags;
4874 	info->ops = &its_msi_domain_ops;
4875 	info->data = its;
4876 	inner_domain->host_data = info;
4877 
4878 	return 0;
4879 }
4880 
4881 static int its_init_vpe_domain(void)
4882 {
4883 	struct its_node *its;
4884 	u32 devid;
4885 	int entries;
4886 
4887 	if (gic_rdists->has_direct_lpi) {
4888 		pr_info("ITS: Using DirectLPI for VPE invalidation\n");
4889 		return 0;
4890 	}
4891 
4892 	/* Any ITS will do, even if not v4 */
4893 	its = list_first_entry(&its_nodes, struct its_node, entry);
4894 
4895 	entries = roundup_pow_of_two(nr_cpu_ids);
4896 	vpe_proxy.vpes = kcalloc(entries, sizeof(*vpe_proxy.vpes),
4897 				 GFP_KERNEL);
4898 	if (!vpe_proxy.vpes)
4899 		return -ENOMEM;
4900 
4901 	/* Use the last possible DevID */
4902 	devid = GENMASK(device_ids(its) - 1, 0);
4903 	vpe_proxy.dev = its_create_device(its, devid, entries, false);
4904 	if (!vpe_proxy.dev) {
4905 		kfree(vpe_proxy.vpes);
4906 		pr_err("ITS: Can't allocate GICv4 proxy device\n");
4907 		return -ENOMEM;
4908 	}
4909 
4910 	BUG_ON(entries > vpe_proxy.dev->nr_ites);
4911 
4912 	raw_spin_lock_init(&vpe_proxy.lock);
4913 	vpe_proxy.next_victim = 0;
4914 	pr_info("ITS: Allocated DevID %x as GICv4 proxy device (%d slots)\n",
4915 		devid, vpe_proxy.dev->nr_ites);
4916 
4917 	return 0;
4918 }
4919 
4920 static int __init its_compute_its_list_map(struct resource *res,
4921 					   void __iomem *its_base)
4922 {
4923 	int its_number;
4924 	u32 ctlr;
4925 
4926 	/*
4927 	 * This is assumed to be done early enough that we're
4928 	 * guaranteed to be single-threaded, hence no
4929 	 * locking. Should this change, we should address
4930 	 * this.
4931 	 */
4932 	its_number = find_first_zero_bit(&its_list_map, GICv4_ITS_LIST_MAX);
4933 	if (its_number >= GICv4_ITS_LIST_MAX) {
4934 		pr_err("ITS@%pa: No ITSList entry available!\n",
4935 		       &res->start);
4936 		return -EINVAL;
4937 	}
4938 
4939 	ctlr = readl_relaxed(its_base + GITS_CTLR);
4940 	ctlr &= ~GITS_CTLR_ITS_NUMBER;
4941 	ctlr |= its_number << GITS_CTLR_ITS_NUMBER_SHIFT;
4942 	writel_relaxed(ctlr, its_base + GITS_CTLR);
4943 	ctlr = readl_relaxed(its_base + GITS_CTLR);
4944 	if ((ctlr & GITS_CTLR_ITS_NUMBER) != (its_number << GITS_CTLR_ITS_NUMBER_SHIFT)) {
4945 		its_number = ctlr & GITS_CTLR_ITS_NUMBER;
4946 		its_number >>= GITS_CTLR_ITS_NUMBER_SHIFT;
4947 	}
4948 
4949 	if (test_and_set_bit(its_number, &its_list_map)) {
4950 		pr_err("ITS@%pa: Duplicate ITSList entry %d\n",
4951 		       &res->start, its_number);
4952 		return -EINVAL;
4953 	}
4954 
4955 	return its_number;
4956 }
4957 
4958 static int __init its_probe_one(struct resource *res,
4959 				struct fwnode_handle *handle, int numa_node)
4960 {
4961 	struct its_node *its;
4962 	void __iomem *its_base;
4963 	u32 val, ctlr;
4964 	u64 baser, tmp, typer;
4965 	struct page *page;
4966 	int err;
4967 
4968 	its_base = ioremap(res->start, SZ_64K);
4969 	if (!its_base) {
4970 		pr_warn("ITS@%pa: Unable to map ITS registers\n", &res->start);
4971 		return -ENOMEM;
4972 	}
4973 
4974 	val = readl_relaxed(its_base + GITS_PIDR2) & GIC_PIDR2_ARCH_MASK;
4975 	if (val != 0x30 && val != 0x40) {
4976 		pr_warn("ITS@%pa: No ITS detected, giving up\n", &res->start);
4977 		err = -ENODEV;
4978 		goto out_unmap;
4979 	}
4980 
4981 	err = its_force_quiescent(its_base);
4982 	if (err) {
4983 		pr_warn("ITS@%pa: Failed to quiesce, giving up\n", &res->start);
4984 		goto out_unmap;
4985 	}
4986 
4987 	pr_info("ITS %pR\n", res);
4988 
4989 	its = kzalloc(sizeof(*its), GFP_KERNEL);
4990 	if (!its) {
4991 		err = -ENOMEM;
4992 		goto out_unmap;
4993 	}
4994 
4995 	raw_spin_lock_init(&its->lock);
4996 	mutex_init(&its->dev_alloc_lock);
4997 	INIT_LIST_HEAD(&its->entry);
4998 	INIT_LIST_HEAD(&its->its_device_list);
4999 	typer = gic_read_typer(its_base + GITS_TYPER);
5000 	its->typer = typer;
5001 	its->base = its_base;
5002 	its->phys_base = res->start;
5003 	if (is_v4(its)) {
5004 		if (!(typer & GITS_TYPER_VMOVP)) {
5005 			err = its_compute_its_list_map(res, its_base);
5006 			if (err < 0)
5007 				goto out_free_its;
5008 
5009 			its->list_nr = err;
5010 
5011 			pr_info("ITS@%pa: Using ITS number %d\n",
5012 				&res->start, err);
5013 		} else {
5014 			pr_info("ITS@%pa: Single VMOVP capable\n", &res->start);
5015 		}
5016 
5017 		if (is_v4_1(its)) {
5018 			u32 svpet = FIELD_GET(GITS_TYPER_SVPET, typer);
5019 
5020 			its->sgir_base = ioremap(res->start + SZ_128K, SZ_64K);
5021 			if (!its->sgir_base) {
5022 				err = -ENOMEM;
5023 				goto out_free_its;
5024 			}
5025 
5026 			its->mpidr = readl_relaxed(its_base + GITS_MPIDR);
5027 
5028 			pr_info("ITS@%pa: Using GICv4.1 mode %08x %08x\n",
5029 				&res->start, its->mpidr, svpet);
5030 		}
5031 	}
5032 
5033 	its->numa_node = numa_node;
5034 
5035 	page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO,
5036 				get_order(ITS_CMD_QUEUE_SZ));
5037 	if (!page) {
5038 		err = -ENOMEM;
5039 		goto out_unmap_sgir;
5040 	}
5041 	its->cmd_base = (void *)page_address(page);
5042 	its->cmd_write = its->cmd_base;
5043 	its->fwnode_handle = handle;
5044 	its->get_msi_base = its_irq_get_msi_base;
5045 	its->msi_domain_flags = IRQ_DOMAIN_FLAG_MSI_REMAP;
5046 
5047 	its_enable_quirks(its);
5048 
5049 	err = its_alloc_tables(its);
5050 	if (err)
5051 		goto out_free_cmd;
5052 
5053 	err = its_alloc_collections(its);
5054 	if (err)
5055 		goto out_free_tables;
5056 
5057 	baser = (virt_to_phys(its->cmd_base)	|
5058 		 GITS_CBASER_RaWaWb		|
5059 		 GITS_CBASER_InnerShareable	|
5060 		 (ITS_CMD_QUEUE_SZ / SZ_4K - 1)	|
5061 		 GITS_CBASER_VALID);
5062 
5063 	gits_write_cbaser(baser, its->base + GITS_CBASER);
5064 	tmp = gits_read_cbaser(its->base + GITS_CBASER);
5065 
5066 	if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) {
5067 		if (!(tmp & GITS_CBASER_SHAREABILITY_MASK)) {
5068 			/*
5069 			 * The HW reports non-shareable, we must
5070 			 * remove the cacheability attributes as
5071 			 * well.
5072 			 */
5073 			baser &= ~(GITS_CBASER_SHAREABILITY_MASK |
5074 				   GITS_CBASER_CACHEABILITY_MASK);
5075 			baser |= GITS_CBASER_nC;
5076 			gits_write_cbaser(baser, its->base + GITS_CBASER);
5077 		}
5078 		pr_info("ITS: using cache flushing for cmd queue\n");
5079 		its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING;
5080 	}
5081 
5082 	gits_write_cwriter(0, its->base + GITS_CWRITER);
5083 	ctlr = readl_relaxed(its->base + GITS_CTLR);
5084 	ctlr |= GITS_CTLR_ENABLE;
5085 	if (is_v4(its))
5086 		ctlr |= GITS_CTLR_ImDe;
5087 	writel_relaxed(ctlr, its->base + GITS_CTLR);
5088 
5089 	err = its_init_domain(handle, its);
5090 	if (err)
5091 		goto out_free_tables;
5092 
5093 	raw_spin_lock(&its_lock);
5094 	list_add(&its->entry, &its_nodes);
5095 	raw_spin_unlock(&its_lock);
5096 
5097 	return 0;
5098 
5099 out_free_tables:
5100 	its_free_tables(its);
5101 out_free_cmd:
5102 	free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ));
5103 out_unmap_sgir:
5104 	if (its->sgir_base)
5105 		iounmap(its->sgir_base);
5106 out_free_its:
5107 	kfree(its);
5108 out_unmap:
5109 	iounmap(its_base);
5110 	pr_err("ITS@%pa: failed probing (%d)\n", &res->start, err);
5111 	return err;
5112 }
5113 
5114 static bool gic_rdists_supports_plpis(void)
5115 {
5116 	return !!(gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER) & GICR_TYPER_PLPIS);
5117 }
5118 
5119 static int redist_disable_lpis(void)
5120 {
5121 	void __iomem *rbase = gic_data_rdist_rd_base();
5122 	u64 timeout = USEC_PER_SEC;
5123 	u64 val;
5124 
5125 	if (!gic_rdists_supports_plpis()) {
5126 		pr_info("CPU%d: LPIs not supported\n", smp_processor_id());
5127 		return -ENXIO;
5128 	}
5129 
5130 	val = readl_relaxed(rbase + GICR_CTLR);
5131 	if (!(val & GICR_CTLR_ENABLE_LPIS))
5132 		return 0;
5133 
5134 	/*
5135 	 * If coming via a CPU hotplug event, we don't need to disable
5136 	 * LPIs before trying to re-enable them. They are already
5137 	 * configured and all is well in the world.
5138 	 *
5139 	 * If running with preallocated tables, there is nothing to do.
5140 	 */
5141 	if (gic_data_rdist()->lpi_enabled ||
5142 	    (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED))
5143 		return 0;
5144 
5145 	/*
5146 	 * From that point on, we only try to do some damage control.
5147 	 */
5148 	pr_warn("GICv3: CPU%d: Booted with LPIs enabled, memory probably corrupted\n",
5149 		smp_processor_id());
5150 	add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
5151 
5152 	/* Disable LPIs */
5153 	val &= ~GICR_CTLR_ENABLE_LPIS;
5154 	writel_relaxed(val, rbase + GICR_CTLR);
5155 
5156 	/* Make sure any change to GICR_CTLR is observable by the GIC */
5157 	dsb(sy);
5158 
5159 	/*
5160 	 * Software must observe RWP==0 after clearing GICR_CTLR.EnableLPIs
5161 	 * from 1 to 0 before programming GICR_PEND{PROP}BASER registers.
5162 	 * Error out if we time out waiting for RWP to clear.
5163 	 */
5164 	while (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_RWP) {
5165 		if (!timeout) {
5166 			pr_err("CPU%d: Timeout while disabling LPIs\n",
5167 			       smp_processor_id());
5168 			return -ETIMEDOUT;
5169 		}
5170 		udelay(1);
5171 		timeout--;
5172 	}
5173 
5174 	/*
5175 	 * After it has been written to 1, it is IMPLEMENTATION
5176 	 * DEFINED whether GICR_CTLR.EnableLPI becomes RES1 or can be
5177 	 * cleared to 0. Error out if clearing the bit failed.
5178 	 */
5179 	if (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_ENABLE_LPIS) {
5180 		pr_err("CPU%d: Failed to disable LPIs\n", smp_processor_id());
5181 		return -EBUSY;
5182 	}
5183 
5184 	return 0;
5185 }
5186 
5187 int its_cpu_init(void)
5188 {
5189 	if (!list_empty(&its_nodes)) {
5190 		int ret;
5191 
5192 		ret = redist_disable_lpis();
5193 		if (ret)
5194 			return ret;
5195 
5196 		its_cpu_init_lpis();
5197 		its_cpu_init_collections();
5198 	}
5199 
5200 	return 0;
5201 }
5202 
5203 static const struct of_device_id its_device_id[] = {
5204 	{	.compatible	= "arm,gic-v3-its",	},
5205 	{},
5206 };
5207 
5208 static int __init its_of_probe(struct device_node *node)
5209 {
5210 	struct device_node *np;
5211 	struct resource res;
5212 
5213 	for (np = of_find_matching_node(node, its_device_id); np;
5214 	     np = of_find_matching_node(np, its_device_id)) {
5215 		if (!of_device_is_available(np))
5216 			continue;
5217 		if (!of_property_read_bool(np, "msi-controller")) {
5218 			pr_warn("%pOF: no msi-controller property, ITS ignored\n",
5219 				np);
5220 			continue;
5221 		}
5222 
5223 		if (of_address_to_resource(np, 0, &res)) {
5224 			pr_warn("%pOF: no regs?\n", np);
5225 			continue;
5226 		}
5227 
5228 		its_probe_one(&res, &np->fwnode, of_node_to_nid(np));
5229 	}
5230 	return 0;
5231 }
5232 
5233 #ifdef CONFIG_ACPI
5234 
5235 #define ACPI_GICV3_ITS_MEM_SIZE (SZ_128K)
5236 
5237 #ifdef CONFIG_ACPI_NUMA
5238 struct its_srat_map {
5239 	/* numa node id */
5240 	u32	numa_node;
5241 	/* GIC ITS ID */
5242 	u32	its_id;
5243 };
5244 
5245 static struct its_srat_map *its_srat_maps __initdata;
5246 static int its_in_srat __initdata;
5247 
5248 static int __init acpi_get_its_numa_node(u32 its_id)
5249 {
5250 	int i;
5251 
5252 	for (i = 0; i < its_in_srat; i++) {
5253 		if (its_id == its_srat_maps[i].its_id)
5254 			return its_srat_maps[i].numa_node;
5255 	}
5256 	return NUMA_NO_NODE;
5257 }
5258 
5259 static int __init gic_acpi_match_srat_its(union acpi_subtable_headers *header,
5260 					  const unsigned long end)
5261 {
5262 	return 0;
5263 }
5264 
5265 static int __init gic_acpi_parse_srat_its(union acpi_subtable_headers *header,
5266 			 const unsigned long end)
5267 {
5268 	int node;
5269 	struct acpi_srat_gic_its_affinity *its_affinity;
5270 
5271 	its_affinity = (struct acpi_srat_gic_its_affinity *)header;
5272 	if (!its_affinity)
5273 		return -EINVAL;
5274 
5275 	if (its_affinity->header.length < sizeof(*its_affinity)) {
5276 		pr_err("SRAT: Invalid header length %d in ITS affinity\n",
5277 			its_affinity->header.length);
5278 		return -EINVAL;
5279 	}
5280 
5281 	/*
5282 	 * Note that in theory a new proximity node could be created by this
5283 	 * entry as it is an SRAT resource allocation structure.
5284 	 * We do not currently support doing so.
5285 	 */
5286 	node = pxm_to_node(its_affinity->proximity_domain);
5287 
5288 	if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) {
5289 		pr_err("SRAT: Invalid NUMA node %d in ITS affinity\n", node);
5290 		return 0;
5291 	}
5292 
5293 	its_srat_maps[its_in_srat].numa_node = node;
5294 	its_srat_maps[its_in_srat].its_id = its_affinity->its_id;
5295 	its_in_srat++;
5296 	pr_info("SRAT: PXM %d -> ITS %d -> Node %d\n",
5297 		its_affinity->proximity_domain, its_affinity->its_id, node);
5298 
5299 	return 0;
5300 }
5301 
5302 static void __init acpi_table_parse_srat_its(void)
5303 {
5304 	int count;
5305 
5306 	count = acpi_table_parse_entries(ACPI_SIG_SRAT,
5307 			sizeof(struct acpi_table_srat),
5308 			ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
5309 			gic_acpi_match_srat_its, 0);
5310 	if (count <= 0)
5311 		return;
5312 
5313 	its_srat_maps = kmalloc_array(count, sizeof(struct its_srat_map),
5314 				      GFP_KERNEL);
5315 	if (!its_srat_maps)
5316 		return;
5317 
5318 	acpi_table_parse_entries(ACPI_SIG_SRAT,
5319 			sizeof(struct acpi_table_srat),
5320 			ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
5321 			gic_acpi_parse_srat_its, 0);
5322 }
5323 
5324 /* free the its_srat_maps after ITS probing */
5325 static void __init acpi_its_srat_maps_free(void)
5326 {
5327 	kfree(its_srat_maps);
5328 }
5329 #else
5330 static void __init acpi_table_parse_srat_its(void)	{ }
5331 static int __init acpi_get_its_numa_node(u32 its_id) { return NUMA_NO_NODE; }
5332 static void __init acpi_its_srat_maps_free(void) { }
5333 #endif
5334 
5335 static int __init gic_acpi_parse_madt_its(union acpi_subtable_headers *header,
5336 					  const unsigned long end)
5337 {
5338 	struct acpi_madt_generic_translator *its_entry;
5339 	struct fwnode_handle *dom_handle;
5340 	struct resource res;
5341 	int err;
5342 
5343 	its_entry = (struct acpi_madt_generic_translator *)header;
5344 	memset(&res, 0, sizeof(res));
5345 	res.start = its_entry->base_address;
5346 	res.end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1;
5347 	res.flags = IORESOURCE_MEM;
5348 
5349 	dom_handle = irq_domain_alloc_fwnode(&res.start);
5350 	if (!dom_handle) {
5351 		pr_err("ITS@%pa: Unable to allocate GICv3 ITS domain token\n",
5352 		       &res.start);
5353 		return -ENOMEM;
5354 	}
5355 
5356 	err = iort_register_domain_token(its_entry->translation_id, res.start,
5357 					 dom_handle);
5358 	if (err) {
5359 		pr_err("ITS@%pa: Unable to register GICv3 ITS domain token (ITS ID %d) to IORT\n",
5360 		       &res.start, its_entry->translation_id);
5361 		goto dom_err;
5362 	}
5363 
5364 	err = its_probe_one(&res, dom_handle,
5365 			acpi_get_its_numa_node(its_entry->translation_id));
5366 	if (!err)
5367 		return 0;
5368 
5369 	iort_deregister_domain_token(its_entry->translation_id);
5370 dom_err:
5371 	irq_domain_free_fwnode(dom_handle);
5372 	return err;
5373 }
5374 
5375 static void __init its_acpi_probe(void)
5376 {
5377 	acpi_table_parse_srat_its();
5378 	acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR,
5379 			      gic_acpi_parse_madt_its, 0);
5380 	acpi_its_srat_maps_free();
5381 }
5382 #else
5383 static void __init its_acpi_probe(void) { }
5384 #endif
5385 
5386 int __init its_init(struct fwnode_handle *handle, struct rdists *rdists,
5387 		    struct irq_domain *parent_domain)
5388 {
5389 	struct device_node *of_node;
5390 	struct its_node *its;
5391 	bool has_v4 = false;
5392 	bool has_v4_1 = false;
5393 	int err;
5394 
5395 	gic_rdists = rdists;
5396 
5397 	its_parent = parent_domain;
5398 	of_node = to_of_node(handle);
5399 	if (of_node)
5400 		its_of_probe(of_node);
5401 	else
5402 		its_acpi_probe();
5403 
5404 	if (list_empty(&its_nodes)) {
5405 		pr_warn("ITS: No ITS available, not enabling LPIs\n");
5406 		return -ENXIO;
5407 	}
5408 
5409 	err = allocate_lpi_tables();
5410 	if (err)
5411 		return err;
5412 
5413 	list_for_each_entry(its, &its_nodes, entry) {
5414 		has_v4 |= is_v4(its);
5415 		has_v4_1 |= is_v4_1(its);
5416 	}
5417 
5418 	/* Don't bother with inconsistent systems */
5419 	if (WARN_ON(!has_v4_1 && rdists->has_rvpeid))
5420 		rdists->has_rvpeid = false;
5421 
5422 	if (has_v4 & rdists->has_vlpis) {
5423 		const struct irq_domain_ops *sgi_ops;
5424 
5425 		if (has_v4_1)
5426 			sgi_ops = &its_sgi_domain_ops;
5427 		else
5428 			sgi_ops = NULL;
5429 
5430 		if (its_init_vpe_domain() ||
5431 		    its_init_v4(parent_domain, &its_vpe_domain_ops, sgi_ops)) {
5432 			rdists->has_vlpis = false;
5433 			pr_err("ITS: Disabling GICv4 support\n");
5434 		}
5435 	}
5436 
5437 	register_syscore_ops(&its_syscore_ops);
5438 
5439 	return 0;
5440 }
5441