1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
4  * Author: Marc Zyngier <marc.zyngier@arm.com>
5  */
6 
7 #include <linux/acpi.h>
8 #include <linux/acpi_iort.h>
9 #include <linux/bitfield.h>
10 #include <linux/bitmap.h>
11 #include <linux/cpu.h>
12 #include <linux/crash_dump.h>
13 #include <linux/delay.h>
14 #include <linux/efi.h>
15 #include <linux/interrupt.h>
16 #include <linux/iommu.h>
17 #include <linux/iopoll.h>
18 #include <linux/irqdomain.h>
19 #include <linux/list.h>
20 #include <linux/log2.h>
21 #include <linux/memblock.h>
22 #include <linux/mm.h>
23 #include <linux/msi.h>
24 #include <linux/of.h>
25 #include <linux/of_address.h>
26 #include <linux/of_irq.h>
27 #include <linux/of_pci.h>
28 #include <linux/of_platform.h>
29 #include <linux/percpu.h>
30 #include <linux/slab.h>
31 #include <linux/syscore_ops.h>
32 
33 #include <linux/irqchip.h>
34 #include <linux/irqchip/arm-gic-v3.h>
35 #include <linux/irqchip/arm-gic-v4.h>
36 
37 #include <asm/cputype.h>
38 #include <asm/exception.h>
39 
40 #include "irq-gic-common.h"
41 
42 #define ITS_FLAGS_CMDQ_NEEDS_FLUSHING		(1ULL << 0)
43 #define ITS_FLAGS_WORKAROUND_CAVIUM_22375	(1ULL << 1)
44 #define ITS_FLAGS_WORKAROUND_CAVIUM_23144	(1ULL << 2)
45 #define ITS_FLAGS_FORCE_NON_SHAREABLE		(1ULL << 3)
46 
47 #define RD_LOCAL_LPI_ENABLED                    BIT(0)
48 #define RD_LOCAL_PENDTABLE_PREALLOCATED         BIT(1)
49 #define RD_LOCAL_MEMRESERVE_DONE                BIT(2)
50 
51 static u32 lpi_id_bits;
52 
53 /*
54  * We allocate memory for PROPBASE to cover 2 ^ lpi_id_bits LPIs to
55  * deal with (one configuration byte per interrupt). PENDBASE has to
56  * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI).
57  */
58 #define LPI_NRBITS		lpi_id_bits
59 #define LPI_PROPBASE_SZ		ALIGN(BIT(LPI_NRBITS), SZ_64K)
60 #define LPI_PENDBASE_SZ		ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K)
61 
62 #define LPI_PROP_DEFAULT_PRIO	GICD_INT_DEF_PRI
63 
64 /*
65  * Collection structure - just an ID, and a redistributor address to
66  * ping. We use one per CPU as a bag of interrupts assigned to this
67  * CPU.
68  */
69 struct its_collection {
70 	u64			target_address;
71 	u16			col_id;
72 };
73 
74 /*
75  * The ITS_BASER structure - contains memory information, cached
76  * value of BASER register configuration and ITS page size.
77  */
78 struct its_baser {
79 	void		*base;
80 	u64		val;
81 	u32		order;
82 	u32		psz;
83 };
84 
85 struct its_device;
86 
87 /*
88  * The ITS structure - contains most of the infrastructure, with the
89  * top-level MSI domain, the command queue, the collections, and the
90  * list of devices writing to it.
91  *
92  * dev_alloc_lock has to be taken for device allocations, while the
93  * spinlock must be taken to parse data structures such as the device
94  * list.
95  */
96 struct its_node {
97 	raw_spinlock_t		lock;
98 	struct mutex		dev_alloc_lock;
99 	struct list_head	entry;
100 	void __iomem		*base;
101 	void __iomem		*sgir_base;
102 	phys_addr_t		phys_base;
103 	struct its_cmd_block	*cmd_base;
104 	struct its_cmd_block	*cmd_write;
105 	struct its_baser	tables[GITS_BASER_NR_REGS];
106 	struct its_collection	*collections;
107 	struct fwnode_handle	*fwnode_handle;
108 	u64			(*get_msi_base)(struct its_device *its_dev);
109 	u64			typer;
110 	u64			cbaser_save;
111 	u32			ctlr_save;
112 	u32			mpidr;
113 	struct list_head	its_device_list;
114 	u64			flags;
115 	unsigned long		list_nr;
116 	int			numa_node;
117 	unsigned int		msi_domain_flags;
118 	u32			pre_its_base; /* for Socionext Synquacer */
119 	int			vlpi_redist_offset;
120 };
121 
122 #define is_v4(its)		(!!((its)->typer & GITS_TYPER_VLPIS))
123 #define is_v4_1(its)		(!!((its)->typer & GITS_TYPER_VMAPP))
124 #define device_ids(its)		(FIELD_GET(GITS_TYPER_DEVBITS, (its)->typer) + 1)
125 
126 #define ITS_ITT_ALIGN		SZ_256
127 
128 /* The maximum number of VPEID bits supported by VLPI commands */
129 #define ITS_MAX_VPEID_BITS						\
130 	({								\
131 		int nvpeid = 16;					\
132 		if (gic_rdists->has_rvpeid &&				\
133 		    gic_rdists->gicd_typer2 & GICD_TYPER2_VIL)		\
134 			nvpeid = 1 + (gic_rdists->gicd_typer2 &		\
135 				      GICD_TYPER2_VID);			\
136 									\
137 		nvpeid;							\
138 	})
139 #define ITS_MAX_VPEID		(1 << (ITS_MAX_VPEID_BITS))
140 
141 /* Convert page order to size in bytes */
142 #define PAGE_ORDER_TO_SIZE(o)	(PAGE_SIZE << (o))
143 
144 struct event_lpi_map {
145 	unsigned long		*lpi_map;
146 	u16			*col_map;
147 	irq_hw_number_t		lpi_base;
148 	int			nr_lpis;
149 	raw_spinlock_t		vlpi_lock;
150 	struct its_vm		*vm;
151 	struct its_vlpi_map	*vlpi_maps;
152 	int			nr_vlpis;
153 };
154 
155 /*
156  * The ITS view of a device - belongs to an ITS, owns an interrupt
157  * translation table, and a list of interrupts.  If it some of its
158  * LPIs are injected into a guest (GICv4), the event_map.vm field
159  * indicates which one.
160  */
161 struct its_device {
162 	struct list_head	entry;
163 	struct its_node		*its;
164 	struct event_lpi_map	event_map;
165 	void			*itt;
166 	u32			nr_ites;
167 	u32			device_id;
168 	bool			shared;
169 };
170 
171 static struct {
172 	raw_spinlock_t		lock;
173 	struct its_device	*dev;
174 	struct its_vpe		**vpes;
175 	int			next_victim;
176 } vpe_proxy;
177 
178 struct cpu_lpi_count {
179 	atomic_t	managed;
180 	atomic_t	unmanaged;
181 };
182 
183 static DEFINE_PER_CPU(struct cpu_lpi_count, cpu_lpi_count);
184 
185 static LIST_HEAD(its_nodes);
186 static DEFINE_RAW_SPINLOCK(its_lock);
187 static struct rdists *gic_rdists;
188 static struct irq_domain *its_parent;
189 
190 static unsigned long its_list_map;
191 static u16 vmovp_seq_num;
192 static DEFINE_RAW_SPINLOCK(vmovp_lock);
193 
194 static DEFINE_IDA(its_vpeid_ida);
195 
196 #define gic_data_rdist()		(raw_cpu_ptr(gic_rdists->rdist))
197 #define gic_data_rdist_cpu(cpu)		(per_cpu_ptr(gic_rdists->rdist, cpu))
198 #define gic_data_rdist_rd_base()	(gic_data_rdist()->rd_base)
199 #define gic_data_rdist_vlpi_base()	(gic_data_rdist_rd_base() + SZ_128K)
200 
201 /*
202  * Skip ITSs that have no vLPIs mapped, unless we're on GICv4.1, as we
203  * always have vSGIs mapped.
204  */
require_its_list_vmovp(struct its_vm * vm,struct its_node * its)205 static bool require_its_list_vmovp(struct its_vm *vm, struct its_node *its)
206 {
207 	return (gic_rdists->has_rvpeid || vm->vlpi_count[its->list_nr]);
208 }
209 
rdists_support_shareable(void)210 static bool rdists_support_shareable(void)
211 {
212 	return !(gic_rdists->flags & RDIST_FLAGS_FORCE_NON_SHAREABLE);
213 }
214 
get_its_list(struct its_vm * vm)215 static u16 get_its_list(struct its_vm *vm)
216 {
217 	struct its_node *its;
218 	unsigned long its_list = 0;
219 
220 	list_for_each_entry(its, &its_nodes, entry) {
221 		if (!is_v4(its))
222 			continue;
223 
224 		if (require_its_list_vmovp(vm, its))
225 			__set_bit(its->list_nr, &its_list);
226 	}
227 
228 	return (u16)its_list;
229 }
230 
its_get_event_id(struct irq_data * d)231 static inline u32 its_get_event_id(struct irq_data *d)
232 {
233 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
234 	return d->hwirq - its_dev->event_map.lpi_base;
235 }
236 
dev_event_to_col(struct its_device * its_dev,u32 event)237 static struct its_collection *dev_event_to_col(struct its_device *its_dev,
238 					       u32 event)
239 {
240 	struct its_node *its = its_dev->its;
241 
242 	return its->collections + its_dev->event_map.col_map[event];
243 }
244 
dev_event_to_vlpi_map(struct its_device * its_dev,u32 event)245 static struct its_vlpi_map *dev_event_to_vlpi_map(struct its_device *its_dev,
246 					       u32 event)
247 {
248 	if (WARN_ON_ONCE(event >= its_dev->event_map.nr_lpis))
249 		return NULL;
250 
251 	return &its_dev->event_map.vlpi_maps[event];
252 }
253 
get_vlpi_map(struct irq_data * d)254 static struct its_vlpi_map *get_vlpi_map(struct irq_data *d)
255 {
256 	if (irqd_is_forwarded_to_vcpu(d)) {
257 		struct its_device *its_dev = irq_data_get_irq_chip_data(d);
258 		u32 event = its_get_event_id(d);
259 
260 		return dev_event_to_vlpi_map(its_dev, event);
261 	}
262 
263 	return NULL;
264 }
265 
vpe_to_cpuid_lock(struct its_vpe * vpe,unsigned long * flags)266 static int vpe_to_cpuid_lock(struct its_vpe *vpe, unsigned long *flags)
267 {
268 	raw_spin_lock_irqsave(&vpe->vpe_lock, *flags);
269 	return vpe->col_idx;
270 }
271 
vpe_to_cpuid_unlock(struct its_vpe * vpe,unsigned long flags)272 static void vpe_to_cpuid_unlock(struct its_vpe *vpe, unsigned long flags)
273 {
274 	raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
275 }
276 
277 static struct irq_chip its_vpe_irq_chip;
278 
irq_to_cpuid_lock(struct irq_data * d,unsigned long * flags)279 static int irq_to_cpuid_lock(struct irq_data *d, unsigned long *flags)
280 {
281 	struct its_vpe *vpe = NULL;
282 	int cpu;
283 
284 	if (d->chip == &its_vpe_irq_chip) {
285 		vpe = irq_data_get_irq_chip_data(d);
286 	} else {
287 		struct its_vlpi_map *map = get_vlpi_map(d);
288 		if (map)
289 			vpe = map->vpe;
290 	}
291 
292 	if (vpe) {
293 		cpu = vpe_to_cpuid_lock(vpe, flags);
294 	} else {
295 		/* Physical LPIs are already locked via the irq_desc lock */
296 		struct its_device *its_dev = irq_data_get_irq_chip_data(d);
297 		cpu = its_dev->event_map.col_map[its_get_event_id(d)];
298 		/* Keep GCC quiet... */
299 		*flags = 0;
300 	}
301 
302 	return cpu;
303 }
304 
irq_to_cpuid_unlock(struct irq_data * d,unsigned long flags)305 static void irq_to_cpuid_unlock(struct irq_data *d, unsigned long flags)
306 {
307 	struct its_vpe *vpe = NULL;
308 
309 	if (d->chip == &its_vpe_irq_chip) {
310 		vpe = irq_data_get_irq_chip_data(d);
311 	} else {
312 		struct its_vlpi_map *map = get_vlpi_map(d);
313 		if (map)
314 			vpe = map->vpe;
315 	}
316 
317 	if (vpe)
318 		vpe_to_cpuid_unlock(vpe, flags);
319 }
320 
valid_col(struct its_collection * col)321 static struct its_collection *valid_col(struct its_collection *col)
322 {
323 	if (WARN_ON_ONCE(col->target_address & GENMASK_ULL(15, 0)))
324 		return NULL;
325 
326 	return col;
327 }
328 
valid_vpe(struct its_node * its,struct its_vpe * vpe)329 static struct its_vpe *valid_vpe(struct its_node *its, struct its_vpe *vpe)
330 {
331 	if (valid_col(its->collections + vpe->col_idx))
332 		return vpe;
333 
334 	return NULL;
335 }
336 
337 /*
338  * ITS command descriptors - parameters to be encoded in a command
339  * block.
340  */
341 struct its_cmd_desc {
342 	union {
343 		struct {
344 			struct its_device *dev;
345 			u32 event_id;
346 		} its_inv_cmd;
347 
348 		struct {
349 			struct its_device *dev;
350 			u32 event_id;
351 		} its_clear_cmd;
352 
353 		struct {
354 			struct its_device *dev;
355 			u32 event_id;
356 		} its_int_cmd;
357 
358 		struct {
359 			struct its_device *dev;
360 			int valid;
361 		} its_mapd_cmd;
362 
363 		struct {
364 			struct its_collection *col;
365 			int valid;
366 		} its_mapc_cmd;
367 
368 		struct {
369 			struct its_device *dev;
370 			u32 phys_id;
371 			u32 event_id;
372 		} its_mapti_cmd;
373 
374 		struct {
375 			struct its_device *dev;
376 			struct its_collection *col;
377 			u32 event_id;
378 		} its_movi_cmd;
379 
380 		struct {
381 			struct its_device *dev;
382 			u32 event_id;
383 		} its_discard_cmd;
384 
385 		struct {
386 			struct its_collection *col;
387 		} its_invall_cmd;
388 
389 		struct {
390 			struct its_vpe *vpe;
391 		} its_vinvall_cmd;
392 
393 		struct {
394 			struct its_vpe *vpe;
395 			struct its_collection *col;
396 			bool valid;
397 		} its_vmapp_cmd;
398 
399 		struct {
400 			struct its_vpe *vpe;
401 			struct its_device *dev;
402 			u32 virt_id;
403 			u32 event_id;
404 			bool db_enabled;
405 		} its_vmapti_cmd;
406 
407 		struct {
408 			struct its_vpe *vpe;
409 			struct its_device *dev;
410 			u32 event_id;
411 			bool db_enabled;
412 		} its_vmovi_cmd;
413 
414 		struct {
415 			struct its_vpe *vpe;
416 			struct its_collection *col;
417 			u16 seq_num;
418 			u16 its_list;
419 		} its_vmovp_cmd;
420 
421 		struct {
422 			struct its_vpe *vpe;
423 		} its_invdb_cmd;
424 
425 		struct {
426 			struct its_vpe *vpe;
427 			u8 sgi;
428 			u8 priority;
429 			bool enable;
430 			bool group;
431 			bool clear;
432 		} its_vsgi_cmd;
433 	};
434 };
435 
436 /*
437  * The ITS command block, which is what the ITS actually parses.
438  */
439 struct its_cmd_block {
440 	union {
441 		u64	raw_cmd[4];
442 		__le64	raw_cmd_le[4];
443 	};
444 };
445 
446 #define ITS_CMD_QUEUE_SZ		SZ_64K
447 #define ITS_CMD_QUEUE_NR_ENTRIES	(ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block))
448 
449 typedef struct its_collection *(*its_cmd_builder_t)(struct its_node *,
450 						    struct its_cmd_block *,
451 						    struct its_cmd_desc *);
452 
453 typedef struct its_vpe *(*its_cmd_vbuilder_t)(struct its_node *,
454 					      struct its_cmd_block *,
455 					      struct its_cmd_desc *);
456 
457 static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l)
458 {
459 	u64 mask = GENMASK_ULL(h, l);
460 	*raw_cmd &= ~mask;
461 	*raw_cmd |= (val << l) & mask;
462 }
463 
its_encode_cmd(struct its_cmd_block * cmd,u8 cmd_nr)464 static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr)
465 {
466 	its_mask_encode(&cmd->raw_cmd[0], cmd_nr, 7, 0);
467 }
468 
its_encode_devid(struct its_cmd_block * cmd,u32 devid)469 static void its_encode_devid(struct its_cmd_block *cmd, u32 devid)
470 {
471 	its_mask_encode(&cmd->raw_cmd[0], devid, 63, 32);
472 }
473 
its_encode_event_id(struct its_cmd_block * cmd,u32 id)474 static void its_encode_event_id(struct its_cmd_block *cmd, u32 id)
475 {
476 	its_mask_encode(&cmd->raw_cmd[1], id, 31, 0);
477 }
478 
its_encode_phys_id(struct its_cmd_block * cmd,u32 phys_id)479 static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id)
480 {
481 	its_mask_encode(&cmd->raw_cmd[1], phys_id, 63, 32);
482 }
483 
its_encode_size(struct its_cmd_block * cmd,u8 size)484 static void its_encode_size(struct its_cmd_block *cmd, u8 size)
485 {
486 	its_mask_encode(&cmd->raw_cmd[1], size, 4, 0);
487 }
488 
its_encode_itt(struct its_cmd_block * cmd,u64 itt_addr)489 static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr)
490 {
491 	its_mask_encode(&cmd->raw_cmd[2], itt_addr >> 8, 51, 8);
492 }
493 
its_encode_valid(struct its_cmd_block * cmd,int valid)494 static void its_encode_valid(struct its_cmd_block *cmd, int valid)
495 {
496 	its_mask_encode(&cmd->raw_cmd[2], !!valid, 63, 63);
497 }
498 
its_encode_target(struct its_cmd_block * cmd,u64 target_addr)499 static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr)
500 {
501 	its_mask_encode(&cmd->raw_cmd[2], target_addr >> 16, 51, 16);
502 }
503 
its_encode_collection(struct its_cmd_block * cmd,u16 col)504 static void its_encode_collection(struct its_cmd_block *cmd, u16 col)
505 {
506 	its_mask_encode(&cmd->raw_cmd[2], col, 15, 0);
507 }
508 
its_encode_vpeid(struct its_cmd_block * cmd,u16 vpeid)509 static void its_encode_vpeid(struct its_cmd_block *cmd, u16 vpeid)
510 {
511 	its_mask_encode(&cmd->raw_cmd[1], vpeid, 47, 32);
512 }
513 
its_encode_virt_id(struct its_cmd_block * cmd,u32 virt_id)514 static void its_encode_virt_id(struct its_cmd_block *cmd, u32 virt_id)
515 {
516 	its_mask_encode(&cmd->raw_cmd[2], virt_id, 31, 0);
517 }
518 
its_encode_db_phys_id(struct its_cmd_block * cmd,u32 db_phys_id)519 static void its_encode_db_phys_id(struct its_cmd_block *cmd, u32 db_phys_id)
520 {
521 	its_mask_encode(&cmd->raw_cmd[2], db_phys_id, 63, 32);
522 }
523 
its_encode_db_valid(struct its_cmd_block * cmd,bool db_valid)524 static void its_encode_db_valid(struct its_cmd_block *cmd, bool db_valid)
525 {
526 	its_mask_encode(&cmd->raw_cmd[2], db_valid, 0, 0);
527 }
528 
its_encode_seq_num(struct its_cmd_block * cmd,u16 seq_num)529 static void its_encode_seq_num(struct its_cmd_block *cmd, u16 seq_num)
530 {
531 	its_mask_encode(&cmd->raw_cmd[0], seq_num, 47, 32);
532 }
533 
its_encode_its_list(struct its_cmd_block * cmd,u16 its_list)534 static void its_encode_its_list(struct its_cmd_block *cmd, u16 its_list)
535 {
536 	its_mask_encode(&cmd->raw_cmd[1], its_list, 15, 0);
537 }
538 
its_encode_vpt_addr(struct its_cmd_block * cmd,u64 vpt_pa)539 static void its_encode_vpt_addr(struct its_cmd_block *cmd, u64 vpt_pa)
540 {
541 	its_mask_encode(&cmd->raw_cmd[3], vpt_pa >> 16, 51, 16);
542 }
543 
its_encode_vpt_size(struct its_cmd_block * cmd,u8 vpt_size)544 static void its_encode_vpt_size(struct its_cmd_block *cmd, u8 vpt_size)
545 {
546 	its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0);
547 }
548 
its_encode_vconf_addr(struct its_cmd_block * cmd,u64 vconf_pa)549 static void its_encode_vconf_addr(struct its_cmd_block *cmd, u64 vconf_pa)
550 {
551 	its_mask_encode(&cmd->raw_cmd[0], vconf_pa >> 16, 51, 16);
552 }
553 
its_encode_alloc(struct its_cmd_block * cmd,bool alloc)554 static void its_encode_alloc(struct its_cmd_block *cmd, bool alloc)
555 {
556 	its_mask_encode(&cmd->raw_cmd[0], alloc, 8, 8);
557 }
558 
its_encode_ptz(struct its_cmd_block * cmd,bool ptz)559 static void its_encode_ptz(struct its_cmd_block *cmd, bool ptz)
560 {
561 	its_mask_encode(&cmd->raw_cmd[0], ptz, 9, 9);
562 }
563 
its_encode_vmapp_default_db(struct its_cmd_block * cmd,u32 vpe_db_lpi)564 static void its_encode_vmapp_default_db(struct its_cmd_block *cmd,
565 					u32 vpe_db_lpi)
566 {
567 	its_mask_encode(&cmd->raw_cmd[1], vpe_db_lpi, 31, 0);
568 }
569 
its_encode_vmovp_default_db(struct its_cmd_block * cmd,u32 vpe_db_lpi)570 static void its_encode_vmovp_default_db(struct its_cmd_block *cmd,
571 					u32 vpe_db_lpi)
572 {
573 	its_mask_encode(&cmd->raw_cmd[3], vpe_db_lpi, 31, 0);
574 }
575 
its_encode_db(struct its_cmd_block * cmd,bool db)576 static void its_encode_db(struct its_cmd_block *cmd, bool db)
577 {
578 	its_mask_encode(&cmd->raw_cmd[2], db, 63, 63);
579 }
580 
its_encode_sgi_intid(struct its_cmd_block * cmd,u8 sgi)581 static void its_encode_sgi_intid(struct its_cmd_block *cmd, u8 sgi)
582 {
583 	its_mask_encode(&cmd->raw_cmd[0], sgi, 35, 32);
584 }
585 
its_encode_sgi_priority(struct its_cmd_block * cmd,u8 prio)586 static void its_encode_sgi_priority(struct its_cmd_block *cmd, u8 prio)
587 {
588 	its_mask_encode(&cmd->raw_cmd[0], prio >> 4, 23, 20);
589 }
590 
its_encode_sgi_group(struct its_cmd_block * cmd,bool grp)591 static void its_encode_sgi_group(struct its_cmd_block *cmd, bool grp)
592 {
593 	its_mask_encode(&cmd->raw_cmd[0], grp, 10, 10);
594 }
595 
its_encode_sgi_clear(struct its_cmd_block * cmd,bool clr)596 static void its_encode_sgi_clear(struct its_cmd_block *cmd, bool clr)
597 {
598 	its_mask_encode(&cmd->raw_cmd[0], clr, 9, 9);
599 }
600 
its_encode_sgi_enable(struct its_cmd_block * cmd,bool en)601 static void its_encode_sgi_enable(struct its_cmd_block *cmd, bool en)
602 {
603 	its_mask_encode(&cmd->raw_cmd[0], en, 8, 8);
604 }
605 
its_fixup_cmd(struct its_cmd_block * cmd)606 static inline void its_fixup_cmd(struct its_cmd_block *cmd)
607 {
608 	/* Let's fixup BE commands */
609 	cmd->raw_cmd_le[0] = cpu_to_le64(cmd->raw_cmd[0]);
610 	cmd->raw_cmd_le[1] = cpu_to_le64(cmd->raw_cmd[1]);
611 	cmd->raw_cmd_le[2] = cpu_to_le64(cmd->raw_cmd[2]);
612 	cmd->raw_cmd_le[3] = cpu_to_le64(cmd->raw_cmd[3]);
613 }
614 
its_build_mapd_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)615 static struct its_collection *its_build_mapd_cmd(struct its_node *its,
616 						 struct its_cmd_block *cmd,
617 						 struct its_cmd_desc *desc)
618 {
619 	unsigned long itt_addr;
620 	u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites);
621 
622 	itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt);
623 	itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN);
624 
625 	its_encode_cmd(cmd, GITS_CMD_MAPD);
626 	its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id);
627 	its_encode_size(cmd, size - 1);
628 	its_encode_itt(cmd, itt_addr);
629 	its_encode_valid(cmd, desc->its_mapd_cmd.valid);
630 
631 	its_fixup_cmd(cmd);
632 
633 	return NULL;
634 }
635 
its_build_mapc_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)636 static struct its_collection *its_build_mapc_cmd(struct its_node *its,
637 						 struct its_cmd_block *cmd,
638 						 struct its_cmd_desc *desc)
639 {
640 	its_encode_cmd(cmd, GITS_CMD_MAPC);
641 	its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id);
642 	its_encode_target(cmd, desc->its_mapc_cmd.col->target_address);
643 	its_encode_valid(cmd, desc->its_mapc_cmd.valid);
644 
645 	its_fixup_cmd(cmd);
646 
647 	return desc->its_mapc_cmd.col;
648 }
649 
its_build_mapti_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)650 static struct its_collection *its_build_mapti_cmd(struct its_node *its,
651 						  struct its_cmd_block *cmd,
652 						  struct its_cmd_desc *desc)
653 {
654 	struct its_collection *col;
655 
656 	col = dev_event_to_col(desc->its_mapti_cmd.dev,
657 			       desc->its_mapti_cmd.event_id);
658 
659 	its_encode_cmd(cmd, GITS_CMD_MAPTI);
660 	its_encode_devid(cmd, desc->its_mapti_cmd.dev->device_id);
661 	its_encode_event_id(cmd, desc->its_mapti_cmd.event_id);
662 	its_encode_phys_id(cmd, desc->its_mapti_cmd.phys_id);
663 	its_encode_collection(cmd, col->col_id);
664 
665 	its_fixup_cmd(cmd);
666 
667 	return valid_col(col);
668 }
669 
its_build_movi_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)670 static struct its_collection *its_build_movi_cmd(struct its_node *its,
671 						 struct its_cmd_block *cmd,
672 						 struct its_cmd_desc *desc)
673 {
674 	struct its_collection *col;
675 
676 	col = dev_event_to_col(desc->its_movi_cmd.dev,
677 			       desc->its_movi_cmd.event_id);
678 
679 	its_encode_cmd(cmd, GITS_CMD_MOVI);
680 	its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id);
681 	its_encode_event_id(cmd, desc->its_movi_cmd.event_id);
682 	its_encode_collection(cmd, desc->its_movi_cmd.col->col_id);
683 
684 	its_fixup_cmd(cmd);
685 
686 	return valid_col(col);
687 }
688 
its_build_discard_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)689 static struct its_collection *its_build_discard_cmd(struct its_node *its,
690 						    struct its_cmd_block *cmd,
691 						    struct its_cmd_desc *desc)
692 {
693 	struct its_collection *col;
694 
695 	col = dev_event_to_col(desc->its_discard_cmd.dev,
696 			       desc->its_discard_cmd.event_id);
697 
698 	its_encode_cmd(cmd, GITS_CMD_DISCARD);
699 	its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id);
700 	its_encode_event_id(cmd, desc->its_discard_cmd.event_id);
701 
702 	its_fixup_cmd(cmd);
703 
704 	return valid_col(col);
705 }
706 
its_build_inv_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)707 static struct its_collection *its_build_inv_cmd(struct its_node *its,
708 						struct its_cmd_block *cmd,
709 						struct its_cmd_desc *desc)
710 {
711 	struct its_collection *col;
712 
713 	col = dev_event_to_col(desc->its_inv_cmd.dev,
714 			       desc->its_inv_cmd.event_id);
715 
716 	its_encode_cmd(cmd, GITS_CMD_INV);
717 	its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
718 	its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
719 
720 	its_fixup_cmd(cmd);
721 
722 	return valid_col(col);
723 }
724 
its_build_int_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)725 static struct its_collection *its_build_int_cmd(struct its_node *its,
726 						struct its_cmd_block *cmd,
727 						struct its_cmd_desc *desc)
728 {
729 	struct its_collection *col;
730 
731 	col = dev_event_to_col(desc->its_int_cmd.dev,
732 			       desc->its_int_cmd.event_id);
733 
734 	its_encode_cmd(cmd, GITS_CMD_INT);
735 	its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
736 	its_encode_event_id(cmd, desc->its_int_cmd.event_id);
737 
738 	its_fixup_cmd(cmd);
739 
740 	return valid_col(col);
741 }
742 
its_build_clear_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)743 static struct its_collection *its_build_clear_cmd(struct its_node *its,
744 						  struct its_cmd_block *cmd,
745 						  struct its_cmd_desc *desc)
746 {
747 	struct its_collection *col;
748 
749 	col = dev_event_to_col(desc->its_clear_cmd.dev,
750 			       desc->its_clear_cmd.event_id);
751 
752 	its_encode_cmd(cmd, GITS_CMD_CLEAR);
753 	its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
754 	its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
755 
756 	its_fixup_cmd(cmd);
757 
758 	return valid_col(col);
759 }
760 
its_build_invall_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)761 static struct its_collection *its_build_invall_cmd(struct its_node *its,
762 						   struct its_cmd_block *cmd,
763 						   struct its_cmd_desc *desc)
764 {
765 	its_encode_cmd(cmd, GITS_CMD_INVALL);
766 	its_encode_collection(cmd, desc->its_invall_cmd.col->col_id);
767 
768 	its_fixup_cmd(cmd);
769 
770 	return desc->its_invall_cmd.col;
771 }
772 
its_build_vinvall_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)773 static struct its_vpe *its_build_vinvall_cmd(struct its_node *its,
774 					     struct its_cmd_block *cmd,
775 					     struct its_cmd_desc *desc)
776 {
777 	its_encode_cmd(cmd, GITS_CMD_VINVALL);
778 	its_encode_vpeid(cmd, desc->its_vinvall_cmd.vpe->vpe_id);
779 
780 	its_fixup_cmd(cmd);
781 
782 	return valid_vpe(its, desc->its_vinvall_cmd.vpe);
783 }
784 
its_build_vmapp_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)785 static struct its_vpe *its_build_vmapp_cmd(struct its_node *its,
786 					   struct its_cmd_block *cmd,
787 					   struct its_cmd_desc *desc)
788 {
789 	struct its_vpe *vpe = valid_vpe(its, desc->its_vmapp_cmd.vpe);
790 	unsigned long vpt_addr, vconf_addr;
791 	u64 target;
792 	bool alloc;
793 
794 	its_encode_cmd(cmd, GITS_CMD_VMAPP);
795 	its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id);
796 	its_encode_valid(cmd, desc->its_vmapp_cmd.valid);
797 
798 	if (!desc->its_vmapp_cmd.valid) {
799 		alloc = !atomic_dec_return(&desc->its_vmapp_cmd.vpe->vmapp_count);
800 		if (is_v4_1(its)) {
801 			its_encode_alloc(cmd, alloc);
802 			/*
803 			 * Unmapping a VPE is self-synchronizing on GICv4.1,
804 			 * no need to issue a VSYNC.
805 			 */
806 			vpe = NULL;
807 		}
808 
809 		goto out;
810 	}
811 
812 	vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page));
813 	target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset;
814 
815 	its_encode_target(cmd, target);
816 	its_encode_vpt_addr(cmd, vpt_addr);
817 	its_encode_vpt_size(cmd, LPI_NRBITS - 1);
818 
819 	alloc = !atomic_fetch_inc(&desc->its_vmapp_cmd.vpe->vmapp_count);
820 
821 	if (!is_v4_1(its))
822 		goto out;
823 
824 	vconf_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->its_vm->vprop_page));
825 
826 	its_encode_alloc(cmd, alloc);
827 
828 	/*
829 	 * GICv4.1 provides a way to get the VLPI state, which needs the vPE
830 	 * to be unmapped first, and in this case, we may remap the vPE
831 	 * back while the VPT is not empty. So we can't assume that the
832 	 * VPT is empty on map. This is why we never advertise PTZ.
833 	 */
834 	its_encode_ptz(cmd, false);
835 	its_encode_vconf_addr(cmd, vconf_addr);
836 	its_encode_vmapp_default_db(cmd, desc->its_vmapp_cmd.vpe->vpe_db_lpi);
837 
838 out:
839 	its_fixup_cmd(cmd);
840 
841 	return vpe;
842 }
843 
its_build_vmapti_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)844 static struct its_vpe *its_build_vmapti_cmd(struct its_node *its,
845 					    struct its_cmd_block *cmd,
846 					    struct its_cmd_desc *desc)
847 {
848 	u32 db;
849 
850 	if (!is_v4_1(its) && desc->its_vmapti_cmd.db_enabled)
851 		db = desc->its_vmapti_cmd.vpe->vpe_db_lpi;
852 	else
853 		db = 1023;
854 
855 	its_encode_cmd(cmd, GITS_CMD_VMAPTI);
856 	its_encode_devid(cmd, desc->its_vmapti_cmd.dev->device_id);
857 	its_encode_vpeid(cmd, desc->its_vmapti_cmd.vpe->vpe_id);
858 	its_encode_event_id(cmd, desc->its_vmapti_cmd.event_id);
859 	its_encode_db_phys_id(cmd, db);
860 	its_encode_virt_id(cmd, desc->its_vmapti_cmd.virt_id);
861 
862 	its_fixup_cmd(cmd);
863 
864 	return valid_vpe(its, desc->its_vmapti_cmd.vpe);
865 }
866 
its_build_vmovi_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)867 static struct its_vpe *its_build_vmovi_cmd(struct its_node *its,
868 					   struct its_cmd_block *cmd,
869 					   struct its_cmd_desc *desc)
870 {
871 	u32 db;
872 
873 	if (!is_v4_1(its) && desc->its_vmovi_cmd.db_enabled)
874 		db = desc->its_vmovi_cmd.vpe->vpe_db_lpi;
875 	else
876 		db = 1023;
877 
878 	its_encode_cmd(cmd, GITS_CMD_VMOVI);
879 	its_encode_devid(cmd, desc->its_vmovi_cmd.dev->device_id);
880 	its_encode_vpeid(cmd, desc->its_vmovi_cmd.vpe->vpe_id);
881 	its_encode_event_id(cmd, desc->its_vmovi_cmd.event_id);
882 	its_encode_db_phys_id(cmd, db);
883 	its_encode_db_valid(cmd, true);
884 
885 	its_fixup_cmd(cmd);
886 
887 	return valid_vpe(its, desc->its_vmovi_cmd.vpe);
888 }
889 
its_build_vmovp_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)890 static struct its_vpe *its_build_vmovp_cmd(struct its_node *its,
891 					   struct its_cmd_block *cmd,
892 					   struct its_cmd_desc *desc)
893 {
894 	u64 target;
895 
896 	target = desc->its_vmovp_cmd.col->target_address + its->vlpi_redist_offset;
897 	its_encode_cmd(cmd, GITS_CMD_VMOVP);
898 	its_encode_seq_num(cmd, desc->its_vmovp_cmd.seq_num);
899 	its_encode_its_list(cmd, desc->its_vmovp_cmd.its_list);
900 	its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id);
901 	its_encode_target(cmd, target);
902 
903 	if (is_v4_1(its)) {
904 		its_encode_db(cmd, true);
905 		its_encode_vmovp_default_db(cmd, desc->its_vmovp_cmd.vpe->vpe_db_lpi);
906 	}
907 
908 	its_fixup_cmd(cmd);
909 
910 	return valid_vpe(its, desc->its_vmovp_cmd.vpe);
911 }
912 
its_build_vinv_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)913 static struct its_vpe *its_build_vinv_cmd(struct its_node *its,
914 					  struct its_cmd_block *cmd,
915 					  struct its_cmd_desc *desc)
916 {
917 	struct its_vlpi_map *map;
918 
919 	map = dev_event_to_vlpi_map(desc->its_inv_cmd.dev,
920 				    desc->its_inv_cmd.event_id);
921 
922 	its_encode_cmd(cmd, GITS_CMD_INV);
923 	its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
924 	its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
925 
926 	its_fixup_cmd(cmd);
927 
928 	return valid_vpe(its, map->vpe);
929 }
930 
its_build_vint_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)931 static struct its_vpe *its_build_vint_cmd(struct its_node *its,
932 					  struct its_cmd_block *cmd,
933 					  struct its_cmd_desc *desc)
934 {
935 	struct its_vlpi_map *map;
936 
937 	map = dev_event_to_vlpi_map(desc->its_int_cmd.dev,
938 				    desc->its_int_cmd.event_id);
939 
940 	its_encode_cmd(cmd, GITS_CMD_INT);
941 	its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
942 	its_encode_event_id(cmd, desc->its_int_cmd.event_id);
943 
944 	its_fixup_cmd(cmd);
945 
946 	return valid_vpe(its, map->vpe);
947 }
948 
its_build_vclear_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)949 static struct its_vpe *its_build_vclear_cmd(struct its_node *its,
950 					    struct its_cmd_block *cmd,
951 					    struct its_cmd_desc *desc)
952 {
953 	struct its_vlpi_map *map;
954 
955 	map = dev_event_to_vlpi_map(desc->its_clear_cmd.dev,
956 				    desc->its_clear_cmd.event_id);
957 
958 	its_encode_cmd(cmd, GITS_CMD_CLEAR);
959 	its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
960 	its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
961 
962 	its_fixup_cmd(cmd);
963 
964 	return valid_vpe(its, map->vpe);
965 }
966 
its_build_invdb_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)967 static struct its_vpe *its_build_invdb_cmd(struct its_node *its,
968 					   struct its_cmd_block *cmd,
969 					   struct its_cmd_desc *desc)
970 {
971 	if (WARN_ON(!is_v4_1(its)))
972 		return NULL;
973 
974 	its_encode_cmd(cmd, GITS_CMD_INVDB);
975 	its_encode_vpeid(cmd, desc->its_invdb_cmd.vpe->vpe_id);
976 
977 	its_fixup_cmd(cmd);
978 
979 	return valid_vpe(its, desc->its_invdb_cmd.vpe);
980 }
981 
its_build_vsgi_cmd(struct its_node * its,struct its_cmd_block * cmd,struct its_cmd_desc * desc)982 static struct its_vpe *its_build_vsgi_cmd(struct its_node *its,
983 					  struct its_cmd_block *cmd,
984 					  struct its_cmd_desc *desc)
985 {
986 	if (WARN_ON(!is_v4_1(its)))
987 		return NULL;
988 
989 	its_encode_cmd(cmd, GITS_CMD_VSGI);
990 	its_encode_vpeid(cmd, desc->its_vsgi_cmd.vpe->vpe_id);
991 	its_encode_sgi_intid(cmd, desc->its_vsgi_cmd.sgi);
992 	its_encode_sgi_priority(cmd, desc->its_vsgi_cmd.priority);
993 	its_encode_sgi_group(cmd, desc->its_vsgi_cmd.group);
994 	its_encode_sgi_clear(cmd, desc->its_vsgi_cmd.clear);
995 	its_encode_sgi_enable(cmd, desc->its_vsgi_cmd.enable);
996 
997 	its_fixup_cmd(cmd);
998 
999 	return valid_vpe(its, desc->its_vsgi_cmd.vpe);
1000 }
1001 
its_cmd_ptr_to_offset(struct its_node * its,struct its_cmd_block * ptr)1002 static u64 its_cmd_ptr_to_offset(struct its_node *its,
1003 				 struct its_cmd_block *ptr)
1004 {
1005 	return (ptr - its->cmd_base) * sizeof(*ptr);
1006 }
1007 
its_queue_full(struct its_node * its)1008 static int its_queue_full(struct its_node *its)
1009 {
1010 	int widx;
1011 	int ridx;
1012 
1013 	widx = its->cmd_write - its->cmd_base;
1014 	ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block);
1015 
1016 	/* This is incredibly unlikely to happen, unless the ITS locks up. */
1017 	if (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx)
1018 		return 1;
1019 
1020 	return 0;
1021 }
1022 
its_allocate_entry(struct its_node * its)1023 static struct its_cmd_block *its_allocate_entry(struct its_node *its)
1024 {
1025 	struct its_cmd_block *cmd;
1026 	u32 count = 1000000;	/* 1s! */
1027 
1028 	while (its_queue_full(its)) {
1029 		count--;
1030 		if (!count) {
1031 			pr_err_ratelimited("ITS queue not draining\n");
1032 			return NULL;
1033 		}
1034 		cpu_relax();
1035 		udelay(1);
1036 	}
1037 
1038 	cmd = its->cmd_write++;
1039 
1040 	/* Handle queue wrapping */
1041 	if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES))
1042 		its->cmd_write = its->cmd_base;
1043 
1044 	/* Clear command  */
1045 	cmd->raw_cmd[0] = 0;
1046 	cmd->raw_cmd[1] = 0;
1047 	cmd->raw_cmd[2] = 0;
1048 	cmd->raw_cmd[3] = 0;
1049 
1050 	return cmd;
1051 }
1052 
its_post_commands(struct its_node * its)1053 static struct its_cmd_block *its_post_commands(struct its_node *its)
1054 {
1055 	u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write);
1056 
1057 	writel_relaxed(wr, its->base + GITS_CWRITER);
1058 
1059 	return its->cmd_write;
1060 }
1061 
its_flush_cmd(struct its_node * its,struct its_cmd_block * cmd)1062 static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd)
1063 {
1064 	/*
1065 	 * Make sure the commands written to memory are observable by
1066 	 * the ITS.
1067 	 */
1068 	if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING)
1069 		gic_flush_dcache_to_poc(cmd, sizeof(*cmd));
1070 	else
1071 		dsb(ishst);
1072 }
1073 
its_wait_for_range_completion(struct its_node * its,u64 prev_idx,struct its_cmd_block * to)1074 static int its_wait_for_range_completion(struct its_node *its,
1075 					 u64	prev_idx,
1076 					 struct its_cmd_block *to)
1077 {
1078 	u64 rd_idx, to_idx, linear_idx;
1079 	u32 count = 1000000;	/* 1s! */
1080 
1081 	/* Linearize to_idx if the command set has wrapped around */
1082 	to_idx = its_cmd_ptr_to_offset(its, to);
1083 	if (to_idx < prev_idx)
1084 		to_idx += ITS_CMD_QUEUE_SZ;
1085 
1086 	linear_idx = prev_idx;
1087 
1088 	while (1) {
1089 		s64 delta;
1090 
1091 		rd_idx = readl_relaxed(its->base + GITS_CREADR);
1092 
1093 		/*
1094 		 * Compute the read pointer progress, taking the
1095 		 * potential wrap-around into account.
1096 		 */
1097 		delta = rd_idx - prev_idx;
1098 		if (rd_idx < prev_idx)
1099 			delta += ITS_CMD_QUEUE_SZ;
1100 
1101 		linear_idx += delta;
1102 		if (linear_idx >= to_idx)
1103 			break;
1104 
1105 		count--;
1106 		if (!count) {
1107 			pr_err_ratelimited("ITS queue timeout (%llu %llu)\n",
1108 					   to_idx, linear_idx);
1109 			return -1;
1110 		}
1111 		prev_idx = rd_idx;
1112 		cpu_relax();
1113 		udelay(1);
1114 	}
1115 
1116 	return 0;
1117 }
1118 
1119 /* Warning, macro hell follows */
1120 #define BUILD_SINGLE_CMD_FUNC(name, buildtype, synctype, buildfn)	\
1121 void name(struct its_node *its,						\
1122 	  buildtype builder,						\
1123 	  struct its_cmd_desc *desc)					\
1124 {									\
1125 	struct its_cmd_block *cmd, *sync_cmd, *next_cmd;		\
1126 	synctype *sync_obj;						\
1127 	unsigned long flags;						\
1128 	u64 rd_idx;							\
1129 									\
1130 	raw_spin_lock_irqsave(&its->lock, flags);			\
1131 									\
1132 	cmd = its_allocate_entry(its);					\
1133 	if (!cmd) {		/* We're soooooo screewed... */		\
1134 		raw_spin_unlock_irqrestore(&its->lock, flags);		\
1135 		return;							\
1136 	}								\
1137 	sync_obj = builder(its, cmd, desc);				\
1138 	its_flush_cmd(its, cmd);					\
1139 									\
1140 	if (sync_obj) {							\
1141 		sync_cmd = its_allocate_entry(its);			\
1142 		if (!sync_cmd)						\
1143 			goto post;					\
1144 									\
1145 		buildfn(its, sync_cmd, sync_obj);			\
1146 		its_flush_cmd(its, sync_cmd);				\
1147 	}								\
1148 									\
1149 post:									\
1150 	rd_idx = readl_relaxed(its->base + GITS_CREADR);		\
1151 	next_cmd = its_post_commands(its);				\
1152 	raw_spin_unlock_irqrestore(&its->lock, flags);			\
1153 									\
1154 	if (its_wait_for_range_completion(its, rd_idx, next_cmd))	\
1155 		pr_err_ratelimited("ITS cmd %ps failed\n", builder);	\
1156 }
1157 
its_build_sync_cmd(struct its_node * its,struct its_cmd_block * sync_cmd,struct its_collection * sync_col)1158 static void its_build_sync_cmd(struct its_node *its,
1159 			       struct its_cmd_block *sync_cmd,
1160 			       struct its_collection *sync_col)
1161 {
1162 	its_encode_cmd(sync_cmd, GITS_CMD_SYNC);
1163 	its_encode_target(sync_cmd, sync_col->target_address);
1164 
1165 	its_fixup_cmd(sync_cmd);
1166 }
1167 
BUILD_SINGLE_CMD_FUNC(its_send_single_command,its_cmd_builder_t,struct its_collection,its_build_sync_cmd)1168 static BUILD_SINGLE_CMD_FUNC(its_send_single_command, its_cmd_builder_t,
1169 			     struct its_collection, its_build_sync_cmd)
1170 
1171 static void its_build_vsync_cmd(struct its_node *its,
1172 				struct its_cmd_block *sync_cmd,
1173 				struct its_vpe *sync_vpe)
1174 {
1175 	its_encode_cmd(sync_cmd, GITS_CMD_VSYNC);
1176 	its_encode_vpeid(sync_cmd, sync_vpe->vpe_id);
1177 
1178 	its_fixup_cmd(sync_cmd);
1179 }
1180 
BUILD_SINGLE_CMD_FUNC(its_send_single_vcommand,its_cmd_vbuilder_t,struct its_vpe,its_build_vsync_cmd)1181 static BUILD_SINGLE_CMD_FUNC(its_send_single_vcommand, its_cmd_vbuilder_t,
1182 			     struct its_vpe, its_build_vsync_cmd)
1183 
1184 static void its_send_int(struct its_device *dev, u32 event_id)
1185 {
1186 	struct its_cmd_desc desc;
1187 
1188 	desc.its_int_cmd.dev = dev;
1189 	desc.its_int_cmd.event_id = event_id;
1190 
1191 	its_send_single_command(dev->its, its_build_int_cmd, &desc);
1192 }
1193 
its_send_clear(struct its_device * dev,u32 event_id)1194 static void its_send_clear(struct its_device *dev, u32 event_id)
1195 {
1196 	struct its_cmd_desc desc;
1197 
1198 	desc.its_clear_cmd.dev = dev;
1199 	desc.its_clear_cmd.event_id = event_id;
1200 
1201 	its_send_single_command(dev->its, its_build_clear_cmd, &desc);
1202 }
1203 
its_send_inv(struct its_device * dev,u32 event_id)1204 static void its_send_inv(struct its_device *dev, u32 event_id)
1205 {
1206 	struct its_cmd_desc desc;
1207 
1208 	desc.its_inv_cmd.dev = dev;
1209 	desc.its_inv_cmd.event_id = event_id;
1210 
1211 	its_send_single_command(dev->its, its_build_inv_cmd, &desc);
1212 }
1213 
its_send_mapd(struct its_device * dev,int valid)1214 static void its_send_mapd(struct its_device *dev, int valid)
1215 {
1216 	struct its_cmd_desc desc;
1217 
1218 	desc.its_mapd_cmd.dev = dev;
1219 	desc.its_mapd_cmd.valid = !!valid;
1220 
1221 	its_send_single_command(dev->its, its_build_mapd_cmd, &desc);
1222 }
1223 
its_send_mapc(struct its_node * its,struct its_collection * col,int valid)1224 static void its_send_mapc(struct its_node *its, struct its_collection *col,
1225 			  int valid)
1226 {
1227 	struct its_cmd_desc desc;
1228 
1229 	desc.its_mapc_cmd.col = col;
1230 	desc.its_mapc_cmd.valid = !!valid;
1231 
1232 	its_send_single_command(its, its_build_mapc_cmd, &desc);
1233 }
1234 
its_send_mapti(struct its_device * dev,u32 irq_id,u32 id)1235 static void its_send_mapti(struct its_device *dev, u32 irq_id, u32 id)
1236 {
1237 	struct its_cmd_desc desc;
1238 
1239 	desc.its_mapti_cmd.dev = dev;
1240 	desc.its_mapti_cmd.phys_id = irq_id;
1241 	desc.its_mapti_cmd.event_id = id;
1242 
1243 	its_send_single_command(dev->its, its_build_mapti_cmd, &desc);
1244 }
1245 
its_send_movi(struct its_device * dev,struct its_collection * col,u32 id)1246 static void its_send_movi(struct its_device *dev,
1247 			  struct its_collection *col, u32 id)
1248 {
1249 	struct its_cmd_desc desc;
1250 
1251 	desc.its_movi_cmd.dev = dev;
1252 	desc.its_movi_cmd.col = col;
1253 	desc.its_movi_cmd.event_id = id;
1254 
1255 	its_send_single_command(dev->its, its_build_movi_cmd, &desc);
1256 }
1257 
its_send_discard(struct its_device * dev,u32 id)1258 static void its_send_discard(struct its_device *dev, u32 id)
1259 {
1260 	struct its_cmd_desc desc;
1261 
1262 	desc.its_discard_cmd.dev = dev;
1263 	desc.its_discard_cmd.event_id = id;
1264 
1265 	its_send_single_command(dev->its, its_build_discard_cmd, &desc);
1266 }
1267 
its_send_invall(struct its_node * its,struct its_collection * col)1268 static void its_send_invall(struct its_node *its, struct its_collection *col)
1269 {
1270 	struct its_cmd_desc desc;
1271 
1272 	desc.its_invall_cmd.col = col;
1273 
1274 	its_send_single_command(its, its_build_invall_cmd, &desc);
1275 }
1276 
its_send_vmapti(struct its_device * dev,u32 id)1277 static void its_send_vmapti(struct its_device *dev, u32 id)
1278 {
1279 	struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
1280 	struct its_cmd_desc desc;
1281 
1282 	desc.its_vmapti_cmd.vpe = map->vpe;
1283 	desc.its_vmapti_cmd.dev = dev;
1284 	desc.its_vmapti_cmd.virt_id = map->vintid;
1285 	desc.its_vmapti_cmd.event_id = id;
1286 	desc.its_vmapti_cmd.db_enabled = map->db_enabled;
1287 
1288 	its_send_single_vcommand(dev->its, its_build_vmapti_cmd, &desc);
1289 }
1290 
its_send_vmovi(struct its_device * dev,u32 id)1291 static void its_send_vmovi(struct its_device *dev, u32 id)
1292 {
1293 	struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
1294 	struct its_cmd_desc desc;
1295 
1296 	desc.its_vmovi_cmd.vpe = map->vpe;
1297 	desc.its_vmovi_cmd.dev = dev;
1298 	desc.its_vmovi_cmd.event_id = id;
1299 	desc.its_vmovi_cmd.db_enabled = map->db_enabled;
1300 
1301 	its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc);
1302 }
1303 
its_send_vmapp(struct its_node * its,struct its_vpe * vpe,bool valid)1304 static void its_send_vmapp(struct its_node *its,
1305 			   struct its_vpe *vpe, bool valid)
1306 {
1307 	struct its_cmd_desc desc;
1308 
1309 	desc.its_vmapp_cmd.vpe = vpe;
1310 	desc.its_vmapp_cmd.valid = valid;
1311 	desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx];
1312 
1313 	its_send_single_vcommand(its, its_build_vmapp_cmd, &desc);
1314 }
1315 
its_send_vmovp(struct its_vpe * vpe)1316 static void its_send_vmovp(struct its_vpe *vpe)
1317 {
1318 	struct its_cmd_desc desc = {};
1319 	struct its_node *its;
1320 	unsigned long flags;
1321 	int col_id = vpe->col_idx;
1322 
1323 	desc.its_vmovp_cmd.vpe = vpe;
1324 
1325 	if (!its_list_map) {
1326 		its = list_first_entry(&its_nodes, struct its_node, entry);
1327 		desc.its_vmovp_cmd.col = &its->collections[col_id];
1328 		its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
1329 		return;
1330 	}
1331 
1332 	/*
1333 	 * Yet another marvel of the architecture. If using the
1334 	 * its_list "feature", we need to make sure that all ITSs
1335 	 * receive all VMOVP commands in the same order. The only way
1336 	 * to guarantee this is to make vmovp a serialization point.
1337 	 *
1338 	 * Wall <-- Head.
1339 	 */
1340 	raw_spin_lock_irqsave(&vmovp_lock, flags);
1341 
1342 	desc.its_vmovp_cmd.seq_num = vmovp_seq_num++;
1343 	desc.its_vmovp_cmd.its_list = get_its_list(vpe->its_vm);
1344 
1345 	/* Emit VMOVPs */
1346 	list_for_each_entry(its, &its_nodes, entry) {
1347 		if (!is_v4(its))
1348 			continue;
1349 
1350 		if (!require_its_list_vmovp(vpe->its_vm, its))
1351 			continue;
1352 
1353 		desc.its_vmovp_cmd.col = &its->collections[col_id];
1354 		its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
1355 	}
1356 
1357 	raw_spin_unlock_irqrestore(&vmovp_lock, flags);
1358 }
1359 
its_send_vinvall(struct its_node * its,struct its_vpe * vpe)1360 static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe)
1361 {
1362 	struct its_cmd_desc desc;
1363 
1364 	desc.its_vinvall_cmd.vpe = vpe;
1365 	its_send_single_vcommand(its, its_build_vinvall_cmd, &desc);
1366 }
1367 
its_send_vinv(struct its_device * dev,u32 event_id)1368 static void its_send_vinv(struct its_device *dev, u32 event_id)
1369 {
1370 	struct its_cmd_desc desc;
1371 
1372 	/*
1373 	 * There is no real VINV command. This is just a normal INV,
1374 	 * with a VSYNC instead of a SYNC.
1375 	 */
1376 	desc.its_inv_cmd.dev = dev;
1377 	desc.its_inv_cmd.event_id = event_id;
1378 
1379 	its_send_single_vcommand(dev->its, its_build_vinv_cmd, &desc);
1380 }
1381 
its_send_vint(struct its_device * dev,u32 event_id)1382 static void its_send_vint(struct its_device *dev, u32 event_id)
1383 {
1384 	struct its_cmd_desc desc;
1385 
1386 	/*
1387 	 * There is no real VINT command. This is just a normal INT,
1388 	 * with a VSYNC instead of a SYNC.
1389 	 */
1390 	desc.its_int_cmd.dev = dev;
1391 	desc.its_int_cmd.event_id = event_id;
1392 
1393 	its_send_single_vcommand(dev->its, its_build_vint_cmd, &desc);
1394 }
1395 
its_send_vclear(struct its_device * dev,u32 event_id)1396 static void its_send_vclear(struct its_device *dev, u32 event_id)
1397 {
1398 	struct its_cmd_desc desc;
1399 
1400 	/*
1401 	 * There is no real VCLEAR command. This is just a normal CLEAR,
1402 	 * with a VSYNC instead of a SYNC.
1403 	 */
1404 	desc.its_clear_cmd.dev = dev;
1405 	desc.its_clear_cmd.event_id = event_id;
1406 
1407 	its_send_single_vcommand(dev->its, its_build_vclear_cmd, &desc);
1408 }
1409 
its_send_invdb(struct its_node * its,struct its_vpe * vpe)1410 static void its_send_invdb(struct its_node *its, struct its_vpe *vpe)
1411 {
1412 	struct its_cmd_desc desc;
1413 
1414 	desc.its_invdb_cmd.vpe = vpe;
1415 	its_send_single_vcommand(its, its_build_invdb_cmd, &desc);
1416 }
1417 
1418 /*
1419  * irqchip functions - assumes MSI, mostly.
1420  */
lpi_write_config(struct irq_data * d,u8 clr,u8 set)1421 static void lpi_write_config(struct irq_data *d, u8 clr, u8 set)
1422 {
1423 	struct its_vlpi_map *map = get_vlpi_map(d);
1424 	irq_hw_number_t hwirq;
1425 	void *va;
1426 	u8 *cfg;
1427 
1428 	if (map) {
1429 		va = page_address(map->vm->vprop_page);
1430 		hwirq = map->vintid;
1431 
1432 		/* Remember the updated property */
1433 		map->properties &= ~clr;
1434 		map->properties |= set | LPI_PROP_GROUP1;
1435 	} else {
1436 		va = gic_rdists->prop_table_va;
1437 		hwirq = d->hwirq;
1438 	}
1439 
1440 	cfg = va + hwirq - 8192;
1441 	*cfg &= ~clr;
1442 	*cfg |= set | LPI_PROP_GROUP1;
1443 
1444 	/*
1445 	 * Make the above write visible to the redistributors.
1446 	 * And yes, we're flushing exactly: One. Single. Byte.
1447 	 * Humpf...
1448 	 */
1449 	if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING)
1450 		gic_flush_dcache_to_poc(cfg, sizeof(*cfg));
1451 	else
1452 		dsb(ishst);
1453 }
1454 
wait_for_syncr(void __iomem * rdbase)1455 static void wait_for_syncr(void __iomem *rdbase)
1456 {
1457 	while (readl_relaxed(rdbase + GICR_SYNCR) & 1)
1458 		cpu_relax();
1459 }
1460 
__direct_lpi_inv(struct irq_data * d,u64 val)1461 static void __direct_lpi_inv(struct irq_data *d, u64 val)
1462 {
1463 	void __iomem *rdbase;
1464 	unsigned long flags;
1465 	int cpu;
1466 
1467 	/* Target the redistributor this LPI is currently routed to */
1468 	cpu = irq_to_cpuid_lock(d, &flags);
1469 	raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
1470 
1471 	rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
1472 	gic_write_lpir(val, rdbase + GICR_INVLPIR);
1473 	wait_for_syncr(rdbase);
1474 
1475 	raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
1476 	irq_to_cpuid_unlock(d, flags);
1477 }
1478 
direct_lpi_inv(struct irq_data * d)1479 static void direct_lpi_inv(struct irq_data *d)
1480 {
1481 	struct its_vlpi_map *map = get_vlpi_map(d);
1482 	u64 val;
1483 
1484 	if (map) {
1485 		struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1486 
1487 		WARN_ON(!is_v4_1(its_dev->its));
1488 
1489 		val  = GICR_INVLPIR_V;
1490 		val |= FIELD_PREP(GICR_INVLPIR_VPEID, map->vpe->vpe_id);
1491 		val |= FIELD_PREP(GICR_INVLPIR_INTID, map->vintid);
1492 	} else {
1493 		val = d->hwirq;
1494 	}
1495 
1496 	__direct_lpi_inv(d, val);
1497 }
1498 
lpi_update_config(struct irq_data * d,u8 clr,u8 set)1499 static void lpi_update_config(struct irq_data *d, u8 clr, u8 set)
1500 {
1501 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1502 
1503 	lpi_write_config(d, clr, set);
1504 	if (gic_rdists->has_direct_lpi &&
1505 	    (is_v4_1(its_dev->its) || !irqd_is_forwarded_to_vcpu(d)))
1506 		direct_lpi_inv(d);
1507 	else if (!irqd_is_forwarded_to_vcpu(d))
1508 		its_send_inv(its_dev, its_get_event_id(d));
1509 	else
1510 		its_send_vinv(its_dev, its_get_event_id(d));
1511 }
1512 
its_vlpi_set_doorbell(struct irq_data * d,bool enable)1513 static void its_vlpi_set_doorbell(struct irq_data *d, bool enable)
1514 {
1515 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1516 	u32 event = its_get_event_id(d);
1517 	struct its_vlpi_map *map;
1518 
1519 	/*
1520 	 * GICv4.1 does away with the per-LPI nonsense, nothing to do
1521 	 * here.
1522 	 */
1523 	if (is_v4_1(its_dev->its))
1524 		return;
1525 
1526 	map = dev_event_to_vlpi_map(its_dev, event);
1527 
1528 	if (map->db_enabled == enable)
1529 		return;
1530 
1531 	map->db_enabled = enable;
1532 
1533 	/*
1534 	 * More fun with the architecture:
1535 	 *
1536 	 * Ideally, we'd issue a VMAPTI to set the doorbell to its LPI
1537 	 * value or to 1023, depending on the enable bit. But that
1538 	 * would be issuing a mapping for an /existing/ DevID+EventID
1539 	 * pair, which is UNPREDICTABLE. Instead, let's issue a VMOVI
1540 	 * to the /same/ vPE, using this opportunity to adjust the
1541 	 * doorbell. Mouahahahaha. We loves it, Precious.
1542 	 */
1543 	its_send_vmovi(its_dev, event);
1544 }
1545 
its_mask_irq(struct irq_data * d)1546 static void its_mask_irq(struct irq_data *d)
1547 {
1548 	if (irqd_is_forwarded_to_vcpu(d))
1549 		its_vlpi_set_doorbell(d, false);
1550 
1551 	lpi_update_config(d, LPI_PROP_ENABLED, 0);
1552 }
1553 
its_unmask_irq(struct irq_data * d)1554 static void its_unmask_irq(struct irq_data *d)
1555 {
1556 	if (irqd_is_forwarded_to_vcpu(d))
1557 		its_vlpi_set_doorbell(d, true);
1558 
1559 	lpi_update_config(d, 0, LPI_PROP_ENABLED);
1560 }
1561 
its_read_lpi_count(struct irq_data * d,int cpu)1562 static __maybe_unused u32 its_read_lpi_count(struct irq_data *d, int cpu)
1563 {
1564 	if (irqd_affinity_is_managed(d))
1565 		return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1566 
1567 	return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1568 }
1569 
its_inc_lpi_count(struct irq_data * d,int cpu)1570 static void its_inc_lpi_count(struct irq_data *d, int cpu)
1571 {
1572 	if (irqd_affinity_is_managed(d))
1573 		atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1574 	else
1575 		atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1576 }
1577 
its_dec_lpi_count(struct irq_data * d,int cpu)1578 static void its_dec_lpi_count(struct irq_data *d, int cpu)
1579 {
1580 	if (irqd_affinity_is_managed(d))
1581 		atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1582 	else
1583 		atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1584 }
1585 
cpumask_pick_least_loaded(struct irq_data * d,const struct cpumask * cpu_mask)1586 static unsigned int cpumask_pick_least_loaded(struct irq_data *d,
1587 					      const struct cpumask *cpu_mask)
1588 {
1589 	unsigned int cpu = nr_cpu_ids, tmp;
1590 	int count = S32_MAX;
1591 
1592 	for_each_cpu(tmp, cpu_mask) {
1593 		int this_count = its_read_lpi_count(d, tmp);
1594 		if (this_count < count) {
1595 			cpu = tmp;
1596 		        count = this_count;
1597 		}
1598 	}
1599 
1600 	return cpu;
1601 }
1602 
1603 /*
1604  * As suggested by Thomas Gleixner in:
1605  * https://lore.kernel.org/r/87h80q2aoc.fsf@nanos.tec.linutronix.de
1606  */
its_select_cpu(struct irq_data * d,const struct cpumask * aff_mask)1607 static int its_select_cpu(struct irq_data *d,
1608 			  const struct cpumask *aff_mask)
1609 {
1610 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1611 	static DEFINE_RAW_SPINLOCK(tmpmask_lock);
1612 	static struct cpumask __tmpmask;
1613 	struct cpumask *tmpmask;
1614 	unsigned long flags;
1615 	int cpu, node;
1616 	node = its_dev->its->numa_node;
1617 	tmpmask = &__tmpmask;
1618 
1619 	raw_spin_lock_irqsave(&tmpmask_lock, flags);
1620 
1621 	if (!irqd_affinity_is_managed(d)) {
1622 		/* First try the NUMA node */
1623 		if (node != NUMA_NO_NODE) {
1624 			/*
1625 			 * Try the intersection of the affinity mask and the
1626 			 * node mask (and the online mask, just to be safe).
1627 			 */
1628 			cpumask_and(tmpmask, cpumask_of_node(node), aff_mask);
1629 			cpumask_and(tmpmask, tmpmask, cpu_online_mask);
1630 
1631 			/*
1632 			 * Ideally, we would check if the mask is empty, and
1633 			 * try again on the full node here.
1634 			 *
1635 			 * But it turns out that the way ACPI describes the
1636 			 * affinity for ITSs only deals about memory, and
1637 			 * not target CPUs, so it cannot describe a single
1638 			 * ITS placed next to two NUMA nodes.
1639 			 *
1640 			 * Instead, just fallback on the online mask. This
1641 			 * diverges from Thomas' suggestion above.
1642 			 */
1643 			cpu = cpumask_pick_least_loaded(d, tmpmask);
1644 			if (cpu < nr_cpu_ids)
1645 				goto out;
1646 
1647 			/* If we can't cross sockets, give up */
1648 			if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144))
1649 				goto out;
1650 
1651 			/* If the above failed, expand the search */
1652 		}
1653 
1654 		/* Try the intersection of the affinity and online masks */
1655 		cpumask_and(tmpmask, aff_mask, cpu_online_mask);
1656 
1657 		/* If that doesn't fly, the online mask is the last resort */
1658 		if (cpumask_empty(tmpmask))
1659 			cpumask_copy(tmpmask, cpu_online_mask);
1660 
1661 		cpu = cpumask_pick_least_loaded(d, tmpmask);
1662 	} else {
1663 		cpumask_copy(tmpmask, aff_mask);
1664 
1665 		/* If we cannot cross sockets, limit the search to that node */
1666 		if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) &&
1667 		    node != NUMA_NO_NODE)
1668 			cpumask_and(tmpmask, tmpmask, cpumask_of_node(node));
1669 
1670 		cpu = cpumask_pick_least_loaded(d, tmpmask);
1671 	}
1672 out:
1673 	raw_spin_unlock_irqrestore(&tmpmask_lock, flags);
1674 
1675 	pr_debug("IRQ%d -> %*pbl CPU%d\n", d->irq, cpumask_pr_args(aff_mask), cpu);
1676 	return cpu;
1677 }
1678 
its_set_affinity(struct irq_data * d,const struct cpumask * mask_val,bool force)1679 static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
1680 			    bool force)
1681 {
1682 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1683 	struct its_collection *target_col;
1684 	u32 id = its_get_event_id(d);
1685 	int cpu, prev_cpu;
1686 
1687 	/* A forwarded interrupt should use irq_set_vcpu_affinity */
1688 	if (irqd_is_forwarded_to_vcpu(d))
1689 		return -EINVAL;
1690 
1691 	prev_cpu = its_dev->event_map.col_map[id];
1692 	its_dec_lpi_count(d, prev_cpu);
1693 
1694 	if (!force)
1695 		cpu = its_select_cpu(d, mask_val);
1696 	else
1697 		cpu = cpumask_pick_least_loaded(d, mask_val);
1698 
1699 	if (cpu < 0 || cpu >= nr_cpu_ids)
1700 		goto err;
1701 
1702 	/* don't set the affinity when the target cpu is same as current one */
1703 	if (cpu != prev_cpu) {
1704 		target_col = &its_dev->its->collections[cpu];
1705 		its_send_movi(its_dev, target_col, id);
1706 		its_dev->event_map.col_map[id] = cpu;
1707 		irq_data_update_effective_affinity(d, cpumask_of(cpu));
1708 	}
1709 
1710 	its_inc_lpi_count(d, cpu);
1711 
1712 	return IRQ_SET_MASK_OK_DONE;
1713 
1714 err:
1715 	its_inc_lpi_count(d, prev_cpu);
1716 	return -EINVAL;
1717 }
1718 
its_irq_get_msi_base(struct its_device * its_dev)1719 static u64 its_irq_get_msi_base(struct its_device *its_dev)
1720 {
1721 	struct its_node *its = its_dev->its;
1722 
1723 	return its->phys_base + GITS_TRANSLATER;
1724 }
1725 
its_irq_compose_msi_msg(struct irq_data * d,struct msi_msg * msg)1726 static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
1727 {
1728 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1729 	struct its_node *its;
1730 	u64 addr;
1731 
1732 	its = its_dev->its;
1733 	addr = its->get_msi_base(its_dev);
1734 
1735 	msg->address_lo		= lower_32_bits(addr);
1736 	msg->address_hi		= upper_32_bits(addr);
1737 	msg->data		= its_get_event_id(d);
1738 
1739 	iommu_dma_compose_msi_msg(irq_data_get_msi_desc(d), msg);
1740 }
1741 
its_irq_set_irqchip_state(struct irq_data * d,enum irqchip_irq_state which,bool state)1742 static int its_irq_set_irqchip_state(struct irq_data *d,
1743 				     enum irqchip_irq_state which,
1744 				     bool state)
1745 {
1746 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1747 	u32 event = its_get_event_id(d);
1748 
1749 	if (which != IRQCHIP_STATE_PENDING)
1750 		return -EINVAL;
1751 
1752 	if (irqd_is_forwarded_to_vcpu(d)) {
1753 		if (state)
1754 			its_send_vint(its_dev, event);
1755 		else
1756 			its_send_vclear(its_dev, event);
1757 	} else {
1758 		if (state)
1759 			its_send_int(its_dev, event);
1760 		else
1761 			its_send_clear(its_dev, event);
1762 	}
1763 
1764 	return 0;
1765 }
1766 
its_irq_retrigger(struct irq_data * d)1767 static int its_irq_retrigger(struct irq_data *d)
1768 {
1769 	return !its_irq_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true);
1770 }
1771 
1772 /*
1773  * Two favourable cases:
1774  *
1775  * (a) Either we have a GICv4.1, and all vPEs have to be mapped at all times
1776  *     for vSGI delivery
1777  *
1778  * (b) Or the ITSs do not use a list map, meaning that VMOVP is cheap enough
1779  *     and we're better off mapping all VPEs always
1780  *
1781  * If neither (a) nor (b) is true, then we map vPEs on demand.
1782  *
1783  */
gic_requires_eager_mapping(void)1784 static bool gic_requires_eager_mapping(void)
1785 {
1786 	if (!its_list_map || gic_rdists->has_rvpeid)
1787 		return true;
1788 
1789 	return false;
1790 }
1791 
its_map_vm(struct its_node * its,struct its_vm * vm)1792 static void its_map_vm(struct its_node *its, struct its_vm *vm)
1793 {
1794 	unsigned long flags;
1795 
1796 	if (gic_requires_eager_mapping())
1797 		return;
1798 
1799 	raw_spin_lock_irqsave(&vmovp_lock, flags);
1800 
1801 	/*
1802 	 * If the VM wasn't mapped yet, iterate over the vpes and get
1803 	 * them mapped now.
1804 	 */
1805 	vm->vlpi_count[its->list_nr]++;
1806 
1807 	if (vm->vlpi_count[its->list_nr] == 1) {
1808 		int i;
1809 
1810 		for (i = 0; i < vm->nr_vpes; i++) {
1811 			struct its_vpe *vpe = vm->vpes[i];
1812 			struct irq_data *d = irq_get_irq_data(vpe->irq);
1813 
1814 			/* Map the VPE to the first possible CPU */
1815 			vpe->col_idx = cpumask_first(cpu_online_mask);
1816 			its_send_vmapp(its, vpe, true);
1817 			its_send_vinvall(its, vpe);
1818 			irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
1819 		}
1820 	}
1821 
1822 	raw_spin_unlock_irqrestore(&vmovp_lock, flags);
1823 }
1824 
its_unmap_vm(struct its_node * its,struct its_vm * vm)1825 static void its_unmap_vm(struct its_node *its, struct its_vm *vm)
1826 {
1827 	unsigned long flags;
1828 
1829 	/* Not using the ITS list? Everything is always mapped. */
1830 	if (gic_requires_eager_mapping())
1831 		return;
1832 
1833 	raw_spin_lock_irqsave(&vmovp_lock, flags);
1834 
1835 	if (!--vm->vlpi_count[its->list_nr]) {
1836 		int i;
1837 
1838 		for (i = 0; i < vm->nr_vpes; i++)
1839 			its_send_vmapp(its, vm->vpes[i], false);
1840 	}
1841 
1842 	raw_spin_unlock_irqrestore(&vmovp_lock, flags);
1843 }
1844 
its_vlpi_map(struct irq_data * d,struct its_cmd_info * info)1845 static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info)
1846 {
1847 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1848 	u32 event = its_get_event_id(d);
1849 
1850 	if (!info->map)
1851 		return -EINVAL;
1852 
1853 	if (!its_dev->event_map.vm) {
1854 		struct its_vlpi_map *maps;
1855 
1856 		maps = kcalloc(its_dev->event_map.nr_lpis, sizeof(*maps),
1857 			       GFP_ATOMIC);
1858 		if (!maps)
1859 			return -ENOMEM;
1860 
1861 		its_dev->event_map.vm = info->map->vm;
1862 		its_dev->event_map.vlpi_maps = maps;
1863 	} else if (its_dev->event_map.vm != info->map->vm) {
1864 		return -EINVAL;
1865 	}
1866 
1867 	/* Get our private copy of the mapping information */
1868 	its_dev->event_map.vlpi_maps[event] = *info->map;
1869 
1870 	if (irqd_is_forwarded_to_vcpu(d)) {
1871 		/* Already mapped, move it around */
1872 		its_send_vmovi(its_dev, event);
1873 	} else {
1874 		/* Ensure all the VPEs are mapped on this ITS */
1875 		its_map_vm(its_dev->its, info->map->vm);
1876 
1877 		/*
1878 		 * Flag the interrupt as forwarded so that we can
1879 		 * start poking the virtual property table.
1880 		 */
1881 		irqd_set_forwarded_to_vcpu(d);
1882 
1883 		/* Write out the property to the prop table */
1884 		lpi_write_config(d, 0xff, info->map->properties);
1885 
1886 		/* Drop the physical mapping */
1887 		its_send_discard(its_dev, event);
1888 
1889 		/* and install the virtual one */
1890 		its_send_vmapti(its_dev, event);
1891 
1892 		/* Increment the number of VLPIs */
1893 		its_dev->event_map.nr_vlpis++;
1894 	}
1895 
1896 	return 0;
1897 }
1898 
its_vlpi_get(struct irq_data * d,struct its_cmd_info * info)1899 static int its_vlpi_get(struct irq_data *d, struct its_cmd_info *info)
1900 {
1901 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1902 	struct its_vlpi_map *map;
1903 
1904 	map = get_vlpi_map(d);
1905 
1906 	if (!its_dev->event_map.vm || !map)
1907 		return -EINVAL;
1908 
1909 	/* Copy our mapping information to the incoming request */
1910 	*info->map = *map;
1911 
1912 	return 0;
1913 }
1914 
its_vlpi_unmap(struct irq_data * d)1915 static int its_vlpi_unmap(struct irq_data *d)
1916 {
1917 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1918 	u32 event = its_get_event_id(d);
1919 
1920 	if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d))
1921 		return -EINVAL;
1922 
1923 	/* Drop the virtual mapping */
1924 	its_send_discard(its_dev, event);
1925 
1926 	/* and restore the physical one */
1927 	irqd_clr_forwarded_to_vcpu(d);
1928 	its_send_mapti(its_dev, d->hwirq, event);
1929 	lpi_update_config(d, 0xff, (LPI_PROP_DEFAULT_PRIO |
1930 				    LPI_PROP_ENABLED |
1931 				    LPI_PROP_GROUP1));
1932 
1933 	/* Potentially unmap the VM from this ITS */
1934 	its_unmap_vm(its_dev->its, its_dev->event_map.vm);
1935 
1936 	/*
1937 	 * Drop the refcount and make the device available again if
1938 	 * this was the last VLPI.
1939 	 */
1940 	if (!--its_dev->event_map.nr_vlpis) {
1941 		its_dev->event_map.vm = NULL;
1942 		kfree(its_dev->event_map.vlpi_maps);
1943 	}
1944 
1945 	return 0;
1946 }
1947 
its_vlpi_prop_update(struct irq_data * d,struct its_cmd_info * info)1948 static int its_vlpi_prop_update(struct irq_data *d, struct its_cmd_info *info)
1949 {
1950 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1951 
1952 	if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d))
1953 		return -EINVAL;
1954 
1955 	if (info->cmd_type == PROP_UPDATE_AND_INV_VLPI)
1956 		lpi_update_config(d, 0xff, info->config);
1957 	else
1958 		lpi_write_config(d, 0xff, info->config);
1959 	its_vlpi_set_doorbell(d, !!(info->config & LPI_PROP_ENABLED));
1960 
1961 	return 0;
1962 }
1963 
its_irq_set_vcpu_affinity(struct irq_data * d,void * vcpu_info)1964 static int its_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
1965 {
1966 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1967 	struct its_cmd_info *info = vcpu_info;
1968 
1969 	/* Need a v4 ITS */
1970 	if (!is_v4(its_dev->its))
1971 		return -EINVAL;
1972 
1973 	guard(raw_spinlock_irq)(&its_dev->event_map.vlpi_lock);
1974 
1975 	/* Unmap request? */
1976 	if (!info)
1977 		return its_vlpi_unmap(d);
1978 
1979 	switch (info->cmd_type) {
1980 	case MAP_VLPI:
1981 		return its_vlpi_map(d, info);
1982 
1983 	case GET_VLPI:
1984 		return its_vlpi_get(d, info);
1985 
1986 	case PROP_UPDATE_VLPI:
1987 	case PROP_UPDATE_AND_INV_VLPI:
1988 		return its_vlpi_prop_update(d, info);
1989 
1990 	default:
1991 		return -EINVAL;
1992 	}
1993 }
1994 
1995 static struct irq_chip its_irq_chip = {
1996 	.name			= "ITS",
1997 	.irq_mask		= its_mask_irq,
1998 	.irq_unmask		= its_unmask_irq,
1999 	.irq_eoi		= irq_chip_eoi_parent,
2000 	.irq_set_affinity	= its_set_affinity,
2001 	.irq_compose_msi_msg	= its_irq_compose_msi_msg,
2002 	.irq_set_irqchip_state	= its_irq_set_irqchip_state,
2003 	.irq_retrigger		= its_irq_retrigger,
2004 	.irq_set_vcpu_affinity	= its_irq_set_vcpu_affinity,
2005 };
2006 
2007 
2008 /*
2009  * How we allocate LPIs:
2010  *
2011  * lpi_range_list contains ranges of LPIs that are to available to
2012  * allocate from. To allocate LPIs, just pick the first range that
2013  * fits the required allocation, and reduce it by the required
2014  * amount. Once empty, remove the range from the list.
2015  *
2016  * To free a range of LPIs, add a free range to the list, sort it and
2017  * merge the result if the new range happens to be adjacent to an
2018  * already free block.
2019  *
2020  * The consequence of the above is that allocation is cost is low, but
2021  * freeing is expensive. We assumes that freeing rarely occurs.
2022  */
2023 #define ITS_MAX_LPI_NRBITS	16 /* 64K LPIs */
2024 
2025 static DEFINE_MUTEX(lpi_range_lock);
2026 static LIST_HEAD(lpi_range_list);
2027 
2028 struct lpi_range {
2029 	struct list_head	entry;
2030 	u32			base_id;
2031 	u32			span;
2032 };
2033 
mk_lpi_range(u32 base,u32 span)2034 static struct lpi_range *mk_lpi_range(u32 base, u32 span)
2035 {
2036 	struct lpi_range *range;
2037 
2038 	range = kmalloc(sizeof(*range), GFP_KERNEL);
2039 	if (range) {
2040 		range->base_id = base;
2041 		range->span = span;
2042 	}
2043 
2044 	return range;
2045 }
2046 
alloc_lpi_range(u32 nr_lpis,u32 * base)2047 static int alloc_lpi_range(u32 nr_lpis, u32 *base)
2048 {
2049 	struct lpi_range *range, *tmp;
2050 	int err = -ENOSPC;
2051 
2052 	mutex_lock(&lpi_range_lock);
2053 
2054 	list_for_each_entry_safe(range, tmp, &lpi_range_list, entry) {
2055 		if (range->span >= nr_lpis) {
2056 			*base = range->base_id;
2057 			range->base_id += nr_lpis;
2058 			range->span -= nr_lpis;
2059 
2060 			if (range->span == 0) {
2061 				list_del(&range->entry);
2062 				kfree(range);
2063 			}
2064 
2065 			err = 0;
2066 			break;
2067 		}
2068 	}
2069 
2070 	mutex_unlock(&lpi_range_lock);
2071 
2072 	pr_debug("ITS: alloc %u:%u\n", *base, nr_lpis);
2073 	return err;
2074 }
2075 
merge_lpi_ranges(struct lpi_range * a,struct lpi_range * b)2076 static void merge_lpi_ranges(struct lpi_range *a, struct lpi_range *b)
2077 {
2078 	if (&a->entry == &lpi_range_list || &b->entry == &lpi_range_list)
2079 		return;
2080 	if (a->base_id + a->span != b->base_id)
2081 		return;
2082 	b->base_id = a->base_id;
2083 	b->span += a->span;
2084 	list_del(&a->entry);
2085 	kfree(a);
2086 }
2087 
free_lpi_range(u32 base,u32 nr_lpis)2088 static int free_lpi_range(u32 base, u32 nr_lpis)
2089 {
2090 	struct lpi_range *new, *old;
2091 
2092 	new = mk_lpi_range(base, nr_lpis);
2093 	if (!new)
2094 		return -ENOMEM;
2095 
2096 	mutex_lock(&lpi_range_lock);
2097 
2098 	list_for_each_entry_reverse(old, &lpi_range_list, entry) {
2099 		if (old->base_id < base)
2100 			break;
2101 	}
2102 	/*
2103 	 * old is the last element with ->base_id smaller than base,
2104 	 * so new goes right after it. If there are no elements with
2105 	 * ->base_id smaller than base, &old->entry ends up pointing
2106 	 * at the head of the list, and inserting new it the start of
2107 	 * the list is the right thing to do in that case as well.
2108 	 */
2109 	list_add(&new->entry, &old->entry);
2110 	/*
2111 	 * Now check if we can merge with the preceding and/or
2112 	 * following ranges.
2113 	 */
2114 	merge_lpi_ranges(old, new);
2115 	merge_lpi_ranges(new, list_next_entry(new, entry));
2116 
2117 	mutex_unlock(&lpi_range_lock);
2118 	return 0;
2119 }
2120 
its_lpi_init(u32 id_bits)2121 static int __init its_lpi_init(u32 id_bits)
2122 {
2123 	u32 lpis = (1UL << id_bits) - 8192;
2124 	u32 numlpis;
2125 	int err;
2126 
2127 	numlpis = 1UL << GICD_TYPER_NUM_LPIS(gic_rdists->gicd_typer);
2128 
2129 	if (numlpis > 2 && !WARN_ON(numlpis > lpis)) {
2130 		lpis = numlpis;
2131 		pr_info("ITS: Using hypervisor restricted LPI range [%u]\n",
2132 			lpis);
2133 	}
2134 
2135 	/*
2136 	 * Initializing the allocator is just the same as freeing the
2137 	 * full range of LPIs.
2138 	 */
2139 	err = free_lpi_range(8192, lpis);
2140 	pr_debug("ITS: Allocator initialized for %u LPIs\n", lpis);
2141 	return err;
2142 }
2143 
its_lpi_alloc(int nr_irqs,u32 * base,int * nr_ids)2144 static unsigned long *its_lpi_alloc(int nr_irqs, u32 *base, int *nr_ids)
2145 {
2146 	unsigned long *bitmap = NULL;
2147 	int err = 0;
2148 
2149 	do {
2150 		err = alloc_lpi_range(nr_irqs, base);
2151 		if (!err)
2152 			break;
2153 
2154 		nr_irqs /= 2;
2155 	} while (nr_irqs > 0);
2156 
2157 	if (!nr_irqs)
2158 		err = -ENOSPC;
2159 
2160 	if (err)
2161 		goto out;
2162 
2163 	bitmap = bitmap_zalloc(nr_irqs, GFP_ATOMIC);
2164 	if (!bitmap)
2165 		goto out;
2166 
2167 	*nr_ids = nr_irqs;
2168 
2169 out:
2170 	if (!bitmap)
2171 		*base = *nr_ids = 0;
2172 
2173 	return bitmap;
2174 }
2175 
its_lpi_free(unsigned long * bitmap,u32 base,u32 nr_ids)2176 static void its_lpi_free(unsigned long *bitmap, u32 base, u32 nr_ids)
2177 {
2178 	WARN_ON(free_lpi_range(base, nr_ids));
2179 	bitmap_free(bitmap);
2180 }
2181 
gic_reset_prop_table(void * va)2182 static void gic_reset_prop_table(void *va)
2183 {
2184 	/* Priority 0xa0, Group-1, disabled */
2185 	memset(va, LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1, LPI_PROPBASE_SZ);
2186 
2187 	/* Make sure the GIC will observe the written configuration */
2188 	gic_flush_dcache_to_poc(va, LPI_PROPBASE_SZ);
2189 }
2190 
its_allocate_prop_table(gfp_t gfp_flags)2191 static struct page *its_allocate_prop_table(gfp_t gfp_flags)
2192 {
2193 	struct page *prop_page;
2194 
2195 	prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ));
2196 	if (!prop_page)
2197 		return NULL;
2198 
2199 	gic_reset_prop_table(page_address(prop_page));
2200 
2201 	return prop_page;
2202 }
2203 
its_free_prop_table(struct page * prop_page)2204 static void its_free_prop_table(struct page *prop_page)
2205 {
2206 	free_pages((unsigned long)page_address(prop_page),
2207 		   get_order(LPI_PROPBASE_SZ));
2208 }
2209 
gic_check_reserved_range(phys_addr_t addr,unsigned long size)2210 static bool gic_check_reserved_range(phys_addr_t addr, unsigned long size)
2211 {
2212 	phys_addr_t start, end, addr_end;
2213 	u64 i;
2214 
2215 	/*
2216 	 * We don't bother checking for a kdump kernel as by
2217 	 * construction, the LPI tables are out of this kernel's
2218 	 * memory map.
2219 	 */
2220 	if (is_kdump_kernel())
2221 		return true;
2222 
2223 	addr_end = addr + size - 1;
2224 
2225 	for_each_reserved_mem_range(i, &start, &end) {
2226 		if (addr >= start && addr_end <= end)
2227 			return true;
2228 	}
2229 
2230 	/* Not found, not a good sign... */
2231 	pr_warn("GICv3: Expected reserved range [%pa:%pa], not found\n",
2232 		&addr, &addr_end);
2233 	add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
2234 	return false;
2235 }
2236 
gic_reserve_range(phys_addr_t addr,unsigned long size)2237 static int gic_reserve_range(phys_addr_t addr, unsigned long size)
2238 {
2239 	if (efi_enabled(EFI_CONFIG_TABLES))
2240 		return efi_mem_reserve_persistent(addr, size);
2241 
2242 	return 0;
2243 }
2244 
its_setup_lpi_prop_table(void)2245 static int __init its_setup_lpi_prop_table(void)
2246 {
2247 	if (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) {
2248 		u64 val;
2249 
2250 		val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
2251 		lpi_id_bits = (val & GICR_PROPBASER_IDBITS_MASK) + 1;
2252 
2253 		gic_rdists->prop_table_pa = val & GENMASK_ULL(51, 12);
2254 		gic_rdists->prop_table_va = memremap(gic_rdists->prop_table_pa,
2255 						     LPI_PROPBASE_SZ,
2256 						     MEMREMAP_WB);
2257 		gic_reset_prop_table(gic_rdists->prop_table_va);
2258 	} else {
2259 		struct page *page;
2260 
2261 		lpi_id_bits = min_t(u32,
2262 				    GICD_TYPER_ID_BITS(gic_rdists->gicd_typer),
2263 				    ITS_MAX_LPI_NRBITS);
2264 		page = its_allocate_prop_table(GFP_NOWAIT);
2265 		if (!page) {
2266 			pr_err("Failed to allocate PROPBASE\n");
2267 			return -ENOMEM;
2268 		}
2269 
2270 		gic_rdists->prop_table_pa = page_to_phys(page);
2271 		gic_rdists->prop_table_va = page_address(page);
2272 		WARN_ON(gic_reserve_range(gic_rdists->prop_table_pa,
2273 					  LPI_PROPBASE_SZ));
2274 	}
2275 
2276 	pr_info("GICv3: using LPI property table @%pa\n",
2277 		&gic_rdists->prop_table_pa);
2278 
2279 	return its_lpi_init(lpi_id_bits);
2280 }
2281 
2282 static const char *its_base_type_string[] = {
2283 	[GITS_BASER_TYPE_DEVICE]	= "Devices",
2284 	[GITS_BASER_TYPE_VCPU]		= "Virtual CPUs",
2285 	[GITS_BASER_TYPE_RESERVED3]	= "Reserved (3)",
2286 	[GITS_BASER_TYPE_COLLECTION]	= "Interrupt Collections",
2287 	[GITS_BASER_TYPE_RESERVED5] 	= "Reserved (5)",
2288 	[GITS_BASER_TYPE_RESERVED6] 	= "Reserved (6)",
2289 	[GITS_BASER_TYPE_RESERVED7] 	= "Reserved (7)",
2290 };
2291 
its_read_baser(struct its_node * its,struct its_baser * baser)2292 static u64 its_read_baser(struct its_node *its, struct its_baser *baser)
2293 {
2294 	u32 idx = baser - its->tables;
2295 
2296 	return gits_read_baser(its->base + GITS_BASER + (idx << 3));
2297 }
2298 
its_write_baser(struct its_node * its,struct its_baser * baser,u64 val)2299 static void its_write_baser(struct its_node *its, struct its_baser *baser,
2300 			    u64 val)
2301 {
2302 	u32 idx = baser - its->tables;
2303 
2304 	gits_write_baser(val, its->base + GITS_BASER + (idx << 3));
2305 	baser->val = its_read_baser(its, baser);
2306 }
2307 
its_setup_baser(struct its_node * its,struct its_baser * baser,u64 cache,u64 shr,u32 order,bool indirect)2308 static int its_setup_baser(struct its_node *its, struct its_baser *baser,
2309 			   u64 cache, u64 shr, u32 order, bool indirect)
2310 {
2311 	u64 val = its_read_baser(its, baser);
2312 	u64 esz = GITS_BASER_ENTRY_SIZE(val);
2313 	u64 type = GITS_BASER_TYPE(val);
2314 	u64 baser_phys, tmp;
2315 	u32 alloc_pages, psz;
2316 	struct page *page;
2317 	void *base;
2318 
2319 	psz = baser->psz;
2320 	alloc_pages = (PAGE_ORDER_TO_SIZE(order) / psz);
2321 	if (alloc_pages > GITS_BASER_PAGES_MAX) {
2322 		pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n",
2323 			&its->phys_base, its_base_type_string[type],
2324 			alloc_pages, GITS_BASER_PAGES_MAX);
2325 		alloc_pages = GITS_BASER_PAGES_MAX;
2326 		order = get_order(GITS_BASER_PAGES_MAX * psz);
2327 	}
2328 
2329 	page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, order);
2330 	if (!page)
2331 		return -ENOMEM;
2332 
2333 	base = (void *)page_address(page);
2334 	baser_phys = virt_to_phys(base);
2335 
2336 	/* Check if the physical address of the memory is above 48bits */
2337 	if (IS_ENABLED(CONFIG_ARM64_64K_PAGES) && (baser_phys >> 48)) {
2338 
2339 		/* 52bit PA is supported only when PageSize=64K */
2340 		if (psz != SZ_64K) {
2341 			pr_err("ITS: no 52bit PA support when psz=%d\n", psz);
2342 			free_pages((unsigned long)base, order);
2343 			return -ENXIO;
2344 		}
2345 
2346 		/* Convert 52bit PA to 48bit field */
2347 		baser_phys = GITS_BASER_PHYS_52_to_48(baser_phys);
2348 	}
2349 
2350 retry_baser:
2351 	val = (baser_phys					 |
2352 		(type << GITS_BASER_TYPE_SHIFT)			 |
2353 		((esz - 1) << GITS_BASER_ENTRY_SIZE_SHIFT)	 |
2354 		((alloc_pages - 1) << GITS_BASER_PAGES_SHIFT)	 |
2355 		cache						 |
2356 		shr						 |
2357 		GITS_BASER_VALID);
2358 
2359 	val |=	indirect ? GITS_BASER_INDIRECT : 0x0;
2360 
2361 	switch (psz) {
2362 	case SZ_4K:
2363 		val |= GITS_BASER_PAGE_SIZE_4K;
2364 		break;
2365 	case SZ_16K:
2366 		val |= GITS_BASER_PAGE_SIZE_16K;
2367 		break;
2368 	case SZ_64K:
2369 		val |= GITS_BASER_PAGE_SIZE_64K;
2370 		break;
2371 	}
2372 
2373 	if (!shr)
2374 		gic_flush_dcache_to_poc(base, PAGE_ORDER_TO_SIZE(order));
2375 
2376 	its_write_baser(its, baser, val);
2377 	tmp = baser->val;
2378 
2379 	if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) {
2380 		/*
2381 		 * Shareability didn't stick. Just use
2382 		 * whatever the read reported, which is likely
2383 		 * to be the only thing this redistributor
2384 		 * supports. If that's zero, make it
2385 		 * non-cacheable as well.
2386 		 */
2387 		shr = tmp & GITS_BASER_SHAREABILITY_MASK;
2388 		if (!shr)
2389 			cache = GITS_BASER_nC;
2390 
2391 		goto retry_baser;
2392 	}
2393 
2394 	if (val != tmp) {
2395 		pr_err("ITS@%pa: %s doesn't stick: %llx %llx\n",
2396 		       &its->phys_base, its_base_type_string[type],
2397 		       val, tmp);
2398 		free_pages((unsigned long)base, order);
2399 		return -ENXIO;
2400 	}
2401 
2402 	baser->order = order;
2403 	baser->base = base;
2404 	baser->psz = psz;
2405 	tmp = indirect ? GITS_LVL1_ENTRY_SIZE : esz;
2406 
2407 	pr_info("ITS@%pa: allocated %d %s @%lx (%s, esz %d, psz %dK, shr %d)\n",
2408 		&its->phys_base, (int)(PAGE_ORDER_TO_SIZE(order) / (int)tmp),
2409 		its_base_type_string[type],
2410 		(unsigned long)virt_to_phys(base),
2411 		indirect ? "indirect" : "flat", (int)esz,
2412 		psz / SZ_1K, (int)shr >> GITS_BASER_SHAREABILITY_SHIFT);
2413 
2414 	return 0;
2415 }
2416 
its_parse_indirect_baser(struct its_node * its,struct its_baser * baser,u32 * order,u32 ids)2417 static bool its_parse_indirect_baser(struct its_node *its,
2418 				     struct its_baser *baser,
2419 				     u32 *order, u32 ids)
2420 {
2421 	u64 tmp = its_read_baser(its, baser);
2422 	u64 type = GITS_BASER_TYPE(tmp);
2423 	u64 esz = GITS_BASER_ENTRY_SIZE(tmp);
2424 	u64 val = GITS_BASER_InnerShareable | GITS_BASER_RaWaWb;
2425 	u32 new_order = *order;
2426 	u32 psz = baser->psz;
2427 	bool indirect = false;
2428 
2429 	/* No need to enable Indirection if memory requirement < (psz*2)bytes */
2430 	if ((esz << ids) > (psz * 2)) {
2431 		/*
2432 		 * Find out whether hw supports a single or two-level table by
2433 		 * table by reading bit at offset '62' after writing '1' to it.
2434 		 */
2435 		its_write_baser(its, baser, val | GITS_BASER_INDIRECT);
2436 		indirect = !!(baser->val & GITS_BASER_INDIRECT);
2437 
2438 		if (indirect) {
2439 			/*
2440 			 * The size of the lvl2 table is equal to ITS page size
2441 			 * which is 'psz'. For computing lvl1 table size,
2442 			 * subtract ID bits that sparse lvl2 table from 'ids'
2443 			 * which is reported by ITS hardware times lvl1 table
2444 			 * entry size.
2445 			 */
2446 			ids -= ilog2(psz / (int)esz);
2447 			esz = GITS_LVL1_ENTRY_SIZE;
2448 		}
2449 	}
2450 
2451 	/*
2452 	 * Allocate as many entries as required to fit the
2453 	 * range of device IDs that the ITS can grok... The ID
2454 	 * space being incredibly sparse, this results in a
2455 	 * massive waste of memory if two-level device table
2456 	 * feature is not supported by hardware.
2457 	 */
2458 	new_order = max_t(u32, get_order(esz << ids), new_order);
2459 	if (new_order > MAX_ORDER) {
2460 		new_order = MAX_ORDER;
2461 		ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / (int)esz);
2462 		pr_warn("ITS@%pa: %s Table too large, reduce ids %llu->%u\n",
2463 			&its->phys_base, its_base_type_string[type],
2464 			device_ids(its), ids);
2465 	}
2466 
2467 	*order = new_order;
2468 
2469 	return indirect;
2470 }
2471 
compute_common_aff(u64 val)2472 static u32 compute_common_aff(u64 val)
2473 {
2474 	u32 aff, clpiaff;
2475 
2476 	aff = FIELD_GET(GICR_TYPER_AFFINITY, val);
2477 	clpiaff = FIELD_GET(GICR_TYPER_COMMON_LPI_AFF, val);
2478 
2479 	return aff & ~(GENMASK(31, 0) >> (clpiaff * 8));
2480 }
2481 
compute_its_aff(struct its_node * its)2482 static u32 compute_its_aff(struct its_node *its)
2483 {
2484 	u64 val;
2485 	u32 svpet;
2486 
2487 	/*
2488 	 * Reencode the ITS SVPET and MPIDR as a GICR_TYPER, and compute
2489 	 * the resulting affinity. We then use that to see if this match
2490 	 * our own affinity.
2491 	 */
2492 	svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer);
2493 	val  = FIELD_PREP(GICR_TYPER_COMMON_LPI_AFF, svpet);
2494 	val |= FIELD_PREP(GICR_TYPER_AFFINITY, its->mpidr);
2495 	return compute_common_aff(val);
2496 }
2497 
find_sibling_its(struct its_node * cur_its)2498 static struct its_node *find_sibling_its(struct its_node *cur_its)
2499 {
2500 	struct its_node *its;
2501 	u32 aff;
2502 
2503 	if (!FIELD_GET(GITS_TYPER_SVPET, cur_its->typer))
2504 		return NULL;
2505 
2506 	aff = compute_its_aff(cur_its);
2507 
2508 	list_for_each_entry(its, &its_nodes, entry) {
2509 		u64 baser;
2510 
2511 		if (!is_v4_1(its) || its == cur_its)
2512 			continue;
2513 
2514 		if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
2515 			continue;
2516 
2517 		if (aff != compute_its_aff(its))
2518 			continue;
2519 
2520 		/* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
2521 		baser = its->tables[2].val;
2522 		if (!(baser & GITS_BASER_VALID))
2523 			continue;
2524 
2525 		return its;
2526 	}
2527 
2528 	return NULL;
2529 }
2530 
its_free_tables(struct its_node * its)2531 static void its_free_tables(struct its_node *its)
2532 {
2533 	int i;
2534 
2535 	for (i = 0; i < GITS_BASER_NR_REGS; i++) {
2536 		if (its->tables[i].base) {
2537 			free_pages((unsigned long)its->tables[i].base,
2538 				   its->tables[i].order);
2539 			its->tables[i].base = NULL;
2540 		}
2541 	}
2542 }
2543 
its_probe_baser_psz(struct its_node * its,struct its_baser * baser)2544 static int its_probe_baser_psz(struct its_node *its, struct its_baser *baser)
2545 {
2546 	u64 psz = SZ_64K;
2547 
2548 	while (psz) {
2549 		u64 val, gpsz;
2550 
2551 		val = its_read_baser(its, baser);
2552 		val &= ~GITS_BASER_PAGE_SIZE_MASK;
2553 
2554 		switch (psz) {
2555 		case SZ_64K:
2556 			gpsz = GITS_BASER_PAGE_SIZE_64K;
2557 			break;
2558 		case SZ_16K:
2559 			gpsz = GITS_BASER_PAGE_SIZE_16K;
2560 			break;
2561 		case SZ_4K:
2562 		default:
2563 			gpsz = GITS_BASER_PAGE_SIZE_4K;
2564 			break;
2565 		}
2566 
2567 		gpsz >>= GITS_BASER_PAGE_SIZE_SHIFT;
2568 
2569 		val |= FIELD_PREP(GITS_BASER_PAGE_SIZE_MASK, gpsz);
2570 		its_write_baser(its, baser, val);
2571 
2572 		if (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser->val) == gpsz)
2573 			break;
2574 
2575 		switch (psz) {
2576 		case SZ_64K:
2577 			psz = SZ_16K;
2578 			break;
2579 		case SZ_16K:
2580 			psz = SZ_4K;
2581 			break;
2582 		case SZ_4K:
2583 		default:
2584 			return -1;
2585 		}
2586 	}
2587 
2588 	baser->psz = psz;
2589 	return 0;
2590 }
2591 
its_alloc_tables(struct its_node * its)2592 static int its_alloc_tables(struct its_node *its)
2593 {
2594 	u64 shr = GITS_BASER_InnerShareable;
2595 	u64 cache = GITS_BASER_RaWaWb;
2596 	int err, i;
2597 
2598 	if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375)
2599 		/* erratum 24313: ignore memory access type */
2600 		cache = GITS_BASER_nCnB;
2601 
2602 	if (its->flags & ITS_FLAGS_FORCE_NON_SHAREABLE) {
2603 		cache = GITS_BASER_nC;
2604 		shr = 0;
2605 	}
2606 
2607 	for (i = 0; i < GITS_BASER_NR_REGS; i++) {
2608 		struct its_baser *baser = its->tables + i;
2609 		u64 val = its_read_baser(its, baser);
2610 		u64 type = GITS_BASER_TYPE(val);
2611 		bool indirect = false;
2612 		u32 order;
2613 
2614 		if (type == GITS_BASER_TYPE_NONE)
2615 			continue;
2616 
2617 		if (its_probe_baser_psz(its, baser)) {
2618 			its_free_tables(its);
2619 			return -ENXIO;
2620 		}
2621 
2622 		order = get_order(baser->psz);
2623 
2624 		switch (type) {
2625 		case GITS_BASER_TYPE_DEVICE:
2626 			indirect = its_parse_indirect_baser(its, baser, &order,
2627 							    device_ids(its));
2628 			break;
2629 
2630 		case GITS_BASER_TYPE_VCPU:
2631 			if (is_v4_1(its)) {
2632 				struct its_node *sibling;
2633 
2634 				WARN_ON(i != 2);
2635 				if ((sibling = find_sibling_its(its))) {
2636 					*baser = sibling->tables[2];
2637 					its_write_baser(its, baser, baser->val);
2638 					continue;
2639 				}
2640 			}
2641 
2642 			indirect = its_parse_indirect_baser(its, baser, &order,
2643 							    ITS_MAX_VPEID_BITS);
2644 			break;
2645 		}
2646 
2647 		err = its_setup_baser(its, baser, cache, shr, order, indirect);
2648 		if (err < 0) {
2649 			its_free_tables(its);
2650 			return err;
2651 		}
2652 
2653 		/* Update settings which will be used for next BASERn */
2654 		cache = baser->val & GITS_BASER_CACHEABILITY_MASK;
2655 		shr = baser->val & GITS_BASER_SHAREABILITY_MASK;
2656 	}
2657 
2658 	return 0;
2659 }
2660 
inherit_vpe_l1_table_from_its(void)2661 static u64 inherit_vpe_l1_table_from_its(void)
2662 {
2663 	struct its_node *its;
2664 	u64 val;
2665 	u32 aff;
2666 
2667 	val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
2668 	aff = compute_common_aff(val);
2669 
2670 	list_for_each_entry(its, &its_nodes, entry) {
2671 		u64 baser, addr;
2672 
2673 		if (!is_v4_1(its))
2674 			continue;
2675 
2676 		if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
2677 			continue;
2678 
2679 		if (aff != compute_its_aff(its))
2680 			continue;
2681 
2682 		/* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
2683 		baser = its->tables[2].val;
2684 		if (!(baser & GITS_BASER_VALID))
2685 			continue;
2686 
2687 		/* We have a winner! */
2688 		gic_data_rdist()->vpe_l1_base = its->tables[2].base;
2689 
2690 		val  = GICR_VPROPBASER_4_1_VALID;
2691 		if (baser & GITS_BASER_INDIRECT)
2692 			val |= GICR_VPROPBASER_4_1_INDIRECT;
2693 		val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE,
2694 				  FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser));
2695 		switch (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser)) {
2696 		case GIC_PAGE_SIZE_64K:
2697 			addr = GITS_BASER_ADDR_48_to_52(baser);
2698 			break;
2699 		default:
2700 			addr = baser & GENMASK_ULL(47, 12);
2701 			break;
2702 		}
2703 		val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, addr >> 12);
2704 		if (rdists_support_shareable()) {
2705 			val |= FIELD_PREP(GICR_VPROPBASER_SHAREABILITY_MASK,
2706 					  FIELD_GET(GITS_BASER_SHAREABILITY_MASK, baser));
2707 			val |= FIELD_PREP(GICR_VPROPBASER_INNER_CACHEABILITY_MASK,
2708 					  FIELD_GET(GITS_BASER_INNER_CACHEABILITY_MASK, baser));
2709 		}
2710 		val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, GITS_BASER_NR_PAGES(baser) - 1);
2711 
2712 		return val;
2713 	}
2714 
2715 	return 0;
2716 }
2717 
inherit_vpe_l1_table_from_rd(cpumask_t ** mask)2718 static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask)
2719 {
2720 	u32 aff;
2721 	u64 val;
2722 	int cpu;
2723 
2724 	val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
2725 	aff = compute_common_aff(val);
2726 
2727 	for_each_possible_cpu(cpu) {
2728 		void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
2729 
2730 		if (!base || cpu == smp_processor_id())
2731 			continue;
2732 
2733 		val = gic_read_typer(base + GICR_TYPER);
2734 		if (aff != compute_common_aff(val))
2735 			continue;
2736 
2737 		/*
2738 		 * At this point, we have a victim. This particular CPU
2739 		 * has already booted, and has an affinity that matches
2740 		 * ours wrt CommonLPIAff. Let's use its own VPROPBASER.
2741 		 * Make sure we don't write the Z bit in that case.
2742 		 */
2743 		val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
2744 		val &= ~GICR_VPROPBASER_4_1_Z;
2745 
2746 		gic_data_rdist()->vpe_l1_base = gic_data_rdist_cpu(cpu)->vpe_l1_base;
2747 		*mask = gic_data_rdist_cpu(cpu)->vpe_table_mask;
2748 
2749 		return val;
2750 	}
2751 
2752 	return 0;
2753 }
2754 
allocate_vpe_l2_table(int cpu,u32 id)2755 static bool allocate_vpe_l2_table(int cpu, u32 id)
2756 {
2757 	void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
2758 	unsigned int psz, esz, idx, npg, gpsz;
2759 	u64 val;
2760 	struct page *page;
2761 	__le64 *table;
2762 
2763 	if (!gic_rdists->has_rvpeid)
2764 		return true;
2765 
2766 	/* Skip non-present CPUs */
2767 	if (!base)
2768 		return true;
2769 
2770 	val  = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
2771 
2772 	esz  = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val) + 1;
2773 	gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
2774 	npg  = FIELD_GET(GICR_VPROPBASER_4_1_SIZE, val) + 1;
2775 
2776 	switch (gpsz) {
2777 	default:
2778 		WARN_ON(1);
2779 		fallthrough;
2780 	case GIC_PAGE_SIZE_4K:
2781 		psz = SZ_4K;
2782 		break;
2783 	case GIC_PAGE_SIZE_16K:
2784 		psz = SZ_16K;
2785 		break;
2786 	case GIC_PAGE_SIZE_64K:
2787 		psz = SZ_64K;
2788 		break;
2789 	}
2790 
2791 	/* Don't allow vpe_id that exceeds single, flat table limit */
2792 	if (!(val & GICR_VPROPBASER_4_1_INDIRECT))
2793 		return (id < (npg * psz / (esz * SZ_8)));
2794 
2795 	/* Compute 1st level table index & check if that exceeds table limit */
2796 	idx = id >> ilog2(psz / (esz * SZ_8));
2797 	if (idx >= (npg * psz / GITS_LVL1_ENTRY_SIZE))
2798 		return false;
2799 
2800 	table = gic_data_rdist_cpu(cpu)->vpe_l1_base;
2801 
2802 	/* Allocate memory for 2nd level table */
2803 	if (!table[idx]) {
2804 		page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(psz));
2805 		if (!page)
2806 			return false;
2807 
2808 		/* Flush Lvl2 table to PoC if hw doesn't support coherency */
2809 		if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
2810 			gic_flush_dcache_to_poc(page_address(page), psz);
2811 
2812 		table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID);
2813 
2814 		/* Flush Lvl1 entry to PoC if hw doesn't support coherency */
2815 		if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
2816 			gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
2817 
2818 		/* Ensure updated table contents are visible to RD hardware */
2819 		dsb(sy);
2820 	}
2821 
2822 	return true;
2823 }
2824 
allocate_vpe_l1_table(void)2825 static int allocate_vpe_l1_table(void)
2826 {
2827 	void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
2828 	u64 val, gpsz, npg, pa;
2829 	unsigned int psz = SZ_64K;
2830 	unsigned int np, epp, esz;
2831 	struct page *page;
2832 
2833 	if (!gic_rdists->has_rvpeid)
2834 		return 0;
2835 
2836 	/*
2837 	 * if VPENDBASER.Valid is set, disable any previously programmed
2838 	 * VPE by setting PendingLast while clearing Valid. This has the
2839 	 * effect of making sure no doorbell will be generated and we can
2840 	 * then safely clear VPROPBASER.Valid.
2841 	 */
2842 	if (gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER) & GICR_VPENDBASER_Valid)
2843 		gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast,
2844 				      vlpi_base + GICR_VPENDBASER);
2845 
2846 	/*
2847 	 * If we can inherit the configuration from another RD, let's do
2848 	 * so. Otherwise, we have to go through the allocation process. We
2849 	 * assume that all RDs have the exact same requirements, as
2850 	 * nothing will work otherwise.
2851 	 */
2852 	val = inherit_vpe_l1_table_from_rd(&gic_data_rdist()->vpe_table_mask);
2853 	if (val & GICR_VPROPBASER_4_1_VALID)
2854 		goto out;
2855 
2856 	gic_data_rdist()->vpe_table_mask = kzalloc(sizeof(cpumask_t), GFP_ATOMIC);
2857 	if (!gic_data_rdist()->vpe_table_mask)
2858 		return -ENOMEM;
2859 
2860 	val = inherit_vpe_l1_table_from_its();
2861 	if (val & GICR_VPROPBASER_4_1_VALID)
2862 		goto out;
2863 
2864 	/* First probe the page size */
2865 	val = FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, GIC_PAGE_SIZE_64K);
2866 	gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
2867 	val = gicr_read_vpropbaser(vlpi_base + GICR_VPROPBASER);
2868 	gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
2869 	esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val);
2870 
2871 	switch (gpsz) {
2872 	default:
2873 		gpsz = GIC_PAGE_SIZE_4K;
2874 		fallthrough;
2875 	case GIC_PAGE_SIZE_4K:
2876 		psz = SZ_4K;
2877 		break;
2878 	case GIC_PAGE_SIZE_16K:
2879 		psz = SZ_16K;
2880 		break;
2881 	case GIC_PAGE_SIZE_64K:
2882 		psz = SZ_64K;
2883 		break;
2884 	}
2885 
2886 	/*
2887 	 * Start populating the register from scratch, including RO fields
2888 	 * (which we want to print in debug cases...)
2889 	 */
2890 	val = 0;
2891 	val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, gpsz);
2892 	val |= FIELD_PREP(GICR_VPROPBASER_4_1_ENTRY_SIZE, esz);
2893 
2894 	/* How many entries per GIC page? */
2895 	esz++;
2896 	epp = psz / (esz * SZ_8);
2897 
2898 	/*
2899 	 * If we need more than just a single L1 page, flag the table
2900 	 * as indirect and compute the number of required L1 pages.
2901 	 */
2902 	if (epp < ITS_MAX_VPEID) {
2903 		int nl2;
2904 
2905 		val |= GICR_VPROPBASER_4_1_INDIRECT;
2906 
2907 		/* Number of L2 pages required to cover the VPEID space */
2908 		nl2 = DIV_ROUND_UP(ITS_MAX_VPEID, epp);
2909 
2910 		/* Number of L1 pages to point to the L2 pages */
2911 		npg = DIV_ROUND_UP(nl2 * SZ_8, psz);
2912 	} else {
2913 		npg = 1;
2914 	}
2915 
2916 	val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, npg - 1);
2917 
2918 	/* Right, that's the number of CPU pages we need for L1 */
2919 	np = DIV_ROUND_UP(npg * psz, PAGE_SIZE);
2920 
2921 	pr_debug("np = %d, npg = %lld, psz = %d, epp = %d, esz = %d\n",
2922 		 np, npg, psz, epp, esz);
2923 	page = alloc_pages(GFP_ATOMIC | __GFP_ZERO, get_order(np * PAGE_SIZE));
2924 	if (!page)
2925 		return -ENOMEM;
2926 
2927 	gic_data_rdist()->vpe_l1_base = page_address(page);
2928 	pa = virt_to_phys(page_address(page));
2929 	WARN_ON(!IS_ALIGNED(pa, psz));
2930 
2931 	val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, pa >> 12);
2932 	if (rdists_support_shareable()) {
2933 		val |= GICR_VPROPBASER_RaWb;
2934 		val |= GICR_VPROPBASER_InnerShareable;
2935 	}
2936 	val |= GICR_VPROPBASER_4_1_Z;
2937 	val |= GICR_VPROPBASER_4_1_VALID;
2938 
2939 out:
2940 	gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
2941 	cpumask_set_cpu(smp_processor_id(), gic_data_rdist()->vpe_table_mask);
2942 
2943 	pr_debug("CPU%d: VPROPBASER = %llx %*pbl\n",
2944 		 smp_processor_id(), val,
2945 		 cpumask_pr_args(gic_data_rdist()->vpe_table_mask));
2946 
2947 	return 0;
2948 }
2949 
its_alloc_collections(struct its_node * its)2950 static int its_alloc_collections(struct its_node *its)
2951 {
2952 	int i;
2953 
2954 	its->collections = kcalloc(nr_cpu_ids, sizeof(*its->collections),
2955 				   GFP_KERNEL);
2956 	if (!its->collections)
2957 		return -ENOMEM;
2958 
2959 	for (i = 0; i < nr_cpu_ids; i++)
2960 		its->collections[i].target_address = ~0ULL;
2961 
2962 	return 0;
2963 }
2964 
its_allocate_pending_table(gfp_t gfp_flags)2965 static struct page *its_allocate_pending_table(gfp_t gfp_flags)
2966 {
2967 	struct page *pend_page;
2968 
2969 	pend_page = alloc_pages(gfp_flags | __GFP_ZERO,
2970 				get_order(LPI_PENDBASE_SZ));
2971 	if (!pend_page)
2972 		return NULL;
2973 
2974 	/* Make sure the GIC will observe the zero-ed page */
2975 	gic_flush_dcache_to_poc(page_address(pend_page), LPI_PENDBASE_SZ);
2976 
2977 	return pend_page;
2978 }
2979 
its_free_pending_table(struct page * pt)2980 static void its_free_pending_table(struct page *pt)
2981 {
2982 	free_pages((unsigned long)page_address(pt), get_order(LPI_PENDBASE_SZ));
2983 }
2984 
2985 /*
2986  * Booting with kdump and LPIs enabled is generally fine. Any other
2987  * case is wrong in the absence of firmware/EFI support.
2988  */
enabled_lpis_allowed(void)2989 static bool enabled_lpis_allowed(void)
2990 {
2991 	phys_addr_t addr;
2992 	u64 val;
2993 
2994 	/* Check whether the property table is in a reserved region */
2995 	val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
2996 	addr = val & GENMASK_ULL(51, 12);
2997 
2998 	return gic_check_reserved_range(addr, LPI_PROPBASE_SZ);
2999 }
3000 
allocate_lpi_tables(void)3001 static int __init allocate_lpi_tables(void)
3002 {
3003 	u64 val;
3004 	int err, cpu;
3005 
3006 	/*
3007 	 * If LPIs are enabled while we run this from the boot CPU,
3008 	 * flag the RD tables as pre-allocated if the stars do align.
3009 	 */
3010 	val = readl_relaxed(gic_data_rdist_rd_base() + GICR_CTLR);
3011 	if ((val & GICR_CTLR_ENABLE_LPIS) && enabled_lpis_allowed()) {
3012 		gic_rdists->flags |= (RDIST_FLAGS_RD_TABLES_PREALLOCATED |
3013 				      RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING);
3014 		pr_info("GICv3: Using preallocated redistributor tables\n");
3015 	}
3016 
3017 	err = its_setup_lpi_prop_table();
3018 	if (err)
3019 		return err;
3020 
3021 	/*
3022 	 * We allocate all the pending tables anyway, as we may have a
3023 	 * mix of RDs that have had LPIs enabled, and some that
3024 	 * don't. We'll free the unused ones as each CPU comes online.
3025 	 */
3026 	for_each_possible_cpu(cpu) {
3027 		struct page *pend_page;
3028 
3029 		pend_page = its_allocate_pending_table(GFP_NOWAIT);
3030 		if (!pend_page) {
3031 			pr_err("Failed to allocate PENDBASE for CPU%d\n", cpu);
3032 			return -ENOMEM;
3033 		}
3034 
3035 		gic_data_rdist_cpu(cpu)->pend_page = pend_page;
3036 	}
3037 
3038 	return 0;
3039 }
3040 
read_vpend_dirty_clear(void __iomem * vlpi_base)3041 static u64 read_vpend_dirty_clear(void __iomem *vlpi_base)
3042 {
3043 	u32 count = 1000000;	/* 1s! */
3044 	bool clean;
3045 	u64 val;
3046 
3047 	do {
3048 		val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
3049 		clean = !(val & GICR_VPENDBASER_Dirty);
3050 		if (!clean) {
3051 			count--;
3052 			cpu_relax();
3053 			udelay(1);
3054 		}
3055 	} while (!clean && count);
3056 
3057 	if (unlikely(!clean))
3058 		pr_err_ratelimited("ITS virtual pending table not cleaning\n");
3059 
3060 	return val;
3061 }
3062 
its_clear_vpend_valid(void __iomem * vlpi_base,u64 clr,u64 set)3063 static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set)
3064 {
3065 	u64 val;
3066 
3067 	/* Make sure we wait until the RD is done with the initial scan */
3068 	val = read_vpend_dirty_clear(vlpi_base);
3069 	val &= ~GICR_VPENDBASER_Valid;
3070 	val &= ~clr;
3071 	val |= set;
3072 	gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
3073 
3074 	val = read_vpend_dirty_clear(vlpi_base);
3075 	if (unlikely(val & GICR_VPENDBASER_Dirty))
3076 		val |= GICR_VPENDBASER_PendingLast;
3077 
3078 	return val;
3079 }
3080 
its_cpu_init_lpis(void)3081 static void its_cpu_init_lpis(void)
3082 {
3083 	void __iomem *rbase = gic_data_rdist_rd_base();
3084 	struct page *pend_page;
3085 	phys_addr_t paddr;
3086 	u64 val, tmp;
3087 
3088 	if (gic_data_rdist()->flags & RD_LOCAL_LPI_ENABLED)
3089 		return;
3090 
3091 	val = readl_relaxed(rbase + GICR_CTLR);
3092 	if ((gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) &&
3093 	    (val & GICR_CTLR_ENABLE_LPIS)) {
3094 		/*
3095 		 * Check that we get the same property table on all
3096 		 * RDs. If we don't, this is hopeless.
3097 		 */
3098 		paddr = gicr_read_propbaser(rbase + GICR_PROPBASER);
3099 		paddr &= GENMASK_ULL(51, 12);
3100 		if (WARN_ON(gic_rdists->prop_table_pa != paddr))
3101 			add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
3102 
3103 		paddr = gicr_read_pendbaser(rbase + GICR_PENDBASER);
3104 		paddr &= GENMASK_ULL(51, 16);
3105 
3106 		WARN_ON(!gic_check_reserved_range(paddr, LPI_PENDBASE_SZ));
3107 		gic_data_rdist()->flags |= RD_LOCAL_PENDTABLE_PREALLOCATED;
3108 
3109 		goto out;
3110 	}
3111 
3112 	pend_page = gic_data_rdist()->pend_page;
3113 	paddr = page_to_phys(pend_page);
3114 
3115 	/* set PROPBASE */
3116 	val = (gic_rdists->prop_table_pa |
3117 	       GICR_PROPBASER_InnerShareable |
3118 	       GICR_PROPBASER_RaWaWb |
3119 	       ((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK));
3120 
3121 	gicr_write_propbaser(val, rbase + GICR_PROPBASER);
3122 	tmp = gicr_read_propbaser(rbase + GICR_PROPBASER);
3123 
3124 	if (!rdists_support_shareable())
3125 		tmp &= ~GICR_PROPBASER_SHAREABILITY_MASK;
3126 
3127 	if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) {
3128 		if (!(tmp & GICR_PROPBASER_SHAREABILITY_MASK)) {
3129 			/*
3130 			 * The HW reports non-shareable, we must
3131 			 * remove the cacheability attributes as
3132 			 * well.
3133 			 */
3134 			val &= ~(GICR_PROPBASER_SHAREABILITY_MASK |
3135 				 GICR_PROPBASER_CACHEABILITY_MASK);
3136 			val |= GICR_PROPBASER_nC;
3137 			gicr_write_propbaser(val, rbase + GICR_PROPBASER);
3138 		}
3139 		pr_info_once("GIC: using cache flushing for LPI property table\n");
3140 		gic_rdists->flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING;
3141 	}
3142 
3143 	/* set PENDBASE */
3144 	val = (page_to_phys(pend_page) |
3145 	       GICR_PENDBASER_InnerShareable |
3146 	       GICR_PENDBASER_RaWaWb);
3147 
3148 	gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
3149 	tmp = gicr_read_pendbaser(rbase + GICR_PENDBASER);
3150 
3151 	if (!rdists_support_shareable())
3152 		tmp &= ~GICR_PENDBASER_SHAREABILITY_MASK;
3153 
3154 	if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) {
3155 		/*
3156 		 * The HW reports non-shareable, we must remove the
3157 		 * cacheability attributes as well.
3158 		 */
3159 		val &= ~(GICR_PENDBASER_SHAREABILITY_MASK |
3160 			 GICR_PENDBASER_CACHEABILITY_MASK);
3161 		val |= GICR_PENDBASER_nC;
3162 		gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
3163 	}
3164 
3165 	/* Enable LPIs */
3166 	val = readl_relaxed(rbase + GICR_CTLR);
3167 	val |= GICR_CTLR_ENABLE_LPIS;
3168 	writel_relaxed(val, rbase + GICR_CTLR);
3169 
3170 out:
3171 	if (gic_rdists->has_vlpis && !gic_rdists->has_rvpeid) {
3172 		void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3173 
3174 		/*
3175 		 * It's possible for CPU to receive VLPIs before it is
3176 		 * scheduled as a vPE, especially for the first CPU, and the
3177 		 * VLPI with INTID larger than 2^(IDbits+1) will be considered
3178 		 * as out of range and dropped by GIC.
3179 		 * So we initialize IDbits to known value to avoid VLPI drop.
3180 		 */
3181 		val = (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
3182 		pr_debug("GICv4: CPU%d: Init IDbits to 0x%llx for GICR_VPROPBASER\n",
3183 			smp_processor_id(), val);
3184 		gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
3185 
3186 		/*
3187 		 * Also clear Valid bit of GICR_VPENDBASER, in case some
3188 		 * ancient programming gets left in and has possibility of
3189 		 * corrupting memory.
3190 		 */
3191 		val = its_clear_vpend_valid(vlpi_base, 0, 0);
3192 	}
3193 
3194 	if (allocate_vpe_l1_table()) {
3195 		/*
3196 		 * If the allocation has failed, we're in massive trouble.
3197 		 * Disable direct injection, and pray that no VM was
3198 		 * already running...
3199 		 */
3200 		gic_rdists->has_rvpeid = false;
3201 		gic_rdists->has_vlpis = false;
3202 	}
3203 
3204 	/* Make sure the GIC has seen the above */
3205 	dsb(sy);
3206 	gic_data_rdist()->flags |= RD_LOCAL_LPI_ENABLED;
3207 	pr_info("GICv3: CPU%d: using %s LPI pending table @%pa\n",
3208 		smp_processor_id(),
3209 		gic_data_rdist()->flags & RD_LOCAL_PENDTABLE_PREALLOCATED ?
3210 		"reserved" : "allocated",
3211 		&paddr);
3212 }
3213 
its_cpu_init_collection(struct its_node * its)3214 static void its_cpu_init_collection(struct its_node *its)
3215 {
3216 	int cpu = smp_processor_id();
3217 	u64 target;
3218 
3219 	/* avoid cross node collections and its mapping */
3220 	if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) {
3221 		struct device_node *cpu_node;
3222 
3223 		cpu_node = of_get_cpu_node(cpu, NULL);
3224 		if (its->numa_node != NUMA_NO_NODE &&
3225 			its->numa_node != of_node_to_nid(cpu_node))
3226 			return;
3227 	}
3228 
3229 	/*
3230 	 * We now have to bind each collection to its target
3231 	 * redistributor.
3232 	 */
3233 	if (gic_read_typer(its->base + GITS_TYPER) & GITS_TYPER_PTA) {
3234 		/*
3235 		 * This ITS wants the physical address of the
3236 		 * redistributor.
3237 		 */
3238 		target = gic_data_rdist()->phys_base;
3239 	} else {
3240 		/* This ITS wants a linear CPU number. */
3241 		target = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
3242 		target = GICR_TYPER_CPU_NUMBER(target) << 16;
3243 	}
3244 
3245 	/* Perform collection mapping */
3246 	its->collections[cpu].target_address = target;
3247 	its->collections[cpu].col_id = cpu;
3248 
3249 	its_send_mapc(its, &its->collections[cpu], 1);
3250 	its_send_invall(its, &its->collections[cpu]);
3251 }
3252 
its_cpu_init_collections(void)3253 static void its_cpu_init_collections(void)
3254 {
3255 	struct its_node *its;
3256 
3257 	raw_spin_lock(&its_lock);
3258 
3259 	list_for_each_entry(its, &its_nodes, entry)
3260 		its_cpu_init_collection(its);
3261 
3262 	raw_spin_unlock(&its_lock);
3263 }
3264 
its_find_device(struct its_node * its,u32 dev_id)3265 static struct its_device *its_find_device(struct its_node *its, u32 dev_id)
3266 {
3267 	struct its_device *its_dev = NULL, *tmp;
3268 	unsigned long flags;
3269 
3270 	raw_spin_lock_irqsave(&its->lock, flags);
3271 
3272 	list_for_each_entry(tmp, &its->its_device_list, entry) {
3273 		if (tmp->device_id == dev_id) {
3274 			its_dev = tmp;
3275 			break;
3276 		}
3277 	}
3278 
3279 	raw_spin_unlock_irqrestore(&its->lock, flags);
3280 
3281 	return its_dev;
3282 }
3283 
its_get_baser(struct its_node * its,u32 type)3284 static struct its_baser *its_get_baser(struct its_node *its, u32 type)
3285 {
3286 	int i;
3287 
3288 	for (i = 0; i < GITS_BASER_NR_REGS; i++) {
3289 		if (GITS_BASER_TYPE(its->tables[i].val) == type)
3290 			return &its->tables[i];
3291 	}
3292 
3293 	return NULL;
3294 }
3295 
its_alloc_table_entry(struct its_node * its,struct its_baser * baser,u32 id)3296 static bool its_alloc_table_entry(struct its_node *its,
3297 				  struct its_baser *baser, u32 id)
3298 {
3299 	struct page *page;
3300 	u32 esz, idx;
3301 	__le64 *table;
3302 
3303 	/* Don't allow device id that exceeds single, flat table limit */
3304 	esz = GITS_BASER_ENTRY_SIZE(baser->val);
3305 	if (!(baser->val & GITS_BASER_INDIRECT))
3306 		return (id < (PAGE_ORDER_TO_SIZE(baser->order) / esz));
3307 
3308 	/* Compute 1st level table index & check if that exceeds table limit */
3309 	idx = id >> ilog2(baser->psz / esz);
3310 	if (idx >= (PAGE_ORDER_TO_SIZE(baser->order) / GITS_LVL1_ENTRY_SIZE))
3311 		return false;
3312 
3313 	table = baser->base;
3314 
3315 	/* Allocate memory for 2nd level table */
3316 	if (!table[idx]) {
3317 		page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO,
3318 					get_order(baser->psz));
3319 		if (!page)
3320 			return false;
3321 
3322 		/* Flush Lvl2 table to PoC if hw doesn't support coherency */
3323 		if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
3324 			gic_flush_dcache_to_poc(page_address(page), baser->psz);
3325 
3326 		table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID);
3327 
3328 		/* Flush Lvl1 entry to PoC if hw doesn't support coherency */
3329 		if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
3330 			gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
3331 
3332 		/* Ensure updated table contents are visible to ITS hardware */
3333 		dsb(sy);
3334 	}
3335 
3336 	return true;
3337 }
3338 
its_alloc_device_table(struct its_node * its,u32 dev_id)3339 static bool its_alloc_device_table(struct its_node *its, u32 dev_id)
3340 {
3341 	struct its_baser *baser;
3342 
3343 	baser = its_get_baser(its, GITS_BASER_TYPE_DEVICE);
3344 
3345 	/* Don't allow device id that exceeds ITS hardware limit */
3346 	if (!baser)
3347 		return (ilog2(dev_id) < device_ids(its));
3348 
3349 	return its_alloc_table_entry(its, baser, dev_id);
3350 }
3351 
its_alloc_vpe_table(u32 vpe_id)3352 static bool its_alloc_vpe_table(u32 vpe_id)
3353 {
3354 	struct its_node *its;
3355 	int cpu;
3356 
3357 	/*
3358 	 * Make sure the L2 tables are allocated on *all* v4 ITSs. We
3359 	 * could try and only do it on ITSs corresponding to devices
3360 	 * that have interrupts targeted at this VPE, but the
3361 	 * complexity becomes crazy (and you have tons of memory
3362 	 * anyway, right?).
3363 	 */
3364 	list_for_each_entry(its, &its_nodes, entry) {
3365 		struct its_baser *baser;
3366 
3367 		if (!is_v4(its))
3368 			continue;
3369 
3370 		baser = its_get_baser(its, GITS_BASER_TYPE_VCPU);
3371 		if (!baser)
3372 			return false;
3373 
3374 		if (!its_alloc_table_entry(its, baser, vpe_id))
3375 			return false;
3376 	}
3377 
3378 	/* Non v4.1? No need to iterate RDs and go back early. */
3379 	if (!gic_rdists->has_rvpeid)
3380 		return true;
3381 
3382 	/*
3383 	 * Make sure the L2 tables are allocated for all copies of
3384 	 * the L1 table on *all* v4.1 RDs.
3385 	 */
3386 	for_each_possible_cpu(cpu) {
3387 		if (!allocate_vpe_l2_table(cpu, vpe_id))
3388 			return false;
3389 	}
3390 
3391 	return true;
3392 }
3393 
its_create_device(struct its_node * its,u32 dev_id,int nvecs,bool alloc_lpis)3394 static struct its_device *its_create_device(struct its_node *its, u32 dev_id,
3395 					    int nvecs, bool alloc_lpis)
3396 {
3397 	struct its_device *dev;
3398 	unsigned long *lpi_map = NULL;
3399 	unsigned long flags;
3400 	u16 *col_map = NULL;
3401 	void *itt;
3402 	int lpi_base;
3403 	int nr_lpis;
3404 	int nr_ites;
3405 	int sz;
3406 
3407 	if (!its_alloc_device_table(its, dev_id))
3408 		return NULL;
3409 
3410 	if (WARN_ON(!is_power_of_2(nvecs)))
3411 		nvecs = roundup_pow_of_two(nvecs);
3412 
3413 	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
3414 	/*
3415 	 * Even if the device wants a single LPI, the ITT must be
3416 	 * sized as a power of two (and you need at least one bit...).
3417 	 */
3418 	nr_ites = max(2, nvecs);
3419 	sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1);
3420 	sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1;
3421 	itt = kzalloc_node(sz, GFP_KERNEL, its->numa_node);
3422 	if (alloc_lpis) {
3423 		lpi_map = its_lpi_alloc(nvecs, &lpi_base, &nr_lpis);
3424 		if (lpi_map)
3425 			col_map = kcalloc(nr_lpis, sizeof(*col_map),
3426 					  GFP_KERNEL);
3427 	} else {
3428 		col_map = kcalloc(nr_ites, sizeof(*col_map), GFP_KERNEL);
3429 		nr_lpis = 0;
3430 		lpi_base = 0;
3431 	}
3432 
3433 	if (!dev || !itt ||  !col_map || (!lpi_map && alloc_lpis)) {
3434 		kfree(dev);
3435 		kfree(itt);
3436 		bitmap_free(lpi_map);
3437 		kfree(col_map);
3438 		return NULL;
3439 	}
3440 
3441 	gic_flush_dcache_to_poc(itt, sz);
3442 
3443 	dev->its = its;
3444 	dev->itt = itt;
3445 	dev->nr_ites = nr_ites;
3446 	dev->event_map.lpi_map = lpi_map;
3447 	dev->event_map.col_map = col_map;
3448 	dev->event_map.lpi_base = lpi_base;
3449 	dev->event_map.nr_lpis = nr_lpis;
3450 	raw_spin_lock_init(&dev->event_map.vlpi_lock);
3451 	dev->device_id = dev_id;
3452 	INIT_LIST_HEAD(&dev->entry);
3453 
3454 	raw_spin_lock_irqsave(&its->lock, flags);
3455 	list_add(&dev->entry, &its->its_device_list);
3456 	raw_spin_unlock_irqrestore(&its->lock, flags);
3457 
3458 	/* Map device to its ITT */
3459 	its_send_mapd(dev, 1);
3460 
3461 	return dev;
3462 }
3463 
its_free_device(struct its_device * its_dev)3464 static void its_free_device(struct its_device *its_dev)
3465 {
3466 	unsigned long flags;
3467 
3468 	raw_spin_lock_irqsave(&its_dev->its->lock, flags);
3469 	list_del(&its_dev->entry);
3470 	raw_spin_unlock_irqrestore(&its_dev->its->lock, flags);
3471 	kfree(its_dev->event_map.col_map);
3472 	kfree(its_dev->itt);
3473 	kfree(its_dev);
3474 }
3475 
its_alloc_device_irq(struct its_device * dev,int nvecs,irq_hw_number_t * hwirq)3476 static int its_alloc_device_irq(struct its_device *dev, int nvecs, irq_hw_number_t *hwirq)
3477 {
3478 	int idx;
3479 
3480 	/* Find a free LPI region in lpi_map and allocate them. */
3481 	idx = bitmap_find_free_region(dev->event_map.lpi_map,
3482 				      dev->event_map.nr_lpis,
3483 				      get_count_order(nvecs));
3484 	if (idx < 0)
3485 		return -ENOSPC;
3486 
3487 	*hwirq = dev->event_map.lpi_base + idx;
3488 
3489 	return 0;
3490 }
3491 
its_msi_prepare(struct irq_domain * domain,struct device * dev,int nvec,msi_alloc_info_t * info)3492 static int its_msi_prepare(struct irq_domain *domain, struct device *dev,
3493 			   int nvec, msi_alloc_info_t *info)
3494 {
3495 	struct its_node *its;
3496 	struct its_device *its_dev;
3497 	struct msi_domain_info *msi_info;
3498 	u32 dev_id;
3499 	int err = 0;
3500 
3501 	/*
3502 	 * We ignore "dev" entirely, and rely on the dev_id that has
3503 	 * been passed via the scratchpad. This limits this domain's
3504 	 * usefulness to upper layers that definitely know that they
3505 	 * are built on top of the ITS.
3506 	 */
3507 	dev_id = info->scratchpad[0].ul;
3508 
3509 	msi_info = msi_get_domain_info(domain);
3510 	its = msi_info->data;
3511 
3512 	if (!gic_rdists->has_direct_lpi &&
3513 	    vpe_proxy.dev &&
3514 	    vpe_proxy.dev->its == its &&
3515 	    dev_id == vpe_proxy.dev->device_id) {
3516 		/* Bad luck. Get yourself a better implementation */
3517 		WARN_ONCE(1, "DevId %x clashes with GICv4 VPE proxy device\n",
3518 			  dev_id);
3519 		return -EINVAL;
3520 	}
3521 
3522 	mutex_lock(&its->dev_alloc_lock);
3523 	its_dev = its_find_device(its, dev_id);
3524 	if (its_dev) {
3525 		/*
3526 		 * We already have seen this ID, probably through
3527 		 * another alias (PCI bridge of some sort). No need to
3528 		 * create the device.
3529 		 */
3530 		its_dev->shared = true;
3531 		pr_debug("Reusing ITT for devID %x\n", dev_id);
3532 		goto out;
3533 	}
3534 
3535 	its_dev = its_create_device(its, dev_id, nvec, true);
3536 	if (!its_dev) {
3537 		err = -ENOMEM;
3538 		goto out;
3539 	}
3540 
3541 	if (info->flags & MSI_ALLOC_FLAGS_PROXY_DEVICE)
3542 		its_dev->shared = true;
3543 
3544 	pr_debug("ITT %d entries, %d bits\n", nvec, ilog2(nvec));
3545 out:
3546 	mutex_unlock(&its->dev_alloc_lock);
3547 	info->scratchpad[0].ptr = its_dev;
3548 	return err;
3549 }
3550 
3551 static struct msi_domain_ops its_msi_domain_ops = {
3552 	.msi_prepare	= its_msi_prepare,
3553 };
3554 
its_irq_gic_domain_alloc(struct irq_domain * domain,unsigned int virq,irq_hw_number_t hwirq)3555 static int its_irq_gic_domain_alloc(struct irq_domain *domain,
3556 				    unsigned int virq,
3557 				    irq_hw_number_t hwirq)
3558 {
3559 	struct irq_fwspec fwspec;
3560 
3561 	if (irq_domain_get_of_node(domain->parent)) {
3562 		fwspec.fwnode = domain->parent->fwnode;
3563 		fwspec.param_count = 3;
3564 		fwspec.param[0] = GIC_IRQ_TYPE_LPI;
3565 		fwspec.param[1] = hwirq;
3566 		fwspec.param[2] = IRQ_TYPE_EDGE_RISING;
3567 	} else if (is_fwnode_irqchip(domain->parent->fwnode)) {
3568 		fwspec.fwnode = domain->parent->fwnode;
3569 		fwspec.param_count = 2;
3570 		fwspec.param[0] = hwirq;
3571 		fwspec.param[1] = IRQ_TYPE_EDGE_RISING;
3572 	} else {
3573 		return -EINVAL;
3574 	}
3575 
3576 	return irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec);
3577 }
3578 
its_irq_domain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * args)3579 static int its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
3580 				unsigned int nr_irqs, void *args)
3581 {
3582 	msi_alloc_info_t *info = args;
3583 	struct its_device *its_dev = info->scratchpad[0].ptr;
3584 	struct its_node *its = its_dev->its;
3585 	struct irq_data *irqd;
3586 	irq_hw_number_t hwirq;
3587 	int err;
3588 	int i;
3589 
3590 	err = its_alloc_device_irq(its_dev, nr_irqs, &hwirq);
3591 	if (err)
3592 		return err;
3593 
3594 	err = iommu_dma_prepare_msi(info->desc, its->get_msi_base(its_dev));
3595 	if (err)
3596 		return err;
3597 
3598 	for (i = 0; i < nr_irqs; i++) {
3599 		err = its_irq_gic_domain_alloc(domain, virq + i, hwirq + i);
3600 		if (err)
3601 			return err;
3602 
3603 		irq_domain_set_hwirq_and_chip(domain, virq + i,
3604 					      hwirq + i, &its_irq_chip, its_dev);
3605 		irqd = irq_get_irq_data(virq + i);
3606 		irqd_set_single_target(irqd);
3607 		irqd_set_affinity_on_activate(irqd);
3608 		irqd_set_resend_when_in_progress(irqd);
3609 		pr_debug("ID:%d pID:%d vID:%d\n",
3610 			 (int)(hwirq + i - its_dev->event_map.lpi_base),
3611 			 (int)(hwirq + i), virq + i);
3612 	}
3613 
3614 	return 0;
3615 }
3616 
its_irq_domain_activate(struct irq_domain * domain,struct irq_data * d,bool reserve)3617 static int its_irq_domain_activate(struct irq_domain *domain,
3618 				   struct irq_data *d, bool reserve)
3619 {
3620 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3621 	u32 event = its_get_event_id(d);
3622 	int cpu;
3623 
3624 	cpu = its_select_cpu(d, cpu_online_mask);
3625 	if (cpu < 0 || cpu >= nr_cpu_ids)
3626 		return -EINVAL;
3627 
3628 	its_inc_lpi_count(d, cpu);
3629 	its_dev->event_map.col_map[event] = cpu;
3630 	irq_data_update_effective_affinity(d, cpumask_of(cpu));
3631 
3632 	/* Map the GIC IRQ and event to the device */
3633 	its_send_mapti(its_dev, d->hwirq, event);
3634 	return 0;
3635 }
3636 
its_irq_domain_deactivate(struct irq_domain * domain,struct irq_data * d)3637 static void its_irq_domain_deactivate(struct irq_domain *domain,
3638 				      struct irq_data *d)
3639 {
3640 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3641 	u32 event = its_get_event_id(d);
3642 
3643 	its_dec_lpi_count(d, its_dev->event_map.col_map[event]);
3644 	/* Stop the delivery of interrupts */
3645 	its_send_discard(its_dev, event);
3646 }
3647 
its_irq_domain_free(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs)3648 static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq,
3649 				unsigned int nr_irqs)
3650 {
3651 	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
3652 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3653 	struct its_node *its = its_dev->its;
3654 	int i;
3655 
3656 	bitmap_release_region(its_dev->event_map.lpi_map,
3657 			      its_get_event_id(irq_domain_get_irq_data(domain, virq)),
3658 			      get_count_order(nr_irqs));
3659 
3660 	for (i = 0; i < nr_irqs; i++) {
3661 		struct irq_data *data = irq_domain_get_irq_data(domain,
3662 								virq + i);
3663 		/* Nuke the entry in the domain */
3664 		irq_domain_reset_irq_data(data);
3665 	}
3666 
3667 	mutex_lock(&its->dev_alloc_lock);
3668 
3669 	/*
3670 	 * If all interrupts have been freed, start mopping the
3671 	 * floor. This is conditioned on the device not being shared.
3672 	 */
3673 	if (!its_dev->shared &&
3674 	    bitmap_empty(its_dev->event_map.lpi_map,
3675 			 its_dev->event_map.nr_lpis)) {
3676 		its_lpi_free(its_dev->event_map.lpi_map,
3677 			     its_dev->event_map.lpi_base,
3678 			     its_dev->event_map.nr_lpis);
3679 
3680 		/* Unmap device/itt */
3681 		its_send_mapd(its_dev, 0);
3682 		its_free_device(its_dev);
3683 	}
3684 
3685 	mutex_unlock(&its->dev_alloc_lock);
3686 
3687 	irq_domain_free_irqs_parent(domain, virq, nr_irqs);
3688 }
3689 
3690 static const struct irq_domain_ops its_domain_ops = {
3691 	.alloc			= its_irq_domain_alloc,
3692 	.free			= its_irq_domain_free,
3693 	.activate		= its_irq_domain_activate,
3694 	.deactivate		= its_irq_domain_deactivate,
3695 };
3696 
3697 /*
3698  * This is insane.
3699  *
3700  * If a GICv4.0 doesn't implement Direct LPIs (which is extremely
3701  * likely), the only way to perform an invalidate is to use a fake
3702  * device to issue an INV command, implying that the LPI has first
3703  * been mapped to some event on that device. Since this is not exactly
3704  * cheap, we try to keep that mapping around as long as possible, and
3705  * only issue an UNMAP if we're short on available slots.
3706  *
3707  * Broken by design(tm).
3708  *
3709  * GICv4.1, on the other hand, mandates that we're able to invalidate
3710  * by writing to a MMIO register. It doesn't implement the whole of
3711  * DirectLPI, but that's good enough. And most of the time, we don't
3712  * even have to invalidate anything, as the redistributor can be told
3713  * whether to generate a doorbell or not (we thus leave it enabled,
3714  * always).
3715  */
its_vpe_db_proxy_unmap_locked(struct its_vpe * vpe)3716 static void its_vpe_db_proxy_unmap_locked(struct its_vpe *vpe)
3717 {
3718 	/* GICv4.1 doesn't use a proxy, so nothing to do here */
3719 	if (gic_rdists->has_rvpeid)
3720 		return;
3721 
3722 	/* Already unmapped? */
3723 	if (vpe->vpe_proxy_event == -1)
3724 		return;
3725 
3726 	its_send_discard(vpe_proxy.dev, vpe->vpe_proxy_event);
3727 	vpe_proxy.vpes[vpe->vpe_proxy_event] = NULL;
3728 
3729 	/*
3730 	 * We don't track empty slots at all, so let's move the
3731 	 * next_victim pointer if we can quickly reuse that slot
3732 	 * instead of nuking an existing entry. Not clear that this is
3733 	 * always a win though, and this might just generate a ripple
3734 	 * effect... Let's just hope VPEs don't migrate too often.
3735 	 */
3736 	if (vpe_proxy.vpes[vpe_proxy.next_victim])
3737 		vpe_proxy.next_victim = vpe->vpe_proxy_event;
3738 
3739 	vpe->vpe_proxy_event = -1;
3740 }
3741 
its_vpe_db_proxy_unmap(struct its_vpe * vpe)3742 static void its_vpe_db_proxy_unmap(struct its_vpe *vpe)
3743 {
3744 	/* GICv4.1 doesn't use a proxy, so nothing to do here */
3745 	if (gic_rdists->has_rvpeid)
3746 		return;
3747 
3748 	if (!gic_rdists->has_direct_lpi) {
3749 		unsigned long flags;
3750 
3751 		raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3752 		its_vpe_db_proxy_unmap_locked(vpe);
3753 		raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3754 	}
3755 }
3756 
its_vpe_db_proxy_map_locked(struct its_vpe * vpe)3757 static void its_vpe_db_proxy_map_locked(struct its_vpe *vpe)
3758 {
3759 	/* GICv4.1 doesn't use a proxy, so nothing to do here */
3760 	if (gic_rdists->has_rvpeid)
3761 		return;
3762 
3763 	/* Already mapped? */
3764 	if (vpe->vpe_proxy_event != -1)
3765 		return;
3766 
3767 	/* This slot was already allocated. Kick the other VPE out. */
3768 	if (vpe_proxy.vpes[vpe_proxy.next_victim])
3769 		its_vpe_db_proxy_unmap_locked(vpe_proxy.vpes[vpe_proxy.next_victim]);
3770 
3771 	/* Map the new VPE instead */
3772 	vpe_proxy.vpes[vpe_proxy.next_victim] = vpe;
3773 	vpe->vpe_proxy_event = vpe_proxy.next_victim;
3774 	vpe_proxy.next_victim = (vpe_proxy.next_victim + 1) % vpe_proxy.dev->nr_ites;
3775 
3776 	vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = vpe->col_idx;
3777 	its_send_mapti(vpe_proxy.dev, vpe->vpe_db_lpi, vpe->vpe_proxy_event);
3778 }
3779 
its_vpe_db_proxy_move(struct its_vpe * vpe,int from,int to)3780 static void its_vpe_db_proxy_move(struct its_vpe *vpe, int from, int to)
3781 {
3782 	unsigned long flags;
3783 	struct its_collection *target_col;
3784 
3785 	/* GICv4.1 doesn't use a proxy, so nothing to do here */
3786 	if (gic_rdists->has_rvpeid)
3787 		return;
3788 
3789 	if (gic_rdists->has_direct_lpi) {
3790 		void __iomem *rdbase;
3791 
3792 		rdbase = per_cpu_ptr(gic_rdists->rdist, from)->rd_base;
3793 		gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
3794 		wait_for_syncr(rdbase);
3795 
3796 		return;
3797 	}
3798 
3799 	raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3800 
3801 	its_vpe_db_proxy_map_locked(vpe);
3802 
3803 	target_col = &vpe_proxy.dev->its->collections[to];
3804 	its_send_movi(vpe_proxy.dev, target_col, vpe->vpe_proxy_event);
3805 	vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = to;
3806 
3807 	raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3808 }
3809 
its_vpe_set_affinity(struct irq_data * d,const struct cpumask * mask_val,bool force)3810 static int its_vpe_set_affinity(struct irq_data *d,
3811 				const struct cpumask *mask_val,
3812 				bool force)
3813 {
3814 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3815 	struct cpumask common, *table_mask;
3816 	unsigned long flags;
3817 	int from, cpu;
3818 
3819 	/*
3820 	 * Check if we're racing against a VPE being destroyed, for
3821 	 * which we don't want to allow a VMOVP.
3822 	 */
3823 	if (!atomic_read(&vpe->vmapp_count))
3824 		return -EINVAL;
3825 
3826 	/*
3827 	 * Changing affinity is mega expensive, so let's be as lazy as
3828 	 * we can and only do it if we really have to. Also, if mapped
3829 	 * into the proxy device, we need to move the doorbell
3830 	 * interrupt to its new location.
3831 	 *
3832 	 * Another thing is that changing the affinity of a vPE affects
3833 	 * *other interrupts* such as all the vLPIs that are routed to
3834 	 * this vPE. This means that the irq_desc lock is not enough to
3835 	 * protect us, and that we must ensure nobody samples vpe->col_idx
3836 	 * during the update, hence the lock below which must also be
3837 	 * taken on any vLPI handling path that evaluates vpe->col_idx.
3838 	 */
3839 	from = vpe_to_cpuid_lock(vpe, &flags);
3840 	table_mask = gic_data_rdist_cpu(from)->vpe_table_mask;
3841 
3842 	/*
3843 	 * If we are offered another CPU in the same GICv4.1 ITS
3844 	 * affinity, pick this one. Otherwise, any CPU will do.
3845 	 */
3846 	if (table_mask && cpumask_and(&common, mask_val, table_mask))
3847 		cpu = cpumask_test_cpu(from, &common) ? from : cpumask_first(&common);
3848 	else
3849 		cpu = cpumask_first(mask_val);
3850 
3851 	if (from == cpu)
3852 		goto out;
3853 
3854 	vpe->col_idx = cpu;
3855 
3856 	its_send_vmovp(vpe);
3857 	its_vpe_db_proxy_move(vpe, from, cpu);
3858 
3859 out:
3860 	irq_data_update_effective_affinity(d, cpumask_of(cpu));
3861 	vpe_to_cpuid_unlock(vpe, flags);
3862 
3863 	return IRQ_SET_MASK_OK_DONE;
3864 }
3865 
its_wait_vpt_parse_complete(void)3866 static void its_wait_vpt_parse_complete(void)
3867 {
3868 	void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3869 	u64 val;
3870 
3871 	if (!gic_rdists->has_vpend_valid_dirty)
3872 		return;
3873 
3874 	WARN_ON_ONCE(readq_relaxed_poll_timeout_atomic(vlpi_base + GICR_VPENDBASER,
3875 						       val,
3876 						       !(val & GICR_VPENDBASER_Dirty),
3877 						       1, 500));
3878 }
3879 
its_vpe_schedule(struct its_vpe * vpe)3880 static void its_vpe_schedule(struct its_vpe *vpe)
3881 {
3882 	void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3883 	u64 val;
3884 
3885 	/* Schedule the VPE */
3886 	val  = virt_to_phys(page_address(vpe->its_vm->vprop_page)) &
3887 		GENMASK_ULL(51, 12);
3888 	val |= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
3889 	if (rdists_support_shareable()) {
3890 		val |= GICR_VPROPBASER_RaWb;
3891 		val |= GICR_VPROPBASER_InnerShareable;
3892 	}
3893 	gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
3894 
3895 	val  = virt_to_phys(page_address(vpe->vpt_page)) &
3896 		GENMASK_ULL(51, 16);
3897 	if (rdists_support_shareable()) {
3898 		val |= GICR_VPENDBASER_RaWaWb;
3899 		val |= GICR_VPENDBASER_InnerShareable;
3900 	}
3901 	/*
3902 	 * There is no good way of finding out if the pending table is
3903 	 * empty as we can race against the doorbell interrupt very
3904 	 * easily. So in the end, vpe->pending_last is only an
3905 	 * indication that the vcpu has something pending, not one
3906 	 * that the pending table is empty. A good implementation
3907 	 * would be able to read its coarse map pretty quickly anyway,
3908 	 * making this a tolerable issue.
3909 	 */
3910 	val |= GICR_VPENDBASER_PendingLast;
3911 	val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0;
3912 	val |= GICR_VPENDBASER_Valid;
3913 	gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
3914 }
3915 
its_vpe_deschedule(struct its_vpe * vpe)3916 static void its_vpe_deschedule(struct its_vpe *vpe)
3917 {
3918 	void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3919 	u64 val;
3920 
3921 	val = its_clear_vpend_valid(vlpi_base, 0, 0);
3922 
3923 	vpe->idai = !!(val & GICR_VPENDBASER_IDAI);
3924 	vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
3925 }
3926 
its_vpe_invall(struct its_vpe * vpe)3927 static void its_vpe_invall(struct its_vpe *vpe)
3928 {
3929 	struct its_node *its;
3930 
3931 	list_for_each_entry(its, &its_nodes, entry) {
3932 		if (!is_v4(its))
3933 			continue;
3934 
3935 		if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr])
3936 			continue;
3937 
3938 		/*
3939 		 * Sending a VINVALL to a single ITS is enough, as all
3940 		 * we need is to reach the redistributors.
3941 		 */
3942 		its_send_vinvall(its, vpe);
3943 		return;
3944 	}
3945 }
3946 
its_vpe_set_vcpu_affinity(struct irq_data * d,void * vcpu_info)3947 static int its_vpe_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
3948 {
3949 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3950 	struct its_cmd_info *info = vcpu_info;
3951 
3952 	switch (info->cmd_type) {
3953 	case SCHEDULE_VPE:
3954 		its_vpe_schedule(vpe);
3955 		return 0;
3956 
3957 	case DESCHEDULE_VPE:
3958 		its_vpe_deschedule(vpe);
3959 		return 0;
3960 
3961 	case COMMIT_VPE:
3962 		its_wait_vpt_parse_complete();
3963 		return 0;
3964 
3965 	case INVALL_VPE:
3966 		its_vpe_invall(vpe);
3967 		return 0;
3968 
3969 	default:
3970 		return -EINVAL;
3971 	}
3972 }
3973 
its_vpe_send_cmd(struct its_vpe * vpe,void (* cmd)(struct its_device *,u32))3974 static void its_vpe_send_cmd(struct its_vpe *vpe,
3975 			     void (*cmd)(struct its_device *, u32))
3976 {
3977 	unsigned long flags;
3978 
3979 	raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3980 
3981 	its_vpe_db_proxy_map_locked(vpe);
3982 	cmd(vpe_proxy.dev, vpe->vpe_proxy_event);
3983 
3984 	raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3985 }
3986 
its_vpe_send_inv(struct irq_data * d)3987 static void its_vpe_send_inv(struct irq_data *d)
3988 {
3989 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3990 
3991 	if (gic_rdists->has_direct_lpi)
3992 		__direct_lpi_inv(d, d->parent_data->hwirq);
3993 	else
3994 		its_vpe_send_cmd(vpe, its_send_inv);
3995 }
3996 
its_vpe_mask_irq(struct irq_data * d)3997 static void its_vpe_mask_irq(struct irq_data *d)
3998 {
3999 	/*
4000 	 * We need to unmask the LPI, which is described by the parent
4001 	 * irq_data. Instead of calling into the parent (which won't
4002 	 * exactly do the right thing, let's simply use the
4003 	 * parent_data pointer. Yes, I'm naughty.
4004 	 */
4005 	lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
4006 	its_vpe_send_inv(d);
4007 }
4008 
its_vpe_unmask_irq(struct irq_data * d)4009 static void its_vpe_unmask_irq(struct irq_data *d)
4010 {
4011 	/* Same hack as above... */
4012 	lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
4013 	its_vpe_send_inv(d);
4014 }
4015 
its_vpe_set_irqchip_state(struct irq_data * d,enum irqchip_irq_state which,bool state)4016 static int its_vpe_set_irqchip_state(struct irq_data *d,
4017 				     enum irqchip_irq_state which,
4018 				     bool state)
4019 {
4020 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4021 
4022 	if (which != IRQCHIP_STATE_PENDING)
4023 		return -EINVAL;
4024 
4025 	if (gic_rdists->has_direct_lpi) {
4026 		void __iomem *rdbase;
4027 
4028 		rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;
4029 		if (state) {
4030 			gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR);
4031 		} else {
4032 			gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
4033 			wait_for_syncr(rdbase);
4034 		}
4035 	} else {
4036 		if (state)
4037 			its_vpe_send_cmd(vpe, its_send_int);
4038 		else
4039 			its_vpe_send_cmd(vpe, its_send_clear);
4040 	}
4041 
4042 	return 0;
4043 }
4044 
its_vpe_retrigger(struct irq_data * d)4045 static int its_vpe_retrigger(struct irq_data *d)
4046 {
4047 	return !its_vpe_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true);
4048 }
4049 
4050 static struct irq_chip its_vpe_irq_chip = {
4051 	.name			= "GICv4-vpe",
4052 	.irq_mask		= its_vpe_mask_irq,
4053 	.irq_unmask		= its_vpe_unmask_irq,
4054 	.irq_eoi		= irq_chip_eoi_parent,
4055 	.irq_set_affinity	= its_vpe_set_affinity,
4056 	.irq_retrigger		= its_vpe_retrigger,
4057 	.irq_set_irqchip_state	= its_vpe_set_irqchip_state,
4058 	.irq_set_vcpu_affinity	= its_vpe_set_vcpu_affinity,
4059 };
4060 
find_4_1_its(void)4061 static struct its_node *find_4_1_its(void)
4062 {
4063 	static struct its_node *its = NULL;
4064 
4065 	if (!its) {
4066 		list_for_each_entry(its, &its_nodes, entry) {
4067 			if (is_v4_1(its))
4068 				return its;
4069 		}
4070 
4071 		/* Oops? */
4072 		its = NULL;
4073 	}
4074 
4075 	return its;
4076 }
4077 
its_vpe_4_1_send_inv(struct irq_data * d)4078 static void its_vpe_4_1_send_inv(struct irq_data *d)
4079 {
4080 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4081 	struct its_node *its;
4082 
4083 	/*
4084 	 * GICv4.1 wants doorbells to be invalidated using the
4085 	 * INVDB command in order to be broadcast to all RDs. Send
4086 	 * it to the first valid ITS, and let the HW do its magic.
4087 	 */
4088 	its = find_4_1_its();
4089 	if (its)
4090 		its_send_invdb(its, vpe);
4091 }
4092 
its_vpe_4_1_mask_irq(struct irq_data * d)4093 static void its_vpe_4_1_mask_irq(struct irq_data *d)
4094 {
4095 	lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
4096 	its_vpe_4_1_send_inv(d);
4097 }
4098 
its_vpe_4_1_unmask_irq(struct irq_data * d)4099 static void its_vpe_4_1_unmask_irq(struct irq_data *d)
4100 {
4101 	lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
4102 	its_vpe_4_1_send_inv(d);
4103 }
4104 
its_vpe_4_1_schedule(struct its_vpe * vpe,struct its_cmd_info * info)4105 static void its_vpe_4_1_schedule(struct its_vpe *vpe,
4106 				 struct its_cmd_info *info)
4107 {
4108 	void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
4109 	u64 val = 0;
4110 
4111 	/* Schedule the VPE */
4112 	val |= GICR_VPENDBASER_Valid;
4113 	val |= info->g0en ? GICR_VPENDBASER_4_1_VGRP0EN : 0;
4114 	val |= info->g1en ? GICR_VPENDBASER_4_1_VGRP1EN : 0;
4115 	val |= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id);
4116 
4117 	gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
4118 }
4119 
its_vpe_4_1_deschedule(struct its_vpe * vpe,struct its_cmd_info * info)4120 static void its_vpe_4_1_deschedule(struct its_vpe *vpe,
4121 				   struct its_cmd_info *info)
4122 {
4123 	void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
4124 	u64 val;
4125 
4126 	if (info->req_db) {
4127 		unsigned long flags;
4128 
4129 		/*
4130 		 * vPE is going to block: make the vPE non-resident with
4131 		 * PendingLast clear and DB set. The GIC guarantees that if
4132 		 * we read-back PendingLast clear, then a doorbell will be
4133 		 * delivered when an interrupt comes.
4134 		 *
4135 		 * Note the locking to deal with the concurrent update of
4136 		 * pending_last from the doorbell interrupt handler that can
4137 		 * run concurrently.
4138 		 */
4139 		raw_spin_lock_irqsave(&vpe->vpe_lock, flags);
4140 		val = its_clear_vpend_valid(vlpi_base,
4141 					    GICR_VPENDBASER_PendingLast,
4142 					    GICR_VPENDBASER_4_1_DB);
4143 		vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
4144 		raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
4145 	} else {
4146 		/*
4147 		 * We're not blocking, so just make the vPE non-resident
4148 		 * with PendingLast set, indicating that we'll be back.
4149 		 */
4150 		val = its_clear_vpend_valid(vlpi_base,
4151 					    0,
4152 					    GICR_VPENDBASER_PendingLast);
4153 		vpe->pending_last = true;
4154 	}
4155 }
4156 
its_vpe_4_1_invall(struct its_vpe * vpe)4157 static void its_vpe_4_1_invall(struct its_vpe *vpe)
4158 {
4159 	void __iomem *rdbase;
4160 	unsigned long flags;
4161 	u64 val;
4162 	int cpu;
4163 
4164 	val  = GICR_INVALLR_V;
4165 	val |= FIELD_PREP(GICR_INVALLR_VPEID, vpe->vpe_id);
4166 
4167 	/* Target the redistributor this vPE is currently known on */
4168 	cpu = vpe_to_cpuid_lock(vpe, &flags);
4169 	raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
4170 	rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
4171 	gic_write_lpir(val, rdbase + GICR_INVALLR);
4172 
4173 	wait_for_syncr(rdbase);
4174 	raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
4175 	vpe_to_cpuid_unlock(vpe, flags);
4176 }
4177 
its_vpe_4_1_set_vcpu_affinity(struct irq_data * d,void * vcpu_info)4178 static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
4179 {
4180 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4181 	struct its_cmd_info *info = vcpu_info;
4182 
4183 	switch (info->cmd_type) {
4184 	case SCHEDULE_VPE:
4185 		its_vpe_4_1_schedule(vpe, info);
4186 		return 0;
4187 
4188 	case DESCHEDULE_VPE:
4189 		its_vpe_4_1_deschedule(vpe, info);
4190 		return 0;
4191 
4192 	case COMMIT_VPE:
4193 		its_wait_vpt_parse_complete();
4194 		return 0;
4195 
4196 	case INVALL_VPE:
4197 		its_vpe_4_1_invall(vpe);
4198 		return 0;
4199 
4200 	default:
4201 		return -EINVAL;
4202 	}
4203 }
4204 
4205 static struct irq_chip its_vpe_4_1_irq_chip = {
4206 	.name			= "GICv4.1-vpe",
4207 	.irq_mask		= its_vpe_4_1_mask_irq,
4208 	.irq_unmask		= its_vpe_4_1_unmask_irq,
4209 	.irq_eoi		= irq_chip_eoi_parent,
4210 	.irq_set_affinity	= its_vpe_set_affinity,
4211 	.irq_set_vcpu_affinity	= its_vpe_4_1_set_vcpu_affinity,
4212 };
4213 
its_configure_sgi(struct irq_data * d,bool clear)4214 static void its_configure_sgi(struct irq_data *d, bool clear)
4215 {
4216 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4217 	struct its_cmd_desc desc;
4218 
4219 	desc.its_vsgi_cmd.vpe = vpe;
4220 	desc.its_vsgi_cmd.sgi = d->hwirq;
4221 	desc.its_vsgi_cmd.priority = vpe->sgi_config[d->hwirq].priority;
4222 	desc.its_vsgi_cmd.enable = vpe->sgi_config[d->hwirq].enabled;
4223 	desc.its_vsgi_cmd.group = vpe->sgi_config[d->hwirq].group;
4224 	desc.its_vsgi_cmd.clear = clear;
4225 
4226 	/*
4227 	 * GICv4.1 allows us to send VSGI commands to any ITS as long as the
4228 	 * destination VPE is mapped there. Since we map them eagerly at
4229 	 * activation time, we're pretty sure the first GICv4.1 ITS will do.
4230 	 */
4231 	its_send_single_vcommand(find_4_1_its(), its_build_vsgi_cmd, &desc);
4232 }
4233 
its_sgi_mask_irq(struct irq_data * d)4234 static void its_sgi_mask_irq(struct irq_data *d)
4235 {
4236 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4237 
4238 	vpe->sgi_config[d->hwirq].enabled = false;
4239 	its_configure_sgi(d, false);
4240 }
4241 
its_sgi_unmask_irq(struct irq_data * d)4242 static void its_sgi_unmask_irq(struct irq_data *d)
4243 {
4244 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4245 
4246 	vpe->sgi_config[d->hwirq].enabled = true;
4247 	its_configure_sgi(d, false);
4248 }
4249 
its_sgi_set_affinity(struct irq_data * d,const struct cpumask * mask_val,bool force)4250 static int its_sgi_set_affinity(struct irq_data *d,
4251 				const struct cpumask *mask_val,
4252 				bool force)
4253 {
4254 	/*
4255 	 * There is no notion of affinity for virtual SGIs, at least
4256 	 * not on the host (since they can only be targeting a vPE).
4257 	 * Tell the kernel we've done whatever it asked for.
4258 	 */
4259 	irq_data_update_effective_affinity(d, mask_val);
4260 	return IRQ_SET_MASK_OK;
4261 }
4262 
its_sgi_set_irqchip_state(struct irq_data * d,enum irqchip_irq_state which,bool state)4263 static int its_sgi_set_irqchip_state(struct irq_data *d,
4264 				     enum irqchip_irq_state which,
4265 				     bool state)
4266 {
4267 	if (which != IRQCHIP_STATE_PENDING)
4268 		return -EINVAL;
4269 
4270 	if (state) {
4271 		struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4272 		struct its_node *its = find_4_1_its();
4273 		u64 val;
4274 
4275 		val  = FIELD_PREP(GITS_SGIR_VPEID, vpe->vpe_id);
4276 		val |= FIELD_PREP(GITS_SGIR_VINTID, d->hwirq);
4277 		writeq_relaxed(val, its->sgir_base + GITS_SGIR - SZ_128K);
4278 	} else {
4279 		its_configure_sgi(d, true);
4280 	}
4281 
4282 	return 0;
4283 }
4284 
its_sgi_get_irqchip_state(struct irq_data * d,enum irqchip_irq_state which,bool * val)4285 static int its_sgi_get_irqchip_state(struct irq_data *d,
4286 				     enum irqchip_irq_state which, bool *val)
4287 {
4288 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4289 	void __iomem *base;
4290 	unsigned long flags;
4291 	u32 count = 1000000;	/* 1s! */
4292 	u32 status;
4293 	int cpu;
4294 
4295 	if (which != IRQCHIP_STATE_PENDING)
4296 		return -EINVAL;
4297 
4298 	/*
4299 	 * Locking galore! We can race against two different events:
4300 	 *
4301 	 * - Concurrent vPE affinity change: we must make sure it cannot
4302 	 *   happen, or we'll talk to the wrong redistributor. This is
4303 	 *   identical to what happens with vLPIs.
4304 	 *
4305 	 * - Concurrent VSGIPENDR access: As it involves accessing two
4306 	 *   MMIO registers, this must be made atomic one way or another.
4307 	 */
4308 	cpu = vpe_to_cpuid_lock(vpe, &flags);
4309 	raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
4310 	base = gic_data_rdist_cpu(cpu)->rd_base + SZ_128K;
4311 	writel_relaxed(vpe->vpe_id, base + GICR_VSGIR);
4312 	do {
4313 		status = readl_relaxed(base + GICR_VSGIPENDR);
4314 		if (!(status & GICR_VSGIPENDR_BUSY))
4315 			goto out;
4316 
4317 		count--;
4318 		if (!count) {
4319 			pr_err_ratelimited("Unable to get SGI status\n");
4320 			goto out;
4321 		}
4322 		cpu_relax();
4323 		udelay(1);
4324 	} while (count);
4325 
4326 out:
4327 	raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
4328 	vpe_to_cpuid_unlock(vpe, flags);
4329 
4330 	if (!count)
4331 		return -ENXIO;
4332 
4333 	*val = !!(status & (1 << d->hwirq));
4334 
4335 	return 0;
4336 }
4337 
its_sgi_set_vcpu_affinity(struct irq_data * d,void * vcpu_info)4338 static int its_sgi_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
4339 {
4340 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4341 	struct its_cmd_info *info = vcpu_info;
4342 
4343 	switch (info->cmd_type) {
4344 	case PROP_UPDATE_VSGI:
4345 		vpe->sgi_config[d->hwirq].priority = info->priority;
4346 		vpe->sgi_config[d->hwirq].group = info->group;
4347 		its_configure_sgi(d, false);
4348 		return 0;
4349 
4350 	default:
4351 		return -EINVAL;
4352 	}
4353 }
4354 
4355 static struct irq_chip its_sgi_irq_chip = {
4356 	.name			= "GICv4.1-sgi",
4357 	.irq_mask		= its_sgi_mask_irq,
4358 	.irq_unmask		= its_sgi_unmask_irq,
4359 	.irq_set_affinity	= its_sgi_set_affinity,
4360 	.irq_set_irqchip_state	= its_sgi_set_irqchip_state,
4361 	.irq_get_irqchip_state	= its_sgi_get_irqchip_state,
4362 	.irq_set_vcpu_affinity	= its_sgi_set_vcpu_affinity,
4363 };
4364 
its_sgi_irq_domain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * args)4365 static int its_sgi_irq_domain_alloc(struct irq_domain *domain,
4366 				    unsigned int virq, unsigned int nr_irqs,
4367 				    void *args)
4368 {
4369 	struct its_vpe *vpe = args;
4370 	int i;
4371 
4372 	/* Yes, we do want 16 SGIs */
4373 	WARN_ON(nr_irqs != 16);
4374 
4375 	for (i = 0; i < 16; i++) {
4376 		vpe->sgi_config[i].priority = 0;
4377 		vpe->sgi_config[i].enabled = false;
4378 		vpe->sgi_config[i].group = false;
4379 
4380 		irq_domain_set_hwirq_and_chip(domain, virq + i, i,
4381 					      &its_sgi_irq_chip, vpe);
4382 		irq_set_status_flags(virq + i, IRQ_DISABLE_UNLAZY);
4383 	}
4384 
4385 	return 0;
4386 }
4387 
its_sgi_irq_domain_free(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs)4388 static void its_sgi_irq_domain_free(struct irq_domain *domain,
4389 				    unsigned int virq,
4390 				    unsigned int nr_irqs)
4391 {
4392 	/* Nothing to do */
4393 }
4394 
its_sgi_irq_domain_activate(struct irq_domain * domain,struct irq_data * d,bool reserve)4395 static int its_sgi_irq_domain_activate(struct irq_domain *domain,
4396 				       struct irq_data *d, bool reserve)
4397 {
4398 	/* Write out the initial SGI configuration */
4399 	its_configure_sgi(d, false);
4400 	return 0;
4401 }
4402 
its_sgi_irq_domain_deactivate(struct irq_domain * domain,struct irq_data * d)4403 static void its_sgi_irq_domain_deactivate(struct irq_domain *domain,
4404 					  struct irq_data *d)
4405 {
4406 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4407 
4408 	/*
4409 	 * The VSGI command is awkward:
4410 	 *
4411 	 * - To change the configuration, CLEAR must be set to false,
4412 	 *   leaving the pending bit unchanged.
4413 	 * - To clear the pending bit, CLEAR must be set to true, leaving
4414 	 *   the configuration unchanged.
4415 	 *
4416 	 * You just can't do both at once, hence the two commands below.
4417 	 */
4418 	vpe->sgi_config[d->hwirq].enabled = false;
4419 	its_configure_sgi(d, false);
4420 	its_configure_sgi(d, true);
4421 }
4422 
4423 static const struct irq_domain_ops its_sgi_domain_ops = {
4424 	.alloc		= its_sgi_irq_domain_alloc,
4425 	.free		= its_sgi_irq_domain_free,
4426 	.activate	= its_sgi_irq_domain_activate,
4427 	.deactivate	= its_sgi_irq_domain_deactivate,
4428 };
4429 
its_vpe_id_alloc(void)4430 static int its_vpe_id_alloc(void)
4431 {
4432 	return ida_simple_get(&its_vpeid_ida, 0, ITS_MAX_VPEID, GFP_KERNEL);
4433 }
4434 
its_vpe_id_free(u16 id)4435 static void its_vpe_id_free(u16 id)
4436 {
4437 	ida_simple_remove(&its_vpeid_ida, id);
4438 }
4439 
its_vpe_init(struct its_vpe * vpe)4440 static int its_vpe_init(struct its_vpe *vpe)
4441 {
4442 	struct page *vpt_page;
4443 	int vpe_id;
4444 
4445 	/* Allocate vpe_id */
4446 	vpe_id = its_vpe_id_alloc();
4447 	if (vpe_id < 0)
4448 		return vpe_id;
4449 
4450 	/* Allocate VPT */
4451 	vpt_page = its_allocate_pending_table(GFP_KERNEL);
4452 	if (!vpt_page) {
4453 		its_vpe_id_free(vpe_id);
4454 		return -ENOMEM;
4455 	}
4456 
4457 	if (!its_alloc_vpe_table(vpe_id)) {
4458 		its_vpe_id_free(vpe_id);
4459 		its_free_pending_table(vpt_page);
4460 		return -ENOMEM;
4461 	}
4462 
4463 	raw_spin_lock_init(&vpe->vpe_lock);
4464 	vpe->vpe_id = vpe_id;
4465 	vpe->vpt_page = vpt_page;
4466 	atomic_set(&vpe->vmapp_count, 0);
4467 	if (!gic_rdists->has_rvpeid)
4468 		vpe->vpe_proxy_event = -1;
4469 
4470 	return 0;
4471 }
4472 
its_vpe_teardown(struct its_vpe * vpe)4473 static void its_vpe_teardown(struct its_vpe *vpe)
4474 {
4475 	its_vpe_db_proxy_unmap(vpe);
4476 	its_vpe_id_free(vpe->vpe_id);
4477 	its_free_pending_table(vpe->vpt_page);
4478 }
4479 
its_vpe_irq_domain_free(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs)4480 static void its_vpe_irq_domain_free(struct irq_domain *domain,
4481 				    unsigned int virq,
4482 				    unsigned int nr_irqs)
4483 {
4484 	struct its_vm *vm = domain->host_data;
4485 	int i;
4486 
4487 	irq_domain_free_irqs_parent(domain, virq, nr_irqs);
4488 
4489 	for (i = 0; i < nr_irqs; i++) {
4490 		struct irq_data *data = irq_domain_get_irq_data(domain,
4491 								virq + i);
4492 		struct its_vpe *vpe = irq_data_get_irq_chip_data(data);
4493 
4494 		BUG_ON(vm != vpe->its_vm);
4495 
4496 		clear_bit(data->hwirq, vm->db_bitmap);
4497 		its_vpe_teardown(vpe);
4498 		irq_domain_reset_irq_data(data);
4499 	}
4500 
4501 	if (bitmap_empty(vm->db_bitmap, vm->nr_db_lpis)) {
4502 		its_lpi_free(vm->db_bitmap, vm->db_lpi_base, vm->nr_db_lpis);
4503 		its_free_prop_table(vm->vprop_page);
4504 	}
4505 }
4506 
its_vpe_irq_domain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * args)4507 static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
4508 				    unsigned int nr_irqs, void *args)
4509 {
4510 	struct irq_chip *irqchip = &its_vpe_irq_chip;
4511 	struct its_vm *vm = args;
4512 	unsigned long *bitmap;
4513 	struct page *vprop_page;
4514 	int base, nr_ids, i, err = 0;
4515 
4516 	bitmap = its_lpi_alloc(roundup_pow_of_two(nr_irqs), &base, &nr_ids);
4517 	if (!bitmap)
4518 		return -ENOMEM;
4519 
4520 	if (nr_ids < nr_irqs) {
4521 		its_lpi_free(bitmap, base, nr_ids);
4522 		return -ENOMEM;
4523 	}
4524 
4525 	vprop_page = its_allocate_prop_table(GFP_KERNEL);
4526 	if (!vprop_page) {
4527 		its_lpi_free(bitmap, base, nr_ids);
4528 		return -ENOMEM;
4529 	}
4530 
4531 	vm->db_bitmap = bitmap;
4532 	vm->db_lpi_base = base;
4533 	vm->nr_db_lpis = nr_ids;
4534 	vm->vprop_page = vprop_page;
4535 
4536 	if (gic_rdists->has_rvpeid)
4537 		irqchip = &its_vpe_4_1_irq_chip;
4538 
4539 	for (i = 0; i < nr_irqs; i++) {
4540 		vm->vpes[i]->vpe_db_lpi = base + i;
4541 		err = its_vpe_init(vm->vpes[i]);
4542 		if (err)
4543 			break;
4544 		err = its_irq_gic_domain_alloc(domain, virq + i,
4545 					       vm->vpes[i]->vpe_db_lpi);
4546 		if (err)
4547 			break;
4548 		irq_domain_set_hwirq_and_chip(domain, virq + i, i,
4549 					      irqchip, vm->vpes[i]);
4550 		set_bit(i, bitmap);
4551 		irqd_set_resend_when_in_progress(irq_get_irq_data(virq + i));
4552 	}
4553 
4554 	if (err)
4555 		its_vpe_irq_domain_free(domain, virq, i);
4556 
4557 	return err;
4558 }
4559 
its_vpe_irq_domain_activate(struct irq_domain * domain,struct irq_data * d,bool reserve)4560 static int its_vpe_irq_domain_activate(struct irq_domain *domain,
4561 				       struct irq_data *d, bool reserve)
4562 {
4563 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4564 	struct its_node *its;
4565 
4566 	/*
4567 	 * If we use the list map, we issue VMAPP on demand... Unless
4568 	 * we're on a GICv4.1 and we eagerly map the VPE on all ITSs
4569 	 * so that VSGIs can work.
4570 	 */
4571 	if (!gic_requires_eager_mapping())
4572 		return 0;
4573 
4574 	/* Map the VPE to the first possible CPU */
4575 	vpe->col_idx = cpumask_first(cpu_online_mask);
4576 
4577 	list_for_each_entry(its, &its_nodes, entry) {
4578 		if (!is_v4(its))
4579 			continue;
4580 
4581 		its_send_vmapp(its, vpe, true);
4582 		its_send_vinvall(its, vpe);
4583 	}
4584 
4585 	irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
4586 
4587 	return 0;
4588 }
4589 
its_vpe_irq_domain_deactivate(struct irq_domain * domain,struct irq_data * d)4590 static void its_vpe_irq_domain_deactivate(struct irq_domain *domain,
4591 					  struct irq_data *d)
4592 {
4593 	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4594 	struct its_node *its;
4595 
4596 	/*
4597 	 * If we use the list map on GICv4.0, we unmap the VPE once no
4598 	 * VLPIs are associated with the VM.
4599 	 */
4600 	if (!gic_requires_eager_mapping())
4601 		return;
4602 
4603 	list_for_each_entry(its, &its_nodes, entry) {
4604 		if (!is_v4(its))
4605 			continue;
4606 
4607 		its_send_vmapp(its, vpe, false);
4608 	}
4609 
4610 	/*
4611 	 * There may be a direct read to the VPT after unmapping the
4612 	 * vPE, to guarantee the validity of this, we make the VPT
4613 	 * memory coherent with the CPU caches here.
4614 	 */
4615 	if (find_4_1_its() && !atomic_read(&vpe->vmapp_count))
4616 		gic_flush_dcache_to_poc(page_address(vpe->vpt_page),
4617 					LPI_PENDBASE_SZ);
4618 }
4619 
4620 static const struct irq_domain_ops its_vpe_domain_ops = {
4621 	.alloc			= its_vpe_irq_domain_alloc,
4622 	.free			= its_vpe_irq_domain_free,
4623 	.activate		= its_vpe_irq_domain_activate,
4624 	.deactivate		= its_vpe_irq_domain_deactivate,
4625 };
4626 
its_force_quiescent(void __iomem * base)4627 static int its_force_quiescent(void __iomem *base)
4628 {
4629 	u32 count = 1000000;	/* 1s */
4630 	u32 val;
4631 
4632 	val = readl_relaxed(base + GITS_CTLR);
4633 	/*
4634 	 * GIC architecture specification requires the ITS to be both
4635 	 * disabled and quiescent for writes to GITS_BASER<n> or
4636 	 * GITS_CBASER to not have UNPREDICTABLE results.
4637 	 */
4638 	if ((val & GITS_CTLR_QUIESCENT) && !(val & GITS_CTLR_ENABLE))
4639 		return 0;
4640 
4641 	/* Disable the generation of all interrupts to this ITS */
4642 	val &= ~(GITS_CTLR_ENABLE | GITS_CTLR_ImDe);
4643 	writel_relaxed(val, base + GITS_CTLR);
4644 
4645 	/* Poll GITS_CTLR and wait until ITS becomes quiescent */
4646 	while (1) {
4647 		val = readl_relaxed(base + GITS_CTLR);
4648 		if (val & GITS_CTLR_QUIESCENT)
4649 			return 0;
4650 
4651 		count--;
4652 		if (!count)
4653 			return -EBUSY;
4654 
4655 		cpu_relax();
4656 		udelay(1);
4657 	}
4658 }
4659 
its_enable_quirk_cavium_22375(void * data)4660 static bool __maybe_unused its_enable_quirk_cavium_22375(void *data)
4661 {
4662 	struct its_node *its = data;
4663 
4664 	/* erratum 22375: only alloc 8MB table size (20 bits) */
4665 	its->typer &= ~GITS_TYPER_DEVBITS;
4666 	its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, 20 - 1);
4667 	its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375;
4668 
4669 	return true;
4670 }
4671 
its_enable_quirk_cavium_23144(void * data)4672 static bool __maybe_unused its_enable_quirk_cavium_23144(void *data)
4673 {
4674 	struct its_node *its = data;
4675 
4676 	its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_23144;
4677 
4678 	return true;
4679 }
4680 
its_enable_quirk_qdf2400_e0065(void * data)4681 static bool __maybe_unused its_enable_quirk_qdf2400_e0065(void *data)
4682 {
4683 	struct its_node *its = data;
4684 
4685 	/* On QDF2400, the size of the ITE is 16Bytes */
4686 	its->typer &= ~GITS_TYPER_ITT_ENTRY_SIZE;
4687 	its->typer |= FIELD_PREP(GITS_TYPER_ITT_ENTRY_SIZE, 16 - 1);
4688 
4689 	return true;
4690 }
4691 
its_irq_get_msi_base_pre_its(struct its_device * its_dev)4692 static u64 its_irq_get_msi_base_pre_its(struct its_device *its_dev)
4693 {
4694 	struct its_node *its = its_dev->its;
4695 
4696 	/*
4697 	 * The Socionext Synquacer SoC has a so-called 'pre-ITS',
4698 	 * which maps 32-bit writes targeted at a separate window of
4699 	 * size '4 << device_id_bits' onto writes to GITS_TRANSLATER
4700 	 * with device ID taken from bits [device_id_bits + 1:2] of
4701 	 * the window offset.
4702 	 */
4703 	return its->pre_its_base + (its_dev->device_id << 2);
4704 }
4705 
its_enable_quirk_socionext_synquacer(void * data)4706 static bool __maybe_unused its_enable_quirk_socionext_synquacer(void *data)
4707 {
4708 	struct its_node *its = data;
4709 	u32 pre_its_window[2];
4710 	u32 ids;
4711 
4712 	if (!fwnode_property_read_u32_array(its->fwnode_handle,
4713 					   "socionext,synquacer-pre-its",
4714 					   pre_its_window,
4715 					   ARRAY_SIZE(pre_its_window))) {
4716 
4717 		its->pre_its_base = pre_its_window[0];
4718 		its->get_msi_base = its_irq_get_msi_base_pre_its;
4719 
4720 		ids = ilog2(pre_its_window[1]) - 2;
4721 		if (device_ids(its) > ids) {
4722 			its->typer &= ~GITS_TYPER_DEVBITS;
4723 			its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, ids - 1);
4724 		}
4725 
4726 		/* the pre-ITS breaks isolation, so disable MSI remapping */
4727 		its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_ISOLATED_MSI;
4728 		return true;
4729 	}
4730 	return false;
4731 }
4732 
its_enable_quirk_hip07_161600802(void * data)4733 static bool __maybe_unused its_enable_quirk_hip07_161600802(void *data)
4734 {
4735 	struct its_node *its = data;
4736 
4737 	/*
4738 	 * Hip07 insists on using the wrong address for the VLPI
4739 	 * page. Trick it into doing the right thing...
4740 	 */
4741 	its->vlpi_redist_offset = SZ_128K;
4742 	return true;
4743 }
4744 
its_enable_rk3588001(void * data)4745 static bool __maybe_unused its_enable_rk3588001(void *data)
4746 {
4747 	struct its_node *its = data;
4748 
4749 	if (!of_machine_is_compatible("rockchip,rk3588") &&
4750 	    !of_machine_is_compatible("rockchip,rk3588s"))
4751 		return false;
4752 
4753 	its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE;
4754 	gic_rdists->flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE;
4755 
4756 	return true;
4757 }
4758 
its_set_non_coherent(void * data)4759 static bool its_set_non_coherent(void *data)
4760 {
4761 	struct its_node *its = data;
4762 
4763 	its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE;
4764 	return true;
4765 }
4766 
4767 static const struct gic_quirk its_quirks[] = {
4768 #ifdef CONFIG_CAVIUM_ERRATUM_22375
4769 	{
4770 		.desc	= "ITS: Cavium errata 22375, 24313",
4771 		.iidr	= 0xa100034c,	/* ThunderX pass 1.x */
4772 		.mask	= 0xffff0fff,
4773 		.init	= its_enable_quirk_cavium_22375,
4774 	},
4775 #endif
4776 #ifdef CONFIG_CAVIUM_ERRATUM_23144
4777 	{
4778 		.desc	= "ITS: Cavium erratum 23144",
4779 		.iidr	= 0xa100034c,	/* ThunderX pass 1.x */
4780 		.mask	= 0xffff0fff,
4781 		.init	= its_enable_quirk_cavium_23144,
4782 	},
4783 #endif
4784 #ifdef CONFIG_QCOM_QDF2400_ERRATUM_0065
4785 	{
4786 		.desc	= "ITS: QDF2400 erratum 0065",
4787 		.iidr	= 0x00001070, /* QDF2400 ITS rev 1.x */
4788 		.mask	= 0xffffffff,
4789 		.init	= its_enable_quirk_qdf2400_e0065,
4790 	},
4791 #endif
4792 #ifdef CONFIG_SOCIONEXT_SYNQUACER_PREITS
4793 	{
4794 		/*
4795 		 * The Socionext Synquacer SoC incorporates ARM's own GIC-500
4796 		 * implementation, but with a 'pre-ITS' added that requires
4797 		 * special handling in software.
4798 		 */
4799 		.desc	= "ITS: Socionext Synquacer pre-ITS",
4800 		.iidr	= 0x0001143b,
4801 		.mask	= 0xffffffff,
4802 		.init	= its_enable_quirk_socionext_synquacer,
4803 	},
4804 #endif
4805 #ifdef CONFIG_HISILICON_ERRATUM_161600802
4806 	{
4807 		.desc	= "ITS: Hip07 erratum 161600802",
4808 		.iidr	= 0x00000004,
4809 		.mask	= 0xffffffff,
4810 		.init	= its_enable_quirk_hip07_161600802,
4811 	},
4812 #endif
4813 #ifdef CONFIG_ROCKCHIP_ERRATUM_3588001
4814 	{
4815 		.desc   = "ITS: Rockchip erratum RK3588001",
4816 		.iidr   = 0x0201743b,
4817 		.mask   = 0xffffffff,
4818 		.init   = its_enable_rk3588001,
4819 	},
4820 #endif
4821 	{
4822 		.desc   = "ITS: non-coherent attribute",
4823 		.property = "dma-noncoherent",
4824 		.init   = its_set_non_coherent,
4825 	},
4826 	{
4827 	}
4828 };
4829 
its_enable_quirks(struct its_node * its)4830 static void its_enable_quirks(struct its_node *its)
4831 {
4832 	u32 iidr = readl_relaxed(its->base + GITS_IIDR);
4833 
4834 	gic_enable_quirks(iidr, its_quirks, its);
4835 
4836 	if (is_of_node(its->fwnode_handle))
4837 		gic_enable_of_quirks(to_of_node(its->fwnode_handle),
4838 				     its_quirks, its);
4839 }
4840 
its_save_disable(void)4841 static int its_save_disable(void)
4842 {
4843 	struct its_node *its;
4844 	int err = 0;
4845 
4846 	raw_spin_lock(&its_lock);
4847 	list_for_each_entry(its, &its_nodes, entry) {
4848 		void __iomem *base;
4849 
4850 		base = its->base;
4851 		its->ctlr_save = readl_relaxed(base + GITS_CTLR);
4852 		err = its_force_quiescent(base);
4853 		if (err) {
4854 			pr_err("ITS@%pa: failed to quiesce: %d\n",
4855 			       &its->phys_base, err);
4856 			writel_relaxed(its->ctlr_save, base + GITS_CTLR);
4857 			goto err;
4858 		}
4859 
4860 		its->cbaser_save = gits_read_cbaser(base + GITS_CBASER);
4861 	}
4862 
4863 err:
4864 	if (err) {
4865 		list_for_each_entry_continue_reverse(its, &its_nodes, entry) {
4866 			void __iomem *base;
4867 
4868 			base = its->base;
4869 			writel_relaxed(its->ctlr_save, base + GITS_CTLR);
4870 		}
4871 	}
4872 	raw_spin_unlock(&its_lock);
4873 
4874 	return err;
4875 }
4876 
its_restore_enable(void)4877 static void its_restore_enable(void)
4878 {
4879 	struct its_node *its;
4880 	int ret;
4881 
4882 	raw_spin_lock(&its_lock);
4883 	list_for_each_entry(its, &its_nodes, entry) {
4884 		void __iomem *base;
4885 		int i;
4886 
4887 		base = its->base;
4888 
4889 		/*
4890 		 * Make sure that the ITS is disabled. If it fails to quiesce,
4891 		 * don't restore it since writing to CBASER or BASER<n>
4892 		 * registers is undefined according to the GIC v3 ITS
4893 		 * Specification.
4894 		 *
4895 		 * Firmware resuming with the ITS enabled is terminally broken.
4896 		 */
4897 		WARN_ON(readl_relaxed(base + GITS_CTLR) & GITS_CTLR_ENABLE);
4898 		ret = its_force_quiescent(base);
4899 		if (ret) {
4900 			pr_err("ITS@%pa: failed to quiesce on resume: %d\n",
4901 			       &its->phys_base, ret);
4902 			continue;
4903 		}
4904 
4905 		gits_write_cbaser(its->cbaser_save, base + GITS_CBASER);
4906 
4907 		/*
4908 		 * Writing CBASER resets CREADR to 0, so make CWRITER and
4909 		 * cmd_write line up with it.
4910 		 */
4911 		its->cmd_write = its->cmd_base;
4912 		gits_write_cwriter(0, base + GITS_CWRITER);
4913 
4914 		/* Restore GITS_BASER from the value cache. */
4915 		for (i = 0; i < GITS_BASER_NR_REGS; i++) {
4916 			struct its_baser *baser = &its->tables[i];
4917 
4918 			if (!(baser->val & GITS_BASER_VALID))
4919 				continue;
4920 
4921 			its_write_baser(its, baser, baser->val);
4922 		}
4923 		writel_relaxed(its->ctlr_save, base + GITS_CTLR);
4924 
4925 		/*
4926 		 * Reinit the collection if it's stored in the ITS. This is
4927 		 * indicated by the col_id being less than the HCC field.
4928 		 * CID < HCC as specified in the GIC v3 Documentation.
4929 		 */
4930 		if (its->collections[smp_processor_id()].col_id <
4931 		    GITS_TYPER_HCC(gic_read_typer(base + GITS_TYPER)))
4932 			its_cpu_init_collection(its);
4933 	}
4934 	raw_spin_unlock(&its_lock);
4935 }
4936 
4937 static struct syscore_ops its_syscore_ops = {
4938 	.suspend = its_save_disable,
4939 	.resume = its_restore_enable,
4940 };
4941 
its_map_one(struct resource * res,int * err)4942 static void __init __iomem *its_map_one(struct resource *res, int *err)
4943 {
4944 	void __iomem *its_base;
4945 	u32 val;
4946 
4947 	its_base = ioremap(res->start, SZ_64K);
4948 	if (!its_base) {
4949 		pr_warn("ITS@%pa: Unable to map ITS registers\n", &res->start);
4950 		*err = -ENOMEM;
4951 		return NULL;
4952 	}
4953 
4954 	val = readl_relaxed(its_base + GITS_PIDR2) & GIC_PIDR2_ARCH_MASK;
4955 	if (val != 0x30 && val != 0x40) {
4956 		pr_warn("ITS@%pa: No ITS detected, giving up\n", &res->start);
4957 		*err = -ENODEV;
4958 		goto out_unmap;
4959 	}
4960 
4961 	*err = its_force_quiescent(its_base);
4962 	if (*err) {
4963 		pr_warn("ITS@%pa: Failed to quiesce, giving up\n", &res->start);
4964 		goto out_unmap;
4965 	}
4966 
4967 	return its_base;
4968 
4969 out_unmap:
4970 	iounmap(its_base);
4971 	return NULL;
4972 }
4973 
its_init_domain(struct its_node * its)4974 static int its_init_domain(struct its_node *its)
4975 {
4976 	struct irq_domain *inner_domain;
4977 	struct msi_domain_info *info;
4978 
4979 	info = kzalloc(sizeof(*info), GFP_KERNEL);
4980 	if (!info)
4981 		return -ENOMEM;
4982 
4983 	info->ops = &its_msi_domain_ops;
4984 	info->data = its;
4985 
4986 	inner_domain = irq_domain_create_hierarchy(its_parent,
4987 						   its->msi_domain_flags, 0,
4988 						   its->fwnode_handle, &its_domain_ops,
4989 						   info);
4990 	if (!inner_domain) {
4991 		kfree(info);
4992 		return -ENOMEM;
4993 	}
4994 
4995 	irq_domain_update_bus_token(inner_domain, DOMAIN_BUS_NEXUS);
4996 
4997 	return 0;
4998 }
4999 
its_init_vpe_domain(void)5000 static int its_init_vpe_domain(void)
5001 {
5002 	struct its_node *its;
5003 	u32 devid;
5004 	int entries;
5005 
5006 	if (gic_rdists->has_direct_lpi) {
5007 		pr_info("ITS: Using DirectLPI for VPE invalidation\n");
5008 		return 0;
5009 	}
5010 
5011 	/* Any ITS will do, even if not v4 */
5012 	its = list_first_entry(&its_nodes, struct its_node, entry);
5013 
5014 	entries = roundup_pow_of_two(nr_cpu_ids);
5015 	vpe_proxy.vpes = kcalloc(entries, sizeof(*vpe_proxy.vpes),
5016 				 GFP_KERNEL);
5017 	if (!vpe_proxy.vpes)
5018 		return -ENOMEM;
5019 
5020 	/* Use the last possible DevID */
5021 	devid = GENMASK(device_ids(its) - 1, 0);
5022 	vpe_proxy.dev = its_create_device(its, devid, entries, false);
5023 	if (!vpe_proxy.dev) {
5024 		kfree(vpe_proxy.vpes);
5025 		pr_err("ITS: Can't allocate GICv4 proxy device\n");
5026 		return -ENOMEM;
5027 	}
5028 
5029 	BUG_ON(entries > vpe_proxy.dev->nr_ites);
5030 
5031 	raw_spin_lock_init(&vpe_proxy.lock);
5032 	vpe_proxy.next_victim = 0;
5033 	pr_info("ITS: Allocated DevID %x as GICv4 proxy device (%d slots)\n",
5034 		devid, vpe_proxy.dev->nr_ites);
5035 
5036 	return 0;
5037 }
5038 
its_compute_its_list_map(struct its_node * its)5039 static int __init its_compute_its_list_map(struct its_node *its)
5040 {
5041 	int its_number;
5042 	u32 ctlr;
5043 
5044 	/*
5045 	 * This is assumed to be done early enough that we're
5046 	 * guaranteed to be single-threaded, hence no
5047 	 * locking. Should this change, we should address
5048 	 * this.
5049 	 */
5050 	its_number = find_first_zero_bit(&its_list_map, GICv4_ITS_LIST_MAX);
5051 	if (its_number >= GICv4_ITS_LIST_MAX) {
5052 		pr_err("ITS@%pa: No ITSList entry available!\n",
5053 		       &its->phys_base);
5054 		return -EINVAL;
5055 	}
5056 
5057 	ctlr = readl_relaxed(its->base + GITS_CTLR);
5058 	ctlr &= ~GITS_CTLR_ITS_NUMBER;
5059 	ctlr |= its_number << GITS_CTLR_ITS_NUMBER_SHIFT;
5060 	writel_relaxed(ctlr, its->base + GITS_CTLR);
5061 	ctlr = readl_relaxed(its->base + GITS_CTLR);
5062 	if ((ctlr & GITS_CTLR_ITS_NUMBER) != (its_number << GITS_CTLR_ITS_NUMBER_SHIFT)) {
5063 		its_number = ctlr & GITS_CTLR_ITS_NUMBER;
5064 		its_number >>= GITS_CTLR_ITS_NUMBER_SHIFT;
5065 	}
5066 
5067 	if (test_and_set_bit(its_number, &its_list_map)) {
5068 		pr_err("ITS@%pa: Duplicate ITSList entry %d\n",
5069 		       &its->phys_base, its_number);
5070 		return -EINVAL;
5071 	}
5072 
5073 	return its_number;
5074 }
5075 
its_probe_one(struct its_node * its)5076 static int __init its_probe_one(struct its_node *its)
5077 {
5078 	u64 baser, tmp;
5079 	struct page *page;
5080 	u32 ctlr;
5081 	int err;
5082 
5083 	its_enable_quirks(its);
5084 
5085 	if (is_v4(its)) {
5086 		if (!(its->typer & GITS_TYPER_VMOVP)) {
5087 			err = its_compute_its_list_map(its);
5088 			if (err < 0)
5089 				goto out;
5090 
5091 			its->list_nr = err;
5092 
5093 			pr_info("ITS@%pa: Using ITS number %d\n",
5094 				&its->phys_base, err);
5095 		} else {
5096 			pr_info("ITS@%pa: Single VMOVP capable\n", &its->phys_base);
5097 		}
5098 
5099 		if (is_v4_1(its)) {
5100 			u32 svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer);
5101 
5102 			its->sgir_base = ioremap(its->phys_base + SZ_128K, SZ_64K);
5103 			if (!its->sgir_base) {
5104 				err = -ENOMEM;
5105 				goto out;
5106 			}
5107 
5108 			its->mpidr = readl_relaxed(its->base + GITS_MPIDR);
5109 
5110 			pr_info("ITS@%pa: Using GICv4.1 mode %08x %08x\n",
5111 				&its->phys_base, its->mpidr, svpet);
5112 		}
5113 	}
5114 
5115 	page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO,
5116 				get_order(ITS_CMD_QUEUE_SZ));
5117 	if (!page) {
5118 		err = -ENOMEM;
5119 		goto out_unmap_sgir;
5120 	}
5121 	its->cmd_base = (void *)page_address(page);
5122 	its->cmd_write = its->cmd_base;
5123 
5124 	err = its_alloc_tables(its);
5125 	if (err)
5126 		goto out_free_cmd;
5127 
5128 	err = its_alloc_collections(its);
5129 	if (err)
5130 		goto out_free_tables;
5131 
5132 	baser = (virt_to_phys(its->cmd_base)	|
5133 		 GITS_CBASER_RaWaWb		|
5134 		 GITS_CBASER_InnerShareable	|
5135 		 (ITS_CMD_QUEUE_SZ / SZ_4K - 1)	|
5136 		 GITS_CBASER_VALID);
5137 
5138 	gits_write_cbaser(baser, its->base + GITS_CBASER);
5139 	tmp = gits_read_cbaser(its->base + GITS_CBASER);
5140 
5141 	if (its->flags & ITS_FLAGS_FORCE_NON_SHAREABLE)
5142 		tmp &= ~GITS_CBASER_SHAREABILITY_MASK;
5143 
5144 	if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) {
5145 		if (!(tmp & GITS_CBASER_SHAREABILITY_MASK)) {
5146 			/*
5147 			 * The HW reports non-shareable, we must
5148 			 * remove the cacheability attributes as
5149 			 * well.
5150 			 */
5151 			baser &= ~(GITS_CBASER_SHAREABILITY_MASK |
5152 				   GITS_CBASER_CACHEABILITY_MASK);
5153 			baser |= GITS_CBASER_nC;
5154 			gits_write_cbaser(baser, its->base + GITS_CBASER);
5155 		}
5156 		pr_info("ITS: using cache flushing for cmd queue\n");
5157 		its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING;
5158 	}
5159 
5160 	gits_write_cwriter(0, its->base + GITS_CWRITER);
5161 	ctlr = readl_relaxed(its->base + GITS_CTLR);
5162 	ctlr |= GITS_CTLR_ENABLE;
5163 	if (is_v4(its))
5164 		ctlr |= GITS_CTLR_ImDe;
5165 	writel_relaxed(ctlr, its->base + GITS_CTLR);
5166 
5167 	err = its_init_domain(its);
5168 	if (err)
5169 		goto out_free_tables;
5170 
5171 	raw_spin_lock(&its_lock);
5172 	list_add(&its->entry, &its_nodes);
5173 	raw_spin_unlock(&its_lock);
5174 
5175 	return 0;
5176 
5177 out_free_tables:
5178 	its_free_tables(its);
5179 out_free_cmd:
5180 	free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ));
5181 out_unmap_sgir:
5182 	if (its->sgir_base)
5183 		iounmap(its->sgir_base);
5184 out:
5185 	pr_err("ITS@%pa: failed probing (%d)\n", &its->phys_base, err);
5186 	return err;
5187 }
5188 
gic_rdists_supports_plpis(void)5189 static bool gic_rdists_supports_plpis(void)
5190 {
5191 	return !!(gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER) & GICR_TYPER_PLPIS);
5192 }
5193 
redist_disable_lpis(void)5194 static int redist_disable_lpis(void)
5195 {
5196 	void __iomem *rbase = gic_data_rdist_rd_base();
5197 	u64 timeout = USEC_PER_SEC;
5198 	u64 val;
5199 
5200 	if (!gic_rdists_supports_plpis()) {
5201 		pr_info("CPU%d: LPIs not supported\n", smp_processor_id());
5202 		return -ENXIO;
5203 	}
5204 
5205 	val = readl_relaxed(rbase + GICR_CTLR);
5206 	if (!(val & GICR_CTLR_ENABLE_LPIS))
5207 		return 0;
5208 
5209 	/*
5210 	 * If coming via a CPU hotplug event, we don't need to disable
5211 	 * LPIs before trying to re-enable them. They are already
5212 	 * configured and all is well in the world.
5213 	 *
5214 	 * If running with preallocated tables, there is nothing to do.
5215 	 */
5216 	if ((gic_data_rdist()->flags & RD_LOCAL_LPI_ENABLED) ||
5217 	    (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED))
5218 		return 0;
5219 
5220 	/*
5221 	 * From that point on, we only try to do some damage control.
5222 	 */
5223 	pr_warn("GICv3: CPU%d: Booted with LPIs enabled, memory probably corrupted\n",
5224 		smp_processor_id());
5225 	add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
5226 
5227 	/* Disable LPIs */
5228 	val &= ~GICR_CTLR_ENABLE_LPIS;
5229 	writel_relaxed(val, rbase + GICR_CTLR);
5230 
5231 	/* Make sure any change to GICR_CTLR is observable by the GIC */
5232 	dsb(sy);
5233 
5234 	/*
5235 	 * Software must observe RWP==0 after clearing GICR_CTLR.EnableLPIs
5236 	 * from 1 to 0 before programming GICR_PEND{PROP}BASER registers.
5237 	 * Error out if we time out waiting for RWP to clear.
5238 	 */
5239 	while (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_RWP) {
5240 		if (!timeout) {
5241 			pr_err("CPU%d: Timeout while disabling LPIs\n",
5242 			       smp_processor_id());
5243 			return -ETIMEDOUT;
5244 		}
5245 		udelay(1);
5246 		timeout--;
5247 	}
5248 
5249 	/*
5250 	 * After it has been written to 1, it is IMPLEMENTATION
5251 	 * DEFINED whether GICR_CTLR.EnableLPI becomes RES1 or can be
5252 	 * cleared to 0. Error out if clearing the bit failed.
5253 	 */
5254 	if (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_ENABLE_LPIS) {
5255 		pr_err("CPU%d: Failed to disable LPIs\n", smp_processor_id());
5256 		return -EBUSY;
5257 	}
5258 
5259 	return 0;
5260 }
5261 
its_cpu_init(void)5262 int its_cpu_init(void)
5263 {
5264 	if (!list_empty(&its_nodes)) {
5265 		int ret;
5266 
5267 		ret = redist_disable_lpis();
5268 		if (ret)
5269 			return ret;
5270 
5271 		its_cpu_init_lpis();
5272 		its_cpu_init_collections();
5273 	}
5274 
5275 	return 0;
5276 }
5277 
rdist_memreserve_cpuhp_cleanup_workfn(struct work_struct * work)5278 static void rdist_memreserve_cpuhp_cleanup_workfn(struct work_struct *work)
5279 {
5280 	cpuhp_remove_state_nocalls(gic_rdists->cpuhp_memreserve_state);
5281 	gic_rdists->cpuhp_memreserve_state = CPUHP_INVALID;
5282 }
5283 
5284 static DECLARE_WORK(rdist_memreserve_cpuhp_cleanup_work,
5285 		    rdist_memreserve_cpuhp_cleanup_workfn);
5286 
its_cpu_memreserve_lpi(unsigned int cpu)5287 static int its_cpu_memreserve_lpi(unsigned int cpu)
5288 {
5289 	struct page *pend_page;
5290 	int ret = 0;
5291 
5292 	/* This gets to run exactly once per CPU */
5293 	if (gic_data_rdist()->flags & RD_LOCAL_MEMRESERVE_DONE)
5294 		return 0;
5295 
5296 	pend_page = gic_data_rdist()->pend_page;
5297 	if (WARN_ON(!pend_page)) {
5298 		ret = -ENOMEM;
5299 		goto out;
5300 	}
5301 	/*
5302 	 * If the pending table was pre-programmed, free the memory we
5303 	 * preemptively allocated. Otherwise, reserve that memory for
5304 	 * later kexecs.
5305 	 */
5306 	if (gic_data_rdist()->flags & RD_LOCAL_PENDTABLE_PREALLOCATED) {
5307 		its_free_pending_table(pend_page);
5308 		gic_data_rdist()->pend_page = NULL;
5309 	} else {
5310 		phys_addr_t paddr = page_to_phys(pend_page);
5311 		WARN_ON(gic_reserve_range(paddr, LPI_PENDBASE_SZ));
5312 	}
5313 
5314 out:
5315 	/* Last CPU being brought up gets to issue the cleanup */
5316 	if (!IS_ENABLED(CONFIG_SMP) ||
5317 	    cpumask_equal(&cpus_booted_once_mask, cpu_possible_mask))
5318 		schedule_work(&rdist_memreserve_cpuhp_cleanup_work);
5319 
5320 	gic_data_rdist()->flags |= RD_LOCAL_MEMRESERVE_DONE;
5321 	return ret;
5322 }
5323 
5324 /* Mark all the BASER registers as invalid before they get reprogrammed */
its_reset_one(struct resource * res)5325 static int __init its_reset_one(struct resource *res)
5326 {
5327 	void __iomem *its_base;
5328 	int err, i;
5329 
5330 	its_base = its_map_one(res, &err);
5331 	if (!its_base)
5332 		return err;
5333 
5334 	for (i = 0; i < GITS_BASER_NR_REGS; i++)
5335 		gits_write_baser(0, its_base + GITS_BASER + (i << 3));
5336 
5337 	iounmap(its_base);
5338 	return 0;
5339 }
5340 
5341 static const struct of_device_id its_device_id[] = {
5342 	{	.compatible	= "arm,gic-v3-its",	},
5343 	{},
5344 };
5345 
its_node_init(struct resource * res,struct fwnode_handle * handle,int numa_node)5346 static struct its_node __init *its_node_init(struct resource *res,
5347 					     struct fwnode_handle *handle, int numa_node)
5348 {
5349 	void __iomem *its_base;
5350 	struct its_node *its;
5351 	int err;
5352 
5353 	its_base = its_map_one(res, &err);
5354 	if (!its_base)
5355 		return NULL;
5356 
5357 	pr_info("ITS %pR\n", res);
5358 
5359 	its = kzalloc(sizeof(*its), GFP_KERNEL);
5360 	if (!its)
5361 		goto out_unmap;
5362 
5363 	raw_spin_lock_init(&its->lock);
5364 	mutex_init(&its->dev_alloc_lock);
5365 	INIT_LIST_HEAD(&its->entry);
5366 	INIT_LIST_HEAD(&its->its_device_list);
5367 
5368 	its->typer = gic_read_typer(its_base + GITS_TYPER);
5369 	its->base = its_base;
5370 	its->phys_base = res->start;
5371 	its->get_msi_base = its_irq_get_msi_base;
5372 	its->msi_domain_flags = IRQ_DOMAIN_FLAG_ISOLATED_MSI;
5373 
5374 	its->numa_node = numa_node;
5375 	its->fwnode_handle = handle;
5376 
5377 	return its;
5378 
5379 out_unmap:
5380 	iounmap(its_base);
5381 	return NULL;
5382 }
5383 
its_node_destroy(struct its_node * its)5384 static void its_node_destroy(struct its_node *its)
5385 {
5386 	iounmap(its->base);
5387 	kfree(its);
5388 }
5389 
its_of_probe(struct device_node * node)5390 static int __init its_of_probe(struct device_node *node)
5391 {
5392 	struct device_node *np;
5393 	struct resource res;
5394 	int err;
5395 
5396 	/*
5397 	 * Make sure *all* the ITS are reset before we probe any, as
5398 	 * they may be sharing memory. If any of the ITS fails to
5399 	 * reset, don't even try to go any further, as this could
5400 	 * result in something even worse.
5401 	 */
5402 	for (np = of_find_matching_node(node, its_device_id); np;
5403 	     np = of_find_matching_node(np, its_device_id)) {
5404 		if (!of_device_is_available(np) ||
5405 		    !of_property_read_bool(np, "msi-controller") ||
5406 		    of_address_to_resource(np, 0, &res))
5407 			continue;
5408 
5409 		err = its_reset_one(&res);
5410 		if (err)
5411 			return err;
5412 	}
5413 
5414 	for (np = of_find_matching_node(node, its_device_id); np;
5415 	     np = of_find_matching_node(np, its_device_id)) {
5416 		struct its_node *its;
5417 
5418 		if (!of_device_is_available(np))
5419 			continue;
5420 		if (!of_property_read_bool(np, "msi-controller")) {
5421 			pr_warn("%pOF: no msi-controller property, ITS ignored\n",
5422 				np);
5423 			continue;
5424 		}
5425 
5426 		if (of_address_to_resource(np, 0, &res)) {
5427 			pr_warn("%pOF: no regs?\n", np);
5428 			continue;
5429 		}
5430 
5431 
5432 		its = its_node_init(&res, &np->fwnode, of_node_to_nid(np));
5433 		if (!its)
5434 			return -ENOMEM;
5435 
5436 		err = its_probe_one(its);
5437 		if (err)  {
5438 			its_node_destroy(its);
5439 			return err;
5440 		}
5441 	}
5442 	return 0;
5443 }
5444 
5445 #ifdef CONFIG_ACPI
5446 
5447 #define ACPI_GICV3_ITS_MEM_SIZE (SZ_128K)
5448 
5449 #ifdef CONFIG_ACPI_NUMA
5450 struct its_srat_map {
5451 	/* numa node id */
5452 	u32	numa_node;
5453 	/* GIC ITS ID */
5454 	u32	its_id;
5455 };
5456 
5457 static struct its_srat_map *its_srat_maps __initdata;
5458 static int its_in_srat __initdata;
5459 
acpi_get_its_numa_node(u32 its_id)5460 static int __init acpi_get_its_numa_node(u32 its_id)
5461 {
5462 	int i;
5463 
5464 	for (i = 0; i < its_in_srat; i++) {
5465 		if (its_id == its_srat_maps[i].its_id)
5466 			return its_srat_maps[i].numa_node;
5467 	}
5468 	return NUMA_NO_NODE;
5469 }
5470 
gic_acpi_match_srat_its(union acpi_subtable_headers * header,const unsigned long end)5471 static int __init gic_acpi_match_srat_its(union acpi_subtable_headers *header,
5472 					  const unsigned long end)
5473 {
5474 	return 0;
5475 }
5476 
gic_acpi_parse_srat_its(union acpi_subtable_headers * header,const unsigned long end)5477 static int __init gic_acpi_parse_srat_its(union acpi_subtable_headers *header,
5478 			 const unsigned long end)
5479 {
5480 	int node;
5481 	struct acpi_srat_gic_its_affinity *its_affinity;
5482 
5483 	its_affinity = (struct acpi_srat_gic_its_affinity *)header;
5484 	if (!its_affinity)
5485 		return -EINVAL;
5486 
5487 	if (its_affinity->header.length < sizeof(*its_affinity)) {
5488 		pr_err("SRAT: Invalid header length %d in ITS affinity\n",
5489 			its_affinity->header.length);
5490 		return -EINVAL;
5491 	}
5492 
5493 	/*
5494 	 * Note that in theory a new proximity node could be created by this
5495 	 * entry as it is an SRAT resource allocation structure.
5496 	 * We do not currently support doing so.
5497 	 */
5498 	node = pxm_to_node(its_affinity->proximity_domain);
5499 
5500 	if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) {
5501 		pr_err("SRAT: Invalid NUMA node %d in ITS affinity\n", node);
5502 		return 0;
5503 	}
5504 
5505 	its_srat_maps[its_in_srat].numa_node = node;
5506 	its_srat_maps[its_in_srat].its_id = its_affinity->its_id;
5507 	its_in_srat++;
5508 	pr_info("SRAT: PXM %d -> ITS %d -> Node %d\n",
5509 		its_affinity->proximity_domain, its_affinity->its_id, node);
5510 
5511 	return 0;
5512 }
5513 
acpi_table_parse_srat_its(void)5514 static void __init acpi_table_parse_srat_its(void)
5515 {
5516 	int count;
5517 
5518 	count = acpi_table_parse_entries(ACPI_SIG_SRAT,
5519 			sizeof(struct acpi_table_srat),
5520 			ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
5521 			gic_acpi_match_srat_its, 0);
5522 	if (count <= 0)
5523 		return;
5524 
5525 	its_srat_maps = kmalloc_array(count, sizeof(struct its_srat_map),
5526 				      GFP_KERNEL);
5527 	if (!its_srat_maps)
5528 		return;
5529 
5530 	acpi_table_parse_entries(ACPI_SIG_SRAT,
5531 			sizeof(struct acpi_table_srat),
5532 			ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
5533 			gic_acpi_parse_srat_its, 0);
5534 }
5535 
5536 /* free the its_srat_maps after ITS probing */
acpi_its_srat_maps_free(void)5537 static void __init acpi_its_srat_maps_free(void)
5538 {
5539 	kfree(its_srat_maps);
5540 }
5541 #else
acpi_table_parse_srat_its(void)5542 static void __init acpi_table_parse_srat_its(void)	{ }
acpi_get_its_numa_node(u32 its_id)5543 static int __init acpi_get_its_numa_node(u32 its_id) { return NUMA_NO_NODE; }
acpi_its_srat_maps_free(void)5544 static void __init acpi_its_srat_maps_free(void) { }
5545 #endif
5546 
gic_acpi_parse_madt_its(union acpi_subtable_headers * header,const unsigned long end)5547 static int __init gic_acpi_parse_madt_its(union acpi_subtable_headers *header,
5548 					  const unsigned long end)
5549 {
5550 	struct acpi_madt_generic_translator *its_entry;
5551 	struct fwnode_handle *dom_handle;
5552 	struct its_node *its;
5553 	struct resource res;
5554 	int err;
5555 
5556 	its_entry = (struct acpi_madt_generic_translator *)header;
5557 	memset(&res, 0, sizeof(res));
5558 	res.start = its_entry->base_address;
5559 	res.end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1;
5560 	res.flags = IORESOURCE_MEM;
5561 
5562 	dom_handle = irq_domain_alloc_fwnode(&res.start);
5563 	if (!dom_handle) {
5564 		pr_err("ITS@%pa: Unable to allocate GICv3 ITS domain token\n",
5565 		       &res.start);
5566 		return -ENOMEM;
5567 	}
5568 
5569 	err = iort_register_domain_token(its_entry->translation_id, res.start,
5570 					 dom_handle);
5571 	if (err) {
5572 		pr_err("ITS@%pa: Unable to register GICv3 ITS domain token (ITS ID %d) to IORT\n",
5573 		       &res.start, its_entry->translation_id);
5574 		goto dom_err;
5575 	}
5576 
5577 	its = its_node_init(&res, dom_handle,
5578 			    acpi_get_its_numa_node(its_entry->translation_id));
5579 	if (!its) {
5580 		err = -ENOMEM;
5581 		goto node_err;
5582 	}
5583 
5584 	err = its_probe_one(its);
5585 	if (!err)
5586 		return 0;
5587 
5588 node_err:
5589 	iort_deregister_domain_token(its_entry->translation_id);
5590 dom_err:
5591 	irq_domain_free_fwnode(dom_handle);
5592 	return err;
5593 }
5594 
its_acpi_reset(union acpi_subtable_headers * header,const unsigned long end)5595 static int __init its_acpi_reset(union acpi_subtable_headers *header,
5596 				 const unsigned long end)
5597 {
5598 	struct acpi_madt_generic_translator *its_entry;
5599 	struct resource res;
5600 
5601 	its_entry = (struct acpi_madt_generic_translator *)header;
5602 	res = (struct resource) {
5603 		.start	= its_entry->base_address,
5604 		.end	= its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1,
5605 		.flags	= IORESOURCE_MEM,
5606 	};
5607 
5608 	return its_reset_one(&res);
5609 }
5610 
its_acpi_probe(void)5611 static void __init its_acpi_probe(void)
5612 {
5613 	acpi_table_parse_srat_its();
5614 	/*
5615 	 * Make sure *all* the ITS are reset before we probe any, as
5616 	 * they may be sharing memory. If any of the ITS fails to
5617 	 * reset, don't even try to go any further, as this could
5618 	 * result in something even worse.
5619 	 */
5620 	if (acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR,
5621 				  its_acpi_reset, 0) > 0)
5622 		acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR,
5623 				      gic_acpi_parse_madt_its, 0);
5624 	acpi_its_srat_maps_free();
5625 }
5626 #else
its_acpi_probe(void)5627 static void __init its_acpi_probe(void) { }
5628 #endif
5629 
its_lpi_memreserve_init(void)5630 int __init its_lpi_memreserve_init(void)
5631 {
5632 	int state;
5633 
5634 	if (!efi_enabled(EFI_CONFIG_TABLES))
5635 		return 0;
5636 
5637 	if (list_empty(&its_nodes))
5638 		return 0;
5639 
5640 	gic_rdists->cpuhp_memreserve_state = CPUHP_INVALID;
5641 	state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
5642 				  "irqchip/arm/gicv3/memreserve:online",
5643 				  its_cpu_memreserve_lpi,
5644 				  NULL);
5645 	if (state < 0)
5646 		return state;
5647 
5648 	gic_rdists->cpuhp_memreserve_state = state;
5649 
5650 	return 0;
5651 }
5652 
its_init(struct fwnode_handle * handle,struct rdists * rdists,struct irq_domain * parent_domain)5653 int __init its_init(struct fwnode_handle *handle, struct rdists *rdists,
5654 		    struct irq_domain *parent_domain)
5655 {
5656 	struct device_node *of_node;
5657 	struct its_node *its;
5658 	bool has_v4 = false;
5659 	bool has_v4_1 = false;
5660 	int err;
5661 
5662 	gic_rdists = rdists;
5663 
5664 	its_parent = parent_domain;
5665 	of_node = to_of_node(handle);
5666 	if (of_node)
5667 		its_of_probe(of_node);
5668 	else
5669 		its_acpi_probe();
5670 
5671 	if (list_empty(&its_nodes)) {
5672 		pr_warn("ITS: No ITS available, not enabling LPIs\n");
5673 		return -ENXIO;
5674 	}
5675 
5676 	err = allocate_lpi_tables();
5677 	if (err)
5678 		return err;
5679 
5680 	list_for_each_entry(its, &its_nodes, entry) {
5681 		has_v4 |= is_v4(its);
5682 		has_v4_1 |= is_v4_1(its);
5683 	}
5684 
5685 	/* Don't bother with inconsistent systems */
5686 	if (WARN_ON(!has_v4_1 && rdists->has_rvpeid))
5687 		rdists->has_rvpeid = false;
5688 
5689 	if (has_v4 & rdists->has_vlpis) {
5690 		const struct irq_domain_ops *sgi_ops;
5691 
5692 		if (has_v4_1)
5693 			sgi_ops = &its_sgi_domain_ops;
5694 		else
5695 			sgi_ops = NULL;
5696 
5697 		if (its_init_vpe_domain() ||
5698 		    its_init_v4(parent_domain, &its_vpe_domain_ops, sgi_ops)) {
5699 			rdists->has_vlpis = false;
5700 			pr_err("ITS: Disabling GICv4 support\n");
5701 		}
5702 	}
5703 
5704 	register_syscore_ops(&its_syscore_ops);
5705 
5706 	return 0;
5707 }
5708